From 9a4a1092fee2bd10dfb7e3a4d0629dd6309533b8 Mon Sep 17 00:00:00 2001 From: Samuel Ortiz Date: Tue, 10 Jan 2017 20:19:01 +0100 Subject: [PATCH 1/5] conmon: Return the exit status code waitpid fills its second argument with a value that contains the process exit code in the 8 least significant bits. Instead of returning the complete value and then convert it from ocid, return the exit status directly by using WEXITSTATUS from conmon. Signed-off-by: Samuel Ortiz --- conmon/conmon.c | 6 ++++-- oci/oci.go | 2 +- utils/utils.go | 5 ----- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/conmon/conmon.c b/conmon/conmon.c index e33dd7bfbfb..3ecc8f0d28f 100644 --- a/conmon/conmon.c +++ b/conmon/conmon.c @@ -303,10 +303,12 @@ int main(int argc, char *argv[]) /* Wait for the container process and record its exit code */ while ((pid = waitpid(-1, &status, 0)) > 0) { - printf("PID %d exited\n", pid); + int exit_status = WEXITSTATUS(status); + + printf("PID %d exited with status %d\n", pid, exit_status); if (pid == cpid) { _cleanup_free_ char *status_str = NULL; - ret = asprintf(&status_str, "%d", status); + ret = asprintf(&status_str, "%d", exit_status); if (ret < 0) { pexit("Failed to allocate memory for status"); } diff --git a/oci/oci.go b/oci/oci.go index eb0edbbbf58..b06ab2d35b7 100644 --- a/oci/oci.go +++ b/oci/oci.go @@ -335,7 +335,7 @@ func (r *Runtime) UpdateStatus(c *Container) error { if err != nil { return fmt.Errorf("status code conversion failed: %v", err) } - c.state.ExitCode = int32(utils.StatusToExitCode(statusCode)) + c.state.ExitCode = int32(statusCode) } return nil diff --git a/utils/utils.go b/utils/utils.go index fd3544fd965..5323d290a73 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -125,8 +125,3 @@ func dockerRemove(container string) error { _, err := ExecCmd("docker", "rm", container) return err } - -// StatusToExitCode converts wait status code to an exit code -func StatusToExitCode(status int) int { - return ((status) & 0xff00) >> 8 -} From 468746aa2843ca1ace15207e8d7564110591b656 Mon Sep 17 00:00:00 2001 From: Samuel Ortiz Date: Tue, 10 Jan 2017 22:48:47 +0100 Subject: [PATCH 2/5] conmon: Use the full PID file path And not a hardcoded "pidfile". Signed-off-by: Samuel Ortiz --- conmon/conmon.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conmon/conmon.c b/conmon/conmon.c index 3ecc8f0d28f..b24dd064705 100644 --- a/conmon/conmon.c +++ b/conmon/conmon.c @@ -200,7 +200,7 @@ int main(int argc, char *argv[]) } /* Read the pid so we can wait for the process to exit */ - g_file_get_contents("pidfile", &contents, NULL, &err); + g_file_get_contents(pid_file, &contents, NULL, &err); if (err) { fprintf(stderr, "Failed to read pidfile: %s\n", err->message); g_error_free(err); From d60d0ac0c3ac241c18eca284ee735fe7fa3f1ec6 Mon Sep 17 00:00:00 2001 From: Samuel Ortiz Date: Tue, 10 Jan 2017 23:08:54 +0100 Subject: [PATCH 3/5] conmon: Use conmon for exec'ing a command Some OCI container runtimes (in particular the hypervisor based ones) will typically create a shim process between the hypervisor and the runtime caller, in order to not rely on the hypervisor process for e.g. forwarding the output streams or getting a command exit code. With these runtimes we need to monitor a different process than the runtime one when executing a command inside a running container. The natural place to do so is conmon and thus we add a new option to conmon for calling the runtime exec command, monitor the PID and then return the running command exit code through the sync pipe to the parent. Signed-off-by: Samuel Ortiz --- conmon/conmon.c | 103 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 74 insertions(+), 29 deletions(-) diff --git a/conmon/conmon.c b/conmon/conmon.c index b24dd064705..279d7a154ec 100644 --- a/conmon/conmon.c +++ b/conmon/conmon.c @@ -76,6 +76,7 @@ static char *runtime_path = NULL; static char *bundle_path = NULL; static char *pid_file = NULL; static bool systemd_cgroup = false; +static bool exec = false; static GOptionEntry entries[] = { { "terminal", 't', 0, G_OPTION_ARG_NONE, &terminal, "Terminal", NULL }, @@ -84,12 +85,13 @@ static GOptionEntry entries[] = { "bundle", 'b', 0, G_OPTION_ARG_STRING, &bundle_path, "Bundle path", NULL }, { "pidfile", 'p', 0, G_OPTION_ARG_STRING, &pid_file, "PID file", NULL }, { "systemd-cgroup", 's', 0, G_OPTION_ARG_NONE, &systemd_cgroup, "Enable systemd cgroup manager", NULL }, + { "exec", 'e', 0, G_OPTION_ARG_NONE, &exec, "Exec a command in a running container", NULL }, { NULL } }; int main(int argc, char *argv[]) { - int ret; + int ret, runtime_status; char cwd[PATH_MAX]; char default_pid_file[PATH_MAX]; GError *err = NULL; @@ -105,7 +107,7 @@ int main(int argc, char *argv[]) struct termios t; struct epoll_event ev; struct epoll_event evlist[MAX_EVENTS]; - int child_pipe = -1; + int sync_pipe_fd = -1; char *sync_pipe, *endptr; int len; GError *error = NULL; @@ -126,7 +128,7 @@ int main(int argc, char *argv[]) if (runtime_path == NULL) nexit("Runtime path not provided. Use --runtime"); - if (bundle_path == NULL) { + if (bundle_path == NULL && !exec) { if (getcwd(cwd, sizeof(cwd)) == NULL) { nexit("Failed to get working directory"); } @@ -147,7 +149,7 @@ int main(int argc, char *argv[]) sync_pipe = getenv("_OCI_SYNCPIPE"); if (sync_pipe) { errno = 0; - child_pipe = strtol(sync_pipe, &endptr, 10); + sync_pipe_fd = strtol(sync_pipe, &endptr, 10); if (errno != 0 || *endptr != '\0') pexit("unable to parse _OCI_SYNCPIPE"); } @@ -184,18 +186,36 @@ int main(int argc, char *argv[]) } - /* Create the container */ cmd = g_string_new(runtime_path); - if (systemd_cgroup) { - g_string_append_printf(cmd, " --systemd-cgroup"); - } - g_string_append_printf(cmd, " create %s --bundle %s --pid-file %s", - cid, bundle_path, pid_file); - if (terminal) { - g_string_append_printf(cmd, " --console %s", slname); + if (!exec) { + /* Create the container */ + if (systemd_cgroup) { + g_string_append_printf(cmd, " --systemd-cgroup"); + } + g_string_append_printf(cmd, " create %s --bundle %s --pid-file %s", + cid, bundle_path, pid_file); + if (terminal) { + g_string_append_printf(cmd, " --console %s", slname); + } + } else { + int i; + + /* Exec the command */ + if (terminal) { + g_string_append_printf(cmd, " exec -d --pid-file %s --console %s %s", + pid_file, slname, cid); + } else { + g_string_append_printf(cmd, " exec -d --pid-file %s %s", + pid_file, cid); + } + + for (i = 1; i < argc; i++) { + g_string_append_printf(cmd, " %s", argv[i]); + } } - ret = system(cmd->str); - if (ret != 0) { + + runtime_status = system(cmd->str); + if (runtime_status != 0) { nexit("Failed to create container"); } @@ -211,9 +231,9 @@ int main(int argc, char *argv[]) printf("container PID: %d\n", cpid); /* Send the container pid back to parent */ - if (child_pipe > 0) { + if (sync_pipe_fd > 0 && !exec) { len = snprintf(buf, BUF_SIZE, "{\"pid\": %d}\n", cpid); - if (len < 0 || write(child_pipe, buf, len) != len) { + if (len < 0 || write(sync_pipe_fd, buf, len) != len) { pexit("unable to send container pid to parent"); } } @@ -307,23 +327,48 @@ int main(int argc, char *argv[]) printf("PID %d exited with status %d\n", pid, exit_status); if (pid == cpid) { - _cleanup_free_ char *status_str = NULL; - ret = asprintf(&status_str, "%d", exit_status); - if (ret < 0) { - pexit("Failed to allocate memory for status"); - } - g_file_set_contents("exit", status_str, - strlen(status_str), &err); - if (err) { - fprintf(stderr, - "Failed to write %s to exit file: %s\n", - status_str, err->message); - g_error_free(err); - exit(1); + if (!exec) { + _cleanup_free_ char *status_str = NULL; + ret = asprintf(&status_str, "%d", exit_status); + if (ret < 0) { + pexit("Failed to allocate memory for status"); + } + g_file_set_contents("exit", status_str, + strlen(status_str), &err); + if (err) { + fprintf(stderr, + "Failed to write %s to exit file: %s\n", + status_str, err->message); + g_error_free(err); + exit(1); + } + } else { + /* Send the command exec exit code back to the parent */ + if (sync_pipe_fd > 0) { + len = snprintf(buf, BUF_SIZE, "{\"exit_code\": %d}\n", exit_status); + if (len < 0 || write(sync_pipe_fd, buf, len) != len) { + pexit("unable to send exit status"); + exit(1); + } + } } break; } } + if (exec && pid < 0 && errno == ECHILD && sync_pipe_fd > 0) { + /* + * waitpid failed and set errno to ECHILD: + * The runtime exec call did not create any child + * process and we can send the system() exit code + * to the parent. + */ + len = snprintf(buf, BUF_SIZE, "{\"exit_code\": %d}\n", WEXITSTATUS(runtime_status)); + if (len < 0 || write(sync_pipe_fd, buf, len) != len) { + pexit("unable to send exit status"); + exit(1); + } + } + return EXIT_SUCCESS; } From 4c7583b4678cc7efe267fbae9cc07bcb22ef0581 Mon Sep 17 00:00:00 2001 From: Samuel Ortiz Date: Tue, 10 Jan 2017 23:18:14 +0100 Subject: [PATCH 4/5] oci: Do not call the container runtime from ExecSync Some OCI container runtimes (in particular the hypervisor based ones) will typically create a shim process between the hypervisor and the runtime caller, in order to not rely on the hypervisor process for e.g. forwarding the output streams or getting a command exit code. When executing a command inside a running container those runtimes will create that shim process and terminate. Therefore calling and monitoring them directly from ExecSync() will fail. Instead we need to have a subreaper calling the runtime and monitoring the shim process. This change uses conmon as the subreaper from ExecSync(), monitors the shim process and read the exec'ed command exit code from the synchronization pipe. Signed-off-by: Samuel Ortiz --- oci/oci.go | 77 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 75 insertions(+), 2 deletions(-) diff --git a/oci/oci.go b/oci/oci.go index b06ab2d35b7..c155ed79cc7 100644 --- a/oci/oci.go +++ b/oci/oci.go @@ -61,6 +61,11 @@ type syncInfo struct { Pid int `json:"pid"` } +// exitCodeInfo is used to return the monitored process exit code to the daemon +type exitCodeInfo struct { + ExitCode int32 `json:"exit_code"` +} + // Name returns the name of the OCI Runtime func (r *Runtime) Name() string { return r.name @@ -177,16 +182,61 @@ func (e ExecSyncError) Error() string { return fmt.Sprintf("command error: %+v, stdout: %s, stderr: %s, exit code %d", e.Err, e.Stdout.Bytes(), e.Stderr.Bytes(), e.ExitCode) } +func prepareExec() (pidFile, parentPipe, childPipe *os.File, err error) { + parentPipe, childPipe, err = os.Pipe() + if err != nil { + return nil, nil, nil, err + } + + pidFile, err = ioutil.TempFile("", "pidfile") + if err != nil { + parentPipe.Close() + childPipe.Close() + return nil, nil, nil, err + } + + return +} + // ExecSync execs a command in a container and returns it's stdout, stderr and return code. func (r *Runtime) ExecSync(c *Container, command []string, timeout int64) (resp *ExecSyncResponse, err error) { - args := []string{"exec", c.name} + pidFile, parentPipe, childPipe, err := prepareExec() + if err != nil { + return nil, ExecSyncError{ + ExitCode: -1, + Err: err, + } + } + defer parentPipe.Close() + defer func() { + if e := os.Remove(pidFile.Name()); e != nil { + logrus.Warnf("could not remove temporary PID file %s", pidFile.Name()) + } + }() + + var args []string + args = append(args, "-c", c.name) + args = append(args, "-r", r.path) + args = append(args, "-p", pidFile.Name()) + args = append(args, "-e") + if c.terminal { + args = append(args, "-t") + } + args = append(args, command...) - cmd := exec.Command(r.Path(), args...) + + cmd := exec.Command(r.conmonPath, args...) + var stdoutBuf, stderrBuf bytes.Buffer cmd.Stdout = &stdoutBuf cmd.Stderr = &stderrBuf + cmd.ExtraFiles = append(cmd.ExtraFiles, childPipe) + // 0, 1 and 2 are stdin, stdout and stderr + cmd.Env = append(r.conmonEnv, fmt.Sprintf("_OCI_SYNCPIPE=%d", 3)) + err = cmd.Start() if err != nil { + childPipe.Close() return nil, ExecSyncError{ Stdout: stdoutBuf, Stderr: stderrBuf, @@ -195,6 +245,9 @@ func (r *Runtime) ExecSync(c *Container, command []string, timeout int64) (resp } } + // We don't need childPipe on the parent side + childPipe.Close() + if timeout > 0 { done := make(chan error, 1) go func() { @@ -260,7 +313,27 @@ func (r *Runtime) ExecSync(c *Container, command []string, timeout int64) (resp Err: err, } } + } + } + var ec *exitCodeInfo + if err := json.NewDecoder(parentPipe).Decode(&ec); err != nil { + return nil, ExecSyncError{ + Stdout: stdoutBuf, + Stderr: stderrBuf, + ExitCode: -1, + Err: err, + } + } + + logrus.Infof("Received container exit code: %v", ec.ExitCode) + + if ec.ExitCode != 0 { + return nil, ExecSyncError{ + Stdout: stdoutBuf, + Stderr: stderrBuf, + ExitCode: ec.ExitCode, + Err: fmt.Errorf("container workload exited with error %v", ec.ExitCode), } } From ce54c1e5e9b38e7f8458ab7d41591e02afd1052e Mon Sep 17 00:00:00 2001 From: Samuel Ortiz Date: Tue, 10 Jan 2017 23:34:03 +0100 Subject: [PATCH 5/5] test: Do not hardcode runc specific output "executable file not found in" is part of a runc specific output when 'runc exec' fails. This prevents the execsync failure to pass when running ocid with other runtimes than runc. Signed-off-by: Samuel Ortiz --- test/ctr.bats | 1 - 1 file changed, 1 deletion(-) diff --git a/test/ctr.bats b/test/ctr.bats index 1d351c8786c..257c43c2cc9 100644 --- a/test/ctr.bats +++ b/test/ctr.bats @@ -387,7 +387,6 @@ function teardown() { run ocic ctr execsync --id "$ctr_id" doesnotexist echo "$output" [ "$status" -ne 0 ] - [[ "$output" =~ "executable file not found in" ]] cleanup_ctrs cleanup_pods