diff --git a/internal/oci/oci.go b/internal/oci/oci.go index b7731443238..087a52b65e1 100644 --- a/internal/oci/oci.go +++ b/internal/oci/oci.go @@ -1,7 +1,6 @@ package oci import ( - "bytes" "fmt" "io" "path/filepath" @@ -11,6 +10,7 @@ import ( "time" "github.com/cri-o/cri-o/pkg/config" + "github.com/cri-o/cri-o/server/cri/types" rspec "github.com/opencontainers/runtime-spec/specs-go" "golang.org/x/net/context" @@ -54,7 +54,7 @@ type RuntimeImpl interface { StartContainer(context.Context, *Container) error ExecContainer(context.Context, *Container, []string, io.Reader, io.WriteCloser, io.WriteCloser, bool, <-chan remotecommand.TerminalSize) error - ExecSyncContainer(context.Context, *Container, []string, int64) (*ExecSyncResponse, error) + ExecSyncContainer(context.Context, *Container, []string, int64) (*types.ExecSyncResponse, error) UpdateContainer(context.Context, *Container, *rspec.LinuxResources) error StopContainer(context.Context, *Container, int64) error DeleteContainer(context.Context, *Container) error @@ -287,7 +287,7 @@ func (r *Runtime) ExecContainer(ctx context.Context, c *Container, cmd []string, } // ExecSyncContainer execs a command in a container and returns it's stdout, stderr and return code. -func (r *Runtime) ExecSyncContainer(ctx context.Context, c *Container, command []string, timeout int64) (*ExecSyncResponse, error) { +func (r *Runtime) ExecSyncContainer(ctx context.Context, c *Container, command []string, timeout int64) (*types.ExecSyncResponse, error) { impl, err := r.RuntimeImpl(c) if err != nil { return nil, err @@ -418,25 +418,6 @@ func (r *Runtime) ReopenContainerLog(ctx context.Context, c *Container) error { return impl.ReopenContainerLog(ctx, c) } -// ExecSyncResponse is returned from ExecSync. -type ExecSyncResponse struct { - Stdout []byte - Stderr []byte - ExitCode int32 -} - -// ExecSyncError wraps command's streams, exit code and error on ExecSync error. -type ExecSyncError struct { - Stdout bytes.Buffer - Stderr bytes.Buffer - ExitCode int32 - Err error -} - -func (e *ExecSyncError) Error() string { - return fmt.Sprintf("command error: %+v, stdout: %s, stderr: %s, exit code %d", e.Err, e.Stdout.Bytes(), e.Stderr.Bytes(), e.ExitCode) -} - // BuildContainerdBinaryName() is responsible for ensuring the binary passed will // be properly converted to the containerd binary naming pattern. // diff --git a/internal/oci/oci_test.go b/internal/oci/oci_test.go index 031e9ae1c0f..d59c53679bf 100644 --- a/internal/oci/oci_test.go +++ b/internal/oci/oci_test.go @@ -175,19 +175,6 @@ var _ = t.Describe("Oci", func() { }) }) - t.Describe("ExecSyncError", func() { - It("should succeed to get the exec sync error", func() { - // Given - sut := oci.ExecSyncError{} - - // When - result := sut.Error() - - // Then - Expect(result).To(ContainSubstring("error")) - }) - }) - t.Describe("BuildContainerdBinaryName", func() { It("Simple binary name (containerd-shim-kata-v2)", func() { binaryName := oci.BuildContainerdBinaryName("containerd-shim-kata-v2") diff --git a/internal/oci/runtime_oci.go b/internal/oci/runtime_oci.go index 4d3e8eed775..c1f722ac31d 100644 --- a/internal/oci/runtime_oci.go +++ b/internal/oci/runtime_oci.go @@ -18,6 +18,7 @@ import ( "github.com/containers/storage/pkg/pools" "github.com/cri-o/cri-o/internal/log" "github.com/cri-o/cri-o/pkg/config" + "github.com/cri-o/cri-o/server/cri/types" "github.com/cri-o/cri-o/server/metrics" "github.com/cri-o/cri-o/utils" "github.com/fsnotify/fsnotify" @@ -70,12 +71,6 @@ type syncInfo struct { Message string `json:"message,omitempty"` } -// exitCodeInfo is used to return the monitored process exit code to the daemon -type exitCodeInfo struct { - ExitCode int32 `json:"exit_code"` - Message string `json:"message,omitempty"` -} - // CreateContainer creates a container. func (r *runtimeOCI) CreateContainer(ctx context.Context, c *Container, cgroupParent string) (retErr error) { if c.Spoofed() { @@ -259,69 +254,6 @@ func (r *runtimeOCI) StartContainer(ctx context.Context, c *Container) error { return nil } -func prepareExec() (pidFileName string, parentPipe, childPipe *os.File, _ error) { - var err error - parentPipe, childPipe, err = os.Pipe() - if err != nil { - return "", nil, nil, err - } - - pidFile, err := ioutil.TempFile("", "pidfile") - if err != nil { - parentPipe.Close() - childPipe.Close() - return "", nil, nil, err - } - pidFile.Close() - pidFileName = pidFile.Name() - - return pidFileName, parentPipe, childPipe, nil -} - -func parseLog(ctx context.Context, l []byte) (stdout, stderr []byte) { - // Split the log on newlines, which is what separates entries. - lines := bytes.SplitAfter(l, []byte{'\n'}) - for _, line := range lines { - // Ignore empty lines. - if len(line) == 0 { - continue - } - - // The format of log lines is "DATE pipe LogTag REST". - parts := bytes.SplitN(line, []byte{' '}, 4) - if len(parts) < 4 { - // Ignore the line if it's formatted incorrectly, but complain - // about it so it can be debugged. - log.Warnf(ctx, "hit invalid log format: %q", string(line)) - continue - } - - pipe := string(parts[1]) - content := parts[3] - - linetype := string(parts[2]) - if linetype == "P" { - contentLen := len(content) - if contentLen > 0 && content[contentLen-1] == '\n' { - content = content[:contentLen-1] - } - } - - switch pipe { - case "stdout": - stdout = append(stdout, content...) - case "stderr": - stderr = append(stderr, content...) - default: - // Complain about unknown pipes. - log.Warnf(ctx, "hit invalid log format [unknown pipe %s]: %q", pipe, string(line)) - continue - } - } - - return stdout, stderr -} - // ExecContainer prepares a streaming endpoint to execute a command in the container. func (r *runtimeOCI) ExecContainer(ctx context.Context, c *Container, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize) error { if c.Spoofed() { @@ -334,12 +266,7 @@ func (r *runtimeOCI) ExecContainer(ctx context.Context, c *Container, cmd []stri } defer os.RemoveAll(processFile) - args := []string{rootFlag, r.root, "exec"} - args = append(args, "--process", processFile, c.ID()) - execCmd := exec.Command(r.path, args...) // nolint: gosec - if v, found := os.LookupEnv("XDG_RUNTIME_DIR"); found { - execCmd.Env = append(execCmd.Env, fmt.Sprintf("XDG_RUNTIME_DIR=%s", v)) - } + execCmd := r.constructExecCommand(ctx, c, processFile, "") var cmdErr, copyError error if tty { cmdErr = ttyCmd(execCmd, stdin, stdout, resize) @@ -393,176 +320,137 @@ func (r *runtimeOCI) ExecContainer(ctx context.Context, c *Container, cmd []stri } // ExecSyncContainer execs a command in a container and returns it's stdout, stderr and return code. -func (r *runtimeOCI) ExecSyncContainer(ctx context.Context, c *Container, command []string, timeout int64) (*ExecSyncResponse, error) { +func (r *runtimeOCI) ExecSyncContainer(ctx context.Context, c *Container, command []string, timeout int64) (*types.ExecSyncResponse, error) { if c.Spoofed() { return nil, nil } - pidFile, parentPipe, childPipe, err := prepareExec() - if err != nil { - return nil, &ExecSyncError{ - ExitCode: -1, - Err: err, - } - } - defer parentPipe.Close() - defer func() { - if e := os.Remove(pidFile); e != nil { - log.Warnf(ctx, "could not remove temporary PID file %s", pidFile) - } - }() - - logFile, err := ioutil.TempFile("", "crio-log-"+c.id) - if err != nil { - return nil, &ExecSyncError{ - ExitCode: -1, - Err: err, - } - } - logFile.Close() - - logPath := logFile.Name() - defer func() { - os.RemoveAll(logPath) - }() - - args := []string{ - "-c", c.id, - "-n", c.name, - "-r", r.path, - "-p", pidFile, - "-e", - "-l", logPath, - "--socket-dir-path", r.config.ContainerAttachSocketDir, - "--log-level", logrus.GetLevel().String(), - } - - if r.config.ConmonSupportsSync() { - args = append(args, "--sync") - } - if c.terminal { - args = append(args, "-t") - } - if timeout > 0 { - args = append(args, "-T", fmt.Sprintf("%d", timeout)) - } - processFile, err := prepareProcessExec(c, command, c.terminal) if err != nil { - return nil, &ExecSyncError{ - ExitCode: -1, - Err: err, - } + return nil, err } defer os.RemoveAll(processFile) - args = append(args, - "--exec-process-spec", processFile, - "--runtime-arg", fmt.Sprintf("%s=%s", rootFlag, r.root)) + pidFile, err := createPidFile() + if err != nil { + return nil, err + } + defer os.RemoveAll(pidFile) - cmd := exec.Command(r.config.Conmon, args...) // nolint: gosec + cmd := r.constructExecCommand(ctx, c, processFile, pidFile) + cmd.SysProcAttr = sysProcAttrPlatform() var stdoutBuf, stderrBuf bytes.Buffer cmd.Stdout = &stdoutBuf cmd.Stderr = &stderrBuf - cmd.ExtraFiles = append(cmd.ExtraFiles, childPipe) - // 0, 1 and 2 are stdin, stdout and stderr - cmd.Env = r.config.ConmonEnv - cmd.Env = append(cmd.Env, fmt.Sprintf("_OCI_SYNCPIPE=%d", 3)) - if v, found := os.LookupEnv("XDG_RUNTIME_DIR"); found { - cmd.Env = append(cmd.Env, fmt.Sprintf("XDG_RUNTIME_DIR=%s", v)) - } err = cmd.Start() if err != nil { - childPipe.Close() - return nil, &ExecSyncError{ - Stdout: stdoutBuf, - Stderr: stderrBuf, - ExitCode: -1, - Err: err, - } + return nil, err } - // We don't need childPipe on the parent side - childPipe.Close() + // wait till the command is done + done := make(chan error, 1) + go func() { + done <- cmd.Wait() + }() + + if timeout > 0 { + select { + case <-time.After(time.Second * time.Duration(timeout)): + // Ensure the process is not left behind + killContainerExecProcess(ctx, pidFile) - // first, wait till the command is done - waitErr := cmd.Wait() - - // regardless of what is in waitErr - // we should attempt to decode the output of the parent pipe - // this allows us to catch TimedOutMessage, which will cause waitErr to not be nil - var ec *exitCodeInfo - decodeErr := json.NewDecoder(parentPipe).Decode(&ec) - if decodeErr == nil { - log.Debugf(ctx, "Received container exit code: %v, message: %s", ec.ExitCode, ec.Message) - - // When we timeout the command in conmon then we should return - // an ExecSyncResponse with a non-zero exit code because - // the prober code in the kubelet checks for it. If we return - // a custom error, then the probes transition into Unknown status - // and the container isn't restarted as expected. - if ec.ExitCode == -1 && ec.Message == conmonconfig.TimedOutMessage { - return &ExecSyncResponse{ + // Make sure the runtime process has been cleaned up + <-done + + // If the command timed out, we should return an ExecSyncResponse with a non-zero exit code because + // the prober code in the kubelet checks for it. If we return a custom error, + // then the probes transition into Unknown status and the container isn't restarted as expected. + return &types.ExecSyncResponse{ Stderr: []byte(conmonconfig.TimedOutMessage), ExitCode: -1, }, nil + case err = <-done: + break } + } else { + err = <-done } - if waitErr != nil { - // if we aren't a ExitError, some I/O problems probably occurred - if _, ok := waitErr.(*exec.ExitError); !ok { - return nil, &ExecSyncError{ - Stdout: stdoutBuf, - Stderr: stderrBuf, - ExitCode: -1, - Err: waitErr, - } + // gather exit code from err + exitCode := int32(0) + if err != nil { + if exitError, ok := err.(*exec.ExitError); ok { + exitCode = int32(exitError.ExitCode()) } } - if decodeErr != nil { - return nil, &ExecSyncError{ - Stdout: stdoutBuf, - Stderr: stderrBuf, - ExitCode: -1, - Err: decodeErr, - } + return &types.ExecSyncResponse{ + Stdout: stdoutBuf.Bytes(), + Stderr: stderrBuf.Bytes(), + ExitCode: exitCode, + }, nil +} + +func (r *runtimeOCI) constructExecCommand(ctx context.Context, c *Container, processFile, pidFile string) *exec.Cmd { + args := []string{rootFlag, r.root, "exec"} + if pidFile != "" { + args = append(args, "--pid-file", pidFile) } + args = append(args, "--process", processFile, c.ID()) + execCmd := exec.CommandContext(ctx, r.path, args...) // nolint: gosec + if v, found := os.LookupEnv("XDG_RUNTIME_DIR"); found { + execCmd.Env = append(execCmd.Env, fmt.Sprintf("XDG_RUNTIME_DIR=%s", v)) + } + return execCmd +} - if ec.ExitCode == -1 { - return nil, &ExecSyncError{ - Stdout: stdoutBuf, - Stderr: stderrBuf, - ExitCode: -1, - Err: fmt.Errorf(ec.Message), - } +func createPidFile() (string, error) { + pidFile, err := ioutil.TempFile("", "pidfile") + if err != nil { + return "", err } + pidFile.Close() + pidFileName := pidFile.Name() - // The actual logged output is not the same as stdoutBuf and stderrBuf, - // which are used for getting error information. For the actual - // ExecSyncResponse we have to read the logfile. - // XXX: Currently runC dups the same console over both stdout and stderr, - // so we can't differentiate between the two. - logBytes, err := ioutil.ReadFile(logPath) + return pidFileName, nil +} + +func killContainerExecProcess(ctx context.Context, pidFile string) { + // Attempt to get the container PID and PGID from the file the runtime should have written. + // TODO(haircommander): There does exist a race that we could time out before the runtime actually creates the file. + // Should we do inotify on this file to ensure it exists? Is that overkill? + ctrPid, ctrPgid, err := pidAndpgidFromFile(pidFile) if err != nil { - return nil, &ExecSyncError{ - Stdout: stdoutBuf, - Stderr: stderrBuf, - ExitCode: -1, - Err: err, + log.Errorf(ctx, "Failed to get pid (%d) or pgid (%d) from file %s: %v", ctrPid, ctrPgid, pidFile, err) + } + + if ctrPgid > 1 { + // First attempt to kill the container group + if err := syscall.Kill(-ctrPgid, syscall.SIGKILL); err != nil { + log.Errorf(ctx, "Failed to kill process group after timeout: %v", err) + } + } else if ctrPid > 0 { + // If that fails, kill the container PID itself + if err := syscall.Kill(ctrPid, syscall.SIGKILL); err != nil { + log.Errorf(ctx, "Failed to kill process after timeout: %v", err) } } +} - // We have to parse the log output into {stdout, stderr} buffers. - stdoutBytes, stderrBytes := parseLog(ctx, logBytes) - return &ExecSyncResponse{ - Stdout: stdoutBytes, - Stderr: stderrBytes, - ExitCode: ec.ExitCode, - }, nil +func pidAndpgidFromFile(pidFile string) (pid, pgid int, _ error) { + // find the pid of the parent process + pidStr, err := ioutil.ReadFile(pidFile) + if err != nil { + return -1, -1, err + } + pid, err = strconv.Atoi(string(pidStr)) + if err != nil { + return -1, -1, err + } + pgid, err = syscall.Getpgid(pid) + return pid, pgid, err } // UpdateContainer updates container resources diff --git a/internal/oci/runtime_vm.go b/internal/oci/runtime_vm.go index 46c3ba9cfc4..6f10cfce133 100644 --- a/internal/oci/runtime_vm.go +++ b/internal/oci/runtime_vm.go @@ -19,6 +19,7 @@ import ( "github.com/containerd/ttrpc" "github.com/containerd/typeurl" "github.com/cri-o/cri-o/internal/log" + "github.com/cri-o/cri-o/server/cri/types" "github.com/cri-o/cri-o/server/metrics" "github.com/cri-o/cri-o/utils" "github.com/cri-o/cri-o/utils/errdefs" @@ -303,7 +304,7 @@ func (r *runtimeVM) ExecContainer(ctx context.Context, c *Container, cmd []strin } // ExecSyncContainer execs a command in a container and returns it's stdout, stderr and return code. -func (r *runtimeVM) ExecSyncContainer(ctx context.Context, c *Container, command []string, timeout int64) (*ExecSyncResponse, error) { +func (r *runtimeVM) ExecSyncContainer(ctx context.Context, c *Container, command []string, timeout int64) (*types.ExecSyncResponse, error) { log.Debugf(ctx, "runtimeVM.ExecSyncContainer() start") defer log.Debugf(ctx, "runtimeVM.ExecSyncContainer() end") @@ -313,13 +314,10 @@ func (r *runtimeVM) ExecSyncContainer(ctx context.Context, c *Container, command exitCode, err := r.execContainerCommon(ctx, c, command, timeout, nil, stdout, stderr, c.terminal, nil) if err != nil { - return nil, &ExecSyncError{ - ExitCode: -1, - Err: errors.Wrap(err, "ExecSyncContainer failed"), - } + return nil, errors.Wrap(err, "ExecSyncContainer failed") } - return &ExecSyncResponse{ + return &types.ExecSyncResponse{ Stdout: stdoutBuf.Bytes(), Stderr: stderrBuf.Bytes(), ExitCode: exitCode, diff --git a/test/mocks/oci/oci.go b/test/mocks/oci/oci.go index ca7d3fca638..8bc8e9838fa 100644 --- a/test/mocks/oci/oci.go +++ b/test/mocks/oci/oci.go @@ -11,6 +11,7 @@ import ( syscall "syscall" oci "github.com/cri-o/cri-o/internal/oci" + types "github.com/cri-o/cri-o/server/cri/types" gomock "github.com/golang/mock/gomock" specs "github.com/opencontainers/runtime-spec/specs-go" remotecommand "k8s.io/client-go/tools/remotecommand" @@ -111,10 +112,10 @@ func (mr *MockRuntimeImplMockRecorder) ExecContainer(arg0, arg1, arg2, arg3, arg } // ExecSyncContainer mocks base method. -func (m *MockRuntimeImpl) ExecSyncContainer(arg0 context.Context, arg1 *oci.Container, arg2 []string, arg3 int64) (*oci.ExecSyncResponse, error) { +func (m *MockRuntimeImpl) ExecSyncContainer(arg0 context.Context, arg1 *oci.Container, arg2 []string, arg3 int64) (*types.ExecSyncResponse, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ExecSyncContainer", arg0, arg1, arg2, arg3) - ret0, _ := ret[0].(*oci.ExecSyncResponse) + ret0, _ := ret[0].(*types.ExecSyncResponse) ret1, _ := ret[1].(error) return ret0, ret1 }