diff --git a/internal/oci/oci.go b/internal/oci/oci.go index c2830cdcda7..5215b0d9e0d 100644 --- a/internal/oci/oci.go +++ b/internal/oci/oci.go @@ -1,7 +1,6 @@ package oci import ( - "bytes" "fmt" "io" "sync" @@ -10,6 +9,7 @@ import ( "github.com/cri-o/cri-o/pkg/annotations" "github.com/cri-o/cri-o/pkg/config" + types "github.com/cri-o/cri-o/server/cri/types" rspec "github.com/opencontainers/runtime-spec/specs-go" "golang.org/x/net/context" @@ -51,9 +51,9 @@ type Runtime struct { type RuntimeImpl interface { CreateContainer(*Container, string) error StartContainer(*Container) error - ExecContainer(*Container, []string, io.Reader, io.WriteCloser, io.WriteCloser, + ExecContainer(context.Context, *Container, []string, io.Reader, io.WriteCloser, io.WriteCloser, bool, <-chan remotecommand.TerminalSize) error - ExecSyncContainer(*Container, []string, int64) (*ExecSyncResponse, error) + ExecSyncContainer(context.Context, *Container, []string, int64) (*types.ExecSyncResponse, error) UpdateContainer(*Container, *rspec.LinuxResources) error StopContainer(context.Context, *Container, int64) error DeleteContainer(*Container) error @@ -305,23 +305,23 @@ func (r *Runtime) StartContainer(c *Container) error { } // ExecContainer prepares a streaming endpoint to execute a command in the container. -func (r *Runtime) ExecContainer(c *Container, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize) error { +func (r *Runtime) ExecContainer(ctx context.Context, c *Container, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize) error { impl, err := r.RuntimeImpl(c) if err != nil { return err } - return impl.ExecContainer(c, cmd, stdin, stdout, stderr, tty, resize) + return impl.ExecContainer(ctx, c, cmd, stdin, stdout, stderr, tty, resize) } // ExecSyncContainer execs a command in a container and returns it's stdout, stderr and return code. -func (r *Runtime) ExecSyncContainer(c *Container, command []string, timeout int64) (*ExecSyncResponse, error) { +func (r *Runtime) ExecSyncContainer(ctx context.Context, c *Container, command []string, timeout int64) (*types.ExecSyncResponse, error) { impl, err := r.RuntimeImpl(c) if err != nil { return nil, err } - return impl.ExecSyncContainer(c, command, timeout) + return impl.ExecSyncContainer(ctx, c, command, timeout) } // UpdateContainer updates container resources @@ -445,22 +445,3 @@ func (r *Runtime) ReopenContainerLog(c *Container) error { return impl.ReopenContainerLog(c) } - -// ExecSyncResponse is returned from ExecSync. -type ExecSyncResponse struct { - Stdout []byte - Stderr []byte - ExitCode int32 -} - -// ExecSyncError wraps command's streams, exit code and error on ExecSync error. -type ExecSyncError struct { - Stdout bytes.Buffer - Stderr bytes.Buffer - ExitCode int32 - Err error -} - -func (e *ExecSyncError) Error() string { - return fmt.Sprintf("command error: %+v, stdout: %s, stderr: %s, exit code %d", e.Err, e.Stdout.Bytes(), e.Stderr.Bytes(), e.ExitCode) -} diff --git a/internal/oci/oci_test.go b/internal/oci/oci_test.go index 3df9721bc2a..0acf673e10f 100644 --- a/internal/oci/oci_test.go +++ b/internal/oci/oci_test.go @@ -142,17 +142,4 @@ var _ = t.Describe("Oci", func() { Expect(allowed).To(Equal(true)) }) }) - - t.Describe("ExecSyncError", func() { - It("should succeed to get the exec sync error", func() { - // Given - sut := oci.ExecSyncError{} - - // When - result := sut.Error() - - // Then - Expect(result).To(ContainSubstring("error")) - }) - }) }) diff --git a/internal/oci/runtime_oci.go b/internal/oci/runtime_oci.go index 35e88752c73..7831d33237f 100644 --- a/internal/oci/runtime_oci.go +++ b/internal/oci/runtime_oci.go @@ -18,6 +18,7 @@ import ( "github.com/containers/storage/pkg/pools" "github.com/cri-o/cri-o/internal/log" "github.com/cri-o/cri-o/pkg/config" + types "github.com/cri-o/cri-o/server/cri/types" "github.com/cri-o/cri-o/utils" "github.com/fsnotify/fsnotify" json "github.com/json-iterator/go" @@ -69,12 +70,6 @@ type syncInfo struct { Message string `json:"message,omitempty"` } -// exitCodeInfo is used to return the monitored process exit code to the daemon -type exitCodeInfo struct { - ExitCode int32 `json:"exit_code"` - Message string `json:"message,omitempty"` -} - // CreateContainer creates a container. func (r *runtimeOCI) CreateContainer(c *Container, cgroupParent string) (retErr error) { if c.Spoofed() { @@ -255,71 +250,8 @@ func (r *runtimeOCI) StartContainer(c *Container) error { return nil } -func prepareExec() (pidFileName string, parentPipe, childPipe *os.File, _ error) { - var err error - parentPipe, childPipe, err = os.Pipe() - if err != nil { - return "", nil, nil, err - } - - pidFile, err := ioutil.TempFile("", "pidfile") - if err != nil { - parentPipe.Close() - childPipe.Close() - return "", nil, nil, err - } - pidFile.Close() - pidFileName = pidFile.Name() - - return pidFileName, parentPipe, childPipe, nil -} - -func parseLog(l []byte) (stdout, stderr []byte) { - // Split the log on newlines, which is what separates entries. - lines := bytes.SplitAfter(l, []byte{'\n'}) - for _, line := range lines { - // Ignore empty lines. - if len(line) == 0 { - continue - } - - // The format of log lines is "DATE pipe LogTag REST". - parts := bytes.SplitN(line, []byte{' '}, 4) - if len(parts) < 4 { - // Ignore the line if it's formatted incorrectly, but complain - // about it so it can be debugged. - logrus.Warnf("hit invalid log format: %q", string(line)) - continue - } - - pipe := string(parts[1]) - content := parts[3] - - linetype := string(parts[2]) - if linetype == "P" { - contentLen := len(content) - if contentLen > 0 && content[contentLen-1] == '\n' { - content = content[:contentLen-1] - } - } - - switch pipe { - case "stdout": - stdout = append(stdout, content...) - case "stderr": - stderr = append(stderr, content...) - default: - // Complain about unknown pipes. - logrus.Warnf("hit invalid log format [unknown pipe %s]: %q", pipe, string(line)) - continue - } - } - - return stdout, stderr -} - // ExecContainer prepares a streaming endpoint to execute a command in the container. -func (r *runtimeOCI) ExecContainer(c *Container, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize) error { +func (r *runtimeOCI) ExecContainer(ctx context.Context, c *Container, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize) error { if c.Spoofed() { return nil } @@ -330,12 +262,7 @@ func (r *runtimeOCI) ExecContainer(c *Container, cmd []string, stdin io.Reader, } defer os.RemoveAll(processFile) - args := []string{rootFlag, r.root, "exec"} - args = append(args, "--process", processFile, c.ID()) - execCmd := exec.Command(r.path, args...) // nolint: gosec - if v, found := os.LookupEnv("XDG_RUNTIME_DIR"); found { - execCmd.Env = append(execCmd.Env, fmt.Sprintf("XDG_RUNTIME_DIR=%s", v)) - } + execCmd := r.constructExecCommand(ctx, c, processFile, "") var cmdErr, copyError error if tty { cmdErr = ttyCmd(execCmd, stdin, stdout, resize) @@ -389,176 +316,137 @@ func (r *runtimeOCI) ExecContainer(c *Container, cmd []string, stdin io.Reader, } // ExecSyncContainer execs a command in a container and returns it's stdout, stderr and return code. -func (r *runtimeOCI) ExecSyncContainer(c *Container, command []string, timeout int64) (*ExecSyncResponse, error) { +func (r *runtimeOCI) ExecSyncContainer(ctx context.Context, c *Container, command []string, timeout int64) (*types.ExecSyncResponse, error) { if c.Spoofed() { return nil, nil } - pidFile, parentPipe, childPipe, err := prepareExec() - if err != nil { - return nil, &ExecSyncError{ - ExitCode: -1, - Err: err, - } - } - defer parentPipe.Close() - defer func() { - if e := os.Remove(pidFile); e != nil { - logrus.Warnf("could not remove temporary PID file %s", pidFile) - } - }() - - logFile, err := ioutil.TempFile("", "crio-log-"+c.id) - if err != nil { - return nil, &ExecSyncError{ - ExitCode: -1, - Err: err, - } - } - logFile.Close() - - logPath := logFile.Name() - defer func() { - os.RemoveAll(logPath) - }() - - args := []string{ - "-c", c.id, - "-n", c.name, - "-r", r.path, - "-p", pidFile, - "-e", - "-l", logPath, - "--socket-dir-path", r.config.ContainerAttachSocketDir, - "--log-level", logrus.GetLevel().String(), - } - - if r.config.ConmonSupportsSync() { - args = append(args, "--sync") - } - if c.terminal { - args = append(args, "-t") - } - if timeout > 0 { - args = append(args, "-T", fmt.Sprintf("%d", timeout)) - } - processFile, err := prepareProcessExec(c, command, c.terminal) if err != nil { - return nil, &ExecSyncError{ - ExitCode: -1, - Err: err, - } + return nil, err } defer os.RemoveAll(processFile) - args = append(args, - "--exec-process-spec", processFile, - "--runtime-arg", fmt.Sprintf("%s=%s", rootFlag, r.root)) + pidFile, err := createPidFile() + if err != nil { + return nil, err + } + defer os.RemoveAll(pidFile) - cmd := exec.Command(r.config.Conmon, args...) // nolint: gosec + cmd := r.constructExecCommand(ctx, c, processFile, pidFile) + cmd.SysProcAttr = sysProcAttrPlatform() var stdoutBuf, stderrBuf bytes.Buffer cmd.Stdout = &stdoutBuf cmd.Stderr = &stderrBuf - cmd.ExtraFiles = append(cmd.ExtraFiles, childPipe) - // 0, 1 and 2 are stdin, stdout and stderr - cmd.Env = r.config.ConmonEnv - cmd.Env = append(cmd.Env, fmt.Sprintf("_OCI_SYNCPIPE=%d", 3)) - if v, found := os.LookupEnv("XDG_RUNTIME_DIR"); found { - cmd.Env = append(cmd.Env, fmt.Sprintf("XDG_RUNTIME_DIR=%s", v)) - } err = cmd.Start() if err != nil { - childPipe.Close() - return nil, &ExecSyncError{ - Stdout: stdoutBuf, - Stderr: stderrBuf, - ExitCode: -1, - Err: err, - } + return nil, err } - // We don't need childPipe on the parent side - childPipe.Close() + // wait till the command is done + done := make(chan error, 1) + go func() { + done <- cmd.Wait() + }() + + if timeout > 0 { + select { + case <-time.After(time.Second * time.Duration(timeout)): + // Ensure the process is not left behind + killContainerExecProcess(ctx, pidFile) - // first, wait till the command is done - waitErr := cmd.Wait() - - // regardless of what is in waitErr - // we should attempt to decode the output of the parent pipe - // this allows us to catch TimedOutMessage, which will cause waitErr to not be nil - var ec *exitCodeInfo - decodeErr := json.NewDecoder(parentPipe).Decode(&ec) - if decodeErr == nil { - logrus.Debugf("Received container exit code: %v, message: %s", ec.ExitCode, ec.Message) - - // When we timeout the command in conmon then we should return - // an ExecSyncResponse with a non-zero exit code because - // the prober code in the kubelet checks for it. If we return - // a custom error, then the probes transition into Unknown status - // and the container isn't restarted as expected. - if ec.ExitCode == -1 && ec.Message == conmonconfig.TimedOutMessage { - return &ExecSyncResponse{ + // Make sure the runtime process has been cleaned up + <-done + + // If the command timed out, we should return an ExecSyncResponse with a non-zero exit code because + // the prober code in the kubelet checks for it. If we return a custom error, + // then the probes transition into Unknown status and the container isn't restarted as expected. + return &types.ExecSyncResponse{ Stderr: []byte(conmonconfig.TimedOutMessage), ExitCode: -1, }, nil + case err = <-done: + break } + } else { + err = <-done } - if waitErr != nil { - // if we aren't a ExitError, some I/O problems probably occurred - if _, ok := waitErr.(*exec.ExitError); !ok { - return nil, &ExecSyncError{ - Stdout: stdoutBuf, - Stderr: stderrBuf, - ExitCode: -1, - Err: waitErr, - } + // gather exit code from err + exitCode := int32(0) + if err != nil { + if exitError, ok := err.(*exec.ExitError); ok { + exitCode = int32(exitError.ExitCode()) } } - if decodeErr != nil { - return nil, &ExecSyncError{ - Stdout: stdoutBuf, - Stderr: stderrBuf, - ExitCode: -1, - Err: decodeErr, - } + return &types.ExecSyncResponse{ + Stdout: stdoutBuf.Bytes(), + Stderr: stderrBuf.Bytes(), + ExitCode: exitCode, + }, nil +} + +func (r *runtimeOCI) constructExecCommand(ctx context.Context, c *Container, processFile, pidFile string) *exec.Cmd { + args := []string{rootFlag, r.root, "exec"} + if pidFile != "" { + args = append(args, "--pid-file", pidFile) } + args = append(args, "--process", processFile, c.ID()) + execCmd := exec.CommandContext(ctx, r.path, args...) // nolint: gosec + if v, found := os.LookupEnv("XDG_RUNTIME_DIR"); found { + execCmd.Env = append(execCmd.Env, fmt.Sprintf("XDG_RUNTIME_DIR=%s", v)) + } + return execCmd +} - if ec.ExitCode == -1 { - return nil, &ExecSyncError{ - Stdout: stdoutBuf, - Stderr: stderrBuf, - ExitCode: -1, - Err: fmt.Errorf(ec.Message), - } +func createPidFile() (string, error) { + pidFile, err := ioutil.TempFile("", "pidfile") + if err != nil { + return "", err } + pidFile.Close() + pidFileName := pidFile.Name() - // The actual logged output is not the same as stdoutBuf and stderrBuf, - // which are used for getting error information. For the actual - // ExecSyncResponse we have to read the logfile. - // XXX: Currently runC dups the same console over both stdout and stderr, - // so we can't differentiate between the two. - logBytes, err := ioutil.ReadFile(logPath) + return pidFileName, nil +} + +func killContainerExecProcess(ctx context.Context, pidFile string) { + // Attempt to get the container PID and PGID from the file the runtime should have written. + // TODO(haircommander): There does exist a race that we could time out before the runtime actually creates the file. + // Should we do inotify on this file to ensure it exists? Is that overkill? + ctrPid, ctrPgid, err := pidAndpgidFromFile(pidFile) if err != nil { - return nil, &ExecSyncError{ - Stdout: stdoutBuf, - Stderr: stderrBuf, - ExitCode: -1, - Err: err, + log.Errorf(ctx, "Failed to get pid (%d) or pgid (%d) from file %s: %v", ctrPid, ctrPgid, pidFile, err) + } + + if ctrPgid > 1 { + // First attempt to kill the container group + if err := syscall.Kill(-ctrPgid, syscall.SIGKILL); err != nil { + log.Errorf(ctx, "Failed to kill process group after timeout: %v", err) + } + } else if ctrPid > 0 { + // If that fails, kill the container PID itself + if err := syscall.Kill(ctrPid, syscall.SIGKILL); err != nil { + log.Errorf(ctx, "Failed to kill process after timeout: %v", err) } } +} - // We have to parse the log output into {stdout, stderr} buffers. - stdoutBytes, stderrBytes := parseLog(logBytes) - return &ExecSyncResponse{ - Stdout: stdoutBytes, - Stderr: stderrBytes, - ExitCode: ec.ExitCode, - }, nil +func pidAndpgidFromFile(pidFile string) (pid, pgid int, _ error) { + // find the pid of the parent process + pidStr, err := ioutil.ReadFile(pidFile) + if err != nil { + return -1, -1, err + } + pid, err = strconv.Atoi(string(pidStr)) + if err != nil { + return -1, -1, err + } + pgid, err = syscall.Getpgid(pid) + return pid, pgid, err } // UpdateContainer updates container resources diff --git a/internal/oci/runtime_vm.go b/internal/oci/runtime_vm.go index 830c3267edb..8b6d56d3e29 100644 --- a/internal/oci/runtime_vm.go +++ b/internal/oci/runtime_vm.go @@ -17,6 +17,7 @@ import ( "github.com/containerd/containerd/runtime/v2/task" "github.com/containerd/ttrpc" "github.com/containers/libpod/v2/pkg/cgroups" + types "github.com/cri-o/cri-o/server/cri/types" "github.com/cri-o/cri-o/utils" "github.com/cri-o/cri-o/utils/errdefs" "github.com/cri-o/cri-o/utils/fifo" @@ -284,7 +285,7 @@ func (r *runtimeVM) StartContainer(c *Container) error { } // ExecContainer prepares a streaming endpoint to execute a command in the container. -func (r *runtimeVM) ExecContainer(c *Container, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize) error { +func (r *runtimeVM) ExecContainer(ctx context.Context, c *Container, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize) error { logrus.Debug("runtimeVM.ExecContainer() start") defer logrus.Debug("runtimeVM.ExecContainer() end") @@ -303,7 +304,7 @@ func (r *runtimeVM) ExecContainer(c *Container, cmd []string, stdin io.Reader, s } // ExecSyncContainer execs a command in a container and returns it's stdout, stderr and return code. -func (r *runtimeVM) ExecSyncContainer(c *Container, command []string, timeout int64) (*ExecSyncResponse, error) { +func (r *runtimeVM) ExecSyncContainer(ctx context.Context, c *Container, command []string, timeout int64) (*types.ExecSyncResponse, error) { logrus.Debug("runtimeVM.ExecSyncContainer() start") defer logrus.Debug("runtimeVM.ExecSyncContainer() end") @@ -313,13 +314,10 @@ func (r *runtimeVM) ExecSyncContainer(c *Container, command []string, timeout in exitCode, err := r.execContainerCommon(c, command, timeout, nil, stdout, stderr, c.terminal, nil) if err != nil { - return nil, &ExecSyncError{ - ExitCode: -1, - Err: errors.Wrap(err, "ExecSyncContainer failed"), - } + return nil, errors.Wrap(err, "ExecSyncContainer failed") } - return &ExecSyncResponse{ + return &types.ExecSyncResponse{ Stdout: stdoutBuf.Bytes(), Stderr: stderrBuf.Bytes(), ExitCode: exitCode, diff --git a/internal/resourcestore/resourcestore_test.go b/internal/resourcestore/resourcestore_test.go index 409e62417c6..e049221a24f 100644 --- a/internal/resourcestore/resourcestore_test.go +++ b/internal/resourcestore/resourcestore_test.go @@ -141,7 +141,7 @@ var _ = t.Describe("ResourceStore", func() { // When go func() { time.Sleep(timeout * 6) - Expect(sut.Put(testName, e, cleaner)).To(BeNil()) + Expect(sut.Put(testName, e, cleanupFuncs)).To(BeNil()) timedOutChan <- true }() diff --git a/server/container_exec.go b/server/container_exec.go index 76ecdd5746e..fde28112ab0 100644 --- a/server/container_exec.go +++ b/server/container_exec.go @@ -38,5 +38,5 @@ func (s StreamService) Exec(containerID string, cmd []string, stdin io.Reader, s return fmt.Errorf("container is not created or running") } - return s.runtimeServer.Runtime().ExecContainer(c, cmd, stdin, stdout, stderr, tty, resize) + return s.runtimeServer.Runtime().ExecContainer(s.ctx, c, cmd, stdin, stdout, stderr, tty, resize) } diff --git a/server/container_execsync.go b/server/container_execsync.go index 316c0c2317a..8fbe8f1decc 100644 --- a/server/container_execsync.go +++ b/server/container_execsync.go @@ -24,7 +24,7 @@ func (s *Server) ExecSync(ctx context.Context, req *pb.ExecSyncRequest) (*pb.Exe return nil, errors.New("exec command cannot be empty") } - execResp, err := s.Runtime().ExecSyncContainer(c, cmd, req.Timeout) + execResp, err := s.Runtime().ExecSyncContainer(ctx, c, cmd, req.Timeout) if err != nil { return nil, err } diff --git a/server/server.go b/server/server.go index 98e5cbd93d7..93c4f4f17db 100644 --- a/server/server.go +++ b/server/server.go @@ -45,6 +45,7 @@ var errSandboxNotCreated = errors.New("sandbox not created") // StreamService implements streaming.Runtime. type StreamService struct { + ctx context.Context runtimeServer *Server // needed by Exec() endpoint streamServer streaming.Server streamServerCloseCh chan struct{} @@ -414,6 +415,7 @@ func New( Certificates: []tls.Certificate{cert}, } } + s.stream.ctx = ctx s.stream.runtimeServer = s s.stream.streamServer, err = streaming.NewServer(streamServerConfig, s.stream) if err != nil { diff --git a/test/mocks/oci/oci.go b/test/mocks/oci/oci.go index 635539bbe3f..75bee1978b6 100644 --- a/test/mocks/oci/oci.go +++ b/test/mocks/oci/oci.go @@ -7,6 +7,7 @@ package ocimock import ( context "context" oci "github.com/cri-o/cri-o/internal/oci" + server "github.com/cri-o/cri-o/server/cri/types" gomock "github.com/golang/mock/gomock" specs "github.com/opencontainers/runtime-spec/specs-go" io "io" @@ -96,32 +97,32 @@ func (mr *MockRuntimeImplMockRecorder) DeleteContainer(arg0 interface{}) *gomock } // ExecContainer mocks base method -func (m *MockRuntimeImpl) ExecContainer(arg0 *oci.Container, arg1 []string, arg2 io.Reader, arg3, arg4 io.WriteCloser, arg5 bool, arg6 <-chan remotecommand.TerminalSize) error { +func (m *MockRuntimeImpl) ExecContainer(arg0 context.Context, arg1 *oci.Container, arg2 []string, arg3 io.Reader, arg4, arg5 io.WriteCloser, arg6 bool, arg7 <-chan remotecommand.TerminalSize) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ExecContainer", arg0, arg1, arg2, arg3, arg4, arg5, arg6) + ret := m.ctrl.Call(m, "ExecContainer", arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7) ret0, _ := ret[0].(error) return ret0 } // ExecContainer indicates an expected call of ExecContainer -func (mr *MockRuntimeImplMockRecorder) ExecContainer(arg0, arg1, arg2, arg3, arg4, arg5, arg6 interface{}) *gomock.Call { +func (mr *MockRuntimeImplMockRecorder) ExecContainer(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExecContainer", reflect.TypeOf((*MockRuntimeImpl)(nil).ExecContainer), arg0, arg1, arg2, arg3, arg4, arg5, arg6) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExecContainer", reflect.TypeOf((*MockRuntimeImpl)(nil).ExecContainer), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7) } // ExecSyncContainer mocks base method -func (m *MockRuntimeImpl) ExecSyncContainer(arg0 *oci.Container, arg1 []string, arg2 int64) (*oci.ExecSyncResponse, error) { +func (m *MockRuntimeImpl) ExecSyncContainer(arg0 context.Context, arg1 *oci.Container, arg2 []string, arg3 int64) (*server.ExecSyncResponse, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ExecSyncContainer", arg0, arg1, arg2) - ret0, _ := ret[0].(*oci.ExecSyncResponse) + ret := m.ctrl.Call(m, "ExecSyncContainer", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(*server.ExecSyncResponse) ret1, _ := ret[1].(error) return ret0, ret1 } // ExecSyncContainer indicates an expected call of ExecSyncContainer -func (mr *MockRuntimeImplMockRecorder) ExecSyncContainer(arg0, arg1, arg2 interface{}) *gomock.Call { +func (mr *MockRuntimeImplMockRecorder) ExecSyncContainer(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExecSyncContainer", reflect.TypeOf((*MockRuntimeImpl)(nil).ExecSyncContainer), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExecSyncContainer", reflect.TypeOf((*MockRuntimeImpl)(nil).ExecSyncContainer), arg0, arg1, arg2, arg3) } // PauseContainer mocks base method