cri-o · haircommander · Jun 11, 2020 · Jun 11, 2020
diff --git a/internal/oci/container.go b/internal/oci/container.go
@@ -344,6 +344,23 @@ func (c *Container) exitFilePath() string {
 	return filepath.Join(c.dir, "exit")
 }
 
+// IsRunning is a function that checks if a specific pid is running
+// it is used to check a container state when we don't want (or don't trust) a `$runtime state` call
+func (c *Container) IsRunning() bool {
+	process, err := os.FindProcess(c.state.Pid)
+	if err != nil {
+		logrus.Infof("container %s not found: %v", c.id, err)
+		return false
+	}
+
+	if err := process.Signal(syscall.Signal(0)); err != nil {
+		logrus.Errorf("container %s not running %v", c.id, err)
+		return false
+	}
+
+	return true
+}
+
 // ShouldBeStopped checks whether the container state is in a place
 // where attempting to stop it makes sense
 // a container is not stoppable if it's paused or stopped

diff --git a/internal/oci/container_test.go b/internal/oci/container_test.go
@@ -238,4 +238,38 @@ var _ = t.Describe("Container", func() {
 		// Then
 		Expect(err).To(BeNil())
 	})
+	It("should be false if pid unintialized", func() {
+		// Given
+		state := &oci.ContainerState{}
+		state.Pid = 0
+		sut.SetState(state)
+		// When
+		err := sut.IsRunning()
+
+		// Then
+		Expect(err).To(Equal(false))
+	})
+	It("should succeed if pid is running", func() {
+		// Given
+		state := &oci.ContainerState{}
+		state.Pid = 1
+		sut.SetState(state)
+		// When
+		err := sut.IsRunning()
+
+		// Then
+		Expect(err).To(Equal(true))
+	})
+	It("should be false if pid is not running", func() {
+		// Given
+		state := &oci.ContainerState{}
+		// the highest allowed pid + 1
+		state.Pid = 4194305
+		sut.SetState(state)
+		// When
+		err := sut.IsRunning()
+
+		// Then
+		Expect(err).To(Equal(false))
+	})
 })
diff --git a/internal/oci/runtime_oci.go b/internal/oci/runtime_oci.go
@@ -705,12 +705,26 @@ func (r *runtimeOCI) UpdateContainerStatus(c *Container) error {
 	}
 
 	stateCmd := func() (*ContainerState, bool, error) {
-		cmd := exec.Command(r.path, rootFlag, r.root, "state", c.id) // nolint: gosec
-		if v, found := os.LookupEnv("XDG_RUNTIME_DIR"); found {
-			cmd.Env = append(cmd.Env, fmt.Sprintf("XDG_RUNTIME_DIR=%s", v))
+		var out []byte
+		var err error
+		// We have seen cases where this state call fails
+		// with little output, but the container is running and
+		// later calls succeed.
+		// There is little harm in trying again
+		for i := 0; i < 3; i++ {
+			out, err = r.attemptUpdateContainerStatus(c)
+			if err == nil {
+				break
+			}
+			logrus.Errorf("%v: attempt %d", err, i)
+
+			if !c.IsRunning() {
+				break
+			}
 		}
-		out, err := cmd.Output()
-		if err != nil {
+		// we check out here, instead of err, because a condition for setting this state
+		// is the container is not running, and the `$runtime state` call failed
+		if out == nil {
 			logrus.Errorf("Failed to update container state for %s: %v", c.id, err)
 			// there are many code paths that could lead to have a bad state in the
 			// underlying runtime.
@@ -798,6 +812,21 @@ func (r *runtimeOCI) UpdateContainerStatus(c *Container) error {
 	return nil
 }
 
+func (r runtimeOCI) attemptUpdateContainerStatus(c *Container) ([]byte, error) {
+	cmd := exec.Command(r.path, rootFlag, r.root, "state", c.id) // nolint: gosec
+	if v, found := os.LookupEnv("XDG_RUNTIME_DIR"); found {
+		cmd.Env = append(cmd.Env, fmt.Sprintf("XDG_RUNTIME_DIR=%s", v))
+	}
+	out, err := cmd.Output()
+	if err != nil {
+		if exitErr, isExitError := err.(*exec.ExitError); isExitError {
+			return nil, errors.Errorf("failed to update container state for %s: stdout: %s, stderr: %s", c.id, string(out), string(exitErr.Stderr))
+		}
+		return nil, errors.Errorf("failed to update container state for %s: %v", c.id, err)
+	}
+	return out, nil
+}
+
 // PauseContainer pauses a container.
 func (r *runtimeOCI) PauseContainer(c *Container) error {
 	c.opLock.Lock()

diff --git a/server/container_execsync.go b/server/container_execsync.go
@@ -3,7 +3,6 @@ package server
 import (
 	"fmt"
 
-	oci "github.com/cri-o/cri-o/internal/oci"
 	"golang.org/x/net/context"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
@@ -17,12 +16,7 @@ func (s *Server) ExecSync(ctx context.Context, req *pb.ExecSyncRequest) (resp *p
 		return nil, status.Errorf(codes.NotFound, "could not find container %q: %v", req.ContainerId, err)
 	}
 
-	if err := s.Runtime().UpdateContainerStatus(c); err != nil {
-		return nil, err
-	}
-
-	cState := c.State()
-	if !(cState.Status == oci.ContainerStateRunning || cState.Status == oci.ContainerStateCreated) {
+	if !c.IsRunning() {
 		return nil, fmt.Errorf("container is not created or running")
 	}