Thanks to visit codestin.com
Credit goes to github.com

Skip to content

[1.22] oci: fix a couple of deadlocks in container stop code #5447

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
openshift-merge-robot merged 3 commits into from
Nov 29, 2021
Merged

[1.22] oci: fix a couple of deadlocks in container stop code #5447

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2c46f98
oci: make some channels buffered
haircommander Nov 1, 2021
7be82a7
oci: always close chControl
haircommander Nov 1, 2021
7f48932
oci: fix deadlock in container stop code
haircommander Nov 3, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 27 additions & 22 deletions internal/oci/container.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ type Container struct {
stopping bool
stopTimeoutChan chan time.Duration
stoppedChan chan struct{}
stopStoppingChan chan struct{}
stopLock sync.Mutex
}

Expand Down Expand Up @@ -117,27 +118,28 @@ func NewContainer(id, name, bundlePath, logPath string, labels, crioAnnotations,
state := &ContainerState{}
state.Created = created
c := &Container{
id: id,
name: name,
bundlePath: bundlePath,
logPath: logPath,
labels: labels,
sandbox: sandbox,
terminal: terminal,
stdin: stdin,
stdinOnce: stdinOnce,
runtimeHandler: runtimeHandler,
metadata: metadata,
annotations: annotations,
crioAnnotations: crioAnnotations,
image: image,
imageName: imageName,
imageRef: imageRef,
dir: dir,
state: state,
stopSignal: stopSignal,
stopTimeoutChan: make(chan time.Duration, 1),
stoppedChan: make(chan struct{}, 1),
id: id,
name: name,
bundlePath: bundlePath,
logPath: logPath,
labels: labels,
sandbox: sandbox,
terminal: terminal,
stdin: stdin,
stdinOnce: stdinOnce,
runtimeHandler: runtimeHandler,
metadata: metadata,
annotations: annotations,
crioAnnotations: crioAnnotations,
image: image,
imageName: imageName,
imageRef: imageRef,
dir: dir,
state: state,
stopSignal: stopSignal,
stopTimeoutChan: make(chan time.Duration, 1),
stoppedChan: make(chan struct{}, 1),
stopStoppingChan: make(chan struct{}, 1),
}
return c, nil
}
Expand Down Expand Up @@ -570,11 +572,14 @@ func (c *Container) SetAsStopping(timeout int64) {
select {
case c.stopTimeoutChan <- time.Duration(timeout) * time.Second:
case <-c.stoppedChan: // This case is to avoid waiting forever once another routine has finished.
return
case <-c.stopStoppingChan: // This case is to avoid deadlocking with SetAsNotStopping.
}
return
}
// Regardless, set the container as actively stopping.
c.stopping = true
// And reset the stopStoppingChan
c.stopStoppingChan = make(chan struct{}, 1)
}

// SetAsNotStopping unsets the stopping field indicating to new callers that the container
Expand Down
7 changes: 3 additions & 4 deletions internal/oci/oci.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,8 +126,9 @@ func (r *Runtime) WaitContainerStateStopped(ctx context.Context, c *Container) e
return nil
}

done := make(chan error)
chControl := make(chan struct{})
done := make(chan error, 1)
chControl := make(chan struct{}, 1)
defer close(chControl)
go func() {
defer close(done)
for {
Expand All @@ -152,10 +153,8 @@ func (r *Runtime) WaitContainerStateStopped(ctx context.Context, c *Container) e
case err = <-done:
break
case <-ctx.Done():
close(chControl)
return ctx.Err()
case <-time.After(time.Duration(r.config.CtrStopTimeout) * time.Second):
close(chControl)
return fmt.Errorf(
"failed to get container stopped status: %ds timeout reached",
r.config.CtrStopTimeout,
Expand Down
3 changes: 3 additions & 0 deletions internal/oci/runtime_oci.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -685,6 +685,9 @@ func (r *runtimeOCI) StopContainer(ctx context.Context, c *Container, timeout in
// Otherwise, we won't actually
// attempt to stop when a new request comes in,
// even though we're not actively stopping anymore.
// Also, close the stopStoppingChan to tell
// routines waiting to change the stop timeout to give up.
close(c.stopStoppingChan)
c.SetAsNotStopping()
}
}()
Expand Down