From c9d3b2c2d03a19cd98d767f799b436d39ee76b6c Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 24 Jun 2025 12:01:08 +0100 Subject: [PATCH 1/2] fix: agentcontainers: fix flake when ctx cancelled while running docker inspect --- agent/agentcontainers/api.go | 12 +++++-- agent/agentcontainers/containers_dockercli.go | 36 ++++++++++++------- 2 files changed, 33 insertions(+), 15 deletions(-) diff --git a/agent/agentcontainers/api.go b/agent/agentcontainers/api.go index 3fc44980568e2..4a8413906e8ce 100644 --- a/agent/agentcontainers/api.go +++ b/agent/agentcontainers/api.go @@ -378,7 +378,11 @@ func (api *API) updaterLoop() { // and anyone looking to interact with the API. api.logger.Debug(api.ctx, "performing initial containers update") if err := api.updateContainers(api.ctx); err != nil { - api.logger.Error(api.ctx, "initial containers update failed", slog.Error(err)) + if errors.Is(err, context.Canceled) { + api.logger.Warn(api.ctx, "initial containers update canceled", slog.Error(err)) + } else { + api.logger.Error(api.ctx, "initial containers update failed", slog.Error(err)) + } } else { api.logger.Debug(api.ctx, "initial containers update complete") } @@ -399,7 +403,11 @@ func (api *API) updaterLoop() { case api.updateTrigger <- done: err := <-done if err != nil { - api.logger.Error(api.ctx, "updater loop ticker failed", slog.Error(err)) + if errors.Is(err, context.Canceled) { + api.logger.Warn(api.ctx, "updater loop ticker canceled", slog.Error(err)) + } else { + api.logger.Error(api.ctx, "updater loop ticker failed", slog.Error(err)) + } } default: api.logger.Debug(api.ctx, "updater loop ticker skipped, update in progress") diff --git a/agent/agentcontainers/containers_dockercli.go b/agent/agentcontainers/containers_dockercli.go index 83463481c97f7..942c1ba4ccd84 100644 --- a/agent/agentcontainers/containers_dockercli.go +++ b/agent/agentcontainers/containers_dockercli.go @@ -311,21 +311,31 @@ func (dcli *dockerCLI) List(ctx context.Context) (codersdk.WorkspaceAgentListCon // container IDs and returns the parsed output. // The stderr output is also returned for logging purposes. func runDockerInspect(ctx context.Context, execer agentexec.Execer, ids ...string) (stdout, stderr []byte, err error) { - var stdoutBuf, stderrBuf bytes.Buffer - cmd := execer.CommandContext(ctx, "docker", append([]string{"inspect"}, ids...)...) - cmd.Stdout = &stdoutBuf - cmd.Stderr = &stderrBuf - err = cmd.Run() - stdout = bytes.TrimSpace(stdoutBuf.Bytes()) - stderr = bytes.TrimSpace(stderrBuf.Bytes()) - if err != nil { - if bytes.Contains(stderr, []byte("No such object:")) { - // This can happen if a container is deleted between the time we check for its existence and the time we inspect it. - return stdout, stderr, nil + select { + case <-ctx.Done(): + // If the context is done, we don't want to run the command. + return []byte{}, []byte{}, ctx.Err() + default: + var stdoutBuf, stderrBuf bytes.Buffer + cmd := execer.CommandContext(ctx, "docker", append([]string{"inspect"}, ids...)...) + cmd.Stdout = &stdoutBuf + cmd.Stderr = &stderrBuf + err = cmd.Run() + stdout = bytes.TrimSpace(stdoutBuf.Bytes()) + stderr = bytes.TrimSpace(stderrBuf.Bytes()) + if err != nil { + if ctx.Err() != nil { + // If the context was canceled, we don't want to return an error. + return stdout, stderr, ctx.Err() + } + if bytes.Contains(stderr, []byte("No such object:")) { + // This can happen if a container is deleted between the time we check for its existence and the time we inspect it. + return stdout, stderr, nil + } + return stdout, stderr, err } - return stdout, stderr, err + return stdout, stderr, nil } - return stdout, stderr, nil } // To avoid a direct dependency on the Docker API, we use the docker CLI From 6f6a29e5221e9c3c538d7acf8c3f5fdb0ec4e535 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 24 Jun 2025 12:24:31 +0100 Subject: [PATCH 2/2] address comments --- agent/agentcontainers/containers_dockercli.go | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/agent/agentcontainers/containers_dockercli.go b/agent/agentcontainers/containers_dockercli.go index 942c1ba4ccd84..58ca3901e2f23 100644 --- a/agent/agentcontainers/containers_dockercli.go +++ b/agent/agentcontainers/containers_dockercli.go @@ -311,31 +311,31 @@ func (dcli *dockerCLI) List(ctx context.Context) (codersdk.WorkspaceAgentListCon // container IDs and returns the parsed output. // The stderr output is also returned for logging purposes. func runDockerInspect(ctx context.Context, execer agentexec.Execer, ids ...string) (stdout, stderr []byte, err error) { - select { - case <-ctx.Done(): + if ctx.Err() != nil { // If the context is done, we don't want to run the command. return []byte{}, []byte{}, ctx.Err() - default: - var stdoutBuf, stderrBuf bytes.Buffer - cmd := execer.CommandContext(ctx, "docker", append([]string{"inspect"}, ids...)...) - cmd.Stdout = &stdoutBuf - cmd.Stderr = &stderrBuf - err = cmd.Run() - stdout = bytes.TrimSpace(stdoutBuf.Bytes()) - stderr = bytes.TrimSpace(stderrBuf.Bytes()) - if err != nil { - if ctx.Err() != nil { - // If the context was canceled, we don't want to return an error. - return stdout, stderr, ctx.Err() - } - if bytes.Contains(stderr, []byte("No such object:")) { - // This can happen if a container is deleted between the time we check for its existence and the time we inspect it. - return stdout, stderr, nil - } - return stdout, stderr, err + } + var stdoutBuf, stderrBuf bytes.Buffer + cmd := execer.CommandContext(ctx, "docker", append([]string{"inspect"}, ids...)...) + cmd.Stdout = &stdoutBuf + cmd.Stderr = &stderrBuf + err = cmd.Run() + stdout = bytes.TrimSpace(stdoutBuf.Bytes()) + stderr = bytes.TrimSpace(stderrBuf.Bytes()) + if err != nil { + if ctx.Err() != nil { + // If the context was canceled while running the command, + // return the context error instead of the command error, + // which is likely to be "signal: killed". + return stdout, stderr, ctx.Err() + } + if bytes.Contains(stderr, []byte("No such object:")) { + // This can happen if a container is deleted between the time we check for its existence and the time we inspect it. + return stdout, stderr, nil } - return stdout, stderr, nil + return stdout, stderr, err } + return stdout, stderr, nil } // To avoid a direct dependency on the Docker API, we use the docker CLI