From 7db75e65b7d28f726d9607f9dc90d152659584a8 Mon Sep 17 00:00:00 2001 From: Ethan Dickson Date: Wed, 15 Jul 2026 07:00:11 +0000 Subject: [PATCH 1/3] fix(coderd/x/chatd): recover timed out agents --- coderd/x/chatd/chatd.go | 49 +++++-- coderd/x/chatd/chatd_internal_test.go | 182 +++++++++++--------------- 2 files changed, 111 insertions(+), 120 deletions(-) diff --git a/coderd/x/chatd/chatd.go b/coderd/x/chatd/chatd.go index 8390441acba..0f3c44f228d 100644 --- a/coderd/x/chatd/chatd.go +++ b/coderd/x/chatd/chatd.go @@ -120,6 +120,12 @@ var ( "to stop the workspace, then start_workspace to start it " + "again", ) + errChatAgentNeverConnected = xerrors.New( + "workspace agent never connected and its connection timeout has " + + "elapsed, so it cannot execute tools. To recover, call " + + "stop_workspace to stop the workspace, then start_workspace " + + "to start it again", + ) errChatDialTimeout = xerrors.New( "connection to the workspace agent timed out. " + "The agent may still be reachable on the next attempt.", @@ -846,17 +852,20 @@ func agentDisconnectedFor(now time.Time, agent database.WorkspaceAgent, inactive return disconnectedFor, true } -func (c *turnWorkspaceContext) latestWorkspaceAgentNeedsRestart( +// latestWorkspaceAgentRecoveryError classifies the latest agent's status after +// a failed dial. It returns the recovery sentinel to surface, or +// errChatDialTimeout when the dial should remain retryable. +func (c *turnWorkspaceContext) latestWorkspaceAgentRecoveryError( ctx context.Context, workspaceID uuid.UUID, -) (bool, error) { +) error { agentID, err := c.latestWorkspaceAgentID(ctx, workspaceID) if err != nil { if xerrors.Is(err, errChatHasNoWorkspaceAgent) { - return false, err + return err } c.server.logger.Warn(ctx, "failed to resolve latest agent for timeout classification", slog.Error(err)) - return false, nil + return errChatDialTimeout } agent, err := c.server.db.GetWorkspaceAgentByID(ctx, agentID) @@ -865,11 +874,26 @@ func (c *turnWorkspaceContext) latestWorkspaceAgentNeedsRestart( slog.F("agent_id", agentID), slog.Error(err), ) - return false, nil + return errChatDialTimeout + } + + now := c.server.clock.Now() + status := agent.Status(now, c.server.agentInactiveDisconnectTimeout) + if status.Status == database.WorkspaceAgentStatusTimeout { + return errChatAgentNeverConnected + } + if status.Status != database.WorkspaceAgentStatusDisconnected || status.DisconnectedAt == nil { + return errChatDialTimeout } - disconnectedFor, disconnected := agentDisconnectedFor(c.server.clock.Now(), agent, c.server.agentInactiveDisconnectTimeout) - return disconnected && disconnectedFor >= agentDisconnectedRecoveryThreshold, nil + disconnectedFor := now.Sub(*status.DisconnectedAt) + if disconnectedFor < 0 { + disconnectedFor = 0 + } + if disconnectedFor >= agentDisconnectedRecoveryThreshold { + return errChatAgentDisconnected + } + return errChatDialTimeout } func (c *turnWorkspaceContext) externalAgentError( @@ -1002,14 +1026,11 @@ func (c *turnWorkspaceContext) getWorkspaceConn(ctx context.Context) (workspaces // propagate unchanged so the chatloop can detect it. if ctx.Err() == nil && errors.Is(context.Cause(dialCtx), errChatDialTimeout) { c.clearCachedWorkspaceState() - needsRestart, statusErr := c.latestWorkspaceAgentNeedsRestart(ctx, chatSnapshot.WorkspaceID.UUID) - if statusErr != nil { - return nil, statusErr - } - if needsRestart { - return nil, c.externalAgentError(ctx, agent, errChatAgentDisconnected) + recoveryErr := c.latestWorkspaceAgentRecoveryError(ctx, chatSnapshot.WorkspaceID.UUID) + if xerrors.Is(recoveryErr, errChatHasNoWorkspaceAgent) { + return nil, recoveryErr } - return nil, c.externalAgentError(ctx, agent, errChatDialTimeout) + return nil, c.externalAgentError(ctx, agent, recoveryErr) } return nil, err } diff --git a/coderd/x/chatd/chatd_internal_test.go b/coderd/x/chatd/chatd_internal_test.go index 4b795308161..972b2465b79 100644 --- a/coderd/x/chatd/chatd_internal_test.go +++ b/coderd/x/chatd/chatd_internal_test.go @@ -344,8 +344,13 @@ func TestChatWorkspaceRecoveryErrorsDifferentiateSignalStrength(t *testing.T) { require.Contains(t, disconnected, "start_workspace") require.NotContains(t, disconnected, "ask_user_question") + neverConnected := errChatAgentNeverConnected.Error() + require.Contains(t, neverConnected, "stop_workspace") + require.Contains(t, neverConnected, "start_workspace") + require.NotContains(t, neverConnected, "ask_user_question") + // Dial timeout alone is a weak signal. The model should not - // escalate to lifecycle tools without DB-confirmed disconnect. + // escalate to lifecycle tools without a DB-confirmed recovery condition. dialTimeout := errChatDialTimeout.Error() require.NotContains(t, dialTimeout, "ask_user_question") require.NotContains(t, dialTimeout, "stop_workspace") @@ -2380,10 +2385,9 @@ func TestGetWorkspaceConn_StatusCheck(t *testing.T) { tests := []testCase{ { - // Agent never connected and the connection timeout - // has elapsed. This should not trigger lifecycle - // recovery because the agent did not connect and - // then disconnect. + // Cache-hit status checks only force a fresh dial for + // disconnected agents. Timeout recovery is classified + // after a failed dial. name: "TimedOutAgentCacheHit", buildAgent: func(now time.Time) database.WorkspaceAgent { return database.WorkspaceAgent{ @@ -2514,34 +2518,76 @@ func TestGetWorkspaceConn_StatusCheck(t *testing.T) { } func TestGetWorkspaceConn_DialTimeoutDisconnectedRecoveryThreshold(t *testing.T) { - // The recovery sentinel requires a failed dial and a fresh - // disconnected status check past the recovery threshold. A - // disconnected DB row alone is not enough to trigger stop/start - // recovery. + // Failed dials use a fresh agent status to distinguish retryable timeouts + // from conditions that require stop/start recovery. t.Parallel() + buildDisconnectedAgent := func(disconnectedFor time.Duration) func(time.Time) database.WorkspaceAgent { + return func(now time.Time) database.WorkspaceAgent { + return database.WorkspaceAgent{ + FirstConnectedAt: sql.NullTime{ + Time: now.Add(-10 * time.Minute), + Valid: true, + }, + LastConnectedAt: sql.NullTime{ + Time: now.Add(-10 * time.Minute), + Valid: true, + }, + DisconnectedAt: sql.NullTime{ + Time: now.Add(-disconnectedFor), + Valid: true, + }, + } + } + } + testCases := []struct { - name string - disconnectedFor time.Duration - wantErr error - wantRecovery bool + name string + buildAgent func(now time.Time) database.WorkspaceAgent + wantErr error }{ { - name: "RecentDisconnectReturnsDialTimeout", - disconnectedFor: agentDisconnectedRecoveryThreshold / 2, - wantErr: errChatDialTimeout, - wantRecovery: false, + name: "RecentDisconnectReturnsDialTimeout", + buildAgent: buildDisconnectedAgent(agentDisconnectedRecoveryThreshold / 2), + wantErr: errChatDialTimeout, + }, + { + name: "PastThresholdEscalates", + buildAgent: buildDisconnectedAgent(agentDisconnectedRecoveryThreshold), + wantErr: errChatAgentDisconnected, + }, + { + name: "NeverConnectedTimeoutEscalates", + buildAgent: func(now time.Time) database.WorkspaceAgent { + return database.WorkspaceAgent{ + CreatedAt: now.Add(-10 * time.Minute), + ConnectionTimeoutSeconds: 60, + } + }, + wantErr: errChatAgentNeverConnected, + }, + { + name: "NeverConnectedWithinTimeoutStaysSoft", + buildAgent: func(now time.Time) database.WorkspaceAgent { + return database.WorkspaceAgent{ + CreatedAt: now.Add(-30 * time.Second), + ConnectionTimeoutSeconds: 120, + } + }, + wantErr: errChatDialTimeout, }, { - name: "PastThresholdEscalates", - disconnectedFor: agentDisconnectedRecoveryThreshold, - wantErr: errChatAgentDisconnected, - wantRecovery: true, + name: "NeverConnectedNoTimeoutStaysSoft", + buildAgent: func(now time.Time) database.WorkspaceAgent { + return database.WorkspaceAgent{ + CreatedAt: now.Add(-10 * time.Minute), + } + }, + wantErr: errChatDialTimeout, }, } for _, tc := range testCases { - tc := tc t.Run(tc.name, func(t *testing.T) { t.Parallel() @@ -2568,27 +2614,14 @@ func TestGetWorkspaceConn_DialTimeoutDisconnectedRecoveryThreshold(t *testing.T) delayTrap := clock.Trap().NewTimer("chatd", dialValidationDelayTimerTag) defer delayTrap.Close() now := clock.Now() - disconnectedAgent := database.WorkspaceAgent{ - ID: agentID, - FirstConnectedAt: sql.NullTime{ - Time: now.Add(-10 * time.Minute), - Valid: true, - }, - LastConnectedAt: sql.NullTime{ - Time: now.Add(-10 * time.Minute), - Valid: true, - }, - DisconnectedAt: sql.NullTime{ - Time: now.Add(-tc.disconnectedFor), - Valid: true, - }, - } + agent := tc.buildAgent(now) + agent.ID = agentID db.EXPECT().GetWorkspaceAgentByID(gomock.Any(), agentID). - Return(disconnectedAgent, nil). + Return(agent, nil). Times(2) db.EXPECT().GetWorkspaceAgentsInLatestBuildByWorkspaceID(gomock.Any(), workspaceID). - Return([]database.WorkspaceAgent{disconnectedAgent}, nil). + Return([]database.WorkspaceAgent{agent}, nil). Times(1) server := &Server{ @@ -2648,11 +2681,12 @@ func TestGetWorkspaceConn_DialTimeoutDisconnectedRecoveryThreshold(t *testing.T) } require.Nil(t, result.conn) require.ErrorIs(t, result.err, tc.wantErr) - if tc.wantRecovery { - require.ErrorIs(t, result.err, errChatAgentDisconnected) - } else { + if !xerrors.Is(tc.wantErr, errChatAgentDisconnected) { require.NotErrorIs(t, result.err, errChatAgentDisconnected) } + if !xerrors.Is(tc.wantErr, errChatAgentNeverConnected) { + require.NotErrorIs(t, result.err, errChatAgentNeverConnected) + } workspaceCtx.mu.Lock() defer workspaceCtx.mu.Unlock() @@ -2880,70 +2914,6 @@ func TestGetWorkspaceConn_DialTimeout(t *testing.T) { require.ErrorIs(t, err, errChatDialTimeout) } -func TestGetWorkspaceConn_DialTimeoutStatusTimeoutDoesNotEscalate(t *testing.T) { - // Agents that never connected are startup failures, not - // disconnected recovery cases. A dial timeout should stay a - // retry/escalation error rather than stop/start guidance. - t.Parallel() - - ctrl := gomock.NewController(t) - db := dbmock.NewMockStore(ctrl) - - workspaceID := uuid.New() - agentID := uuid.New() - chat := database.Chat{ - ID: uuid.New(), - WorkspaceID: uuid.NullUUID{ - UUID: workspaceID, - Valid: true, - }, - AgentID: uuid.NullUUID{ - UUID: agentID, - Valid: true, - }, - } - - timedOutAgent := database.WorkspaceAgent{ - ID: agentID, - CreatedAt: time.Now().Add(-10 * time.Minute), - ConnectionTimeoutSeconds: 60, - } - - db.EXPECT().GetWorkspaceAgentByID(gomock.Any(), agentID). - Return(timedOutAgent, nil). - Times(2) - db.EXPECT().GetWorkspaceAgentsInLatestBuildByWorkspaceID(gomock.Any(), workspaceID). - Return([]database.WorkspaceAgent{timedOutAgent}, nil). - Times(1) - - server := &Server{ - db: db, - clock: quartz.NewReal(), - agentInactiveDisconnectTimeout: 30 * time.Second, - dialTimeout: 10 * time.Millisecond, - } - server.agentConnFn = func(ctx context.Context, _ uuid.UUID) (workspacesdk.AgentConn, func(), error) { - <-ctx.Done() - return nil, nil, ctx.Err() - } - - chatStateMu := &sync.Mutex{} - currentChat := chat - workspaceCtx := turnWorkspaceContext{ - server: server, - chatStateMu: chatStateMu, - currentChat: ¤tChat, - loadChatSnapshot: func(context.Context, uuid.UUID) (database.Chat, error) { return database.Chat{}, nil }, - } - defer workspaceCtx.close() - - ctx := testutil.Context(t, testutil.WaitShort) - gotConn, err := workspaceCtx.getWorkspaceConn(ctx) - require.Nil(t, gotConn) - require.ErrorIs(t, err, errChatDialTimeout) - require.NotErrorIs(t, err, errChatAgentDisconnected) -} - func TestGetWorkspaceConn_DialTimeoutParentCanceled(t *testing.T) { // When the parent context is canceled, the parent's error // must propagate unchanged (not wrapped as a dial timeout). From b120fb201d61204d5b214f48d7854044dd3d3b15 Mon Sep 17 00:00:00 2001 From: Ethan Dickson Date: Wed, 15 Jul 2026 07:08:57 +0000 Subject: [PATCH 2/3] style(coderd/x/chatd): remove redundant comments --- coderd/x/chatd/chatd.go | 3 --- coderd/x/chatd/chatd_internal_test.go | 7 ------- 2 files changed, 10 deletions(-) diff --git a/coderd/x/chatd/chatd.go b/coderd/x/chatd/chatd.go index 0f3c44f228d..c083ce67b4d 100644 --- a/coderd/x/chatd/chatd.go +++ b/coderd/x/chatd/chatd.go @@ -852,9 +852,6 @@ func agentDisconnectedFor(now time.Time, agent database.WorkspaceAgent, inactive return disconnectedFor, true } -// latestWorkspaceAgentRecoveryError classifies the latest agent's status after -// a failed dial. It returns the recovery sentinel to surface, or -// errChatDialTimeout when the dial should remain retryable. func (c *turnWorkspaceContext) latestWorkspaceAgentRecoveryError( ctx context.Context, workspaceID uuid.UUID, diff --git a/coderd/x/chatd/chatd_internal_test.go b/coderd/x/chatd/chatd_internal_test.go index 972b2465b79..c86c0fec360 100644 --- a/coderd/x/chatd/chatd_internal_test.go +++ b/coderd/x/chatd/chatd_internal_test.go @@ -349,8 +349,6 @@ func TestChatWorkspaceRecoveryErrorsDifferentiateSignalStrength(t *testing.T) { require.Contains(t, neverConnected, "start_workspace") require.NotContains(t, neverConnected, "ask_user_question") - // Dial timeout alone is a weak signal. The model should not - // escalate to lifecycle tools without a DB-confirmed recovery condition. dialTimeout := errChatDialTimeout.Error() require.NotContains(t, dialTimeout, "ask_user_question") require.NotContains(t, dialTimeout, "stop_workspace") @@ -2385,9 +2383,6 @@ func TestGetWorkspaceConn_StatusCheck(t *testing.T) { tests := []testCase{ { - // Cache-hit status checks only force a fresh dial for - // disconnected agents. Timeout recovery is classified - // after a failed dial. name: "TimedOutAgentCacheHit", buildAgent: func(now time.Time) database.WorkspaceAgent { return database.WorkspaceAgent{ @@ -2518,8 +2513,6 @@ func TestGetWorkspaceConn_StatusCheck(t *testing.T) { } func TestGetWorkspaceConn_DialTimeoutDisconnectedRecoveryThreshold(t *testing.T) { - // Failed dials use a fresh agent status to distinguish retryable timeouts - // from conditions that require stop/start recovery. t.Parallel() buildDisconnectedAgent := func(disconnectedFor time.Duration) func(time.Time) database.WorkspaceAgent { From dfb9104f6593b0de918b37056f5d765bfee5dfc0 Mon Sep 17 00:00:00 2001 From: Ethan Dickson Date: Wed, 15 Jul 2026 07:37:32 +0000 Subject: [PATCH 3/3] fix(coderd/x/chatd): classify recovery against latest agent --- coderd/x/chatd/chatd.go | 30 ++++++-------- coderd/x/chatd/chatd_internal_test.go | 60 ++++++++++++++++++--------- 2 files changed, 53 insertions(+), 37 deletions(-) diff --git a/coderd/x/chatd/chatd.go b/coderd/x/chatd/chatd.go index c083ce67b4d..09bade41a8a 100644 --- a/coderd/x/chatd/chatd.go +++ b/coderd/x/chatd/chatd.go @@ -876,21 +876,19 @@ func (c *turnWorkspaceContext) latestWorkspaceAgentRecoveryError( now := c.server.clock.Now() status := agent.Status(now, c.server.agentInactiveDisconnectTimeout) + recoveryErr := errChatDialTimeout if status.Status == database.WorkspaceAgentStatusTimeout { - return errChatAgentNeverConnected - } - if status.Status != database.WorkspaceAgentStatusDisconnected || status.DisconnectedAt == nil { - return errChatDialTimeout - } - - disconnectedFor := now.Sub(*status.DisconnectedAt) - if disconnectedFor < 0 { - disconnectedFor = 0 - } - if disconnectedFor >= agentDisconnectedRecoveryThreshold { - return errChatAgentDisconnected + recoveryErr = errChatAgentNeverConnected + } else if status.Status == database.WorkspaceAgentStatusDisconnected && status.DisconnectedAt != nil { + disconnectedFor := now.Sub(*status.DisconnectedAt) + if disconnectedFor < 0 { + disconnectedFor = 0 + } + if disconnectedFor >= agentDisconnectedRecoveryThreshold { + recoveryErr = errChatAgentDisconnected + } } - return errChatDialTimeout + return c.externalAgentError(ctx, agent, recoveryErr) } func (c *turnWorkspaceContext) externalAgentError( @@ -1023,11 +1021,7 @@ func (c *turnWorkspaceContext) getWorkspaceConn(ctx context.Context) (workspaces // propagate unchanged so the chatloop can detect it. if ctx.Err() == nil && errors.Is(context.Cause(dialCtx), errChatDialTimeout) { c.clearCachedWorkspaceState() - recoveryErr := c.latestWorkspaceAgentRecoveryError(ctx, chatSnapshot.WorkspaceID.UUID) - if xerrors.Is(recoveryErr, errChatHasNoWorkspaceAgent) { - return nil, recoveryErr - } - return nil, c.externalAgentError(ctx, agent, recoveryErr) + return nil, c.latestWorkspaceAgentRecoveryError(ctx, chatSnapshot.WorkspaceID.UUID) } return nil, err } diff --git a/coderd/x/chatd/chatd_internal_test.go b/coderd/x/chatd/chatd_internal_test.go index c86c0fec360..916df0f80fb 100644 --- a/coderd/x/chatd/chatd_internal_test.go +++ b/coderd/x/chatd/chatd_internal_test.go @@ -2534,10 +2534,18 @@ func TestGetWorkspaceConn_DialTimeoutDisconnectedRecoveryThreshold(t *testing.T) } } + buildTimedOutAgent := func(now time.Time) database.WorkspaceAgent { + return database.WorkspaceAgent{ + CreatedAt: now.Add(-10 * time.Minute), + ConnectionTimeoutSeconds: 60, + } + } + testCases := []struct { - name string - buildAgent func(now time.Time) database.WorkspaceAgent - wantErr error + name string + buildAgent func(now time.Time) database.WorkspaceAgent + staleExternalBinding bool + wantErr error }{ { name: "RecentDisconnectReturnsDialTimeout", @@ -2550,14 +2558,15 @@ func TestGetWorkspaceConn_DialTimeoutDisconnectedRecoveryThreshold(t *testing.T) wantErr: errChatAgentDisconnected, }, { - name: "NeverConnectedTimeoutEscalates", - buildAgent: func(now time.Time) database.WorkspaceAgent { - return database.WorkspaceAgent{ - CreatedAt: now.Add(-10 * time.Minute), - ConnectionTimeoutSeconds: 60, - } - }, - wantErr: errChatAgentNeverConnected, + name: "NeverConnectedTimeoutEscalates", + buildAgent: buildTimedOutAgent, + wantErr: errChatAgentNeverConnected, + }, + { + name: "StaleExternalBindingUsesLatestInternalAgent", + buildAgent: buildTimedOutAgent, + staleExternalBinding: true, + wantErr: errChatAgentNeverConnected, }, { name: "NeverConnectedWithinTimeoutStaysSoft", @@ -2607,15 +2616,28 @@ func TestGetWorkspaceConn_DialTimeoutDisconnectedRecoveryThreshold(t *testing.T) delayTrap := clock.Trap().NewTimer("chatd", dialValidationDelayTimerTag) defer delayTrap.Close() now := clock.Now() - agent := tc.buildAgent(now) - agent.ID = agentID - - db.EXPECT().GetWorkspaceAgentByID(gomock.Any(), agentID). - Return(agent, nil). - Times(2) - db.EXPECT().GetWorkspaceAgentsInLatestBuildByWorkspaceID(gomock.Any(), workspaceID). - Return([]database.WorkspaceAgent{agent}, nil). + boundAgent := tc.buildAgent(now) + boundAgent.ID = agentID + latestAgent := boundAgent + latestAgent.ID = uuid.New() + latestAgentLookups := 1 + if tc.staleExternalBinding { + boundAgent.ResourceID = uuid.New() + latestAgent.ResourceID = uuid.Nil + latestAgentLookups++ + db.EXPECT().GetWorkspaceResourceByID(gomock.Any(), boundAgent.ResourceID). + Return(database.WorkspaceResource{Type: chattool.ExternalAgentResourceType}, nil). + AnyTimes() + } + db.EXPECT().GetWorkspaceAgentByID(gomock.Any(), boundAgent.ID). + Return(boundAgent, nil). Times(1) + db.EXPECT().GetWorkspaceAgentByID(gomock.Any(), latestAgent.ID). + Return(latestAgent, nil). + Times(1) + db.EXPECT().GetWorkspaceAgentsInLatestBuildByWorkspaceID(gomock.Any(), workspaceID). + Return([]database.WorkspaceAgent{latestAgent}, nil). + Times(latestAgentLookups) server := &Server{ db: db,