diff --git a/coderd/x/chatd/active_turn_debug.go b/coderd/x/chatd/active_turn_debug.go index 2fbd653a04130..d4ebf0c4b17e7 100644 --- a/coderd/x/chatd/active_turn_debug.go +++ b/coderd/x/chatd/active_turn_debug.go @@ -9,6 +9,7 @@ import ( "cdr.dev/slog/v3" "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/x/chatd/chatdebug" + "github.com/coder/coder/v2/coderd/x/chatd/mcpclient" ) type runnerDebugTurn struct { @@ -49,7 +50,14 @@ func (d *runnerDebugTurn) Ensure( d.mu.Lock() defer d.mu.Unlock() + return d.ensureLocked(ctx, chat, debug) +} +func (d *runnerDebugTurn) ensureLocked( + ctx context.Context, + chat database.Chat, + debug *generationDebug, +) context.Context { // Check finalized/disabled before created: once the turn is // finalized, new contexts must not be attributed to the // finalized run, even if it was created earlier. @@ -68,6 +76,17 @@ func (d *runnerDebugTurn) Ensure( seedSummary := chatdebug.SeedSummary( chatdebug.TruncateLabel(debug.TriggerLabel, chatdebug.MaxLabelLength), ) + // Carry per-server MCP connect outcomes (and their dropped + // count) stashed by RecordMCPConnectSummaries before the run + // existed, so slow or failing servers appear in the run instead + // of as a silent gap before the first step. Seeded keys survive + // FinalizeRun's summary aggregation. + for key, stashed := range d.seedSummary { + if seedSummary == nil { + seedSummary = make(map[string]any, len(d.seedSummary)) + } + seedSummary[key] = stashed + } rootChatID := uuid.Nil if chat.RootChatID.Valid { rootChatID = chat.RootChatID.UUID @@ -121,6 +140,69 @@ func (d *runnerDebugTurn) Context(ctx context.Context) context.Context { return d.contextLocked(ctx) } +// RecordMCPConnectSummaries merges one preparation's per-server MCP +// connect outcomes into the mcp_connect summary key, creating the +// debug run if it does not exist yet. Preparation invokes it as soon +// as its MCP connect phase completes, so every attempt is recorded: +// preparations that fail after connecting, decision errors, and +// actions that never reach Ensure (local tool execution, +// requires-action, turn finishing). Creating the run here matters +// when the first preparation connects and then fails: no model or +// compaction action ever runs Ensure, and Finalize discards the +// stash of a never-created run. +func (d *runnerDebugTurn) RecordMCPConnectSummaries( + ctx context.Context, + chat database.Chat, + debug *generationDebug, + summaries []mcpclient.ConnectSummary, +) { + if d == nil || len(summaries) == 0 { + return + } + d.mu.Lock() + defer d.mu.Unlock() + if d.disabled || d.finalized { + return + } + // Merge before ensuring so the outcomes ride the seed summary + // when this call is the one that creates the run. + d.mergeMCPConnectSummariesLocked(summaries) + d.ensureLocked(ctx, chat, debug) +} + +// maxMCPConnectSummaryEntries bounds the retained per-preparation MCP +// connect outcomes. A turn may run up to 1,200 generation steps, each +// reconnecting to every selected server, so unbounded retention could +// grow one run summary to megabytes of database and API payload. The +// newest entries win because the tail of the history is what shows a +// server that degraded mid-turn; the mcp_connect_dropped count keeps +// the truncation visible. +const maxMCPConnectSummaryEntries = 100 + +// mergeMCPConnectSummariesLocked appends a preparation's per-server +// MCP connect outcomes to the mcp_connect summary key. chatd +// reconnects to every configured MCP server on each generation step +// while the run is created only once, so without this merge only +// one preparation's outcomes would survive to the finalized run and +// a server that degrades mid-turn would still be reported as +// connected. +func (d *runnerDebugTurn) mergeMCPConnectSummariesLocked(summaries []mcpclient.ConnectSummary) { + if len(summaries) == 0 { + return + } + if d.seedSummary == nil { + d.seedSummary = make(map[string]any, 1) + } + existing, _ := d.seedSummary["mcp_connect"].([]mcpclient.ConnectSummary) + existing = append(existing, summaries...) + if over := len(existing) - maxMCPConnectSummaryEntries; over > 0 { + existing = existing[over:] + dropped, _ := d.seedSummary["mcp_connect_dropped"].(int) + d.seedSummary["mcp_connect_dropped"] = dropped + over + } + d.seedSummary["mcp_connect"] = existing +} + func (d *runnerDebugTurn) contextLocked(ctx context.Context) context.Context { if !d.created || d.runContext.RunID == uuid.Nil { return ctx diff --git a/coderd/x/chatd/active_turn_debug_internal_test.go b/coderd/x/chatd/active_turn_debug_internal_test.go index f599021853eaf..1a02b604577c1 100644 --- a/coderd/x/chatd/active_turn_debug_internal_test.go +++ b/coderd/x/chatd/active_turn_debug_internal_test.go @@ -3,6 +3,8 @@ package chatd import ( "context" "database/sql" + "encoding/json" + "fmt" "testing" "github.com/google/uuid" @@ -12,6 +14,7 @@ import ( "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/database/dbmock" "github.com/coder/coder/v2/coderd/x/chatd/chatdebug" + "github.com/coder/coder/v2/coderd/x/chatd/mcpclient" "github.com/coder/coder/v2/testutil" ) @@ -153,3 +156,179 @@ func TestRunnerDebugTurnFinalizeOnce(t *testing.T) { turn.Finalize(ctx) turn.Finalize(ctx) } + +func TestRunnerDebugTurnRecordMCPConnectSummaries(t *testing.T) { + t.Parallel() + + ctx := testutil.Context(t, testutil.WaitLong) + runnerCtx, cancel := context.WithCancel(ctx) + defer cancel() + ctrl := gomock.NewController(t) + db := dbmock.NewMockStore(ctrl) + chatID := uuid.New() + runID := uuid.New() + configID := uuid.New() + svc := chatdebug.NewService(db, testutil.Logger(t), nil) + turn := newRunnerDebugTurn(runnerCtx, testutil.Logger(t)) + + var seededSummary []byte + db.EXPECT().InsertChatDebugRun(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, params database.InsertChatDebugRunParams) (database.ChatDebugRun, error) { + require.True(t, params.Summary.Valid) + seededSummary = params.Summary.RawMessage + return database.ChatDebugRun{ + ID: runID, + ChatID: chatID, + Kind: string(chatdebug.KindChatTurn), + Status: string(chatdebug.StatusInProgress), + }, nil + }). + Times(1) + db.EXPECT().GetChatDebugStepsByRunID(gomock.Any(), runID).Return(nil, nil).Times(1) + var finalSummary []byte + db.EXPECT().UpdateChatDebugRun(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, params database.UpdateChatDebugRunParams) (database.ChatDebugRun, error) { + require.True(t, params.Summary.Valid) + finalSummary = params.Summary.RawMessage + return database.ChatDebugRun{ID: runID, ChatID: chatID}, nil + }).Times(1) + + debug := generationDebug{ + Enabled: true, + Service: svc, + TriggerMessageID: 1, + ModelConfig: database.ChatModelConfig{ID: uuid.New()}, + } + first := []mcpclient.ConnectSummary{{ + ConfigID: configID, + Slug: "registry", + Outcome: mcpclient.ConnectOutcomeConnected, + DurationMS: 17, + ToolCount: 1, + }} + second := []mcpclient.ConnectSummary{{ + ConfigID: configID, + Slug: "registry", + Outcome: mcpclient.ConnectOutcomeTimeout, + DurationMS: 10000, + Error: "connect: context deadline exceeded", + }} + + // Preparation records each attempt as soon as its connect phase + // completes; the first record creates the run with the outcome + // already seeded, and a later Ensure must not create a second + // run. + turn.RecordMCPConnectSummaries(ctx, database.Chat{ID: chatID}, &debug, first) + turn.Ensure(ctx, database.Chat{ID: chatID}, &debug) + // A later generation step reconnects and reports a degraded + // outcome for the same server; its action may never reach + // Ensure, and the outcome must still survive to the finalized + // summary. + turn.RecordMCPConnectSummaries(ctx, database.Chat{ID: chatID}, &debug, second) + turn.RecordOutcome(chatdebug.StatusCompleted) + turn.Finalize(ctx) + + var seeded struct { + MCPConnect []mcpclient.ConnectSummary `json:"mcp_connect"` + } + require.NoError(t, json.Unmarshal(seededSummary, &seeded)) + require.Len(t, seeded.MCPConnect, 1) + require.Equal(t, mcpclient.ConnectOutcomeConnected, seeded.MCPConnect[0].Outcome) + + var summary struct { + MCPConnect []mcpclient.ConnectSummary `json:"mcp_connect"` + } + require.NoError(t, json.Unmarshal(finalSummary, &summary)) + require.Len(t, summary.MCPConnect, 2) + require.Equal(t, mcpclient.ConnectOutcomeConnected, summary.MCPConnect[0].Outcome) + require.Equal(t, mcpclient.ConnectOutcomeTimeout, summary.MCPConnect[1].Outcome) +} + +func TestRunnerDebugTurnBoundsMCPConnectSummaries(t *testing.T) { + t.Parallel() + + ctx := testutil.Context(t, testutil.WaitLong) + runnerCtx, cancel := context.WithCancel(ctx) + defer cancel() + ctrl := gomock.NewController(t) + db := dbmock.NewMockStore(ctrl) + chatID := uuid.New() + runID := uuid.New() + svc := chatdebug.NewService(db, testutil.Logger(t), nil) + turn := newRunnerDebugTurn(runnerCtx, testutil.Logger(t)) + + var seededSummary []byte + db.EXPECT().InsertChatDebugRun(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, params database.InsertChatDebugRunParams) (database.ChatDebugRun, error) { + require.True(t, params.Summary.Valid) + seededSummary = params.Summary.RawMessage + return database.ChatDebugRun{ + ID: runID, + ChatID: chatID, + Kind: string(chatdebug.KindChatTurn), + Status: string(chatdebug.StatusInProgress), + }, nil + }). + Times(1) + db.EXPECT().GetChatDebugStepsByRunID(gomock.Any(), runID).Return(nil, nil).Times(1) + var finalSummary []byte + db.EXPECT().UpdateChatDebugRun(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, params database.UpdateChatDebugRunParams) (database.ChatDebugRun, error) { + require.True(t, params.Summary.Valid) + finalSummary = params.Summary.RawMessage + return database.ChatDebugRun{ID: runID, ChatID: chatID}, nil + }).Times(1) + + debug := generationDebug{ + Enabled: true, + Service: svc, + TriggerMessageID: 1, + ModelConfig: database.ChatModelConfig{ID: uuid.New()}, + } + // 25 preparations against 5 servers produce 125 outcomes, + // overflowing the cap. The first record creates the run. + for prep := 0; prep < 25; prep++ { + batch := make([]mcpclient.ConnectSummary, 5) + for server := range batch { + batch[server] = mcpclient.ConnectSummary{ + ConfigID: uuid.New(), + Slug: fmt.Sprintf("server-%d", server), + Outcome: mcpclient.ConnectOutcomeConnected, + DurationMS: int64(prep), + } + } + turn.RecordMCPConnectSummaries(ctx, database.Chat{ID: chatID}, &debug, batch) + } + + type boundedSummary struct { + MCPConnect []mcpclient.ConnectSummary `json:"mcp_connect"` + MCPConnectDropped int `json:"mcp_connect_dropped"` + } + // The run was created by the first record, so the seed carries + // only that preparation's outcomes. + var seeded boundedSummary + require.NoError(t, json.Unmarshal(seededSummary, &seeded)) + require.Len(t, seeded.MCPConnect, 5) + require.Zero(t, seeded.MCPConnectDropped) + + // One more preparation still respects the cap and grows the + // dropped count. + turn.RecordMCPConnectSummaries(ctx, database.Chat{ID: chatID}, &debug, []mcpclient.ConnectSummary{{ + ConfigID: uuid.New(), + Slug: "server-0", + Outcome: mcpclient.ConnectOutcomeTimeout, + DurationMS: 10000, + }}) + turn.RecordOutcome(chatdebug.StatusCompleted) + turn.Finalize(ctx) + + var final boundedSummary + require.NoError(t, json.Unmarshal(finalSummary, &final)) + require.Len(t, final.MCPConnect, maxMCPConnectSummaryEntries) + require.Equal(t, 26, final.MCPConnectDropped) + // The newest outcomes win: the oldest five preparations were + // dropped, so the retained history starts at preparation 5 and + // ends with the timeout recorded above. + require.Equal(t, int64(5), final.MCPConnect[0].DurationMS) + require.Equal(t, mcpclient.ConnectOutcomeTimeout, final.MCPConnect[maxMCPConnectSummaryEntries-1].Outcome) +} diff --git a/coderd/x/chatd/chatd.go b/coderd/x/chatd/chatd.go index fbb3a22bf368f..1c3570f9360c5 100644 --- a/coderd/x/chatd/chatd.go +++ b/coderd/x/chatd/chatd.go @@ -84,6 +84,12 @@ const ( DefaultChatHeartbeatInterval = 30 * time.Second maxChatSteps = 1200 + // slowPrepareThreshold is the generation-preparation duration + // above which a warning is logged. Preparation runs before + // every generation step, so sustained slowness (workspace + // dials, MCP connects) taxes the whole turn. + slowPrepareThreshold = 30 * time.Second + // maxConcurrentRecordingUploads caps the number of recording // stop-and-store operations that can run concurrently. Each // slot buffers up to MaxRecordingSize + MaxThumbnailSize diff --git a/coderd/x/chatd/chatd_test.go b/coderd/x/chatd/chatd_test.go index bfc2c946e2c72..1e10b9c323293 100644 --- a/coderd/x/chatd/chatd_test.go +++ b/coderd/x/chatd/chatd_test.go @@ -11193,6 +11193,429 @@ func TestMCPServerToolInvocation(t *testing.T) { "MCP tool result should be persisted as a tool message in the database") } +func TestActiveServer_ChatTurnDebugRunRecordsMCPConnectPerPreparation(t *testing.T) { + t.Parallel() + + db, ps := dbtestutil.NewDB(t) + ctx := testutil.Context(t, testutil.WaitLong) + + mcpSrv := newTestMCPServer("test-mcp") + addTestMCPTextTool(mcpSrv, "echo", "Echoes the input", "echo: ") + mcpTS := httptest.NewServer(testMCPHTTPHandler(mcpSrv)) + t.Cleanup(mcpTS.Close) + + var callCount atomic.Int32 + openAIURL := chattest.NewOpenAI(t, func(req *chattest.OpenAIRequest) chattest.OpenAIResponse { + if !req.Stream { + return chattest.OpenAINonStreamingResponse("title") + } + if callCount.Add(1) == 1 { + return chattest.OpenAIStreamingResponse( + chattest.OpenAIToolCallChunk( + "test-mcp__echo", + `{"input":"hello from LLM"}`, + ), + ) + } + return chattest.OpenAIStreamingResponse( + chattest.OpenAITextChunks("Got it!")..., + ) + }) + + user, org, model := seedChatDependenciesWithProvider(t, db, "openai-compat", openAIURL) + mcpConfig := dbgen.MCPServerConfig(t, db, database.MCPServerConfig{ + OrganizationID: org.ID, + DisplayName: "Test MCP", + Slug: "test-mcp", + Url: mcpTS.URL, + CreatedBy: uuid.NullUUID{UUID: user.ID, Valid: true}, + UpdatedBy: uuid.NullUUID{UUID: user.ID, Valid: true}, + }) + + server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) { + withoutMCPToolSearch(cfg) + cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(chattest.NewMockAIBridgeTransport(t, openAIURL)) + cfg.AlwaysEnableDebugLogs = true + }) + + chat, err := server.CreateChat(ctx, chatd.CreateOptions{ + OrganizationID: org.ID, + OwnerID: user.ID, + Title: "mcp-connect-debug-test", + ModelConfigID: model.ID, + MCPServerIDs: []uuid.UUID{mcpConfig.ID}, + InitialUserContent: []codersdk.ChatMessagePart{ + codersdk.ChatMessageText("Echo something via MCP."), + }, + }) + require.NoError(t, err) + + waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusWaiting) + require.NoError(t, server.Close()) + debugCtx := testutil.Context(t, testutil.WaitLong) + + var chatTurnRuns []database.ChatDebugRun + testutil.Eventually(debugCtx, t, func(ctx context.Context) bool { + runs, runsErr := db.GetChatDebugRunsByChatID(ctx, database.GetChatDebugRunsByChatIDParams{ + ChatID: chat.ID, + LimitVal: 100, + }) + if runsErr != nil { + return false + } + chatTurnRuns = chatTurnRuns[:0] + for _, run := range runs { + if run.Kind == string(codersdk.ChatDebugRunKindChatTurn) { + chatTurnRuns = append(chatTurnRuns, run) + } + } + return len(chatTurnRuns) == 1 && chatTurnRuns[0].FinishedAt.Valid + }, testutil.IntervalFast) + + require.Len(t, chatTurnRuns, 1) + var summary struct { + MCPConnect []struct { + Slug string `json:"slug"` + Outcome string `json:"outcome"` + } `json:"mcp_connect"` + } + require.NoError(t, json.Unmarshal(chatTurnRuns[0].Summary, &summary)) + // A tool-call turn runs at least three preparations: the + // assistant step that requests the tool, the local tool + // execution step, and the assistant step that consumes the + // result. Each preparation reconnects to the MCP server and + // must contribute its connect outcome to the finalized run. + require.GreaterOrEqual(t, len(summary.MCPConnect), 3) + for _, entry := range summary.MCPConnect { + require.Equal(t, "test-mcp", entry.Slug) + require.Equal(t, "connected", entry.Outcome) + } +} + +func TestActiveServer_ChatTurnDebugRunRecordsMCPConnectOnDecisionError(t *testing.T) { + t.Parallel() + + db, ps := dbtestutil.NewDB(t) + ctx := testutil.Context(t, testutil.WaitLong) + + mcpSrv := newTestMCPServer("test-mcp") + addTestMCPTextTool(mcpSrv, "echo", "Echoes the input", "echo: ") + mcpTS := httptest.NewServer(testMCPHTTPHandler(mcpSrv)) + t.Cleanup(mcpTS.Close) + + // Every streamed assistant response is a high-usage tool call, + // so the turn compacts once and the post-compaction assistant + // is still over the context limit: the next decision fails + // terminally with the still-over-limit error. + var streamCount atomic.Int32 + anthropicURL := chattest.NewAnthropic(t, func(req *chattest.AnthropicRequest) chattest.AnthropicResponse { + if !req.Stream { + if strings.Contains(anthropicRequestBody(t, *req), "You are performing a context compaction") { + return anthropicCompactionResponse("summary text for compaction") + } + return chattest.AnthropicNonStreamingResponse("title") + } + if streamCount.Add(1) == 1 { + return highUsageReadFileResponse("/tmp/a.txt") + } + return highUsageReadFileResponse("/tmp/b.txt") + }) + + user, org, model := seedAnthropicChatDependencies(t, db, anthropicURL) + model = updateChatModelCompressionThreshold(t, db, model, 100, 70) + mcpConfig := dbgen.MCPServerConfig(t, db, database.MCPServerConfig{ + OrganizationID: org.ID, + DisplayName: "Test MCP", + Slug: "test-mcp", + Url: mcpTS.URL, + CreatedBy: uuid.NullUUID{UUID: user.ID, Valid: true}, + UpdatedBy: uuid.NullUUID{UUID: user.ID, Valid: true}, + }) + ws, dbAgent := seedWorkspaceWithAgent(t, db, user.ID) + + ctrl := gomock.NewController(t) + mockConn := agentconnmock.NewMockAgentConn(ctrl) + setupToolExecutionAgentConn(t, mockConn) + mockConn.EXPECT().ReadFileLines(gomock.Any(), "/tmp/a.txt", int64(1), int64(0), gomock.Any()). + Return(workspacesdk.ReadFileLinesResponse{Success: true, FileSize: 12, TotalLines: 1, LinesRead: 1, Content: "1\tpackage main"}, nil). + Times(1) + mockConn.EXPECT().ReadFileLines(gomock.Any(), "/tmp/b.txt", int64(1), int64(0), gomock.Any()). + Return(workspacesdk.ReadFileLinesResponse{Success: true, FileSize: 12, TotalLines: 1, LinesRead: 1, Content: "1\tpackage main"}, nil). + Times(1) + + server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) { + withoutMCPToolSearch(cfg) + cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(chattest.NewMockAIBridgeTransport(t, anthropicURL, chattest.WithPreservePath())) + cfg.AlwaysEnableDebugLogs = true + cfg.AgentConn = func(_ context.Context, agentID uuid.UUID) (workspacesdk.AgentConn, func(), error) { + require.Equal(t, dbAgent.ID, agentID) + return mockConn, func() {}, nil + } + }) + + chat, err := server.CreateChat(ctx, chatd.CreateOptions{ + OrganizationID: org.ID, + OwnerID: user.ID, + WorkspaceID: uuid.NullUUID{UUID: ws.ID, Valid: true}, + AgentID: uuid.NullUUID{UUID: dbAgent.ID, Valid: true}, + Title: "mcp-connect-decision-error-test", + ModelConfigID: model.ID, + MCPServerIDs: []uuid.UUID{mcpConfig.ID}, + InitialUserContent: []codersdk.ChatMessagePart{ + codersdk.ChatMessageText("read the files"), + }, + }) + require.NoError(t, err) + + waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusError) + require.NoError(t, server.Close()) + debugCtx := testutil.Context(t, testutil.WaitLong) + + var chatTurnRuns []database.ChatDebugRun + testutil.Eventually(debugCtx, t, func(ctx context.Context) bool { + runs, runsErr := db.GetChatDebugRunsByChatID(ctx, database.GetChatDebugRunsByChatIDParams{ + ChatID: chat.ID, + LimitVal: 100, + }) + if runsErr != nil { + return false + } + chatTurnRuns = chatTurnRuns[:0] + for _, run := range runs { + if run.Kind == string(codersdk.ChatDebugRunKindChatTurn) { + chatTurnRuns = append(chatTurnRuns, run) + } + } + return len(chatTurnRuns) == 1 && chatTurnRuns[0].FinishedAt.Valid + }, testutil.IntervalFast) + + require.Len(t, chatTurnRuns, 1) + var summary struct { + MCPConnect []struct { + Slug string `json:"slug"` + Outcome string `json:"outcome"` + } `json:"mcp_connect"` + } + require.NoError(t, json.Unmarshal(chatTurnRuns[0].Summary, &summary)) + // The turn performs six preparations: the first assistant step, + // its tool execution, the compaction, the post-compaction + // assistant step, its tool execution, and the preparation whose + // decision fails with still-over-limit. Each one must + // contribute its connect outcome, including the final + // preparation that exits through the decision error. + require.GreaterOrEqual(t, len(summary.MCPConnect), 6) + for _, entry := range summary.MCPConnect { + require.Equal(t, "test-mcp", entry.Slug) + require.Equal(t, "connected", entry.Outcome) + } +} + +// overrideReadFailStore wraps a database.Store so +// GetChatOrganizationModelOverride fails once the test arms it, +// making a later generation preparation fail after its MCP connect +// phase has already completed. +type overrideReadFailStore struct { + database.Store + fail *atomic.Bool +} + +func (s *overrideReadFailStore) GetChatOrganizationModelOverride(ctx context.Context, arg database.GetChatOrganizationModelOverrideParams) (database.ChatOrganizationModelOverride, error) { + if s.fail.Load() { + return database.ChatOrganizationModelOverride{}, xerrors.New("injected compaction override read failure") + } + return s.Store.GetChatOrganizationModelOverride(ctx, arg) +} + +func TestActiveServer_ChatTurnDebugRunRecordsMCPConnectOnPrepareError(t *testing.T) { + t.Parallel() + + db, ps := dbtestutil.NewDB(t) + ctx := testutil.Context(t, testutil.WaitLong) + + mcpSrv := newTestMCPServer("test-mcp") + addTestMCPTextTool(mcpSrv, "echo", "Echoes the input", "echo: ") + mcpTS := httptest.NewServer(testMCPHTTPHandler(mcpSrv)) + t.Cleanup(mcpTS.Close) + + // Serving the first assistant stream arms the failure, so the + // next preparation completes its MCP connect phase and then + // fails reading the compaction override before returning a + // prepared generation. + var failOverrideReads atomic.Bool + openAIURL := chattest.NewOpenAI(t, func(req *chattest.OpenAIRequest) chattest.OpenAIResponse { + if !req.Stream { + return chattest.OpenAINonStreamingResponse("title") + } + failOverrideReads.Store(true) + return chattest.OpenAIStreamingResponse( + chattest.OpenAITextChunks("Done!")..., + ) + }) + + user, org, model := seedChatDependenciesWithProvider(t, db, "openai-compat", openAIURL) + mcpConfig := dbgen.MCPServerConfig(t, db, database.MCPServerConfig{ + OrganizationID: org.ID, + DisplayName: "Test MCP", + Slug: "test-mcp", + Url: mcpTS.URL, + CreatedBy: uuid.NullUUID{UUID: user.ID, Valid: true}, + UpdatedBy: uuid.NullUUID{UUID: user.ID, Valid: true}, + }) + + server := newActiveTestServer(t, &overrideReadFailStore{Store: db, fail: &failOverrideReads}, ps, func(cfg *chatd.Config) { + withoutMCPToolSearch(cfg) + cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(chattest.NewMockAIBridgeTransport(t, openAIURL)) + cfg.AlwaysEnableDebugLogs = true + }) + + chat, err := server.CreateChat(ctx, chatd.CreateOptions{ + OrganizationID: org.ID, + OwnerID: user.ID, + Title: "mcp-connect-prepare-error-test", + ModelConfigID: model.ID, + MCPServerIDs: []uuid.UUID{mcpConfig.ID}, + InitialUserContent: []codersdk.ChatMessagePart{ + codersdk.ChatMessageText("Say done."), + }, + }) + require.NoError(t, err) + + waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusError) + require.NoError(t, server.Close()) + debugCtx := testutil.Context(t, testutil.WaitLong) + + var chatTurnRuns []database.ChatDebugRun + testutil.Eventually(debugCtx, t, func(ctx context.Context) bool { + runs, runsErr := db.GetChatDebugRunsByChatID(ctx, database.GetChatDebugRunsByChatIDParams{ + ChatID: chat.ID, + LimitVal: 100, + }) + if runsErr != nil { + return false + } + chatTurnRuns = chatTurnRuns[:0] + for _, run := range runs { + if run.Kind == string(codersdk.ChatDebugRunKindChatTurn) { + chatTurnRuns = append(chatTurnRuns, run) + } + } + return len(chatTurnRuns) == 1 && chatTurnRuns[0].FinishedAt.Valid + }, testutil.IntervalFast) + + require.Len(t, chatTurnRuns, 1) + var summary struct { + MCPConnect []struct { + Slug string `json:"slug"` + Outcome string `json:"outcome"` + } `json:"mcp_connect"` + } + require.NoError(t, json.Unmarshal(chatTurnRuns[0].Summary, &summary)) + // The preparation feeding the assistant step records one + // outcome. The following preparation connects to the MCP server + // and then fails reading the compaction override; its attempts + // must still contribute their connect outcomes even though + // preparation never returns a prepared generation. + require.GreaterOrEqual(t, len(summary.MCPConnect), 2) + for _, entry := range summary.MCPConnect { + require.Equal(t, "test-mcp", entry.Slug) + require.Equal(t, "connected", entry.Outcome) + } +} + +func TestActiveServer_ChatTurnDebugRunRecordsMCPConnectOnFirstPrepareError(t *testing.T) { + t.Parallel() + + db, ps := dbtestutil.NewDB(t) + ctx := testutil.Context(t, testutil.WaitLong) + + mcpSrv := newTestMCPServer("test-mcp") + addTestMCPTextTool(mcpSrv, "echo", "Echoes the input", "echo: ") + mcpTS := httptest.NewServer(testMCPHTTPHandler(mcpSrv)) + t.Cleanup(mcpTS.Close) + + openAIURL := chattest.NewOpenAI(t, func(req *chattest.OpenAIRequest) chattest.OpenAIResponse { + if !req.Stream { + return chattest.OpenAINonStreamingResponse("title") + } + return chattest.OpenAIStreamingResponse( + chattest.OpenAITextChunks("Done!")..., + ) + }) + + user, org, model := seedChatDependenciesWithProvider(t, db, "openai-compat", openAIURL) + mcpConfig := dbgen.MCPServerConfig(t, db, database.MCPServerConfig{ + OrganizationID: org.ID, + DisplayName: "Test MCP", + Slug: "test-mcp", + Url: mcpTS.URL, + CreatedBy: uuid.NullUUID{UUID: user.ID, Valid: true}, + UpdatedBy: uuid.NullUUID{UUID: user.ID, Valid: true}, + }) + + // Armed from the start: the very first preparation connects to + // the MCP server and then fails reading the compaction override, + // so no model or compaction action ever creates the debug run + // through Ensure. + var failOverrideReads atomic.Bool + failOverrideReads.Store(true) + server := newActiveTestServer(t, &overrideReadFailStore{Store: db, fail: &failOverrideReads}, ps, func(cfg *chatd.Config) { + withoutMCPToolSearch(cfg) + cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(chattest.NewMockAIBridgeTransport(t, openAIURL)) + cfg.AlwaysEnableDebugLogs = true + }) + + chat, err := server.CreateChat(ctx, chatd.CreateOptions{ + OrganizationID: org.ID, + OwnerID: user.ID, + Title: "mcp-connect-first-prepare-error-test", + ModelConfigID: model.ID, + MCPServerIDs: []uuid.UUID{mcpConfig.ID}, + InitialUserContent: []codersdk.ChatMessagePart{ + codersdk.ChatMessageText("Say done."), + }, + }) + require.NoError(t, err) + + waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusError) + require.NoError(t, server.Close()) + debugCtx := testutil.Context(t, testutil.WaitLong) + + var chatTurnRuns []database.ChatDebugRun + testutil.Eventually(debugCtx, t, func(ctx context.Context) bool { + runs, runsErr := db.GetChatDebugRunsByChatID(ctx, database.GetChatDebugRunsByChatIDParams{ + ChatID: chat.ID, + LimitVal: 100, + }) + if runsErr != nil { + return false + } + chatTurnRuns = chatTurnRuns[:0] + for _, run := range runs { + if run.Kind == string(codersdk.ChatDebugRunKindChatTurn) { + chatTurnRuns = append(chatTurnRuns, run) + } + } + return len(chatTurnRuns) == 1 && chatTurnRuns[0].FinishedAt.Valid + }, testutil.IntervalFast) + + require.Len(t, chatTurnRuns, 1) + var summary struct { + MCPConnect []struct { + Slug string `json:"slug"` + Outcome string `json:"outcome"` + } `json:"mcp_connect"` + } + require.NoError(t, json.Unmarshal(chatTurnRuns[0].Summary, &summary)) + // Every preparation fails after its connect phase, so the run + // exists only because recording the outcomes created it; it must + // carry at least the first attempt's connect outcome. + require.GreaterOrEqual(t, len(summary.MCPConnect), 1) + for _, entry := range summary.MCPConnect { + require.Equal(t, "test-mcp", entry.Slug) + require.Equal(t, "connected", entry.Outcome) + } +} + func TestPlanModeRootChatApprovedExternalMCPToolInvocation(t *testing.T) { t.Parallel() diff --git a/coderd/x/chatd/generation.go b/coderd/x/chatd/generation.go index 33eb7ff7c33f4..bff6f21480db1 100644 --- a/coderd/x/chatd/generation.go +++ b/coderd/x/chatd/generation.go @@ -24,6 +24,7 @@ import ( "github.com/coder/coder/v2/coderd/x/chatd/chatretry" "github.com/coder/coder/v2/coderd/x/chatd/chatstate" "github.com/coder/coder/v2/coderd/x/chatd/chattool" + "github.com/coder/coder/v2/coderd/x/chatd/mcpclient" "github.com/coder/coder/v2/coderd/x/chatd/messagepartbuffer" "github.com/coder/coder/v2/codersdk" "github.com/coder/coder/v2/codersdk/x/agenthooks" @@ -34,6 +35,19 @@ import ( type generationPrepareInput struct { Chat database.Chat Messages []database.ChatMessage + // RecordMCPConnectSummaries receives the preparation's per-server + // MCP connect outcomes as soon as the connect phase completes, + // with the debug context needed to create the run when no action + // ever reaches Ensure. Preparation invokes it directly (rather + // than returning the outcomes) so attempts that fail after + // connecting still record before their error discards the + // prepared state. + RecordMCPConnectSummaries func( + ctx context.Context, + chat database.Chat, + debug *generationDebug, + summaries []mcpclient.ConnectSummary, + ) } // generationPrepared contains the side-effect inputs for a generation task. @@ -441,8 +455,9 @@ func (s *taskStarter) StartGeneration(ctx context.Context, input chatWorkerTaskS } } prepareInput := generationPrepareInput{ - Chat: chat, - Messages: messages, + Chat: chat, + Messages: messages, + RecordMCPConnectSummaries: input.DebugTurn.RecordMCPConnectSummaries, } prepared, err := retryGenerationPhase(ctx, s, "prepare", func() (generationPrepared, error) { return s.server.prepareGeneration(ctx, prepareInput) diff --git a/coderd/x/chatd/generation_preparer.go b/coderd/x/chatd/generation_preparer.go index bb79819a3f56a..ff49843deb647 100644 --- a/coderd/x/chatd/generation_preparer.go +++ b/coderd/x/chatd/generation_preparer.go @@ -76,6 +76,15 @@ func (server *Server) prepareGeneration( slog.F("owner_id", chat.OwnerID), ) + prepStart := server.clock.Now() + defer func() { + if prepDuration := server.clock.Since(prepStart); prepDuration >= slowPrepareThreshold { + logger.Warn(ctx, "slow generation preparation", + slog.F("duration", prepDuration), + ) + } + }() + var ( promptRows []database.ChatMessage mcpConfigs []database.MCPServerConfig @@ -263,6 +272,7 @@ func (server *Server) prepareGeneration( prompt []fantasy.Message instruction string mcpTools []fantasy.AgentTool + mcpSummaries []mcpclient.ConnectSummary mcpCleanup func() workspaceMCPTools []fantasy.AgentTool workspaceSkills []chattool.SkillMeta @@ -304,6 +314,29 @@ func (server *Server) prepareGeneration( } } + // Build the debug context before the connect phase so its + // outcomes can be recorded with run-creation context even when a + // later preparation step fails. + triggerMessageID, historyTipMessageID, triggerLabel := deriveChatDebugSeed(promptRows) + debugSvc := server.existingDebugService() + var debug *generationDebug + if resolved.debugEnabled { + if debugSvc == nil { + cleanup() + return generationPrepared{}, xerrors.New("chat debug service missing after enablement check") + } + debug = &generationDebug{ + Enabled: true, + Service: debugSvc, + Provider: resolved.resolvedProvider, + Model: resolved.resolvedModel, + TriggerMessageID: triggerMessageID, + HistoryTipMessageID: historyTipMessageID, + TriggerLabel: triggerLabel, + ModelConfig: modelConfig, + } + } + var g2 errgroup.Group g2.Go(func() error { var err error @@ -336,7 +369,7 @@ func (server *Server) prepareGeneration( logger.Warn(ctx, "failed to load MCP user tokens", slog.Error(tokenErr)) } mcpTokens = server.refreshExpiredMCPTokens(ctx, logger, mcpConnectConfigs, mcpTokens) - mcpTools, mcpCleanup = mcpclient.ConnectAll( + mcpTools, mcpSummaries, mcpCleanup = mcpclient.ConnectAll( ctx, logger, mcpConnectConfigs, @@ -365,9 +398,16 @@ func (server *Server) prepareGeneration( return nil }) } - if err := g2.Wait(); err != nil { + g2Err := g2.Wait() + // Record connect outcomes before acting on any preparation error: + // ConnectAll has already run, so a failure below (or in g2 itself) + // would otherwise discard this attempt's outcomes. + if debug != nil && input.RecordMCPConnectSummaries != nil && len(mcpSummaries) > 0 { + input.RecordMCPConnectSummaries(ctx, chat, debug, mcpSummaries) + } + if g2Err != nil { cleanup() - return generationPrepared{}, err + return generationPrepared{}, g2Err } if mcpCleanup != nil { @@ -635,26 +675,6 @@ func (server *Server) prepareGeneration( } } - triggerMessageID, historyTipMessageID, triggerLabel := deriveChatDebugSeed(promptRows) - debugSvc := server.existingDebugService() - var debug *generationDebug - if resolved.debugEnabled { - if debugSvc == nil { - cleanup() - return generationPrepared{}, xerrors.New("chat debug service missing after enablement check") - } - debug = &generationDebug{ - Enabled: true, - Service: debugSvc, - Provider: resolved.resolvedProvider, - Model: resolved.resolvedModel, - TriggerMessageID: triggerMessageID, - HistoryTipMessageID: historyTipMessageID, - TriggerLabel: triggerLabel, - ModelConfig: modelConfig, - } - } - compactionToolCallID := "chat_summarized_" + uuid.NewString() effectiveThreshold := modelConfig.CompressionThreshold if override, ok := server.resolveUserCompactionThreshold(ctx, chat.OwnerID, modelConfig.ID); ok { diff --git a/coderd/x/chatd/mcpclient/coder_headers_test.go b/coderd/x/chatd/mcpclient/coder_headers_test.go index a034c38bdb7f9..1835e10f732ff 100644 --- a/coderd/x/chatd/mcpclient/coder_headers_test.go +++ b/coderd/x/chatd/mcpclient/coder_headers_test.go @@ -63,7 +63,7 @@ func TestConnectAll_ForwardCoderHeaders_DefaultOff(t *testing.T) { chatprovider.HeaderCoderWorkspaceID: uuid.NewString(), } - tools, cleanup := mcpclient.ConnectAll( + tools, _, cleanup := mcpclient.ConnectAll( ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, coderHeaders, ) @@ -113,7 +113,7 @@ func TestConnectAll_ForwardCoderHeaders_Enabled(t *testing.T) { WorkspaceID: uuid.NullUUID{UUID: workspaceID, Valid: true}, }) - tools, cleanup := mcpclient.ConnectAll( + tools, _, cleanup := mcpclient.ConnectAll( ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, coderHeaders, ) @@ -156,7 +156,7 @@ func TestConnectAll_ForwardCoderHeaders_RootChat(t *testing.T) { OwnerID: ownerID, }) - tools, cleanup := mcpclient.ConnectAll( + tools, _, cleanup := mcpclient.ConnectAll( ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, coderHeaders, ) @@ -202,7 +202,7 @@ func TestConnectAll_ForwardCoderHeaders_WithAPIKeyAuth(t *testing.T) { OwnerID: ownerID, }) - tools, cleanup := mcpclient.ConnectAll( + tools, _, cleanup := mcpclient.ConnectAll( ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, coderHeaders, ) @@ -252,7 +252,7 @@ func TestConnectAll_ForwardCoderHeaders_WithOAuth2(t *testing.T) { chatprovider.HeaderCoderOwnerID: ownerID, } - tools, cleanup := mcpclient.ConnectAll( + tools, _, cleanup := mcpclient.ConnectAll( ctx, logger, []database.MCPServerConfig{cfg}, []database.MCPServerUserToken{token}, @@ -304,7 +304,7 @@ func TestConnectAll_ForwardCoderHeaders_WithCustomHeaders(t *testing.T) { OwnerID: ownerID, }) - tools, cleanup := mcpclient.ConnectAll( + tools, _, cleanup := mcpclient.ConnectAll( ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, coderHeaders, ) diff --git a/coderd/x/chatd/mcpclient/export_test.go b/coderd/x/chatd/mcpclient/export_test.go index 3bfc1ae506507..53c3bf0d10aa0 100644 --- a/coderd/x/chatd/mcpclient/export_test.go +++ b/coderd/x/chatd/mcpclient/export_test.go @@ -24,7 +24,7 @@ func ConnectAllForTest( configs []database.MCPServerConfig, timeout time.Duration, reaperDone func(), -) ([]fantasy.AgentTool, func()) { +) ([]fantasy.AgentTool, []ConnectSummary, func()) { return connectAllWithHooks( ctx, logger, configs, nil, uuid.Nil, nil, nil, timeout, connectHooks{reaperDone: reaperDone}, @@ -34,3 +34,10 @@ func ConnectAllForTest( // BuildAuthHeadersForTest exposes buildAuthHeaders for external // tests. var BuildAuthHeadersForTest = buildAuthHeaders + +// SummaryErrorForTest exposes summaryError for external tests. +var SummaryErrorForTest = summaryError + +// MaxSummaryErrorLenForTest exposes the persisted-error byte cap for +// external tests. +const MaxSummaryErrorLenForTest = maxSummaryErrorLen diff --git a/coderd/x/chatd/mcpclient/mcpclient.go b/coderd/x/chatd/mcpclient/mcpclient.go index e54e5a2d5ffd7..4eb90dda904a6 100644 --- a/coderd/x/chatd/mcpclient/mcpclient.go +++ b/coderd/x/chatd/mcpclient/mcpclient.go @@ -14,6 +14,7 @@ import ( "strings" "sync" "time" + "unicode/utf8" "charm.land/fantasy" "github.com/google/uuid" @@ -58,6 +59,46 @@ const connectTimeout = 10 * time.Second // take before being canceled. const toolCallTimeout = 60 * time.Second +// slowConnectThreshold is the successful-connect duration above +// which a warning is logged. Connects happen on every generation +// step, so a consistently slow server taxes the whole chat even +// when it stays inside the connect budget. +const slowConnectThreshold = 5 * time.Second + +// ConnectOutcome classifies the result of one MCP server connect +// attempt for logs and chat debug runs. +type ConnectOutcome string + +const ( + // ConnectOutcomeConnected means tools were discovered and the + // session is live. + ConnectOutcomeConnected ConnectOutcome = "connected" + // ConnectOutcomeTimeout means the connect budget elapsed before + // the server completed the handshake and tool listing. + ConnectOutcomeTimeout ConnectOutcome = "timeout" + // ConnectOutcomeError means the handshake or tool listing + // failed. + ConnectOutcomeError ConnectOutcome = "error" + // ConnectOutcomeNoTools means the server connected but no tools + // survived allow/deny filtering, so the session was closed. + ConnectOutcomeNoTools ConnectOutcome = "no_tools" +) + +// ConnectSummary describes one MCP server connect attempt. It is +// logged and recorded into chat debug runs so slow or failing +// servers are visible instead of appearing as silent gaps in the +// turn timeline. +type ConnectSummary struct { + ConfigID uuid.UUID `json:"config_id"` + Slug string `json:"slug"` + Outcome ConnectOutcome `json:"outcome"` + DurationMS int64 `json:"duration_ms"` + ToolCount int `json:"tool_count,omitempty"` + // Error is the redacted, size-bounded connect error, present + // unless the outcome is connected or no_tools. + Error string `json:"error,omitempty"` +} + // UserOIDCTokenSource resolves the OIDC access token for the calling // user. Implementations attempt to refresh tokens that are expired // or close to expiring and MUST return ("", nil) when the user has @@ -74,8 +115,9 @@ type UserOIDCTokenSource interface { // their tools, and returns them as fantasy.AgentTool values. // Tools are sorted by their prefixed name so callers // receive a deterministic order. It skips servers that fail to -// connect and logs warnings. The returned cleanup function -// must be called to close all connections. +// connect and logs warnings; per-server outcomes are returned as +// ConnectSummary values sorted by slug. The returned cleanup +// function must be called to close all connections. func ConnectAll( ctx context.Context, logger slog.Logger, @@ -84,7 +126,7 @@ func ConnectAll( userID uuid.UUID, oidcSrc UserOIDCTokenSource, coderHeaders map[string]string, -) ([]fantasy.AgentTool, func()) { +) ([]fantasy.AgentTool, []ConnectSummary, func()) { return connectAllWithHooks( ctx, logger, configs, tokens, userID, oidcSrc, coderHeaders, connectTimeout, connectHooks{}, @@ -110,7 +152,7 @@ func connectAllWithHooks( coderHeaders map[string]string, timeout time.Duration, hooks connectHooks, -) ([]fantasy.AgentTool, func()) { +) ([]fantasy.AgentTool, []ConnectSummary, func()) { // Index tokens by server config ID so auth header // construction is O(1) per server. tokensByConfigID := make( @@ -121,9 +163,10 @@ func connectAllWithHooks( } var ( - mu sync.Mutex - sessions []*mcp.ClientSession - tools []fantasy.AgentTool + mu sync.Mutex + sessions []*mcp.ClientSession + tools []fantasy.AgentTool + summaries []ConnectSummary ) // Build cleanup eagerly so it always closes any sessions @@ -150,28 +193,59 @@ func connectAllWithHooks( } eg.Go(func() error { + start := time.Now() serverTools, session, connectErr := connectOne( ctx, logger, cfg, tokensByConfigID, userID, oidcSrc, coderHeaders, timeout, hooks, ) + duration := time.Since(start) + summary := ConnectSummary{ + ConfigID: cfg.ID, + Slug: cfg.Slug, + DurationMS: duration.Milliseconds(), + ToolCount: len(serverTools), + } + switch { + case connectErr != nil && errors.Is(connectErr, context.DeadlineExceeded): + summary.Outcome = ConnectOutcomeTimeout + summary.Error = summaryError(connectErr) + case connectErr != nil: + summary.Outcome = ConnectOutcomeError + summary.Error = summaryError(connectErr) + case len(serverTools) == 0: + summary.Outcome = ConnectOutcomeNoTools + default: + summary.Outcome = ConnectOutcomeConnected + } + if connectErr != nil { logger.Warn(ctx, "skipping MCP server due to connection failure", slog.F("server_slug", cfg.Slug), slog.F("server_url", RedactURL(cfg.Url)), + slog.F("duration", duration), slog.F("error", redactErrorURL(connectErr)), ) - // Connection failures are not propagated — the - // LLM simply won't have this server's tools. - return nil + } else if duration >= slowConnectThreshold { + logger.Warn(ctx, + "slow MCP server connect", + slog.F("server_slug", cfg.Slug), + slog.F("server_url", RedactURL(cfg.Url)), + slog.F("duration", duration), + ) } mu.Lock() - if session != nil { - sessions = append(sessions, session) + summaries = append(summaries, summary) + if connectErr == nil { + if session != nil { + sessions = append(sessions, session) + } + tools = append(tools, serverTools...) } - tools = append(tools, serverTools...) mu.Unlock() + // Connection failures are not propagated; the + // LLM simply won't have this server's tools. return nil }) } @@ -180,6 +254,15 @@ func connectAllWithHooks( // discarded. _ = eg.Wait() + // Sort summaries for deterministic ordering regardless of + // goroutine completion order. + slices.SortFunc(summaries, func(a, b ConnectSummary) int { + return cmp.Or( + cmp.Compare(a.Slug, b.Slug), + cmp.Compare(a.ConfigID.String(), b.ConfigID.String()), + ) + }) + // Sort tools by prefixed name for deterministic ordering // regardless of goroutine completion order. Ties, possible // when the __ separator produces ambiguous prefixed names, @@ -230,7 +313,7 @@ func connectAllWithHooks( } } - return tools, cleanup + return tools, summaries, cleanup } // connectOne establishes a connection to a single MCP server, @@ -579,6 +662,28 @@ func redactErrorURL(err error) string { return err.Error() } +// maxSummaryErrorLen bounds the persisted connect error in bytes. +// Protocol errors can embed arbitrarily large remote-controlled +// response bodies, so without this cap a single retained summary +// could inflate the JSONB row and every debug-runs payload +// regardless of the entry-count cap. +const maxSummaryErrorLen = 512 + +// summaryError renders a connect error for the persisted summary: +// credential-bearing URLs are redacted and the result is truncated +// to maxSummaryErrorLen bytes on a rune boundary. +func summaryError(err error) string { + msg := redactErrorURL(err) + if len(msg) <= maxSummaryErrorLen { + return msg + } + cut := maxSummaryErrorLen + for cut > 0 && !utf8.RuneStart(msg[cut]) { + cut-- + } + return msg[:cut] + "... (truncated)" +} + // MCPToolIdentifier is implemented by tools that originate from // an MCP server config and can report the config's database ID. type MCPToolIdentifier interface { diff --git a/coderd/x/chatd/mcpclient/mcpclient_connect_test.go b/coderd/x/chatd/mcpclient/mcpclient_connect_test.go index 2152786bcadda..a04e3b1c50308 100644 --- a/coderd/x/chatd/mcpclient/mcpclient_connect_test.go +++ b/coderd/x/chatd/mcpclient/mcpclient_connect_test.go @@ -82,7 +82,7 @@ func TestConnectAll_BlackHoledServerBudget(t *testing.T) { timeout := 1 * time.Second start := time.Now() - tools, cleanup := mcpclient.ConnectAllForTest(ctx, logger, + tools, summaries, cleanup := mcpclient.ConnectAllForTest(ctx, logger, []database.MCPServerConfig{ makeConfig("blackhole", bh.url()), makeConfig("healthy", healthy.URL), @@ -99,6 +99,17 @@ func TestConnectAll_BlackHoledServerBudget(t *testing.T) { "ConnectAll took %s, budget was %s", elapsed, timeout) require.Equal(t, []string{"healthy__echo"}, toolNames(tools)) + // Per-server outcomes are summarized for logs and debug runs, + // sorted by slug. + require.Len(t, summaries, 2) + require.Equal(t, "blackhole", summaries[0].Slug) + require.Equal(t, mcpclient.ConnectOutcomeTimeout, summaries[0].Outcome) + require.NotEmpty(t, summaries[0].Error) + require.GreaterOrEqual(t, summaries[0].DurationMS, timeout.Milliseconds()) + require.Equal(t, "healthy", summaries[1].Slug) + require.Equal(t, mcpclient.ConnectOutcomeConnected, summaries[1].Outcome) + require.Equal(t, 1, summaries[1].ToolCount) + // Terminating the black-holed connections unblocks the // abandoned connect goroutine; the reaper must then drain its // result and exit. @@ -131,7 +142,7 @@ func TestConnectAll_SlowServerStillConnects(t *testing.T) { t.Cleanup(ts.Close) cfg := makeConfig("slow", ts.URL) - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) require.Equal(t, []string{"slow__echo"}, toolNames(tools)) @@ -163,7 +174,7 @@ func TestConnectAll_LateServerReaped(t *testing.T) { timeout := 500 * time.Millisecond start := time.Now() - tools, cleanup := mcpclient.ConnectAllForTest(ctx, logger, + tools, summaries, cleanup := mcpclient.ConnectAllForTest(ctx, logger, []database.MCPServerConfig{makeConfig("late", ts.URL)}, timeout, func() { reaperDone <- struct{}{} }, @@ -174,6 +185,8 @@ func TestConnectAll_LateServerReaped(t *testing.T) { require.Less(t, elapsed, 4*timeout, "ConnectAll took %s, budget was %s", elapsed, timeout) require.Empty(t, tools) + require.Len(t, summaries, 1) + require.Equal(t, mcpclient.ConnectOutcomeTimeout, summaries[0].Outcome) select { case <-reaperDone: @@ -223,7 +236,7 @@ func TestConnectAll_CleanupPromptWhenServerWedges(t *testing.T) { t.Cleanup(release) cfg := makeConfig("wedge", ts.URL) - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) require.Equal(t, []string{"wedge__echo"}, toolNames(tools)) start := time.Now() @@ -277,11 +290,13 @@ func TestConnectAll_NoToolsWedgedCloseWithinBudget(t *testing.T) { cfg := makeConfig("notools", ts.URL) start := time.Now() - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, summaries, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) elapsed := time.Since(start) t.Cleanup(cleanup) require.Empty(t, tools) + require.Len(t, summaries, 1) + require.Equal(t, mcpclient.ConnectOutcomeNoTools, summaries[0].Outcome) require.Less(t, elapsed, 5*time.Second, "ConnectAll took %s with a wedged no-tools teardown", elapsed) diff --git a/coderd/x/chatd/mcpclient/mcpclient_test.go b/coderd/x/chatd/mcpclient/mcpclient_test.go index c844b68b1913e..bd9a9b2643bbe 100644 --- a/coderd/x/chatd/mcpclient/mcpclient_test.go +++ b/coderd/x/chatd/mcpclient/mcpclient_test.go @@ -17,6 +17,7 @@ import ( "github.com/modelcontextprotocol/go-sdk/mcp" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "golang.org/x/xerrors" "cdr.dev/slog/v3/sloggers/slogtest" "github.com/coder/coder/v2/coderd/database" @@ -134,7 +135,7 @@ func TestConnectAll_DiscoverTools(t *testing.T) { ts := newTestMCPServer(t, echoTool(), greetTool()) cfg := makeConfig("myserver", ts.URL) - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) // Two tools should be discovered, namespaced with the server slug. @@ -161,7 +162,7 @@ func TestConnectAll_SanitizesDottedSlug(t *testing.T) { // Use a dotted slug like awslabs.* MCP servers ship with. // Dots violate Bedrock's tool name pattern ^[a-zA-Z0-9_-]{1,128}$. cfg := makeConfig("awslabs.aws-documentation-mcp-server", ts.URL) - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) require.Len(t, tools, 1) @@ -200,7 +201,7 @@ func TestConnectAll_TruncationCollisionWarning(t *testing.T) { cfg1 := makeConfig(slug1, ts.URL) cfg2 := makeConfig(slug2, ts.URL) - tools, cleanup := mcpclient.ConnectAll( + tools, _, cleanup := mcpclient.ConnectAll( ctx, logger, []database.MCPServerConfig{cfg1, cfg2}, nil, uuid.Nil, nil, nil, @@ -222,7 +223,7 @@ func TestConnectAll_CallTool(t *testing.T) { ts := newTestMCPServer(t, echoTool()) cfg := makeConfig("srv", ts.URL) - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) require.Len(t, tools, 1) @@ -248,7 +249,7 @@ func TestConnectAll_ToolAllowList(t *testing.T) { // Only allow the "echo" tool. cfg.ToolAllowList = []string{"echo"} - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) require.Len(t, tools, 1) @@ -266,7 +267,7 @@ func TestConnectAll_ToolDenyList(t *testing.T) { // Deny the "greet" tool, so only "echo" remains. cfg.ToolDenyList = []string{"greet"} - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) require.Len(t, tools, 1) @@ -280,7 +281,7 @@ func TestConnectAll_ConnectionFailure(t *testing.T) { cfg := makeConfig("bad", "http://127.0.0.1:0/does-not-exist") - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) assert.Empty(t, tools, "no tools should be returned for an unreachable server") @@ -297,7 +298,7 @@ func TestConnectAll_MultipleServers(t *testing.T) { cfg1 := makeConfig("alpha", ts1.URL) cfg2 := makeConfig("beta", ts2.URL) - tools, cleanup := mcpclient.ConnectAll( + tools, _, cleanup := mcpclient.ConnectAll( ctx, logger, []database.MCPServerConfig{cfg1, cfg2}, nil, @@ -323,7 +324,7 @@ func TestConnectAll_NoToolsAfterFiltering(t *testing.T) { cfg := makeConfig("filtered", ts.URL) cfg.ToolAllowList = []string{"greet"} - tools, cleanup := mcpclient.ConnectAll( + tools, _, cleanup := mcpclient.ConnectAll( ctx, logger, []database.MCPServerConfig{cfg}, @@ -348,7 +349,7 @@ func TestConnectAll_DeterministicOrder(t *testing.T) { ts2 := newTestMCPServer(t, makeTool("alpha")) ts3 := newTestMCPServer(t, makeTool("middle")) - tools, cleanup := mcpclient.ConnectAll( + tools, _, cleanup := mcpclient.ConnectAll( ctx, logger, []database.MCPServerConfig{ @@ -379,7 +380,7 @@ func TestConnectAll_DeterministicOrder(t *testing.T) { multi := newTestMCPServer(t, makeTool("zeta"), makeTool("beta")) other := newTestMCPServer(t, makeTool("gamma")) - tools, cleanup := mcpclient.ConnectAll( + tools, _, cleanup := mcpclient.ConnectAll( ctx, logger, []database.MCPServerConfig{ @@ -416,7 +417,7 @@ func TestConnectAll_DeterministicOrder(t *testing.T) { cfg2 := makeConfig("a__b", ts2.URL) cfg2.ID = uuid.MustParse("00000000-0000-0000-0000-000000000001") - tools, cleanup := mcpclient.ConnectAll( + tools, _, cleanup := mcpclient.ConnectAll( ctx, logger, []database.MCPServerConfig{cfg1, cfg2}, @@ -479,7 +480,7 @@ func TestConnectAll_AuthHeaders(t *testing.T) { TokenType: "Bearer", } - tools, cleanup := mcpclient.ConnectAll( + tools, _, cleanup := mcpclient.ConnectAll( ctx, logger, []database.MCPServerConfig{cfg}, []database.MCPServerUserToken{token}, @@ -539,7 +540,7 @@ func TestConnectAll_DisabledServer(t *testing.T) { cfg := makeConfig("disabled", ts.URL) cfg.Enabled = false - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) assert.Empty(t, tools) } @@ -554,7 +555,7 @@ func TestConnectAll_CallToolInvalidInput(t *testing.T) { ts := newTestMCPServer(t, echoTool()) cfg := makeConfig("srv", ts.URL) - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) require.Len(t, tools, 1) @@ -579,7 +580,7 @@ func TestConnectAll_ToolInfoParameters(t *testing.T) { ts := newTestMCPServer(t, echoTool()) cfg := makeConfig("srv", ts.URL) - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) require.Len(t, tools, 1) @@ -630,7 +631,7 @@ func TestConnectAll_NilRequiredBecomesEmptySlice(t *testing.T) { ts := newTestMCPServer(t, noRequiredTool) cfg := makeConfig("srv", ts.URL) - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) require.Len(t, tools, 1) @@ -677,7 +678,7 @@ func TestConnectAll_APIKeyAuth(t *testing.T) { cfg.APIKeyHeader = "X-API-Key" cfg.APIKeyValue = "secret-123" - tools, cleanup := mcpclient.ConnectAll( + tools, _, cleanup := mcpclient.ConnectAll( ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil, @@ -732,7 +733,7 @@ func TestConnectAll_CustomHeadersAuth(t *testing.T) { cfg.AuthType = "custom_headers" cfg.CustomHeaders = `{"X-Custom-Auth":"custom-val"}` - tools, cleanup := mcpclient.ConnectAll( + tools, _, cleanup := mcpclient.ConnectAll( ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil, @@ -770,7 +771,7 @@ func TestConnectAll_CustomHeadersInvalidJSON(t *testing.T) { cfg.AuthType = "custom_headers" cfg.CustomHeaders = "{not json}" - tools, cleanup := mcpclient.ConnectAll( + tools, _, cleanup := mcpclient.ConnectAll( ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil, @@ -827,7 +828,7 @@ func TestConnectAll_UserOIDCAuth(t *testing.T) { userID := uuid.New() src := staticOIDCSource{token: "fake-oidc-token"} - tools, cleanup := mcpclient.ConnectAll( + tools, _, cleanup := mcpclient.ConnectAll( ctx, logger, []database.MCPServerConfig{cfg}, nil, userID, src, nil, ) @@ -883,7 +884,7 @@ func TestConnectAll_UserOIDCAuth_NoLink(t *testing.T) { cfg.AuthType = "user_oidc" src := staticOIDCSource{token: "", err: nil} - tools, cleanup := mcpclient.ConnectAll( + tools, _, cleanup := mcpclient.ConnectAll( ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.New(), src, nil, ) @@ -919,7 +920,7 @@ func TestConnectAll_UserOIDCAuth_NilSource(t *testing.T) { cfg := makeConfig("oidc-nilsrc", ts.URL) cfg.AuthType = "user_oidc" - tools, cleanup := mcpclient.ConnectAll( + tools, _, cleanup := mcpclient.ConnectAll( ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.New(), nil, nil, ) @@ -945,7 +946,7 @@ func TestConnectAll_ParallelConnections(t *testing.T) { cfg2 := makeConfig("srv2", ts2.URL) cfg3 := makeConfig("srv3", ts3.URL) - tools, cleanup := mcpclient.ConnectAll( + tools, _, cleanup := mcpclient.ConnectAll( ctx, logger, []database.MCPServerConfig{cfg1, cfg2, cfg3}, nil, @@ -986,6 +987,37 @@ func TestRedactURL(t *testing.T) { } } +func TestSummaryErrorTruncation(t *testing.T) { + t.Parallel() + + t.Run("short errors pass through", func(t *testing.T) { + t.Parallel() + got := mcpclient.SummaryErrorForTest(xerrors.New("connect refused")) + require.Equal(t, "connect refused", got) + }) + + t.Run("large remote-controlled errors are bounded", func(t *testing.T) { + t.Parallel() + huge := strings.Repeat("x", 1<<20) + got := mcpclient.SummaryErrorForTest(xerrors.New(huge)) + require.Len(t, got, mcpclient.MaxSummaryErrorLenForTest+len("... (truncated)")) + require.True(t, strings.HasSuffix(got, "... (truncated)")) + require.True(t, strings.HasPrefix(got, "xxx")) + }) + + t.Run("truncation lands on a rune boundary", func(t *testing.T) { + t.Parallel() + // A two-byte rune straddles the cap boundary, so the cut + // must back up instead of splitting it. + msg := strings.Repeat("a", mcpclient.MaxSummaryErrorLenForTest-1) + + strings.Repeat("é", 20) + got := mcpclient.SummaryErrorForTest(xerrors.New(msg)) + require.True(t, utf8.ValidString(got)) + require.True(t, strings.HasSuffix(got, "... (truncated)")) + require.Len(t, got, mcpclient.MaxSummaryErrorLenForTest-1+len("... (truncated)")) + }) +} + func TestConnectAll_ExpiredToken(t *testing.T) { t.Parallel() ctx := context.Background() @@ -1011,7 +1043,7 @@ func TestConnectAll_ExpiredToken(t *testing.T) { Expiry: sql.NullTime{Time: time.Now().Add(-1 * time.Hour), Valid: true}, } - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, []database.MCPServerUserToken{token}, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, []database.MCPServerUserToken{token}, uuid.Nil, nil, nil) t.Cleanup(cleanup) // The server accepts any auth, so the tool is still discovered @@ -1044,7 +1076,7 @@ func TestConnectAll_EmptyAccessToken(t *testing.T) { TokenType: "Bearer", } - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, []database.MCPServerUserToken{token}, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, []database.MCPServerUserToken{token}, uuid.Nil, nil, nil) t.Cleanup(cleanup) // Tool is still discovered (server doesn't require auth), but @@ -1074,7 +1106,7 @@ func TestConnectAll_MCPToolIdentifier(t *testing.T) { Enabled: true, } - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) require.Len(t, tools, 1) @@ -1116,7 +1148,7 @@ func TestConnectAll_MCPToolIdentifier_MultipleServers(t *testing.T) { Enabled: true, } - tools, cleanup := mcpclient.ConnectAll( + tools, _, cleanup := mcpclient.ConnectAll( ctx, logger, []database.MCPServerConfig{cfg1, cfg2}, nil, @@ -1171,7 +1203,7 @@ func TestConnectAll_EmbeddedResourceText(t *testing.T) { }) cfg := makeConfig("embed-txt", ts.URL) - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) require.Len(t, tools, 1) @@ -1234,7 +1266,7 @@ func TestConnectAll_EmbeddedResourceBlob(t *testing.T) { }) cfg := makeConfig("embed-blob", ts.URL) - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) require.Len(t, tools, 1) @@ -1309,7 +1341,7 @@ func TestConnectAll_ResourceLink(t *testing.T) { }) cfg := makeConfig("res-link", ts.URL) - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) require.Len(t, tools, 1) @@ -1351,7 +1383,7 @@ func TestConnectAll_CallToolError(t *testing.T) { }) cfg := makeConfig("err-srv", ts.URL) - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) require.Len(t, tools, 1) @@ -1375,7 +1407,7 @@ func TestModelIntent_Info_WrapsSchema(t *testing.T) { cfg := makeConfig("intent-srv", ts.URL) cfg.ModelIntent = true - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) require.Len(t, tools, 1) @@ -1411,7 +1443,7 @@ func TestModelIntent_Info_NoWrapWhenDisabled(t *testing.T) { cfg := makeConfig("no-intent", ts.URL) cfg.ModelIntent = false - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) require.Len(t, tools, 1) @@ -1434,7 +1466,7 @@ func TestModelIntent_Run_UnwrapsProperties(t *testing.T) { cfg := makeConfig("unwrap-srv", ts.URL) cfg.ModelIntent = true - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) require.Len(t, tools, 1) @@ -1459,7 +1491,7 @@ func TestModelIntent_Run_UnwrapsFlat(t *testing.T) { cfg := makeConfig("flat-srv", ts.URL) cfg.ModelIntent = true - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) require.Len(t, tools, 1) @@ -1484,7 +1516,7 @@ func TestModelIntent_Run_PassthroughWhenDisabled(t *testing.T) { cfg := makeConfig("pass-srv", ts.URL) cfg.ModelIntent = false - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) require.Len(t, tools, 1) @@ -1509,7 +1541,7 @@ func TestModelIntent_Run_FallbackOnBadJSON(t *testing.T) { cfg := makeConfig("bad-srv", ts.URL) cfg.ModelIntent = true - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) require.Len(t, tools, 1) diff --git a/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/DebugPanel.stories.tsx b/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/DebugPanel.stories.tsx index b452a6fe30b5c..a9bfc81bff5bf 100644 --- a/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/DebugPanel.stories.tsx +++ b/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/DebugPanel.stories.tsx @@ -712,6 +712,93 @@ export const RunWithNoSteps: Story = { }, }; +const mcpConnectRunId = "run-mcp-connect"; +const mcpConnectSummary = { + first_message: "MCP connect probe", + mcp_connect: [ + { + config_id: "b8f9f3f2-4a3f-4f8a-9c5e-2f4f4be00001", + slug: "linear", + outcome: "connected", + duration_ms: 320, + tool_count: 12, + }, + { + config_id: "b8f9f3f2-4a3f-4f8a-9c5e-2f4f4be00002", + slug: "registry", + outcome: "timeout", + duration_ms: 10000, + error: "connect: context deadline exceeded", + }, + // The same server reported again by a later generation + // step; entries accumulate across the turn's preparations. + { + config_id: "b8f9f3f2-4a3f-4f8a-9c5e-2f4f4be00002", + slug: "registry", + outcome: "connected", + duration_ms: 45, + tool_count: 3, + }, + ], + // Entries beyond the retention cap; the card renders a + // truncation notice for them. + mcp_connect_dropped: 4, +}; + +export const RunWithMCPConnectSummary: Story = { + parameters: { + queries: [ + { + key: chatDebugRunsKey(CHAT_ID), + data: [ + buildRunSummary({ + id: mcpConnectRunId, + summary: mcpConnectSummary, + }), + ], + }, + { + key: chatDebugRunKey(CHAT_ID, mcpConnectRunId), + data: { + ...MockRun, + id: mcpConnectRunId, + summary: mcpConnectSummary, + steps: [], + }, + }, + ], + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + const user = userEvent.setup(); + + const runTrigger = await canvas.findByRole("button", { + name: /MCP connect probe/i, + }); + await user.click(runTrigger); + + const section = await canvas.findByRole("region", { + name: /MCP server connections/i, + }); + const mcp = within(section); + await waitFor(() => { + expect(mcp.getByText("linear")).toBeVisible(); + expect(mcp.getAllByText("connected")).toHaveLength(2); + expect(mcp.getByText("320ms")).toBeVisible(); + expect(mcp.getByText("12 tools")).toBeVisible(); + expect(mcp.getAllByText("registry")).toHaveLength(2); + expect(mcp.getByText("timeout")).toBeVisible(); + expect(mcp.getByText("10.0s")).toBeVisible(); + expect(mcp.getByText("connect: context deadline exceeded")).toBeVisible(); + expect(mcp.getByText("45ms")).toBeVisible(); + expect(mcp.getByText("3 tools")).toBeVisible(); + expect( + mcp.getByText("4 earlier connection samples omitted"), + ).toBeVisible(); + }); + }, +}; + // --------------------------------------------------------------------------- // Core state stories. // --------------------------------------------------------------------------- @@ -1219,6 +1306,46 @@ export const CompactionAndTitleGenerationBadges: Story = { }, }; +export const NonChatTurnKindShownWithFirstMessage: Story = { + parameters: { + queries: [ + { + key: chatDebugRunsKey(CHAT_ID), + data: [ + buildRunSummary({ + id: "run-title-with-label", + kind: "title_generation", + status: "error", + model: "gpt-4o-mini", + summary: { first_message: "Summarize my workspace" }, + }), + buildRunSummary({ + id: "run-turn-with-label", + kind: "chat_turn", + status: "completed", + summary: { first_message: "Fix the login bug" }, + }), + ], + }, + ], + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + // A failed title generation with a first_message label must + // still identify itself by kind so it is distinguishable from + // a failed chat turn. + const titleRun = await canvas.findByRole("button", { + name: /Summarize my workspace/i, + }); + await expect(within(titleRun).getByText("Title Generation")).toBeVisible(); + // Chat turns keep their metadata kind-free. + const chatRun = await canvas.findByRole("button", { + name: /Fix the login bug/i, + }); + expect(within(chatRun).queryByText("Chat Turn")).toBeNull(); + }, +}; + export const LongRawPayloads: Story = { parameters: { queries: [ diff --git a/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/DebugRunCard.tsx b/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/DebugRunCard.tsx index 2831806256379..492857e821ad3 100644 --- a/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/DebugRunCard.tsx +++ b/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/DebugRunCard.tsx @@ -1,6 +1,6 @@ import { saveAs } from "file-saver"; import { ChevronDownIcon, DownloadIcon } from "lucide-react"; -import { type FC, useState } from "react"; +import { type FC, useId, useState } from "react"; import { useQuery } from "react-query"; import { toast } from "sonner"; import { getErrorDetail, getErrorMessage } from "#/api/errors"; @@ -49,6 +49,17 @@ const getDurationLabel = (startedAt: string, finishedAt?: string): string => { return durationMs !== null ? compactDuration(durationMs) : "-"; }; +const getMCPOutcomeBadgeVariant = (outcome: string) => { + switch (outcome) { + case "connected": + return "green"; + case "no_tools": + return "default"; + default: + return "destructive"; + } +}; + export const DebugRunCard: FC = ({ run, chatId, @@ -57,6 +68,7 @@ export const DebugRunCard: FC = ({ }) => { const [isExpanded, setIsExpanded] = useState(false); const [isExporting, setIsExporting] = useState(false); + const mcpConnectHeadingId = useId(); const runDetailQuery = useQuery({ ...chatDebugRun(chatId, run.id), enabled: isVisible && isExpanded, @@ -85,7 +97,16 @@ export const DebugRunCard: FC = ({ // Step count from detail or summary. const stepCount = steps.length > 0 ? steps.length : summaryVm.stepCount; const durationLabel = getDurationLabel(run.started_at, run.finished_at); + // Non-chat-turn runs (title generation, quickgen, compaction) + // usually carry a first_message label that hides the kind, so + // surface the kind in the metadata; otherwise a failed title + // generation is indistinguishable from a failed chat turn. + const kindLabel = + run.kind !== "chat_turn" && summaryVm.primaryLabel.trim() + ? getRunKindLabel(run.kind) + : undefined; const metadataItems = [ + kindLabel, modelLabel || undefined, stepCount !== undefined && stepCount > 0 ? `${stepCount} ${stepCount === 1 ? "step" : "steps"}` @@ -200,6 +221,65 @@ export const DebugRunCard: FC = ({

) : null} + {summaryVm.mcpConnect.length > 0 ? ( +
+

+ MCP server connections +

+
    + {summaryVm.mcpConnect.map((server, index) => ( +
  • + + {server.slug} + + + {server.outcome} + + {server.durationMs !== undefined ? ( + + {compactDuration(server.durationMs)} + + ) : null} + {server.toolCount !== undefined && + server.toolCount > 0 ? ( + + {server.toolCount}{" "} + {server.toolCount === 1 ? "tool" : "tools"} + + ) : null} + {server.error ? ( + + {server.error} + + ) : null} +
    • + ))} +
+ {summaryVm.mcpConnectDropped > 0 ? ( +

+ {summaryVm.mcpConnectDropped} earlier connection{" "} + {summaryVm.mcpConnectDropped === 1 ? "sample" : "samples"}{" "} + omitted +

+ ) : null} +
+ ) : null} {steps.map((step) => ( ))} diff --git a/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/debugPanelUtils.test.ts b/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/debugPanelUtils.test.ts index 0b0866d6456ca..b663aa7bef981 100644 --- a/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/debugPanelUtils.test.ts +++ b/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/debugPanelUtils.test.ts @@ -826,6 +826,8 @@ describe("coerceRunSummary", () => { stepCount: 3, totalInputTokens: 120, totalOutputTokens: 45, + mcpConnect: [], + mcpConnectDropped: 0, warnings: [], }); }); @@ -851,10 +853,70 @@ describe("coerceRunSummary", () => { stepCount: undefined, totalInputTokens: undefined, totalOutputTokens: undefined, + mcpConnect: [], + mcpConnectDropped: 0, warnings: [], }); }); + it("coerces MCP connect summaries and drops malformed entries", () => { + const summary = coerceRunSummary({ + mcp_connect: [ + { + slug: "registry", + outcome: "timeout", + duration_ms: 10000, + error: "connect: context deadline exceeded", + }, + { + slug: "linear", + outcome: "connected", + duration_ms: 320, + tool_count: 12, + }, + { outcome: "error" }, + "not-a-record", + ], + }); + + expect(summary.mcpConnect).toEqual([ + { + slug: "registry", + outcome: "timeout", + durationMs: 10000, + toolCount: undefined, + error: "connect: context deadline exceeded", + }, + { + slug: "linear", + outcome: "connected", + durationMs: 320, + toolCount: 12, + error: undefined, + }, + ]); + }); + + it("returns an empty MCP connect list for non-array values", () => { + expect(coerceRunSummary({ mcp_connect: "oops" }).mcpConnect).toEqual([]); + }); + + it("coerces the dropped MCP connect sample count", () => { + expect( + coerceRunSummary({ mcp_connect_dropped: 25 }).mcpConnectDropped, + ).toBe(25); + expect(coerceRunSummary({ mcpConnectDropped: "7" }).mcpConnectDropped).toBe( + 7, + ); + expect( + coerceRunSummary({ mcp_connect_dropped: -3 }).mcpConnectDropped, + ).toBe(0); + expect( + coerceRunSummary({ mcp_connect_dropped: "oops" }).mcpConnectDropped, + ).toBe(0); + expect(coerceRunSummary({}).mcpConnectDropped).toBe(0); + }); + it("unwraps JSON-string payloads before coercing", () => { const summary = coerceRunSummary( JSON.stringify({ diff --git a/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/debugPanelUtils.ts b/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/debugPanelUtils.ts index 9a9e570364d2d..da532e32841c7 100644 --- a/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/debugPanelUtils.ts +++ b/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/debugPanelUtils.ts @@ -436,6 +436,14 @@ export const compactDuration = (ms: number): string => { // View-model types for coerced debug payloads. // --------------------------------------------------------------------------- +export interface MCPConnectSummaryViewModel { + slug: string; + outcome: string; + durationMs: number | undefined; + toolCount: number | undefined; + error: string | undefined; +} + interface RunSummaryViewModel { primaryLabel: string; endpointLabel: string | undefined; @@ -444,6 +452,8 @@ interface RunSummaryViewModel { stepCount: number | undefined; totalInputTokens: number | undefined; totalOutputTokens: number | undefined; + mcpConnect: MCPConnectSummaryViewModel[]; + mcpConnectDropped: number; warnings: string[]; } @@ -868,6 +878,32 @@ const extractKnownFields = ( // Public coercion: run summary. // --------------------------------------------------------------------------- +const coerceMCPConnectSummaries = ( + value: unknown, +): MCPConnectSummaryViewModel[] => { + if (!Array.isArray(value)) { + return []; + } + const result: MCPConnectSummaryViewModel[] = []; + for (const entry of value) { + if (!isRecord(entry)) { + continue; + } + const slug = toOptionalString(entry.slug); + if (!slug) { + continue; + } + result.push({ + slug, + outcome: toOptionalString(entry.outcome) ?? "unknown", + durationMs: toFiniteNumber(entry.duration_ms), + toolCount: toFiniteNumber(entry.tool_count), + error: toOptionalString(entry.error), + }); + } + return result; +}; + export const coerceRunSummary = (data: unknown): RunSummaryViewModel => { const defaults: RunSummaryViewModel = { primaryLabel: "", @@ -877,6 +913,8 @@ export const coerceRunSummary = (data: unknown): RunSummaryViewModel => { stepCount: undefined, totalInputTokens: undefined, totalOutputTokens: undefined, + mcpConnect: [], + mcpConnectDropped: 0, warnings: [], }; const parsed = deepParse(data); @@ -892,6 +930,9 @@ export const coerceRunSummary = (data: unknown): RunSummaryViewModel => { "primaryLabel", ), ); + const mcpConnectDropped = toFiniteNumber( + pickField(parsed, "mcp_connect_dropped", "mcpConnectDropped"), + ); return { primaryLabel: firstMessage ?? "", endpointLabel: toOptionalString( @@ -924,6 +965,13 @@ export const coerceRunSummary = (data: unknown): RunSummaryViewModel => { "completionTokens", ), ), + mcpConnect: coerceMCPConnectSummaries( + pickField(parsed, "mcp_connect", "mcpConnect"), + ), + mcpConnectDropped: + mcpConnectDropped !== undefined && mcpConnectDropped > 0 + ? Math.floor(mcpConnectDropped) + : 0, warnings: [], }; };