From d9f25e66a144dbf0b5673c7750b8e450f666ee43 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 20 Aug 2026 16:33:48 +0000 Subject: [PATCH 1/8] feat(coderd/x/chatd): add MCP connect and generation-prep observability Chat turns that stall in MCP connects or preparation were invisible: the debug UI showed a silent gap before the first step and logs had no durations. - ConnectAll returns per-server ConnectSummary values (outcome, duration, tool count, redacted error), logs connect duration on failure, and warns on successful connects slower than 5s. - Connect summaries are seeded into the chat debug run summary (mcp_connect key) and rendered in the debug panel run card as a per-server list with outcome badges, durations, and errors. - prepareGeneration warns when preparation exceeds 30s. --- coderd/x/chatd/active_turn_debug.go | 10 ++ coderd/x/chatd/chatd.go | 6 + coderd/x/chatd/generation.go | 5 + coderd/x/chatd/generation_preparer.go | 13 ++- .../x/chatd/mcpclient/coder_headers_test.go | 12 +- coderd/x/chatd/mcpclient/export_test.go | 2 +- coderd/x/chatd/mcpclient/mcpclient.go | 110 +++++++++++++++--- .../chatd/mcpclient/mcpclient_connect_test.go | 25 +++- coderd/x/chatd/mcpclient/mcpclient_test.go | 76 ++++++------ .../DebugPanel/DebugPanel.stories.tsx | 70 +++++++++++ .../RightPanel/DebugPanel/DebugRunCard.tsx | 66 ++++++++++- .../DebugPanel/debugPanelUtils.test.ts | 44 +++++++ .../RightPanel/DebugPanel/debugPanelUtils.ts | 39 +++++++ 13 files changed, 412 insertions(+), 66 deletions(-) diff --git a/coderd/x/chatd/active_turn_debug.go b/coderd/x/chatd/active_turn_debug.go index 2fbd653a04130..d9c2721496d61 100644 --- a/coderd/x/chatd/active_turn_debug.go +++ b/coderd/x/chatd/active_turn_debug.go @@ -68,6 +68,16 @@ func (d *runnerDebugTurn) Ensure( seedSummary := chatdebug.SeedSummary( chatdebug.TruncateLabel(debug.TriggerLabel, chatdebug.MaxLabelLength), ) + // Seed per-server MCP connect outcomes so slow or failing + // servers appear in the run instead of as a silent gap before + // the first step. Seeded keys survive FinalizeRun's summary + // aggregation. + if len(debug.MCPConnectSummaries) > 0 { + if seedSummary == nil { + seedSummary = make(map[string]any, 1) + } + seedSummary["mcp_connect"] = debug.MCPConnectSummaries + } rootChatID := uuid.Nil if chat.RootChatID.Valid { rootChatID = chat.RootChatID.UUID diff --git a/coderd/x/chatd/chatd.go b/coderd/x/chatd/chatd.go index fbb3a22bf368f..1c3570f9360c5 100644 --- a/coderd/x/chatd/chatd.go +++ b/coderd/x/chatd/chatd.go @@ -84,6 +84,12 @@ const ( DefaultChatHeartbeatInterval = 30 * time.Second maxChatSteps = 1200 + // slowPrepareThreshold is the generation-preparation duration + // above which a warning is logged. Preparation runs before + // every generation step, so sustained slowness (workspace + // dials, MCP connects) taxes the whole turn. + slowPrepareThreshold = 30 * time.Second + // maxConcurrentRecordingUploads caps the number of recording // stop-and-store operations that can run concurrently. Each // slot buffers up to MaxRecordingSize + MaxThumbnailSize diff --git a/coderd/x/chatd/generation.go b/coderd/x/chatd/generation.go index 33eb7ff7c33f4..5843e822966d2 100644 --- a/coderd/x/chatd/generation.go +++ b/coderd/x/chatd/generation.go @@ -24,6 +24,7 @@ import ( "github.com/coder/coder/v2/coderd/x/chatd/chatretry" "github.com/coder/coder/v2/coderd/x/chatd/chatstate" "github.com/coder/coder/v2/coderd/x/chatd/chattool" + "github.com/coder/coder/v2/coderd/x/chatd/mcpclient" "github.com/coder/coder/v2/coderd/x/chatd/messagepartbuffer" "github.com/coder/coder/v2/codersdk" "github.com/coder/coder/v2/codersdk/x/agenthooks" @@ -95,6 +96,10 @@ type generationDebug struct { HistoryTipMessageID int64 TriggerLabel string ModelConfig database.ChatModelConfig + // MCPConnectSummaries carries per-server MCP connect outcomes + // from turn preparation into the debug run summary so slow or + // failing servers are visible in the debug UI. + MCPConnectSummaries []mcpclient.ConnectSummary } // generationOutcome describes a completed generation outcome. diff --git a/coderd/x/chatd/generation_preparer.go b/coderd/x/chatd/generation_preparer.go index bb79819a3f56a..b6f1d26a07dcc 100644 --- a/coderd/x/chatd/generation_preparer.go +++ b/coderd/x/chatd/generation_preparer.go @@ -76,6 +76,15 @@ func (server *Server) prepareGeneration( slog.F("owner_id", chat.OwnerID), ) + prepStart := server.clock.Now() + defer func() { + if prepDuration := server.clock.Since(prepStart); prepDuration >= slowPrepareThreshold { + logger.Warn(ctx, "slow generation preparation", + slog.F("duration", prepDuration), + ) + } + }() + var ( promptRows []database.ChatMessage mcpConfigs []database.MCPServerConfig @@ -263,6 +272,7 @@ func (server *Server) prepareGeneration( prompt []fantasy.Message instruction string mcpTools []fantasy.AgentTool + mcpSummaries []mcpclient.ConnectSummary mcpCleanup func() workspaceMCPTools []fantasy.AgentTool workspaceSkills []chattool.SkillMeta @@ -336,7 +346,7 @@ func (server *Server) prepareGeneration( logger.Warn(ctx, "failed to load MCP user tokens", slog.Error(tokenErr)) } mcpTokens = server.refreshExpiredMCPTokens(ctx, logger, mcpConnectConfigs, mcpTokens) - mcpTools, mcpCleanup = mcpclient.ConnectAll( + mcpTools, mcpSummaries, mcpCleanup = mcpclient.ConnectAll( ctx, logger, mcpConnectConfigs, @@ -652,6 +662,7 @@ func (server *Server) prepareGeneration( HistoryTipMessageID: historyTipMessageID, TriggerLabel: triggerLabel, ModelConfig: modelConfig, + MCPConnectSummaries: mcpSummaries, } } diff --git a/coderd/x/chatd/mcpclient/coder_headers_test.go b/coderd/x/chatd/mcpclient/coder_headers_test.go index a034c38bdb7f9..1835e10f732ff 100644 --- a/coderd/x/chatd/mcpclient/coder_headers_test.go +++ b/coderd/x/chatd/mcpclient/coder_headers_test.go @@ -63,7 +63,7 @@ func TestConnectAll_ForwardCoderHeaders_DefaultOff(t *testing.T) { chatprovider.HeaderCoderWorkspaceID: uuid.NewString(), } - tools, cleanup := mcpclient.ConnectAll( + tools, _, cleanup := mcpclient.ConnectAll( ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, coderHeaders, ) @@ -113,7 +113,7 @@ func TestConnectAll_ForwardCoderHeaders_Enabled(t *testing.T) { WorkspaceID: uuid.NullUUID{UUID: workspaceID, Valid: true}, }) - tools, cleanup := mcpclient.ConnectAll( + tools, _, cleanup := mcpclient.ConnectAll( ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, coderHeaders, ) @@ -156,7 +156,7 @@ func TestConnectAll_ForwardCoderHeaders_RootChat(t *testing.T) { OwnerID: ownerID, }) - tools, cleanup := mcpclient.ConnectAll( + tools, _, cleanup := mcpclient.ConnectAll( ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, coderHeaders, ) @@ -202,7 +202,7 @@ func TestConnectAll_ForwardCoderHeaders_WithAPIKeyAuth(t *testing.T) { OwnerID: ownerID, }) - tools, cleanup := mcpclient.ConnectAll( + tools, _, cleanup := mcpclient.ConnectAll( ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, coderHeaders, ) @@ -252,7 +252,7 @@ func TestConnectAll_ForwardCoderHeaders_WithOAuth2(t *testing.T) { chatprovider.HeaderCoderOwnerID: ownerID, } - tools, cleanup := mcpclient.ConnectAll( + tools, _, cleanup := mcpclient.ConnectAll( ctx, logger, []database.MCPServerConfig{cfg}, []database.MCPServerUserToken{token}, @@ -304,7 +304,7 @@ func TestConnectAll_ForwardCoderHeaders_WithCustomHeaders(t *testing.T) { OwnerID: ownerID, }) - tools, cleanup := mcpclient.ConnectAll( + tools, _, cleanup := mcpclient.ConnectAll( ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, coderHeaders, ) diff --git a/coderd/x/chatd/mcpclient/export_test.go b/coderd/x/chatd/mcpclient/export_test.go index 3bfc1ae506507..d5ec3bb687111 100644 --- a/coderd/x/chatd/mcpclient/export_test.go +++ b/coderd/x/chatd/mcpclient/export_test.go @@ -24,7 +24,7 @@ func ConnectAllForTest( configs []database.MCPServerConfig, timeout time.Duration, reaperDone func(), -) ([]fantasy.AgentTool, func()) { +) ([]fantasy.AgentTool, []ConnectSummary, func()) { return connectAllWithHooks( ctx, logger, configs, nil, uuid.Nil, nil, nil, timeout, connectHooks{reaperDone: reaperDone}, diff --git a/coderd/x/chatd/mcpclient/mcpclient.go b/coderd/x/chatd/mcpclient/mcpclient.go index e54e5a2d5ffd7..f6538f9c07b48 100644 --- a/coderd/x/chatd/mcpclient/mcpclient.go +++ b/coderd/x/chatd/mcpclient/mcpclient.go @@ -58,6 +58,46 @@ const connectTimeout = 10 * time.Second // take before being canceled. const toolCallTimeout = 60 * time.Second +// slowConnectThreshold is the successful-connect duration above +// which a warning is logged. Connects happen on every generation +// step, so a consistently slow server taxes the whole chat even +// when it stays inside the connect budget. +const slowConnectThreshold = 5 * time.Second + +// ConnectOutcome classifies the result of one MCP server connect +// attempt for logs and chat debug runs. +type ConnectOutcome string + +const ( + // ConnectOutcomeConnected means tools were discovered and the + // session is live. + ConnectOutcomeConnected ConnectOutcome = "connected" + // ConnectOutcomeTimeout means the connect budget elapsed before + // the server completed the handshake and tool listing. + ConnectOutcomeTimeout ConnectOutcome = "timeout" + // ConnectOutcomeError means the handshake or tool listing + // failed. + ConnectOutcomeError ConnectOutcome = "error" + // ConnectOutcomeNoTools means the server connected but no tools + // survived allow/deny filtering, so the session was closed. + ConnectOutcomeNoTools ConnectOutcome = "no_tools" +) + +// ConnectSummary describes one MCP server connect attempt. It is +// logged and recorded into chat debug runs so slow or failing +// servers are visible instead of appearing as silent gaps in the +// turn timeline. +type ConnectSummary struct { + ConfigID uuid.UUID `json:"config_id"` + Slug string `json:"slug"` + Outcome ConnectOutcome `json:"outcome"` + DurationMS int64 `json:"duration_ms"` + ToolCount int `json:"tool_count,omitempty"` + // Error is the redacted connect error, present unless the + // outcome is connected or no_tools. + Error string `json:"error,omitempty"` +} + // UserOIDCTokenSource resolves the OIDC access token for the calling // user. Implementations attempt to refresh tokens that are expired // or close to expiring and MUST return ("", nil) when the user has @@ -74,8 +114,9 @@ type UserOIDCTokenSource interface { // their tools, and returns them as fantasy.AgentTool values. // Tools are sorted by their prefixed name so callers // receive a deterministic order. It skips servers that fail to -// connect and logs warnings. The returned cleanup function -// must be called to close all connections. +// connect and logs warnings; per-server outcomes are returned as +// ConnectSummary values sorted by slug. The returned cleanup +// function must be called to close all connections. func ConnectAll( ctx context.Context, logger slog.Logger, @@ -84,7 +125,7 @@ func ConnectAll( userID uuid.UUID, oidcSrc UserOIDCTokenSource, coderHeaders map[string]string, -) ([]fantasy.AgentTool, func()) { +) ([]fantasy.AgentTool, []ConnectSummary, func()) { return connectAllWithHooks( ctx, logger, configs, tokens, userID, oidcSrc, coderHeaders, connectTimeout, connectHooks{}, @@ -110,7 +151,7 @@ func connectAllWithHooks( coderHeaders map[string]string, timeout time.Duration, hooks connectHooks, -) ([]fantasy.AgentTool, func()) { +) ([]fantasy.AgentTool, []ConnectSummary, func()) { // Index tokens by server config ID so auth header // construction is O(1) per server. tokensByConfigID := make( @@ -121,9 +162,10 @@ func connectAllWithHooks( } var ( - mu sync.Mutex - sessions []*mcp.ClientSession - tools []fantasy.AgentTool + mu sync.Mutex + sessions []*mcp.ClientSession + tools []fantasy.AgentTool + summaries []ConnectSummary ) // Build cleanup eagerly so it always closes any sessions @@ -150,28 +192,59 @@ func connectAllWithHooks( } eg.Go(func() error { + start := time.Now() serverTools, session, connectErr := connectOne( ctx, logger, cfg, tokensByConfigID, userID, oidcSrc, coderHeaders, timeout, hooks, ) + duration := time.Since(start) + summary := ConnectSummary{ + ConfigID: cfg.ID, + Slug: cfg.Slug, + DurationMS: duration.Milliseconds(), + ToolCount: len(serverTools), + } + switch { + case connectErr != nil && errors.Is(connectErr, context.DeadlineExceeded): + summary.Outcome = ConnectOutcomeTimeout + summary.Error = redactErrorURL(connectErr) + case connectErr != nil: + summary.Outcome = ConnectOutcomeError + summary.Error = redactErrorURL(connectErr) + case len(serverTools) == 0: + summary.Outcome = ConnectOutcomeNoTools + default: + summary.Outcome = ConnectOutcomeConnected + } + if connectErr != nil { logger.Warn(ctx, "skipping MCP server due to connection failure", slog.F("server_slug", cfg.Slug), slog.F("server_url", RedactURL(cfg.Url)), + slog.F("duration", duration), slog.F("error", redactErrorURL(connectErr)), ) - // Connection failures are not propagated — the - // LLM simply won't have this server's tools. - return nil + } else if duration >= slowConnectThreshold { + logger.Warn(ctx, + "slow MCP server connect", + slog.F("server_slug", cfg.Slug), + slog.F("server_url", RedactURL(cfg.Url)), + slog.F("duration", duration), + ) } mu.Lock() - if session != nil { - sessions = append(sessions, session) + summaries = append(summaries, summary) + if connectErr == nil { + if session != nil { + sessions = append(sessions, session) + } + tools = append(tools, serverTools...) } - tools = append(tools, serverTools...) mu.Unlock() + // Connection failures are not propagated; the + // LLM simply won't have this server's tools. return nil }) } @@ -180,6 +253,15 @@ func connectAllWithHooks( // discarded. _ = eg.Wait() + // Sort summaries for deterministic ordering regardless of + // goroutine completion order. + slices.SortFunc(summaries, func(a, b ConnectSummary) int { + return cmp.Or( + cmp.Compare(a.Slug, b.Slug), + cmp.Compare(a.ConfigID.String(), b.ConfigID.String()), + ) + }) + // Sort tools by prefixed name for deterministic ordering // regardless of goroutine completion order. Ties, possible // when the __ separator produces ambiguous prefixed names, @@ -230,7 +312,7 @@ func connectAllWithHooks( } } - return tools, cleanup + return tools, summaries, cleanup } // connectOne establishes a connection to a single MCP server, diff --git a/coderd/x/chatd/mcpclient/mcpclient_connect_test.go b/coderd/x/chatd/mcpclient/mcpclient_connect_test.go index 2152786bcadda..a04e3b1c50308 100644 --- a/coderd/x/chatd/mcpclient/mcpclient_connect_test.go +++ b/coderd/x/chatd/mcpclient/mcpclient_connect_test.go @@ -82,7 +82,7 @@ func TestConnectAll_BlackHoledServerBudget(t *testing.T) { timeout := 1 * time.Second start := time.Now() - tools, cleanup := mcpclient.ConnectAllForTest(ctx, logger, + tools, summaries, cleanup := mcpclient.ConnectAllForTest(ctx, logger, []database.MCPServerConfig{ makeConfig("blackhole", bh.url()), makeConfig("healthy", healthy.URL), @@ -99,6 +99,17 @@ func TestConnectAll_BlackHoledServerBudget(t *testing.T) { "ConnectAll took %s, budget was %s", elapsed, timeout) require.Equal(t, []string{"healthy__echo"}, toolNames(tools)) + // Per-server outcomes are summarized for logs and debug runs, + // sorted by slug. + require.Len(t, summaries, 2) + require.Equal(t, "blackhole", summaries[0].Slug) + require.Equal(t, mcpclient.ConnectOutcomeTimeout, summaries[0].Outcome) + require.NotEmpty(t, summaries[0].Error) + require.GreaterOrEqual(t, summaries[0].DurationMS, timeout.Milliseconds()) + require.Equal(t, "healthy", summaries[1].Slug) + require.Equal(t, mcpclient.ConnectOutcomeConnected, summaries[1].Outcome) + require.Equal(t, 1, summaries[1].ToolCount) + // Terminating the black-holed connections unblocks the // abandoned connect goroutine; the reaper must then drain its // result and exit. @@ -131,7 +142,7 @@ func TestConnectAll_SlowServerStillConnects(t *testing.T) { t.Cleanup(ts.Close) cfg := makeConfig("slow", ts.URL) - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) require.Equal(t, []string{"slow__echo"}, toolNames(tools)) @@ -163,7 +174,7 @@ func TestConnectAll_LateServerReaped(t *testing.T) { timeout := 500 * time.Millisecond start := time.Now() - tools, cleanup := mcpclient.ConnectAllForTest(ctx, logger, + tools, summaries, cleanup := mcpclient.ConnectAllForTest(ctx, logger, []database.MCPServerConfig{makeConfig("late", ts.URL)}, timeout, func() { reaperDone <- struct{}{} }, @@ -174,6 +185,8 @@ func TestConnectAll_LateServerReaped(t *testing.T) { require.Less(t, elapsed, 4*timeout, "ConnectAll took %s, budget was %s", elapsed, timeout) require.Empty(t, tools) + require.Len(t, summaries, 1) + require.Equal(t, mcpclient.ConnectOutcomeTimeout, summaries[0].Outcome) select { case <-reaperDone: @@ -223,7 +236,7 @@ func TestConnectAll_CleanupPromptWhenServerWedges(t *testing.T) { t.Cleanup(release) cfg := makeConfig("wedge", ts.URL) - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) require.Equal(t, []string{"wedge__echo"}, toolNames(tools)) start := time.Now() @@ -277,11 +290,13 @@ func TestConnectAll_NoToolsWedgedCloseWithinBudget(t *testing.T) { cfg := makeConfig("notools", ts.URL) start := time.Now() - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, summaries, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) elapsed := time.Since(start) t.Cleanup(cleanup) require.Empty(t, tools) + require.Len(t, summaries, 1) + require.Equal(t, mcpclient.ConnectOutcomeNoTools, summaries[0].Outcome) require.Less(t, elapsed, 5*time.Second, "ConnectAll took %s with a wedged no-tools teardown", elapsed) diff --git a/coderd/x/chatd/mcpclient/mcpclient_test.go b/coderd/x/chatd/mcpclient/mcpclient_test.go index c844b68b1913e..82434a9ace5e8 100644 --- a/coderd/x/chatd/mcpclient/mcpclient_test.go +++ b/coderd/x/chatd/mcpclient/mcpclient_test.go @@ -134,7 +134,7 @@ func TestConnectAll_DiscoverTools(t *testing.T) { ts := newTestMCPServer(t, echoTool(), greetTool()) cfg := makeConfig("myserver", ts.URL) - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) // Two tools should be discovered, namespaced with the server slug. @@ -161,7 +161,7 @@ func TestConnectAll_SanitizesDottedSlug(t *testing.T) { // Use a dotted slug like awslabs.* MCP servers ship with. // Dots violate Bedrock's tool name pattern ^[a-zA-Z0-9_-]{1,128}$. cfg := makeConfig("awslabs.aws-documentation-mcp-server", ts.URL) - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) require.Len(t, tools, 1) @@ -200,7 +200,7 @@ func TestConnectAll_TruncationCollisionWarning(t *testing.T) { cfg1 := makeConfig(slug1, ts.URL) cfg2 := makeConfig(slug2, ts.URL) - tools, cleanup := mcpclient.ConnectAll( + tools, _, cleanup := mcpclient.ConnectAll( ctx, logger, []database.MCPServerConfig{cfg1, cfg2}, nil, uuid.Nil, nil, nil, @@ -222,7 +222,7 @@ func TestConnectAll_CallTool(t *testing.T) { ts := newTestMCPServer(t, echoTool()) cfg := makeConfig("srv", ts.URL) - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) require.Len(t, tools, 1) @@ -248,7 +248,7 @@ func TestConnectAll_ToolAllowList(t *testing.T) { // Only allow the "echo" tool. cfg.ToolAllowList = []string{"echo"} - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) require.Len(t, tools, 1) @@ -266,7 +266,7 @@ func TestConnectAll_ToolDenyList(t *testing.T) { // Deny the "greet" tool, so only "echo" remains. cfg.ToolDenyList = []string{"greet"} - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) require.Len(t, tools, 1) @@ -280,7 +280,7 @@ func TestConnectAll_ConnectionFailure(t *testing.T) { cfg := makeConfig("bad", "http://127.0.0.1:0/does-not-exist") - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) assert.Empty(t, tools, "no tools should be returned for an unreachable server") @@ -297,7 +297,7 @@ func TestConnectAll_MultipleServers(t *testing.T) { cfg1 := makeConfig("alpha", ts1.URL) cfg2 := makeConfig("beta", ts2.URL) - tools, cleanup := mcpclient.ConnectAll( + tools, _, cleanup := mcpclient.ConnectAll( ctx, logger, []database.MCPServerConfig{cfg1, cfg2}, nil, @@ -323,7 +323,7 @@ func TestConnectAll_NoToolsAfterFiltering(t *testing.T) { cfg := makeConfig("filtered", ts.URL) cfg.ToolAllowList = []string{"greet"} - tools, cleanup := mcpclient.ConnectAll( + tools, _, cleanup := mcpclient.ConnectAll( ctx, logger, []database.MCPServerConfig{cfg}, @@ -348,7 +348,7 @@ func TestConnectAll_DeterministicOrder(t *testing.T) { ts2 := newTestMCPServer(t, makeTool("alpha")) ts3 := newTestMCPServer(t, makeTool("middle")) - tools, cleanup := mcpclient.ConnectAll( + tools, _, cleanup := mcpclient.ConnectAll( ctx, logger, []database.MCPServerConfig{ @@ -379,7 +379,7 @@ func TestConnectAll_DeterministicOrder(t *testing.T) { multi := newTestMCPServer(t, makeTool("zeta"), makeTool("beta")) other := newTestMCPServer(t, makeTool("gamma")) - tools, cleanup := mcpclient.ConnectAll( + tools, _, cleanup := mcpclient.ConnectAll( ctx, logger, []database.MCPServerConfig{ @@ -416,7 +416,7 @@ func TestConnectAll_DeterministicOrder(t *testing.T) { cfg2 := makeConfig("a__b", ts2.URL) cfg2.ID = uuid.MustParse("00000000-0000-0000-0000-000000000001") - tools, cleanup := mcpclient.ConnectAll( + tools, _, cleanup := mcpclient.ConnectAll( ctx, logger, []database.MCPServerConfig{cfg1, cfg2}, @@ -479,7 +479,7 @@ func TestConnectAll_AuthHeaders(t *testing.T) { TokenType: "Bearer", } - tools, cleanup := mcpclient.ConnectAll( + tools, _, cleanup := mcpclient.ConnectAll( ctx, logger, []database.MCPServerConfig{cfg}, []database.MCPServerUserToken{token}, @@ -539,7 +539,7 @@ func TestConnectAll_DisabledServer(t *testing.T) { cfg := makeConfig("disabled", ts.URL) cfg.Enabled = false - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) assert.Empty(t, tools) } @@ -554,7 +554,7 @@ func TestConnectAll_CallToolInvalidInput(t *testing.T) { ts := newTestMCPServer(t, echoTool()) cfg := makeConfig("srv", ts.URL) - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) require.Len(t, tools, 1) @@ -579,7 +579,7 @@ func TestConnectAll_ToolInfoParameters(t *testing.T) { ts := newTestMCPServer(t, echoTool()) cfg := makeConfig("srv", ts.URL) - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) require.Len(t, tools, 1) @@ -630,7 +630,7 @@ func TestConnectAll_NilRequiredBecomesEmptySlice(t *testing.T) { ts := newTestMCPServer(t, noRequiredTool) cfg := makeConfig("srv", ts.URL) - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) require.Len(t, tools, 1) @@ -677,7 +677,7 @@ func TestConnectAll_APIKeyAuth(t *testing.T) { cfg.APIKeyHeader = "X-API-Key" cfg.APIKeyValue = "secret-123" - tools, cleanup := mcpclient.ConnectAll( + tools, _, cleanup := mcpclient.ConnectAll( ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil, @@ -732,7 +732,7 @@ func TestConnectAll_CustomHeadersAuth(t *testing.T) { cfg.AuthType = "custom_headers" cfg.CustomHeaders = `{"X-Custom-Auth":"custom-val"}` - tools, cleanup := mcpclient.ConnectAll( + tools, _, cleanup := mcpclient.ConnectAll( ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil, @@ -770,7 +770,7 @@ func TestConnectAll_CustomHeadersInvalidJSON(t *testing.T) { cfg.AuthType = "custom_headers" cfg.CustomHeaders = "{not json}" - tools, cleanup := mcpclient.ConnectAll( + tools, _, cleanup := mcpclient.ConnectAll( ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil, @@ -827,7 +827,7 @@ func TestConnectAll_UserOIDCAuth(t *testing.T) { userID := uuid.New() src := staticOIDCSource{token: "fake-oidc-token"} - tools, cleanup := mcpclient.ConnectAll( + tools, _, cleanup := mcpclient.ConnectAll( ctx, logger, []database.MCPServerConfig{cfg}, nil, userID, src, nil, ) @@ -883,7 +883,7 @@ func TestConnectAll_UserOIDCAuth_NoLink(t *testing.T) { cfg.AuthType = "user_oidc" src := staticOIDCSource{token: "", err: nil} - tools, cleanup := mcpclient.ConnectAll( + tools, _, cleanup := mcpclient.ConnectAll( ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.New(), src, nil, ) @@ -919,7 +919,7 @@ func TestConnectAll_UserOIDCAuth_NilSource(t *testing.T) { cfg := makeConfig("oidc-nilsrc", ts.URL) cfg.AuthType = "user_oidc" - tools, cleanup := mcpclient.ConnectAll( + tools, _, cleanup := mcpclient.ConnectAll( ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.New(), nil, nil, ) @@ -945,7 +945,7 @@ func TestConnectAll_ParallelConnections(t *testing.T) { cfg2 := makeConfig("srv2", ts2.URL) cfg3 := makeConfig("srv3", ts3.URL) - tools, cleanup := mcpclient.ConnectAll( + tools, _, cleanup := mcpclient.ConnectAll( ctx, logger, []database.MCPServerConfig{cfg1, cfg2, cfg3}, nil, @@ -1011,7 +1011,7 @@ func TestConnectAll_ExpiredToken(t *testing.T) { Expiry: sql.NullTime{Time: time.Now().Add(-1 * time.Hour), Valid: true}, } - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, []database.MCPServerUserToken{token}, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, []database.MCPServerUserToken{token}, uuid.Nil, nil, nil) t.Cleanup(cleanup) // The server accepts any auth, so the tool is still discovered @@ -1044,7 +1044,7 @@ func TestConnectAll_EmptyAccessToken(t *testing.T) { TokenType: "Bearer", } - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, []database.MCPServerUserToken{token}, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, []database.MCPServerUserToken{token}, uuid.Nil, nil, nil) t.Cleanup(cleanup) // Tool is still discovered (server doesn't require auth), but @@ -1074,7 +1074,7 @@ func TestConnectAll_MCPToolIdentifier(t *testing.T) { Enabled: true, } - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) require.Len(t, tools, 1) @@ -1116,7 +1116,7 @@ func TestConnectAll_MCPToolIdentifier_MultipleServers(t *testing.T) { Enabled: true, } - tools, cleanup := mcpclient.ConnectAll( + tools, _, cleanup := mcpclient.ConnectAll( ctx, logger, []database.MCPServerConfig{cfg1, cfg2}, nil, @@ -1171,7 +1171,7 @@ func TestConnectAll_EmbeddedResourceText(t *testing.T) { }) cfg := makeConfig("embed-txt", ts.URL) - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) require.Len(t, tools, 1) @@ -1234,7 +1234,7 @@ func TestConnectAll_EmbeddedResourceBlob(t *testing.T) { }) cfg := makeConfig("embed-blob", ts.URL) - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) require.Len(t, tools, 1) @@ -1309,7 +1309,7 @@ func TestConnectAll_ResourceLink(t *testing.T) { }) cfg := makeConfig("res-link", ts.URL) - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) require.Len(t, tools, 1) @@ -1351,7 +1351,7 @@ func TestConnectAll_CallToolError(t *testing.T) { }) cfg := makeConfig("err-srv", ts.URL) - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) require.Len(t, tools, 1) @@ -1375,7 +1375,7 @@ func TestModelIntent_Info_WrapsSchema(t *testing.T) { cfg := makeConfig("intent-srv", ts.URL) cfg.ModelIntent = true - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) require.Len(t, tools, 1) @@ -1411,7 +1411,7 @@ func TestModelIntent_Info_NoWrapWhenDisabled(t *testing.T) { cfg := makeConfig("no-intent", ts.URL) cfg.ModelIntent = false - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) require.Len(t, tools, 1) @@ -1434,7 +1434,7 @@ func TestModelIntent_Run_UnwrapsProperties(t *testing.T) { cfg := makeConfig("unwrap-srv", ts.URL) cfg.ModelIntent = true - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) require.Len(t, tools, 1) @@ -1459,7 +1459,7 @@ func TestModelIntent_Run_UnwrapsFlat(t *testing.T) { cfg := makeConfig("flat-srv", ts.URL) cfg.ModelIntent = true - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) require.Len(t, tools, 1) @@ -1484,7 +1484,7 @@ func TestModelIntent_Run_PassthroughWhenDisabled(t *testing.T) { cfg := makeConfig("pass-srv", ts.URL) cfg.ModelIntent = false - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) require.Len(t, tools, 1) @@ -1509,7 +1509,7 @@ func TestModelIntent_Run_FallbackOnBadJSON(t *testing.T) { cfg := makeConfig("bad-srv", ts.URL) cfg.ModelIntent = true - tools, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) + tools, _, cleanup := mcpclient.ConnectAll(ctx, logger, []database.MCPServerConfig{cfg}, nil, uuid.Nil, nil, nil) t.Cleanup(cleanup) require.Len(t, tools, 1) diff --git a/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/DebugPanel.stories.tsx b/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/DebugPanel.stories.tsx index b452a6fe30b5c..a905a6a4c1111 100644 --- a/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/DebugPanel.stories.tsx +++ b/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/DebugPanel.stories.tsx @@ -712,6 +712,76 @@ export const RunWithNoSteps: Story = { }, }; +const mcpConnectRunId = "run-mcp-connect"; +const mcpConnectSummary = { + first_message: "MCP connect probe", + mcp_connect: [ + { + config_id: "b8f9f3f2-4a3f-4f8a-9c5e-2f4f4be00001", + slug: "linear", + outcome: "connected", + duration_ms: 320, + tool_count: 12, + }, + { + config_id: "b8f9f3f2-4a3f-4f8a-9c5e-2f4f4be00002", + slug: "registry", + outcome: "timeout", + duration_ms: 10000, + error: "connect: context deadline exceeded", + }, + ], +}; + +export const RunWithMCPConnectSummary: Story = { + parameters: { + queries: [ + { + key: chatDebugRunsKey(CHAT_ID), + data: [ + buildRunSummary({ + id: mcpConnectRunId, + summary: mcpConnectSummary, + }), + ], + }, + { + key: chatDebugRunKey(CHAT_ID, mcpConnectRunId), + data: { + ...MockRun, + id: mcpConnectRunId, + summary: mcpConnectSummary, + steps: [], + }, + }, + ], + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + const user = userEvent.setup(); + + const runTrigger = await canvas.findByRole("button", { + name: /MCP connect probe/i, + }); + await user.click(runTrigger); + + const section = await canvas.findByRole("region", { + name: /MCP server connections/i, + }); + const mcp = within(section); + await waitFor(() => { + expect(mcp.getByText("linear")).toBeVisible(); + expect(mcp.getByText("connected")).toBeVisible(); + expect(mcp.getByText("320ms")).toBeVisible(); + expect(mcp.getByText("12 tools")).toBeVisible(); + expect(mcp.getByText("registry")).toBeVisible(); + expect(mcp.getByText("timeout")).toBeVisible(); + expect(mcp.getByText("10.0s")).toBeVisible(); + expect(mcp.getByText("connect: context deadline exceeded")).toBeVisible(); + }); + }, +}; + // --------------------------------------------------------------------------- // Core state stories. // --------------------------------------------------------------------------- diff --git a/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/DebugRunCard.tsx b/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/DebugRunCard.tsx index 2831806256379..99887422fc319 100644 --- a/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/DebugRunCard.tsx +++ b/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/DebugRunCard.tsx @@ -1,6 +1,6 @@ import { saveAs } from "file-saver"; import { ChevronDownIcon, DownloadIcon } from "lucide-react"; -import { type FC, useState } from "react"; +import { type FC, useId, useState } from "react"; import { useQuery } from "react-query"; import { toast } from "sonner"; import { getErrorDetail, getErrorMessage } from "#/api/errors"; @@ -49,6 +49,17 @@ const getDurationLabel = (startedAt: string, finishedAt?: string): string => { return durationMs !== null ? compactDuration(durationMs) : "-"; }; +const getMCPOutcomeBadgeVariant = (outcome: string) => { + switch (outcome) { + case "connected": + return "green"; + case "no_tools": + return "default"; + default: + return "destructive"; + } +}; + export const DebugRunCard: FC = ({ run, chatId, @@ -57,6 +68,7 @@ export const DebugRunCard: FC = ({ }) => { const [isExpanded, setIsExpanded] = useState(false); const [isExporting, setIsExporting] = useState(false); + const mcpConnectHeadingId = useId(); const runDetailQuery = useQuery({ ...chatDebugRun(chatId, run.id), enabled: isVisible && isExpanded, @@ -200,6 +212,58 @@ export const DebugRunCard: FC = ({

) : null} + {summaryVm.mcpConnect.length > 0 ? ( +
+

+ MCP server connections +

+
    + {summaryVm.mcpConnect.map((server) => ( +
  • + + {server.slug} + + + {server.outcome} + + {server.durationMs !== undefined ? ( + + {compactDuration(server.durationMs)} + + ) : null} + {server.toolCount !== undefined && + server.toolCount > 0 ? ( + + {server.toolCount}{" "} + {server.toolCount === 1 ? "tool" : "tools"} + + ) : null} + {server.error ? ( + + {server.error} + + ) : null} +
    • + ))} +
+
+ ) : null} {steps.map((step) => ( ))} diff --git a/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/debugPanelUtils.test.ts b/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/debugPanelUtils.test.ts index 0b0866d6456ca..5d9684d7bf637 100644 --- a/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/debugPanelUtils.test.ts +++ b/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/debugPanelUtils.test.ts @@ -826,6 +826,7 @@ describe("coerceRunSummary", () => { stepCount: 3, totalInputTokens: 120, totalOutputTokens: 45, + mcpConnect: [], warnings: [], }); }); @@ -851,10 +852,53 @@ describe("coerceRunSummary", () => { stepCount: undefined, totalInputTokens: undefined, totalOutputTokens: undefined, + mcpConnect: [], warnings: [], }); }); + it("coerces MCP connect summaries and drops malformed entries", () => { + const summary = coerceRunSummary({ + mcp_connect: [ + { + slug: "registry", + outcome: "timeout", + duration_ms: 10000, + error: "connect: context deadline exceeded", + }, + { + slug: "linear", + outcome: "connected", + duration_ms: 320, + tool_count: 12, + }, + { outcome: "error" }, + "not-a-record", + ], + }); + + expect(summary.mcpConnect).toEqual([ + { + slug: "registry", + outcome: "timeout", + durationMs: 10000, + toolCount: undefined, + error: "connect: context deadline exceeded", + }, + { + slug: "linear", + outcome: "connected", + durationMs: 320, + toolCount: 12, + error: undefined, + }, + ]); + }); + + it("returns an empty MCP connect list for non-array values", () => { + expect(coerceRunSummary({ mcp_connect: "oops" }).mcpConnect).toEqual([]); + }); + it("unwraps JSON-string payloads before coercing", () => { const summary = coerceRunSummary( JSON.stringify({ diff --git a/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/debugPanelUtils.ts b/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/debugPanelUtils.ts index 9a9e570364d2d..39dd375bba076 100644 --- a/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/debugPanelUtils.ts +++ b/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/debugPanelUtils.ts @@ -436,6 +436,14 @@ export const compactDuration = (ms: number): string => { // View-model types for coerced debug payloads. // --------------------------------------------------------------------------- +export interface MCPConnectSummaryViewModel { + slug: string; + outcome: string; + durationMs: number | undefined; + toolCount: number | undefined; + error: string | undefined; +} + interface RunSummaryViewModel { primaryLabel: string; endpointLabel: string | undefined; @@ -444,6 +452,7 @@ interface RunSummaryViewModel { stepCount: number | undefined; totalInputTokens: number | undefined; totalOutputTokens: number | undefined; + mcpConnect: MCPConnectSummaryViewModel[]; warnings: string[]; } @@ -868,6 +877,32 @@ const extractKnownFields = ( // Public coercion: run summary. // --------------------------------------------------------------------------- +const coerceMCPConnectSummaries = ( + value: unknown, +): MCPConnectSummaryViewModel[] => { + if (!Array.isArray(value)) { + return []; + } + const result: MCPConnectSummaryViewModel[] = []; + for (const entry of value) { + if (!isRecord(entry)) { + continue; + } + const slug = toOptionalString(entry.slug); + if (!slug) { + continue; + } + result.push({ + slug, + outcome: toOptionalString(entry.outcome) ?? "unknown", + durationMs: toFiniteNumber(entry.duration_ms), + toolCount: toFiniteNumber(entry.tool_count), + error: toOptionalString(entry.error), + }); + } + return result; +}; + export const coerceRunSummary = (data: unknown): RunSummaryViewModel => { const defaults: RunSummaryViewModel = { primaryLabel: "", @@ -877,6 +912,7 @@ export const coerceRunSummary = (data: unknown): RunSummaryViewModel => { stepCount: undefined, totalInputTokens: undefined, totalOutputTokens: undefined, + mcpConnect: [], warnings: [], }; const parsed = deepParse(data); @@ -924,6 +960,9 @@ export const coerceRunSummary = (data: unknown): RunSummaryViewModel => { "completionTokens", ), ), + mcpConnect: coerceMCPConnectSummaries( + pickField(parsed, "mcp_connect", "mcpConnect"), + ), warnings: [], }; }; From ab9dab51c9cc0115bf2d4815b60e105fe7fb42f0 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 20 Aug 2026 16:47:40 +0000 Subject: [PATCH 2/8] feat(site/src/pages/AgentsPage): label non-chat-turn debug runs by kind A failed title generation or quickgen run with a first_message label was indistinguishable from a failed chat turn in the debug panel. Show the run kind in the card metadata whenever a non-chat-turn run carries a first_message label. --- .../DebugPanel/DebugPanel.stories.tsx | 40 +++++++++++++++++++ .../RightPanel/DebugPanel/DebugRunCard.tsx | 9 +++++ 2 files changed, 49 insertions(+) diff --git a/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/DebugPanel.stories.tsx b/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/DebugPanel.stories.tsx index a905a6a4c1111..131299431ad63 100644 --- a/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/DebugPanel.stories.tsx +++ b/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/DebugPanel.stories.tsx @@ -1289,6 +1289,46 @@ export const CompactionAndTitleGenerationBadges: Story = { }, }; +export const NonChatTurnKindShownWithFirstMessage: Story = { + parameters: { + queries: [ + { + key: chatDebugRunsKey(CHAT_ID), + data: [ + buildRunSummary({ + id: "run-title-with-label", + kind: "title_generation", + status: "error", + model: "gpt-4o-mini", + summary: { first_message: "Summarize my workspace" }, + }), + buildRunSummary({ + id: "run-turn-with-label", + kind: "chat_turn", + status: "completed", + summary: { first_message: "Fix the login bug" }, + }), + ], + }, + ], + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + // A failed title generation with a first_message label must + // still identify itself by kind so it is distinguishable from + // a failed chat turn. + const titleRun = await canvas.findByRole("button", { + name: /Summarize my workspace/i, + }); + await expect(within(titleRun).getByText("Title Generation")).toBeVisible(); + // Chat turns keep their metadata kind-free. + const chatRun = await canvas.findByRole("button", { + name: /Fix the login bug/i, + }); + expect(within(chatRun).queryByText("Chat Turn")).toBeNull(); + }, +}; + export const LongRawPayloads: Story = { parameters: { queries: [ diff --git a/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/DebugRunCard.tsx b/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/DebugRunCard.tsx index 99887422fc319..895d47ea2c703 100644 --- a/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/DebugRunCard.tsx +++ b/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/DebugRunCard.tsx @@ -97,7 +97,16 @@ export const DebugRunCard: FC = ({ // Step count from detail or summary. const stepCount = steps.length > 0 ? steps.length : summaryVm.stepCount; const durationLabel = getDurationLabel(run.started_at, run.finished_at); + // Non-chat-turn runs (title generation, quickgen, compaction) + // usually carry a first_message label that hides the kind, so + // surface the kind in the metadata; otherwise a failed title + // generation is indistinguishable from a failed chat turn. + const kindLabel = + run.kind !== "chat_turn" && summaryVm.primaryLabel.trim() + ? getRunKindLabel(run.kind) + : undefined; const metadataItems = [ + kindLabel, modelLabel || undefined, stepCount !== undefined && stepCount > 0 ? `${stepCount} ${stepCount === 1 ? "step" : "steps"}` From 304f86705e6508c21c2f6006162e592d8cc32bc3 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Mon, 24 Aug 2026 10:42:38 +0000 Subject: [PATCH 3/8] fix(coderd/x/chatd): record MCP connect outcomes from every generation preparation --- coderd/x/chatd/active_turn_debug.go | 20 +++++ .../chatd/active_turn_debug_internal_test.go | 73 +++++++++++++++++++ .../DebugPanel/DebugPanel.stories.tsx | 15 +++- .../RightPanel/DebugPanel/DebugRunCard.tsx | 4 +- 4 files changed, 108 insertions(+), 4 deletions(-) diff --git a/coderd/x/chatd/active_turn_debug.go b/coderd/x/chatd/active_turn_debug.go index d9c2721496d61..716e14afce753 100644 --- a/coderd/x/chatd/active_turn_debug.go +++ b/coderd/x/chatd/active_turn_debug.go @@ -9,6 +9,7 @@ import ( "cdr.dev/slog/v3" "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/x/chatd/chatdebug" + "github.com/coder/coder/v2/coderd/x/chatd/mcpclient" ) type runnerDebugTurn struct { @@ -57,6 +58,7 @@ func (d *runnerDebugTurn) Ensure( return ctx } if d.created { + d.mergeMCPConnectSummariesLocked(debug) return d.contextLocked(ctx) } if debug == nil || !debug.Enabled || debug.Service == nil || @@ -131,6 +133,24 @@ func (d *runnerDebugTurn) Context(ctx context.Context) context.Context { return d.contextLocked(ctx) } +// mergeMCPConnectSummariesLocked appends a later preparation's +// per-server MCP connect outcomes to the seeded mcp_connect key. +// chatd reconnects to every configured MCP server on each +// generation step while the run is created only once, so without +// this merge only the first preparation's outcomes would survive +// to the finalized run and a server that degrades mid-turn would +// still be reported as connected. +func (d *runnerDebugTurn) mergeMCPConnectSummariesLocked(debug *generationDebug) { + if debug == nil || len(debug.MCPConnectSummaries) == 0 { + return + } + if d.seedSummary == nil { + d.seedSummary = make(map[string]any, 1) + } + existing, _ := d.seedSummary["mcp_connect"].([]mcpclient.ConnectSummary) + d.seedSummary["mcp_connect"] = append(existing, debug.MCPConnectSummaries...) +} + func (d *runnerDebugTurn) contextLocked(ctx context.Context) context.Context { if !d.created || d.runContext.RunID == uuid.Nil { return ctx diff --git a/coderd/x/chatd/active_turn_debug_internal_test.go b/coderd/x/chatd/active_turn_debug_internal_test.go index f599021853eaf..3b3e515675ad1 100644 --- a/coderd/x/chatd/active_turn_debug_internal_test.go +++ b/coderd/x/chatd/active_turn_debug_internal_test.go @@ -3,6 +3,7 @@ package chatd import ( "context" "database/sql" + "encoding/json" "testing" "github.com/google/uuid" @@ -12,6 +13,7 @@ import ( "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/database/dbmock" "github.com/coder/coder/v2/coderd/x/chatd/chatdebug" + "github.com/coder/coder/v2/coderd/x/chatd/mcpclient" "github.com/coder/coder/v2/testutil" ) @@ -153,3 +155,74 @@ func TestRunnerDebugTurnFinalizeOnce(t *testing.T) { turn.Finalize(ctx) turn.Finalize(ctx) } + +func TestRunnerDebugTurnEnsureMergesMCPConnectSummaries(t *testing.T) { + t.Parallel() + + ctx := testutil.Context(t, testutil.WaitLong) + runnerCtx, cancel := context.WithCancel(ctx) + defer cancel() + ctrl := gomock.NewController(t) + db := dbmock.NewMockStore(ctrl) + chatID := uuid.New() + runID := uuid.New() + configID := uuid.New() + svc := chatdebug.NewService(db, testutil.Logger(t), nil) + turn := newRunnerDebugTurn(runnerCtx, testutil.Logger(t)) + + db.EXPECT().InsertChatDebugRun(gomock.Any(), gomock.Any()). + Return(database.ChatDebugRun{ + ID: runID, + ChatID: chatID, + Kind: string(chatdebug.KindChatTurn), + Status: string(chatdebug.StatusInProgress), + }, nil). + Times(1) + db.EXPECT().GetChatDebugStepsByRunID(gomock.Any(), runID).Return(nil, nil).Times(1) + var finalSummary []byte + db.EXPECT().UpdateChatDebugRun(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, params database.UpdateChatDebugRunParams) (database.ChatDebugRun, error) { + require.True(t, params.Summary.Valid) + finalSummary = params.Summary.RawMessage + return database.ChatDebugRun{ID: runID, ChatID: chatID}, nil + }).Times(1) + + base := generationDebug{ + Enabled: true, + Service: svc, + TriggerMessageID: 1, + ModelConfig: database.ChatModelConfig{ID: uuid.New()}, + } + first := base + first.MCPConnectSummaries = []mcpclient.ConnectSummary{{ + ConfigID: configID, + Slug: "registry", + Outcome: mcpclient.ConnectOutcomeConnected, + DurationMS: 17, + ToolCount: 1, + }} + second := base + second.MCPConnectSummaries = []mcpclient.ConnectSummary{{ + ConfigID: configID, + Slug: "registry", + Outcome: mcpclient.ConnectOutcomeTimeout, + DurationMS: 10000, + Error: "connect: context deadline exceeded", + }} + + turn.Ensure(ctx, database.Chat{ID: chatID}, &first) + // A later generation step reconnects and reports a degraded + // outcome for the same server; it must survive to the + // finalized summary. + turn.Ensure(ctx, database.Chat{ID: chatID}, &second) + turn.RecordOutcome(chatdebug.StatusCompleted) + turn.Finalize(ctx) + + var summary struct { + MCPConnect []mcpclient.ConnectSummary `json:"mcp_connect"` + } + require.NoError(t, json.Unmarshal(finalSummary, &summary)) + require.Len(t, summary.MCPConnect, 2) + require.Equal(t, mcpclient.ConnectOutcomeConnected, summary.MCPConnect[0].Outcome) + require.Equal(t, mcpclient.ConnectOutcomeTimeout, summary.MCPConnect[1].Outcome) +} diff --git a/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/DebugPanel.stories.tsx b/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/DebugPanel.stories.tsx index 131299431ad63..e59c896024a30 100644 --- a/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/DebugPanel.stories.tsx +++ b/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/DebugPanel.stories.tsx @@ -730,6 +730,15 @@ const mcpConnectSummary = { duration_ms: 10000, error: "connect: context deadline exceeded", }, + // The same server reported again by a later generation + // step; entries accumulate across the turn's preparations. + { + config_id: "b8f9f3f2-4a3f-4f8a-9c5e-2f4f4be00002", + slug: "registry", + outcome: "connected", + duration_ms: 45, + tool_count: 3, + }, ], }; @@ -771,13 +780,15 @@ export const RunWithMCPConnectSummary: Story = { const mcp = within(section); await waitFor(() => { expect(mcp.getByText("linear")).toBeVisible(); - expect(mcp.getByText("connected")).toBeVisible(); + expect(mcp.getAllByText("connected")).toHaveLength(2); expect(mcp.getByText("320ms")).toBeVisible(); expect(mcp.getByText("12 tools")).toBeVisible(); - expect(mcp.getByText("registry")).toBeVisible(); + expect(mcp.getAllByText("registry")).toHaveLength(2); expect(mcp.getByText("timeout")).toBeVisible(); expect(mcp.getByText("10.0s")).toBeVisible(); expect(mcp.getByText("connect: context deadline exceeded")).toBeVisible(); + expect(mcp.getByText("45ms")).toBeVisible(); + expect(mcp.getByText("3 tools")).toBeVisible(); }); }, }; diff --git a/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/DebugRunCard.tsx b/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/DebugRunCard.tsx index 895d47ea2c703..b7d6facb4b706 100644 --- a/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/DebugRunCard.tsx +++ b/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/DebugRunCard.tsx @@ -233,9 +233,9 @@ export const DebugRunCard: FC = ({ MCP server connections
    - {summaryVm.mcpConnect.map((server) => ( + {summaryVm.mcpConnect.map((server, index) => (
  • From e3ed5ed4f177cefb4c284c333d09fa9168bdba56 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Tue, 25 Aug 2026 10:08:56 +0000 Subject: [PATCH 4/8] fix(coderd/x/chatd): record MCP connect outcomes at the generation dispatch point --- coderd/x/chatd/active_turn_debug.go | 47 ++++++--- .../chatd/active_turn_debug_internal_test.go | 37 +++++-- coderd/x/chatd/chatd_test.go | 99 +++++++++++++++++++ coderd/x/chatd/generation.go | 6 ++ 4 files changed, 165 insertions(+), 24 deletions(-) diff --git a/coderd/x/chatd/active_turn_debug.go b/coderd/x/chatd/active_turn_debug.go index 716e14afce753..be68df03af433 100644 --- a/coderd/x/chatd/active_turn_debug.go +++ b/coderd/x/chatd/active_turn_debug.go @@ -58,7 +58,6 @@ func (d *runnerDebugTurn) Ensure( return ctx } if d.created { - d.mergeMCPConnectSummariesLocked(debug) return d.contextLocked(ctx) } if debug == nil || !debug.Enabled || debug.Service == nil || @@ -70,15 +69,16 @@ func (d *runnerDebugTurn) Ensure( seedSummary := chatdebug.SeedSummary( chatdebug.TruncateLabel(debug.TriggerLabel, chatdebug.MaxLabelLength), ) - // Seed per-server MCP connect outcomes so slow or failing - // servers appear in the run instead of as a silent gap before - // the first step. Seeded keys survive FinalizeRun's summary - // aggregation. - if len(debug.MCPConnectSummaries) > 0 { + // Carry per-server MCP connect outcomes stashed by + // RecordMCPConnectSummaries before the run existed, so slow or + // failing servers appear in the run instead of as a silent gap + // before the first step. Seeded keys survive FinalizeRun's + // summary aggregation. + if stashed, ok := d.seedSummary["mcp_connect"]; ok { if seedSummary == nil { seedSummary = make(map[string]any, 1) } - seedSummary["mcp_connect"] = debug.MCPConnectSummaries + seedSummary["mcp_connect"] = stashed } rootChatID := uuid.Nil if chat.RootChatID.Valid { @@ -133,13 +133,32 @@ func (d *runnerDebugTurn) Context(ctx context.Context) context.Context { return d.contextLocked(ctx) } -// mergeMCPConnectSummariesLocked appends a later preparation's -// per-server MCP connect outcomes to the seeded mcp_connect key. -// chatd reconnects to every configured MCP server on each -// generation step while the run is created only once, so without -// this merge only the first preparation's outcomes would survive -// to the finalized run and a server that degrades mid-turn would -// still be reported as connected. +// RecordMCPConnectSummaries merges one preparation's per-server MCP +// connect outcomes into the mcp_connect summary key. It runs at the +// generation dispatch point so every preparation is recorded, +// including ones feeding actions that never reach Ensure (local +// tool execution, requires-action, turn finishing). Outcomes +// recorded before the run exists are stashed and seeded into the +// run when Ensure creates it. +func (d *runnerDebugTurn) RecordMCPConnectSummaries(debug *generationDebug) { + if d == nil { + return + } + d.mu.Lock() + defer d.mu.Unlock() + if d.disabled || d.finalized { + return + } + d.mergeMCPConnectSummariesLocked(debug) +} + +// mergeMCPConnectSummariesLocked appends a preparation's per-server +// MCP connect outcomes to the mcp_connect summary key. chatd +// reconnects to every configured MCP server on each generation step +// while the run is created only once, so without this merge only +// one preparation's outcomes would survive to the finalized run and +// a server that degrades mid-turn would still be reported as +// connected. func (d *runnerDebugTurn) mergeMCPConnectSummariesLocked(debug *generationDebug) { if debug == nil || len(debug.MCPConnectSummaries) == 0 { return diff --git a/coderd/x/chatd/active_turn_debug_internal_test.go b/coderd/x/chatd/active_turn_debug_internal_test.go index 3b3e515675ad1..e038b56990846 100644 --- a/coderd/x/chatd/active_turn_debug_internal_test.go +++ b/coderd/x/chatd/active_turn_debug_internal_test.go @@ -156,7 +156,7 @@ func TestRunnerDebugTurnFinalizeOnce(t *testing.T) { turn.Finalize(ctx) } -func TestRunnerDebugTurnEnsureMergesMCPConnectSummaries(t *testing.T) { +func TestRunnerDebugTurnRecordMCPConnectSummaries(t *testing.T) { t.Parallel() ctx := testutil.Context(t, testutil.WaitLong) @@ -170,13 +170,18 @@ func TestRunnerDebugTurnEnsureMergesMCPConnectSummaries(t *testing.T) { svc := chatdebug.NewService(db, testutil.Logger(t), nil) turn := newRunnerDebugTurn(runnerCtx, testutil.Logger(t)) + var seededSummary []byte db.EXPECT().InsertChatDebugRun(gomock.Any(), gomock.Any()). - Return(database.ChatDebugRun{ - ID: runID, - ChatID: chatID, - Kind: string(chatdebug.KindChatTurn), - Status: string(chatdebug.StatusInProgress), - }, nil). + DoAndReturn(func(_ context.Context, params database.InsertChatDebugRunParams) (database.ChatDebugRun, error) { + require.True(t, params.Summary.Valid) + seededSummary = params.Summary.RawMessage + return database.ChatDebugRun{ + ID: runID, + ChatID: chatID, + Kind: string(chatdebug.KindChatTurn), + Status: string(chatdebug.StatusInProgress), + }, nil + }). Times(1) db.EXPECT().GetChatDebugStepsByRunID(gomock.Any(), runID).Return(nil, nil).Times(1) var finalSummary []byte @@ -210,14 +215,26 @@ func TestRunnerDebugTurnEnsureMergesMCPConnectSummaries(t *testing.T) { Error: "connect: context deadline exceeded", }} + // The dispatch point records each preparation before its action + // runs, so the first outcome lands before Ensure creates the + // run and must be seeded into it. + turn.RecordMCPConnectSummaries(&first) turn.Ensure(ctx, database.Chat{ID: chatID}, &first) // A later generation step reconnects and reports a degraded - // outcome for the same server; it must survive to the - // finalized summary. - turn.Ensure(ctx, database.Chat{ID: chatID}, &second) + // outcome for the same server; its action may never reach + // Ensure, and the outcome must still survive to the finalized + // summary. + turn.RecordMCPConnectSummaries(&second) turn.RecordOutcome(chatdebug.StatusCompleted) turn.Finalize(ctx) + var seeded struct { + MCPConnect []mcpclient.ConnectSummary `json:"mcp_connect"` + } + require.NoError(t, json.Unmarshal(seededSummary, &seeded)) + require.Len(t, seeded.MCPConnect, 1) + require.Equal(t, mcpclient.ConnectOutcomeConnected, seeded.MCPConnect[0].Outcome) + var summary struct { MCPConnect []mcpclient.ConnectSummary `json:"mcp_connect"` } diff --git a/coderd/x/chatd/chatd_test.go b/coderd/x/chatd/chatd_test.go index bfc2c946e2c72..dfefa3cc20e07 100644 --- a/coderd/x/chatd/chatd_test.go +++ b/coderd/x/chatd/chatd_test.go @@ -11193,6 +11193,105 @@ func TestMCPServerToolInvocation(t *testing.T) { "MCP tool result should be persisted as a tool message in the database") } +func TestActiveServer_ChatTurnDebugRunRecordsMCPConnectPerPreparation(t *testing.T) { + t.Parallel() + + db, ps := dbtestutil.NewDB(t) + ctx := testutil.Context(t, testutil.WaitLong) + + mcpSrv := newTestMCPServer("test-mcp") + addTestMCPTextTool(mcpSrv, "echo", "Echoes the input", "echo: ") + mcpTS := httptest.NewServer(testMCPHTTPHandler(mcpSrv)) + t.Cleanup(mcpTS.Close) + + var callCount atomic.Int32 + openAIURL := chattest.NewOpenAI(t, func(req *chattest.OpenAIRequest) chattest.OpenAIResponse { + if !req.Stream { + return chattest.OpenAINonStreamingResponse("title") + } + if callCount.Add(1) == 1 { + return chattest.OpenAIStreamingResponse( + chattest.OpenAIToolCallChunk( + "test-mcp__echo", + `{"input":"hello from LLM"}`, + ), + ) + } + return chattest.OpenAIStreamingResponse( + chattest.OpenAITextChunks("Got it!")..., + ) + }) + + user, org, model := seedChatDependenciesWithProvider(t, db, "openai-compat", openAIURL) + mcpConfig := dbgen.MCPServerConfig(t, db, database.MCPServerConfig{ + OrganizationID: org.ID, + DisplayName: "Test MCP", + Slug: "test-mcp", + Url: mcpTS.URL, + CreatedBy: uuid.NullUUID{UUID: user.ID, Valid: true}, + UpdatedBy: uuid.NullUUID{UUID: user.ID, Valid: true}, + }) + + server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) { + withoutMCPToolSearch(cfg) + cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(chattest.NewMockAIBridgeTransport(t, openAIURL)) + cfg.AlwaysEnableDebugLogs = true + }) + + chat, err := server.CreateChat(ctx, chatd.CreateOptions{ + OrganizationID: org.ID, + OwnerID: user.ID, + Title: "mcp-connect-debug-test", + ModelConfigID: model.ID, + MCPServerIDs: []uuid.UUID{mcpConfig.ID}, + InitialUserContent: []codersdk.ChatMessagePart{ + codersdk.ChatMessageText("Echo something via MCP."), + }, + }) + require.NoError(t, err) + + waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusWaiting) + require.NoError(t, server.Close()) + debugCtx := testutil.Context(t, testutil.WaitLong) + + var chatTurnRuns []database.ChatDebugRun + testutil.Eventually(debugCtx, t, func(ctx context.Context) bool { + runs, runsErr := db.GetChatDebugRunsByChatID(ctx, database.GetChatDebugRunsByChatIDParams{ + ChatID: chat.ID, + LimitVal: 100, + }) + if runsErr != nil { + return false + } + chatTurnRuns = chatTurnRuns[:0] + for _, run := range runs { + if run.Kind == string(codersdk.ChatDebugRunKindChatTurn) { + chatTurnRuns = append(chatTurnRuns, run) + } + } + return len(chatTurnRuns) == 1 && chatTurnRuns[0].FinishedAt.Valid + }, testutil.IntervalFast) + + require.Len(t, chatTurnRuns, 1) + var summary struct { + MCPConnect []struct { + Slug string `json:"slug"` + Outcome string `json:"outcome"` + } `json:"mcp_connect"` + } + require.NoError(t, json.Unmarshal(chatTurnRuns[0].Summary, &summary)) + // A tool-call turn runs at least three preparations: the + // assistant step that requests the tool, the local tool + // execution step, and the assistant step that consumes the + // result. Each preparation reconnects to the MCP server and + // must contribute its connect outcome to the finalized run. + require.GreaterOrEqual(t, len(summary.MCPConnect), 3) + for _, entry := range summary.MCPConnect { + require.Equal(t, "test-mcp", entry.Slug) + require.Equal(t, "connected", entry.Outcome) + } +} + func TestPlanModeRootChatApprovedExternalMCPToolInvocation(t *testing.T) { t.Parallel() diff --git a/coderd/x/chatd/generation.go b/coderd/x/chatd/generation.go index 5843e822966d2..f326a6d2ad3c7 100644 --- a/coderd/x/chatd/generation.go +++ b/coderd/x/chatd/generation.go @@ -495,6 +495,12 @@ func (s *taskStarter) StartGeneration(ctx context.Context, input chatWorkerTaskS return s.finishGenerationError(ctx, machine, input, err, generationAttemptNotRequired) } + // Every preparation reconnects to MCP, including ones whose + // action below never reaches Ensure; record each + // preparation's connect outcomes at the single dispatch + // point. + input.DebugTurn.RecordMCPConnectSummaries(prepared.Debug) + var actionErr error switch decision.kind { case generationActionEnterRequiresAction: From 937fa5e8ed4841a661f4a9fb14640515456ea4ff Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Tue, 25 Aug 2026 10:29:28 +0000 Subject: [PATCH 5/8] fix(coderd/x/chatd): record MCP connect outcomes before generation decisions --- coderd/x/chatd/chatd_test.go | 118 +++++++++++++++++++++++++++++++++++ coderd/x/chatd/generation.go | 11 ++-- 2 files changed, 123 insertions(+), 6 deletions(-) diff --git a/coderd/x/chatd/chatd_test.go b/coderd/x/chatd/chatd_test.go index dfefa3cc20e07..b1a766461e37e 100644 --- a/coderd/x/chatd/chatd_test.go +++ b/coderd/x/chatd/chatd_test.go @@ -11292,6 +11292,124 @@ func TestActiveServer_ChatTurnDebugRunRecordsMCPConnectPerPreparation(t *testing } } +func TestActiveServer_ChatTurnDebugRunRecordsMCPConnectOnDecisionError(t *testing.T) { + t.Parallel() + + db, ps := dbtestutil.NewDB(t) + ctx := testutil.Context(t, testutil.WaitLong) + + mcpSrv := newTestMCPServer("test-mcp") + addTestMCPTextTool(mcpSrv, "echo", "Echoes the input", "echo: ") + mcpTS := httptest.NewServer(testMCPHTTPHandler(mcpSrv)) + t.Cleanup(mcpTS.Close) + + // Every streamed assistant response is a high-usage tool call, + // so the turn compacts once and the post-compaction assistant + // is still over the context limit: the next decision fails + // terminally with the still-over-limit error. + var streamCount atomic.Int32 + anthropicURL := chattest.NewAnthropic(t, func(req *chattest.AnthropicRequest) chattest.AnthropicResponse { + if !req.Stream { + if strings.Contains(anthropicRequestBody(t, *req), "You are performing a context compaction") { + return anthropicCompactionResponse("summary text for compaction") + } + return chattest.AnthropicNonStreamingResponse("title") + } + if streamCount.Add(1) == 1 { + return highUsageReadFileResponse("/tmp/a.txt") + } + return highUsageReadFileResponse("/tmp/b.txt") + }) + + user, org, model := seedAnthropicChatDependencies(t, db, anthropicURL) + model = updateChatModelCompressionThreshold(t, db, model, 100, 70) + mcpConfig := dbgen.MCPServerConfig(t, db, database.MCPServerConfig{ + OrganizationID: org.ID, + DisplayName: "Test MCP", + Slug: "test-mcp", + Url: mcpTS.URL, + CreatedBy: uuid.NullUUID{UUID: user.ID, Valid: true}, + UpdatedBy: uuid.NullUUID{UUID: user.ID, Valid: true}, + }) + ws, dbAgent := seedWorkspaceWithAgent(t, db, user.ID) + + ctrl := gomock.NewController(t) + mockConn := agentconnmock.NewMockAgentConn(ctrl) + setupToolExecutionAgentConn(t, mockConn) + mockConn.EXPECT().ReadFileLines(gomock.Any(), "/tmp/a.txt", int64(1), int64(0), gomock.Any()). + Return(workspacesdk.ReadFileLinesResponse{Success: true, FileSize: 12, TotalLines: 1, LinesRead: 1, Content: "1\tpackage main"}, nil). + Times(1) + mockConn.EXPECT().ReadFileLines(gomock.Any(), "/tmp/b.txt", int64(1), int64(0), gomock.Any()). + Return(workspacesdk.ReadFileLinesResponse{Success: true, FileSize: 12, TotalLines: 1, LinesRead: 1, Content: "1\tpackage main"}, nil). + Times(1) + + server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) { + withoutMCPToolSearch(cfg) + cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(chattest.NewMockAIBridgeTransport(t, anthropicURL, chattest.WithPreservePath())) + cfg.AlwaysEnableDebugLogs = true + cfg.AgentConn = func(_ context.Context, agentID uuid.UUID) (workspacesdk.AgentConn, func(), error) { + require.Equal(t, dbAgent.ID, agentID) + return mockConn, func() {}, nil + } + }) + + chat, err := server.CreateChat(ctx, chatd.CreateOptions{ + OrganizationID: org.ID, + OwnerID: user.ID, + WorkspaceID: uuid.NullUUID{UUID: ws.ID, Valid: true}, + AgentID: uuid.NullUUID{UUID: dbAgent.ID, Valid: true}, + Title: "mcp-connect-decision-error-test", + ModelConfigID: model.ID, + MCPServerIDs: []uuid.UUID{mcpConfig.ID}, + InitialUserContent: []codersdk.ChatMessagePart{ + codersdk.ChatMessageText("read the files"), + }, + }) + require.NoError(t, err) + + waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusError) + require.NoError(t, server.Close()) + debugCtx := testutil.Context(t, testutil.WaitLong) + + var chatTurnRuns []database.ChatDebugRun + testutil.Eventually(debugCtx, t, func(ctx context.Context) bool { + runs, runsErr := db.GetChatDebugRunsByChatID(ctx, database.GetChatDebugRunsByChatIDParams{ + ChatID: chat.ID, + LimitVal: 100, + }) + if runsErr != nil { + return false + } + chatTurnRuns = chatTurnRuns[:0] + for _, run := range runs { + if run.Kind == string(codersdk.ChatDebugRunKindChatTurn) { + chatTurnRuns = append(chatTurnRuns, run) + } + } + return len(chatTurnRuns) == 1 && chatTurnRuns[0].FinishedAt.Valid + }, testutil.IntervalFast) + + require.Len(t, chatTurnRuns, 1) + var summary struct { + MCPConnect []struct { + Slug string `json:"slug"` + Outcome string `json:"outcome"` + } `json:"mcp_connect"` + } + require.NoError(t, json.Unmarshal(chatTurnRuns[0].Summary, &summary)) + // The turn performs six preparations: the first assistant step, + // its tool execution, the compaction, the post-compaction + // assistant step, its tool execution, and the preparation whose + // decision fails with still-over-limit. Each one must + // contribute its connect outcome, including the final + // preparation that exits through the decision error. + require.GreaterOrEqual(t, len(summary.MCPConnect), 6) + for _, entry := range summary.MCPConnect { + require.Equal(t, "test-mcp", entry.Slug) + require.Equal(t, "connected", entry.Outcome) + } +} + func TestPlanModeRootChatApprovedExternalMCPToolInvocation(t *testing.T) { t.Parallel() diff --git a/coderd/x/chatd/generation.go b/coderd/x/chatd/generation.go index f326a6d2ad3c7..d8785da92ce43 100644 --- a/coderd/x/chatd/generation.go +++ b/coderd/x/chatd/generation.go @@ -459,6 +459,11 @@ func (s *taskStarter) StartGeneration(ctx context.Context, input chatWorkerTaskS return s.finishGenerationError(ctx, machine, input, err, generationAttemptNotRequired) } cleanup := prepared.Cleanup + // Every preparation reconnects to MCP; record its connect + // outcomes as soon as preparation succeeds so exits that + // never reach the action dispatch (decision errors and + // actions that skip Ensure) still contribute to the run. + input.DebugTurn.RecordMCPConnectSummaries(prepared.Debug) var decision generationDecision if input.StopNudges.consume(stopNudgeKey(prepared.Messages)) { decision = generationDecision{kind: generationActionGenerateAssistant} @@ -495,12 +500,6 @@ func (s *taskStarter) StartGeneration(ctx context.Context, input chatWorkerTaskS return s.finishGenerationError(ctx, machine, input, err, generationAttemptNotRequired) } - // Every preparation reconnects to MCP, including ones whose - // action below never reaches Ensure; record each - // preparation's connect outcomes at the single dispatch - // point. - input.DebugTurn.RecordMCPConnectSummaries(prepared.Debug) - var actionErr error switch decision.kind { case generationActionEnterRequiresAction: From 08221f02f00fee37df95e4af0a0cab2649ca4ed6 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 27 Aug 2026 00:30:39 +0000 Subject: [PATCH 6/8] fix(coderd/x/chatd): record failing-preparation MCP connect outcomes and bound retention --- coderd/x/chatd/active_turn_debug.go | 54 ++++++--- .../chatd/active_turn_debug_internal_test.go | 109 +++++++++++++++-- coderd/x/chatd/chatd_test.go | 112 ++++++++++++++++++ coderd/x/chatd/generation.go | 20 ++-- coderd/x/chatd/generation_preparer.go | 12 +- 5 files changed, 263 insertions(+), 44 deletions(-) diff --git a/coderd/x/chatd/active_turn_debug.go b/coderd/x/chatd/active_turn_debug.go index be68df03af433..01c30d86910a2 100644 --- a/coderd/x/chatd/active_turn_debug.go +++ b/coderd/x/chatd/active_turn_debug.go @@ -69,16 +69,16 @@ func (d *runnerDebugTurn) Ensure( seedSummary := chatdebug.SeedSummary( chatdebug.TruncateLabel(debug.TriggerLabel, chatdebug.MaxLabelLength), ) - // Carry per-server MCP connect outcomes stashed by - // RecordMCPConnectSummaries before the run existed, so slow or - // failing servers appear in the run instead of as a silent gap - // before the first step. Seeded keys survive FinalizeRun's - // summary aggregation. - if stashed, ok := d.seedSummary["mcp_connect"]; ok { + // Carry per-server MCP connect outcomes (and their dropped + // count) stashed by RecordMCPConnectSummaries before the run + // existed, so slow or failing servers appear in the run instead + // of as a silent gap before the first step. Seeded keys survive + // FinalizeRun's summary aggregation. + for key, stashed := range d.seedSummary { if seedSummary == nil { - seedSummary = make(map[string]any, 1) + seedSummary = make(map[string]any, len(d.seedSummary)) } - seedSummary["mcp_connect"] = stashed + seedSummary[key] = stashed } rootChatID := uuid.Nil if chat.RootChatID.Valid { @@ -134,13 +134,14 @@ func (d *runnerDebugTurn) Context(ctx context.Context) context.Context { } // RecordMCPConnectSummaries merges one preparation's per-server MCP -// connect outcomes into the mcp_connect summary key. It runs at the -// generation dispatch point so every preparation is recorded, -// including ones feeding actions that never reach Ensure (local -// tool execution, requires-action, turn finishing). Outcomes -// recorded before the run exists are stashed and seeded into the -// run when Ensure creates it. -func (d *runnerDebugTurn) RecordMCPConnectSummaries(debug *generationDebug) { +// connect outcomes into the mcp_connect summary key. Preparation +// invokes it as soon as its MCP connect phase completes, so every +// attempt is recorded: preparations that fail after connecting, +// decision errors, and actions that never reach Ensure (local tool +// execution, requires-action, turn finishing). Outcomes recorded +// before the run exists are stashed and seeded into the run when +// Ensure creates it. +func (d *runnerDebugTurn) RecordMCPConnectSummaries(summaries []mcpclient.ConnectSummary) { if d == nil { return } @@ -149,9 +150,18 @@ func (d *runnerDebugTurn) RecordMCPConnectSummaries(debug *generationDebug) { if d.disabled || d.finalized { return } - d.mergeMCPConnectSummariesLocked(debug) + d.mergeMCPConnectSummariesLocked(summaries) } +// maxMCPConnectSummaryEntries bounds the retained per-preparation MCP +// connect outcomes. A turn may run up to 1,200 generation steps, each +// reconnecting to every selected server, so unbounded retention could +// grow one run summary to megabytes of database and API payload. The +// newest entries win because the tail of the history is what shows a +// server that degraded mid-turn; the mcp_connect_dropped count keeps +// the truncation visible. +const maxMCPConnectSummaryEntries = 100 + // mergeMCPConnectSummariesLocked appends a preparation's per-server // MCP connect outcomes to the mcp_connect summary key. chatd // reconnects to every configured MCP server on each generation step @@ -159,15 +169,21 @@ func (d *runnerDebugTurn) RecordMCPConnectSummaries(debug *generationDebug) { // one preparation's outcomes would survive to the finalized run and // a server that degrades mid-turn would still be reported as // connected. -func (d *runnerDebugTurn) mergeMCPConnectSummariesLocked(debug *generationDebug) { - if debug == nil || len(debug.MCPConnectSummaries) == 0 { +func (d *runnerDebugTurn) mergeMCPConnectSummariesLocked(summaries []mcpclient.ConnectSummary) { + if len(summaries) == 0 { return } if d.seedSummary == nil { d.seedSummary = make(map[string]any, 1) } existing, _ := d.seedSummary["mcp_connect"].([]mcpclient.ConnectSummary) - d.seedSummary["mcp_connect"] = append(existing, debug.MCPConnectSummaries...) + existing = append(existing, summaries...) + if over := len(existing) - maxMCPConnectSummaryEntries; over > 0 { + existing = existing[over:] + dropped, _ := d.seedSummary["mcp_connect_dropped"].(int) + d.seedSummary["mcp_connect_dropped"] = dropped + over + } + d.seedSummary["mcp_connect"] = existing } func (d *runnerDebugTurn) contextLocked(ctx context.Context) context.Context { diff --git a/coderd/x/chatd/active_turn_debug_internal_test.go b/coderd/x/chatd/active_turn_debug_internal_test.go index e038b56990846..59ed5de4b9701 100644 --- a/coderd/x/chatd/active_turn_debug_internal_test.go +++ b/coderd/x/chatd/active_turn_debug_internal_test.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "encoding/json" + "fmt" "testing" "github.com/google/uuid" @@ -192,22 +193,20 @@ func TestRunnerDebugTurnRecordMCPConnectSummaries(t *testing.T) { return database.ChatDebugRun{ID: runID, ChatID: chatID}, nil }).Times(1) - base := generationDebug{ + debug := generationDebug{ Enabled: true, Service: svc, TriggerMessageID: 1, ModelConfig: database.ChatModelConfig{ID: uuid.New()}, } - first := base - first.MCPConnectSummaries = []mcpclient.ConnectSummary{{ + first := []mcpclient.ConnectSummary{{ ConfigID: configID, Slug: "registry", Outcome: mcpclient.ConnectOutcomeConnected, DurationMS: 17, ToolCount: 1, }} - second := base - second.MCPConnectSummaries = []mcpclient.ConnectSummary{{ + second := []mcpclient.ConnectSummary{{ ConfigID: configID, Slug: "registry", Outcome: mcpclient.ConnectOutcomeTimeout, @@ -215,16 +214,16 @@ func TestRunnerDebugTurnRecordMCPConnectSummaries(t *testing.T) { Error: "connect: context deadline exceeded", }} - // The dispatch point records each preparation before its action - // runs, so the first outcome lands before Ensure creates the - // run and must be seeded into it. - turn.RecordMCPConnectSummaries(&first) - turn.Ensure(ctx, database.Chat{ID: chatID}, &first) + // Preparation records each attempt as soon as its connect phase + // completes, so the first outcome lands before Ensure creates + // the run and must be seeded into it. + turn.RecordMCPConnectSummaries(first) + turn.Ensure(ctx, database.Chat{ID: chatID}, &debug) // A later generation step reconnects and reports a degraded // outcome for the same server; its action may never reach // Ensure, and the outcome must still survive to the finalized // summary. - turn.RecordMCPConnectSummaries(&second) + turn.RecordMCPConnectSummaries(second) turn.RecordOutcome(chatdebug.StatusCompleted) turn.Finalize(ctx) @@ -243,3 +242,91 @@ func TestRunnerDebugTurnRecordMCPConnectSummaries(t *testing.T) { require.Equal(t, mcpclient.ConnectOutcomeConnected, summary.MCPConnect[0].Outcome) require.Equal(t, mcpclient.ConnectOutcomeTimeout, summary.MCPConnect[1].Outcome) } + +func TestRunnerDebugTurnBoundsMCPConnectSummaries(t *testing.T) { + t.Parallel() + + ctx := testutil.Context(t, testutil.WaitLong) + runnerCtx, cancel := context.WithCancel(ctx) + defer cancel() + ctrl := gomock.NewController(t) + db := dbmock.NewMockStore(ctrl) + chatID := uuid.New() + runID := uuid.New() + svc := chatdebug.NewService(db, testutil.Logger(t), nil) + turn := newRunnerDebugTurn(runnerCtx, testutil.Logger(t)) + + var seededSummary []byte + db.EXPECT().InsertChatDebugRun(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, params database.InsertChatDebugRunParams) (database.ChatDebugRun, error) { + require.True(t, params.Summary.Valid) + seededSummary = params.Summary.RawMessage + return database.ChatDebugRun{ + ID: runID, + ChatID: chatID, + Kind: string(chatdebug.KindChatTurn), + Status: string(chatdebug.StatusInProgress), + }, nil + }). + Times(1) + db.EXPECT().GetChatDebugStepsByRunID(gomock.Any(), runID).Return(nil, nil).Times(1) + var finalSummary []byte + db.EXPECT().UpdateChatDebugRun(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, params database.UpdateChatDebugRunParams) (database.ChatDebugRun, error) { + require.True(t, params.Summary.Valid) + finalSummary = params.Summary.RawMessage + return database.ChatDebugRun{ID: runID, ChatID: chatID}, nil + }).Times(1) + + // 25 preparations against 5 servers produce 125 outcomes, + // overflowing the cap before the run exists. + for prep := 0; prep < 25; prep++ { + batch := make([]mcpclient.ConnectSummary, 5) + for server := range batch { + batch[server] = mcpclient.ConnectSummary{ + ConfigID: uuid.New(), + Slug: fmt.Sprintf("server-%d", server), + Outcome: mcpclient.ConnectOutcomeConnected, + DurationMS: int64(prep), + } + } + turn.RecordMCPConnectSummaries(batch) + } + + debug := generationDebug{ + Enabled: true, + Service: svc, + TriggerMessageID: 1, + ModelConfig: database.ChatModelConfig{ID: uuid.New()}, + } + turn.Ensure(ctx, database.Chat{ID: chatID}, &debug) + + type boundedSummary struct { + MCPConnect []mcpclient.ConnectSummary `json:"mcp_connect"` + MCPConnectDropped int `json:"mcp_connect_dropped"` + } + var seeded boundedSummary + require.NoError(t, json.Unmarshal(seededSummary, &seeded)) + require.Len(t, seeded.MCPConnect, maxMCPConnectSummaryEntries) + require.Equal(t, 25, seeded.MCPConnectDropped) + // The newest outcomes win: the five oldest preparations were + // dropped, so the retained history starts at preparation 5. + require.Equal(t, int64(5), seeded.MCPConnect[0].DurationMS) + + // A preparation recorded after the run exists still respects + // the cap and grows the dropped count. + turn.RecordMCPConnectSummaries([]mcpclient.ConnectSummary{{ + ConfigID: uuid.New(), + Slug: "server-0", + Outcome: mcpclient.ConnectOutcomeTimeout, + DurationMS: 10000, + }}) + turn.RecordOutcome(chatdebug.StatusCompleted) + turn.Finalize(ctx) + + var final boundedSummary + require.NoError(t, json.Unmarshal(finalSummary, &final)) + require.Len(t, final.MCPConnect, maxMCPConnectSummaryEntries) + require.Equal(t, 26, final.MCPConnectDropped) + require.Equal(t, mcpclient.ConnectOutcomeTimeout, final.MCPConnect[maxMCPConnectSummaryEntries-1].Outcome) +} diff --git a/coderd/x/chatd/chatd_test.go b/coderd/x/chatd/chatd_test.go index b1a766461e37e..8dd69b11b7ed0 100644 --- a/coderd/x/chatd/chatd_test.go +++ b/coderd/x/chatd/chatd_test.go @@ -11410,6 +11410,118 @@ func TestActiveServer_ChatTurnDebugRunRecordsMCPConnectOnDecisionError(t *testin } } +// overrideReadFailStore wraps a database.Store so +// GetChatOrganizationModelOverride fails once the test arms it, +// making a later generation preparation fail after its MCP connect +// phase has already completed. +type overrideReadFailStore struct { + database.Store + fail *atomic.Bool +} + +func (s *overrideReadFailStore) GetChatOrganizationModelOverride(ctx context.Context, arg database.GetChatOrganizationModelOverrideParams) (database.ChatOrganizationModelOverride, error) { + if s.fail.Load() { + return database.ChatOrganizationModelOverride{}, xerrors.New("injected compaction override read failure") + } + return s.Store.GetChatOrganizationModelOverride(ctx, arg) +} + +func TestActiveServer_ChatTurnDebugRunRecordsMCPConnectOnPrepareError(t *testing.T) { + t.Parallel() + + db, ps := dbtestutil.NewDB(t) + ctx := testutil.Context(t, testutil.WaitLong) + + mcpSrv := newTestMCPServer("test-mcp") + addTestMCPTextTool(mcpSrv, "echo", "Echoes the input", "echo: ") + mcpTS := httptest.NewServer(testMCPHTTPHandler(mcpSrv)) + t.Cleanup(mcpTS.Close) + + // Serving the first assistant stream arms the failure, so the + // next preparation completes its MCP connect phase and then + // fails reading the compaction override before returning a + // prepared generation. + var failOverrideReads atomic.Bool + openAIURL := chattest.NewOpenAI(t, func(req *chattest.OpenAIRequest) chattest.OpenAIResponse { + if !req.Stream { + return chattest.OpenAINonStreamingResponse("title") + } + failOverrideReads.Store(true) + return chattest.OpenAIStreamingResponse( + chattest.OpenAITextChunks("Done!")..., + ) + }) + + user, org, model := seedChatDependenciesWithProvider(t, db, "openai-compat", openAIURL) + mcpConfig := dbgen.MCPServerConfig(t, db, database.MCPServerConfig{ + OrganizationID: org.ID, + DisplayName: "Test MCP", + Slug: "test-mcp", + Url: mcpTS.URL, + CreatedBy: uuid.NullUUID{UUID: user.ID, Valid: true}, + UpdatedBy: uuid.NullUUID{UUID: user.ID, Valid: true}, + }) + + server := newActiveTestServer(t, &overrideReadFailStore{Store: db, fail: &failOverrideReads}, ps, func(cfg *chatd.Config) { + withoutMCPToolSearch(cfg) + cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(chattest.NewMockAIBridgeTransport(t, openAIURL)) + cfg.AlwaysEnableDebugLogs = true + }) + + chat, err := server.CreateChat(ctx, chatd.CreateOptions{ + OrganizationID: org.ID, + OwnerID: user.ID, + Title: "mcp-connect-prepare-error-test", + ModelConfigID: model.ID, + MCPServerIDs: []uuid.UUID{mcpConfig.ID}, + InitialUserContent: []codersdk.ChatMessagePart{ + codersdk.ChatMessageText("Say done."), + }, + }) + require.NoError(t, err) + + waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusError) + require.NoError(t, server.Close()) + debugCtx := testutil.Context(t, testutil.WaitLong) + + var chatTurnRuns []database.ChatDebugRun + testutil.Eventually(debugCtx, t, func(ctx context.Context) bool { + runs, runsErr := db.GetChatDebugRunsByChatID(ctx, database.GetChatDebugRunsByChatIDParams{ + ChatID: chat.ID, + LimitVal: 100, + }) + if runsErr != nil { + return false + } + chatTurnRuns = chatTurnRuns[:0] + for _, run := range runs { + if run.Kind == string(codersdk.ChatDebugRunKindChatTurn) { + chatTurnRuns = append(chatTurnRuns, run) + } + } + return len(chatTurnRuns) == 1 && chatTurnRuns[0].FinishedAt.Valid + }, testutil.IntervalFast) + + require.Len(t, chatTurnRuns, 1) + var summary struct { + MCPConnect []struct { + Slug string `json:"slug"` + Outcome string `json:"outcome"` + } `json:"mcp_connect"` + } + require.NoError(t, json.Unmarshal(chatTurnRuns[0].Summary, &summary)) + // The preparation feeding the assistant step records one + // outcome. The following preparation connects to the MCP server + // and then fails reading the compaction override; its attempts + // must still contribute their connect outcomes even though + // preparation never returns a prepared generation. + require.GreaterOrEqual(t, len(summary.MCPConnect), 2) + for _, entry := range summary.MCPConnect { + require.Equal(t, "test-mcp", entry.Slug) + require.Equal(t, "connected", entry.Outcome) + } +} + func TestPlanModeRootChatApprovedExternalMCPToolInvocation(t *testing.T) { t.Parallel() diff --git a/coderd/x/chatd/generation.go b/coderd/x/chatd/generation.go index d8785da92ce43..22b6c20cf6b6a 100644 --- a/coderd/x/chatd/generation.go +++ b/coderd/x/chatd/generation.go @@ -35,6 +35,12 @@ import ( type generationPrepareInput struct { Chat database.Chat Messages []database.ChatMessage + // RecordMCPConnectSummaries receives the preparation's per-server + // MCP connect outcomes as soon as the connect phase completes. + // Preparation invokes it directly (rather than returning the + // outcomes) so attempts that fail after connecting still record + // before their error discards the prepared state. + RecordMCPConnectSummaries func([]mcpclient.ConnectSummary) } // generationPrepared contains the side-effect inputs for a generation task. @@ -96,10 +102,6 @@ type generationDebug struct { HistoryTipMessageID int64 TriggerLabel string ModelConfig database.ChatModelConfig - // MCPConnectSummaries carries per-server MCP connect outcomes - // from turn preparation into the debug run summary so slow or - // failing servers are visible in the debug UI. - MCPConnectSummaries []mcpclient.ConnectSummary } // generationOutcome describes a completed generation outcome. @@ -446,8 +448,9 @@ func (s *taskStarter) StartGeneration(ctx context.Context, input chatWorkerTaskS } } prepareInput := generationPrepareInput{ - Chat: chat, - Messages: messages, + Chat: chat, + Messages: messages, + RecordMCPConnectSummaries: input.DebugTurn.RecordMCPConnectSummaries, } prepared, err := retryGenerationPhase(ctx, s, "prepare", func() (generationPrepared, error) { return s.server.prepareGeneration(ctx, prepareInput) @@ -459,11 +462,6 @@ func (s *taskStarter) StartGeneration(ctx context.Context, input chatWorkerTaskS return s.finishGenerationError(ctx, machine, input, err, generationAttemptNotRequired) } cleanup := prepared.Cleanup - // Every preparation reconnects to MCP; record its connect - // outcomes as soon as preparation succeeds so exits that - // never reach the action dispatch (decision errors and - // actions that skip Ensure) still contribute to the run. - input.DebugTurn.RecordMCPConnectSummaries(prepared.Debug) var decision generationDecision if input.StopNudges.consume(stopNudgeKey(prepared.Messages)) { decision = generationDecision{kind: generationActionGenerateAssistant} diff --git a/coderd/x/chatd/generation_preparer.go b/coderd/x/chatd/generation_preparer.go index b6f1d26a07dcc..3d8a1153e5925 100644 --- a/coderd/x/chatd/generation_preparer.go +++ b/coderd/x/chatd/generation_preparer.go @@ -375,9 +375,16 @@ func (server *Server) prepareGeneration( return nil }) } - if err := g2.Wait(); err != nil { + g2Err := g2.Wait() + // Record connect outcomes before acting on any preparation error: + // ConnectAll has already run, so a failure below (or in g2 itself) + // would otherwise discard this attempt's outcomes. + if resolved.debugEnabled && input.RecordMCPConnectSummaries != nil && len(mcpSummaries) > 0 { + input.RecordMCPConnectSummaries(mcpSummaries) + } + if g2Err != nil { cleanup() - return generationPrepared{}, err + return generationPrepared{}, g2Err } if mcpCleanup != nil { @@ -662,7 +669,6 @@ func (server *Server) prepareGeneration( HistoryTipMessageID: historyTipMessageID, TriggerLabel: triggerLabel, ModelConfig: modelConfig, - MCPConnectSummaries: mcpSummaries, } } From 2b39852bd11b2063bdb2bc12702d3935aecf1e69 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 27 Aug 2026 01:00:47 +0000 Subject: [PATCH 7/8] fix: create chat debug runs from recorded MCP outcomes and surface the dropped count --- coderd/x/chatd/active_turn_debug.go | 35 +++++-- .../chatd/active_turn_debug_internal_test.go | 48 +++++----- coderd/x/chatd/chatd_test.go | 94 +++++++++++++++++++ coderd/x/chatd/generation.go | 17 +++- coderd/x/chatd/generation_preparer.go | 47 +++++----- .../DebugPanel/DebugPanel.stories.tsx | 6 ++ .../RightPanel/DebugPanel/DebugRunCard.tsx | 7 ++ .../DebugPanel/debugPanelUtils.test.ts | 18 ++++ .../RightPanel/DebugPanel/debugPanelUtils.ts | 9 ++ 9 files changed, 222 insertions(+), 59 deletions(-) diff --git a/coderd/x/chatd/active_turn_debug.go b/coderd/x/chatd/active_turn_debug.go index 01c30d86910a2..d4ebf0c4b17e7 100644 --- a/coderd/x/chatd/active_turn_debug.go +++ b/coderd/x/chatd/active_turn_debug.go @@ -50,7 +50,14 @@ func (d *runnerDebugTurn) Ensure( d.mu.Lock() defer d.mu.Unlock() + return d.ensureLocked(ctx, chat, debug) +} +func (d *runnerDebugTurn) ensureLocked( + ctx context.Context, + chat database.Chat, + debug *generationDebug, +) context.Context { // Check finalized/disabled before created: once the turn is // finalized, new contexts must not be attributed to the // finalized run, even if it was created earlier. @@ -134,15 +141,22 @@ func (d *runnerDebugTurn) Context(ctx context.Context) context.Context { } // RecordMCPConnectSummaries merges one preparation's per-server MCP -// connect outcomes into the mcp_connect summary key. Preparation -// invokes it as soon as its MCP connect phase completes, so every -// attempt is recorded: preparations that fail after connecting, -// decision errors, and actions that never reach Ensure (local tool -// execution, requires-action, turn finishing). Outcomes recorded -// before the run exists are stashed and seeded into the run when -// Ensure creates it. -func (d *runnerDebugTurn) RecordMCPConnectSummaries(summaries []mcpclient.ConnectSummary) { - if d == nil { +// connect outcomes into the mcp_connect summary key, creating the +// debug run if it does not exist yet. Preparation invokes it as soon +// as its MCP connect phase completes, so every attempt is recorded: +// preparations that fail after connecting, decision errors, and +// actions that never reach Ensure (local tool execution, +// requires-action, turn finishing). Creating the run here matters +// when the first preparation connects and then fails: no model or +// compaction action ever runs Ensure, and Finalize discards the +// stash of a never-created run. +func (d *runnerDebugTurn) RecordMCPConnectSummaries( + ctx context.Context, + chat database.Chat, + debug *generationDebug, + summaries []mcpclient.ConnectSummary, +) { + if d == nil || len(summaries) == 0 { return } d.mu.Lock() @@ -150,7 +164,10 @@ func (d *runnerDebugTurn) RecordMCPConnectSummaries(summaries []mcpclient.Connec if d.disabled || d.finalized { return } + // Merge before ensuring so the outcomes ride the seed summary + // when this call is the one that creates the run. d.mergeMCPConnectSummariesLocked(summaries) + d.ensureLocked(ctx, chat, debug) } // maxMCPConnectSummaryEntries bounds the retained per-preparation MCP diff --git a/coderd/x/chatd/active_turn_debug_internal_test.go b/coderd/x/chatd/active_turn_debug_internal_test.go index 59ed5de4b9701..1a02b604577c1 100644 --- a/coderd/x/chatd/active_turn_debug_internal_test.go +++ b/coderd/x/chatd/active_turn_debug_internal_test.go @@ -215,15 +215,16 @@ func TestRunnerDebugTurnRecordMCPConnectSummaries(t *testing.T) { }} // Preparation records each attempt as soon as its connect phase - // completes, so the first outcome lands before Ensure creates - // the run and must be seeded into it. - turn.RecordMCPConnectSummaries(first) + // completes; the first record creates the run with the outcome + // already seeded, and a later Ensure must not create a second + // run. + turn.RecordMCPConnectSummaries(ctx, database.Chat{ID: chatID}, &debug, first) turn.Ensure(ctx, database.Chat{ID: chatID}, &debug) // A later generation step reconnects and reports a degraded // outcome for the same server; its action may never reach // Ensure, and the outcome must still survive to the finalized // summary. - turn.RecordMCPConnectSummaries(second) + turn.RecordMCPConnectSummaries(ctx, database.Chat{ID: chatID}, &debug, second) turn.RecordOutcome(chatdebug.StatusCompleted) turn.Finalize(ctx) @@ -278,8 +279,14 @@ func TestRunnerDebugTurnBoundsMCPConnectSummaries(t *testing.T) { return database.ChatDebugRun{ID: runID, ChatID: chatID}, nil }).Times(1) + debug := generationDebug{ + Enabled: true, + Service: svc, + TriggerMessageID: 1, + ModelConfig: database.ChatModelConfig{ID: uuid.New()}, + } // 25 preparations against 5 servers produce 125 outcomes, - // overflowing the cap before the run exists. + // overflowing the cap. The first record creates the run. for prep := 0; prep < 25; prep++ { batch := make([]mcpclient.ConnectSummary, 5) for server := range batch { @@ -290,32 +297,23 @@ func TestRunnerDebugTurnBoundsMCPConnectSummaries(t *testing.T) { DurationMS: int64(prep), } } - turn.RecordMCPConnectSummaries(batch) + turn.RecordMCPConnectSummaries(ctx, database.Chat{ID: chatID}, &debug, batch) } - debug := generationDebug{ - Enabled: true, - Service: svc, - TriggerMessageID: 1, - ModelConfig: database.ChatModelConfig{ID: uuid.New()}, - } - turn.Ensure(ctx, database.Chat{ID: chatID}, &debug) - type boundedSummary struct { MCPConnect []mcpclient.ConnectSummary `json:"mcp_connect"` MCPConnectDropped int `json:"mcp_connect_dropped"` } + // The run was created by the first record, so the seed carries + // only that preparation's outcomes. var seeded boundedSummary require.NoError(t, json.Unmarshal(seededSummary, &seeded)) - require.Len(t, seeded.MCPConnect, maxMCPConnectSummaryEntries) - require.Equal(t, 25, seeded.MCPConnectDropped) - // The newest outcomes win: the five oldest preparations were - // dropped, so the retained history starts at preparation 5. - require.Equal(t, int64(5), seeded.MCPConnect[0].DurationMS) - - // A preparation recorded after the run exists still respects - // the cap and grows the dropped count. - turn.RecordMCPConnectSummaries([]mcpclient.ConnectSummary{{ + require.Len(t, seeded.MCPConnect, 5) + require.Zero(t, seeded.MCPConnectDropped) + + // One more preparation still respects the cap and grows the + // dropped count. + turn.RecordMCPConnectSummaries(ctx, database.Chat{ID: chatID}, &debug, []mcpclient.ConnectSummary{{ ConfigID: uuid.New(), Slug: "server-0", Outcome: mcpclient.ConnectOutcomeTimeout, @@ -328,5 +326,9 @@ func TestRunnerDebugTurnBoundsMCPConnectSummaries(t *testing.T) { require.NoError(t, json.Unmarshal(finalSummary, &final)) require.Len(t, final.MCPConnect, maxMCPConnectSummaryEntries) require.Equal(t, 26, final.MCPConnectDropped) + // The newest outcomes win: the oldest five preparations were + // dropped, so the retained history starts at preparation 5 and + // ends with the timeout recorded above. + require.Equal(t, int64(5), final.MCPConnect[0].DurationMS) require.Equal(t, mcpclient.ConnectOutcomeTimeout, final.MCPConnect[maxMCPConnectSummaryEntries-1].Outcome) } diff --git a/coderd/x/chatd/chatd_test.go b/coderd/x/chatd/chatd_test.go index 8dd69b11b7ed0..1e10b9c323293 100644 --- a/coderd/x/chatd/chatd_test.go +++ b/coderd/x/chatd/chatd_test.go @@ -11522,6 +11522,100 @@ func TestActiveServer_ChatTurnDebugRunRecordsMCPConnectOnPrepareError(t *testing } } +func TestActiveServer_ChatTurnDebugRunRecordsMCPConnectOnFirstPrepareError(t *testing.T) { + t.Parallel() + + db, ps := dbtestutil.NewDB(t) + ctx := testutil.Context(t, testutil.WaitLong) + + mcpSrv := newTestMCPServer("test-mcp") + addTestMCPTextTool(mcpSrv, "echo", "Echoes the input", "echo: ") + mcpTS := httptest.NewServer(testMCPHTTPHandler(mcpSrv)) + t.Cleanup(mcpTS.Close) + + openAIURL := chattest.NewOpenAI(t, func(req *chattest.OpenAIRequest) chattest.OpenAIResponse { + if !req.Stream { + return chattest.OpenAINonStreamingResponse("title") + } + return chattest.OpenAIStreamingResponse( + chattest.OpenAITextChunks("Done!")..., + ) + }) + + user, org, model := seedChatDependenciesWithProvider(t, db, "openai-compat", openAIURL) + mcpConfig := dbgen.MCPServerConfig(t, db, database.MCPServerConfig{ + OrganizationID: org.ID, + DisplayName: "Test MCP", + Slug: "test-mcp", + Url: mcpTS.URL, + CreatedBy: uuid.NullUUID{UUID: user.ID, Valid: true}, + UpdatedBy: uuid.NullUUID{UUID: user.ID, Valid: true}, + }) + + // Armed from the start: the very first preparation connects to + // the MCP server and then fails reading the compaction override, + // so no model or compaction action ever creates the debug run + // through Ensure. + var failOverrideReads atomic.Bool + failOverrideReads.Store(true) + server := newActiveTestServer(t, &overrideReadFailStore{Store: db, fail: &failOverrideReads}, ps, func(cfg *chatd.Config) { + withoutMCPToolSearch(cfg) + cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(chattest.NewMockAIBridgeTransport(t, openAIURL)) + cfg.AlwaysEnableDebugLogs = true + }) + + chat, err := server.CreateChat(ctx, chatd.CreateOptions{ + OrganizationID: org.ID, + OwnerID: user.ID, + Title: "mcp-connect-first-prepare-error-test", + ModelConfigID: model.ID, + MCPServerIDs: []uuid.UUID{mcpConfig.ID}, + InitialUserContent: []codersdk.ChatMessagePart{ + codersdk.ChatMessageText("Say done."), + }, + }) + require.NoError(t, err) + + waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusError) + require.NoError(t, server.Close()) + debugCtx := testutil.Context(t, testutil.WaitLong) + + var chatTurnRuns []database.ChatDebugRun + testutil.Eventually(debugCtx, t, func(ctx context.Context) bool { + runs, runsErr := db.GetChatDebugRunsByChatID(ctx, database.GetChatDebugRunsByChatIDParams{ + ChatID: chat.ID, + LimitVal: 100, + }) + if runsErr != nil { + return false + } + chatTurnRuns = chatTurnRuns[:0] + for _, run := range runs { + if run.Kind == string(codersdk.ChatDebugRunKindChatTurn) { + chatTurnRuns = append(chatTurnRuns, run) + } + } + return len(chatTurnRuns) == 1 && chatTurnRuns[0].FinishedAt.Valid + }, testutil.IntervalFast) + + require.Len(t, chatTurnRuns, 1) + var summary struct { + MCPConnect []struct { + Slug string `json:"slug"` + Outcome string `json:"outcome"` + } `json:"mcp_connect"` + } + require.NoError(t, json.Unmarshal(chatTurnRuns[0].Summary, &summary)) + // Every preparation fails after its connect phase, so the run + // exists only because recording the outcomes created it; it must + // carry at least the first attempt's connect outcome. + require.GreaterOrEqual(t, len(summary.MCPConnect), 1) + for _, entry := range summary.MCPConnect { + require.Equal(t, "test-mcp", entry.Slug) + require.Equal(t, "connected", entry.Outcome) + } +} + func TestPlanModeRootChatApprovedExternalMCPToolInvocation(t *testing.T) { t.Parallel() diff --git a/coderd/x/chatd/generation.go b/coderd/x/chatd/generation.go index 22b6c20cf6b6a..bff6f21480db1 100644 --- a/coderd/x/chatd/generation.go +++ b/coderd/x/chatd/generation.go @@ -36,11 +36,18 @@ type generationPrepareInput struct { Chat database.Chat Messages []database.ChatMessage // RecordMCPConnectSummaries receives the preparation's per-server - // MCP connect outcomes as soon as the connect phase completes. - // Preparation invokes it directly (rather than returning the - // outcomes) so attempts that fail after connecting still record - // before their error discards the prepared state. - RecordMCPConnectSummaries func([]mcpclient.ConnectSummary) + // MCP connect outcomes as soon as the connect phase completes, + // with the debug context needed to create the run when no action + // ever reaches Ensure. Preparation invokes it directly (rather + // than returning the outcomes) so attempts that fail after + // connecting still record before their error discards the + // prepared state. + RecordMCPConnectSummaries func( + ctx context.Context, + chat database.Chat, + debug *generationDebug, + summaries []mcpclient.ConnectSummary, + ) } // generationPrepared contains the side-effect inputs for a generation task. diff --git a/coderd/x/chatd/generation_preparer.go b/coderd/x/chatd/generation_preparer.go index 3d8a1153e5925..ff49843deb647 100644 --- a/coderd/x/chatd/generation_preparer.go +++ b/coderd/x/chatd/generation_preparer.go @@ -314,6 +314,29 @@ func (server *Server) prepareGeneration( } } + // Build the debug context before the connect phase so its + // outcomes can be recorded with run-creation context even when a + // later preparation step fails. + triggerMessageID, historyTipMessageID, triggerLabel := deriveChatDebugSeed(promptRows) + debugSvc := server.existingDebugService() + var debug *generationDebug + if resolved.debugEnabled { + if debugSvc == nil { + cleanup() + return generationPrepared{}, xerrors.New("chat debug service missing after enablement check") + } + debug = &generationDebug{ + Enabled: true, + Service: debugSvc, + Provider: resolved.resolvedProvider, + Model: resolved.resolvedModel, + TriggerMessageID: triggerMessageID, + HistoryTipMessageID: historyTipMessageID, + TriggerLabel: triggerLabel, + ModelConfig: modelConfig, + } + } + var g2 errgroup.Group g2.Go(func() error { var err error @@ -379,8 +402,8 @@ func (server *Server) prepareGeneration( // Record connect outcomes before acting on any preparation error: // ConnectAll has already run, so a failure below (or in g2 itself) // would otherwise discard this attempt's outcomes. - if resolved.debugEnabled && input.RecordMCPConnectSummaries != nil && len(mcpSummaries) > 0 { - input.RecordMCPConnectSummaries(mcpSummaries) + if debug != nil && input.RecordMCPConnectSummaries != nil && len(mcpSummaries) > 0 { + input.RecordMCPConnectSummaries(ctx, chat, debug, mcpSummaries) } if g2Err != nil { cleanup() @@ -652,26 +675,6 @@ func (server *Server) prepareGeneration( } } - triggerMessageID, historyTipMessageID, triggerLabel := deriveChatDebugSeed(promptRows) - debugSvc := server.existingDebugService() - var debug *generationDebug - if resolved.debugEnabled { - if debugSvc == nil { - cleanup() - return generationPrepared{}, xerrors.New("chat debug service missing after enablement check") - } - debug = &generationDebug{ - Enabled: true, - Service: debugSvc, - Provider: resolved.resolvedProvider, - Model: resolved.resolvedModel, - TriggerMessageID: triggerMessageID, - HistoryTipMessageID: historyTipMessageID, - TriggerLabel: triggerLabel, - ModelConfig: modelConfig, - } - } - compactionToolCallID := "chat_summarized_" + uuid.NewString() effectiveThreshold := modelConfig.CompressionThreshold if override, ok := server.resolveUserCompactionThreshold(ctx, chat.OwnerID, modelConfig.ID); ok { diff --git a/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/DebugPanel.stories.tsx b/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/DebugPanel.stories.tsx index e59c896024a30..a9bfc81bff5bf 100644 --- a/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/DebugPanel.stories.tsx +++ b/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/DebugPanel.stories.tsx @@ -740,6 +740,9 @@ const mcpConnectSummary = { tool_count: 3, }, ], + // Entries beyond the retention cap; the card renders a + // truncation notice for them. + mcp_connect_dropped: 4, }; export const RunWithMCPConnectSummary: Story = { @@ -789,6 +792,9 @@ export const RunWithMCPConnectSummary: Story = { expect(mcp.getByText("connect: context deadline exceeded")).toBeVisible(); expect(mcp.getByText("45ms")).toBeVisible(); expect(mcp.getByText("3 tools")).toBeVisible(); + expect( + mcp.getByText("4 earlier connection samples omitted"), + ).toBeVisible(); }); }, }; diff --git a/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/DebugRunCard.tsx b/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/DebugRunCard.tsx index b7d6facb4b706..492857e821ad3 100644 --- a/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/DebugRunCard.tsx +++ b/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/DebugRunCard.tsx @@ -271,6 +271,13 @@ export const DebugRunCard: FC = ({
    • ))}
+ {summaryVm.mcpConnectDropped > 0 ? ( +

+ {summaryVm.mcpConnectDropped} earlier connection{" "} + {summaryVm.mcpConnectDropped === 1 ? "sample" : "samples"}{" "} + omitted +

+ ) : null} ) : null} {steps.map((step) => ( diff --git a/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/debugPanelUtils.test.ts b/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/debugPanelUtils.test.ts index 5d9684d7bf637..b663aa7bef981 100644 --- a/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/debugPanelUtils.test.ts +++ b/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/debugPanelUtils.test.ts @@ -827,6 +827,7 @@ describe("coerceRunSummary", () => { totalInputTokens: 120, totalOutputTokens: 45, mcpConnect: [], + mcpConnectDropped: 0, warnings: [], }); }); @@ -853,6 +854,7 @@ describe("coerceRunSummary", () => { totalInputTokens: undefined, totalOutputTokens: undefined, mcpConnect: [], + mcpConnectDropped: 0, warnings: [], }); }); @@ -899,6 +901,22 @@ describe("coerceRunSummary", () => { expect(coerceRunSummary({ mcp_connect: "oops" }).mcpConnect).toEqual([]); }); + it("coerces the dropped MCP connect sample count", () => { + expect( + coerceRunSummary({ mcp_connect_dropped: 25 }).mcpConnectDropped, + ).toBe(25); + expect(coerceRunSummary({ mcpConnectDropped: "7" }).mcpConnectDropped).toBe( + 7, + ); + expect( + coerceRunSummary({ mcp_connect_dropped: -3 }).mcpConnectDropped, + ).toBe(0); + expect( + coerceRunSummary({ mcp_connect_dropped: "oops" }).mcpConnectDropped, + ).toBe(0); + expect(coerceRunSummary({}).mcpConnectDropped).toBe(0); + }); + it("unwraps JSON-string payloads before coercing", () => { const summary = coerceRunSummary( JSON.stringify({ diff --git a/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/debugPanelUtils.ts b/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/debugPanelUtils.ts index 39dd375bba076..da532e32841c7 100644 --- a/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/debugPanelUtils.ts +++ b/site/src/pages/AgentsPage/components/RightPanel/DebugPanel/debugPanelUtils.ts @@ -453,6 +453,7 @@ interface RunSummaryViewModel { totalInputTokens: number | undefined; totalOutputTokens: number | undefined; mcpConnect: MCPConnectSummaryViewModel[]; + mcpConnectDropped: number; warnings: string[]; } @@ -913,6 +914,7 @@ export const coerceRunSummary = (data: unknown): RunSummaryViewModel => { totalInputTokens: undefined, totalOutputTokens: undefined, mcpConnect: [], + mcpConnectDropped: 0, warnings: [], }; const parsed = deepParse(data); @@ -928,6 +930,9 @@ export const coerceRunSummary = (data: unknown): RunSummaryViewModel => { "primaryLabel", ), ); + const mcpConnectDropped = toFiniteNumber( + pickField(parsed, "mcp_connect_dropped", "mcpConnectDropped"), + ); return { primaryLabel: firstMessage ?? "", endpointLabel: toOptionalString( @@ -963,6 +968,10 @@ export const coerceRunSummary = (data: unknown): RunSummaryViewModel => { mcpConnect: coerceMCPConnectSummaries( pickField(parsed, "mcp_connect", "mcpConnect"), ), + mcpConnectDropped: + mcpConnectDropped !== undefined && mcpConnectDropped > 0 + ? Math.floor(mcpConnectDropped) + : 0, warnings: [], }; }; From 7bb3c7297a52579e272bae1bbe648634c00e0e5a Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 27 Aug 2026 01:15:00 +0000 Subject: [PATCH 8/8] fix(coderd/x/chatd/mcpclient): bound persisted MCP connect error size --- coderd/x/chatd/mcpclient/export_test.go | 7 +++++ coderd/x/chatd/mcpclient/mcpclient.go | 31 ++++++++++++++++++--- coderd/x/chatd/mcpclient/mcpclient_test.go | 32 ++++++++++++++++++++++ 3 files changed, 66 insertions(+), 4 deletions(-) diff --git a/coderd/x/chatd/mcpclient/export_test.go b/coderd/x/chatd/mcpclient/export_test.go index d5ec3bb687111..53c3bf0d10aa0 100644 --- a/coderd/x/chatd/mcpclient/export_test.go +++ b/coderd/x/chatd/mcpclient/export_test.go @@ -34,3 +34,10 @@ func ConnectAllForTest( // BuildAuthHeadersForTest exposes buildAuthHeaders for external // tests. var BuildAuthHeadersForTest = buildAuthHeaders + +// SummaryErrorForTest exposes summaryError for external tests. +var SummaryErrorForTest = summaryError + +// MaxSummaryErrorLenForTest exposes the persisted-error byte cap for +// external tests. +const MaxSummaryErrorLenForTest = maxSummaryErrorLen diff --git a/coderd/x/chatd/mcpclient/mcpclient.go b/coderd/x/chatd/mcpclient/mcpclient.go index f6538f9c07b48..4eb90dda904a6 100644 --- a/coderd/x/chatd/mcpclient/mcpclient.go +++ b/coderd/x/chatd/mcpclient/mcpclient.go @@ -14,6 +14,7 @@ import ( "strings" "sync" "time" + "unicode/utf8" "charm.land/fantasy" "github.com/google/uuid" @@ -93,8 +94,8 @@ type ConnectSummary struct { Outcome ConnectOutcome `json:"outcome"` DurationMS int64 `json:"duration_ms"` ToolCount int `json:"tool_count,omitempty"` - // Error is the redacted connect error, present unless the - // outcome is connected or no_tools. + // Error is the redacted, size-bounded connect error, present + // unless the outcome is connected or no_tools. Error string `json:"error,omitempty"` } @@ -207,10 +208,10 @@ func connectAllWithHooks( switch { case connectErr != nil && errors.Is(connectErr, context.DeadlineExceeded): summary.Outcome = ConnectOutcomeTimeout - summary.Error = redactErrorURL(connectErr) + summary.Error = summaryError(connectErr) case connectErr != nil: summary.Outcome = ConnectOutcomeError - summary.Error = redactErrorURL(connectErr) + summary.Error = summaryError(connectErr) case len(serverTools) == 0: summary.Outcome = ConnectOutcomeNoTools default: @@ -661,6 +662,28 @@ func redactErrorURL(err error) string { return err.Error() } +// maxSummaryErrorLen bounds the persisted connect error in bytes. +// Protocol errors can embed arbitrarily large remote-controlled +// response bodies, so without this cap a single retained summary +// could inflate the JSONB row and every debug-runs payload +// regardless of the entry-count cap. +const maxSummaryErrorLen = 512 + +// summaryError renders a connect error for the persisted summary: +// credential-bearing URLs are redacted and the result is truncated +// to maxSummaryErrorLen bytes on a rune boundary. +func summaryError(err error) string { + msg := redactErrorURL(err) + if len(msg) <= maxSummaryErrorLen { + return msg + } + cut := maxSummaryErrorLen + for cut > 0 && !utf8.RuneStart(msg[cut]) { + cut-- + } + return msg[:cut] + "... (truncated)" +} + // MCPToolIdentifier is implemented by tools that originate from // an MCP server config and can report the config's database ID. type MCPToolIdentifier interface { diff --git a/coderd/x/chatd/mcpclient/mcpclient_test.go b/coderd/x/chatd/mcpclient/mcpclient_test.go index 82434a9ace5e8..bd9a9b2643bbe 100644 --- a/coderd/x/chatd/mcpclient/mcpclient_test.go +++ b/coderd/x/chatd/mcpclient/mcpclient_test.go @@ -17,6 +17,7 @@ import ( "github.com/modelcontextprotocol/go-sdk/mcp" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "golang.org/x/xerrors" "cdr.dev/slog/v3/sloggers/slogtest" "github.com/coder/coder/v2/coderd/database" @@ -986,6 +987,37 @@ func TestRedactURL(t *testing.T) { } } +func TestSummaryErrorTruncation(t *testing.T) { + t.Parallel() + + t.Run("short errors pass through", func(t *testing.T) { + t.Parallel() + got := mcpclient.SummaryErrorForTest(xerrors.New("connect refused")) + require.Equal(t, "connect refused", got) + }) + + t.Run("large remote-controlled errors are bounded", func(t *testing.T) { + t.Parallel() + huge := strings.Repeat("x", 1<<20) + got := mcpclient.SummaryErrorForTest(xerrors.New(huge)) + require.Len(t, got, mcpclient.MaxSummaryErrorLenForTest+len("... (truncated)")) + require.True(t, strings.HasSuffix(got, "... (truncated)")) + require.True(t, strings.HasPrefix(got, "xxx")) + }) + + t.Run("truncation lands on a rune boundary", func(t *testing.T) { + t.Parallel() + // A two-byte rune straddles the cap boundary, so the cut + // must back up instead of splitting it. + msg := strings.Repeat("a", mcpclient.MaxSummaryErrorLenForTest-1) + + strings.Repeat("é", 20) + got := mcpclient.SummaryErrorForTest(xerrors.New(msg)) + require.True(t, utf8.ValidString(got)) + require.True(t, strings.HasSuffix(got, "... (truncated)")) + require.Len(t, got, mcpclient.MaxSummaryErrorLenForTest-1+len("... (truncated)")) + }) +} + func TestConnectAll_ExpiredToken(t *testing.T) { t.Parallel() ctx := context.Background()