From 7afa2c9185f3b47d814ddfbd6f7cf187bb7f6dbf Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 20 Aug 2026 16:55:20 +0000 Subject: [PATCH] feat(coderd/x/chatd): skip MCP servers with recently timed-out connects chatd reconnects to every configured MCP server on every generation step, so a black-holed server (even with the connect budget enforced) still costs the full budget per step of every chat on the pod. Cache connect timeouts per process and skip the server for 60 seconds, keyed by config ID and updated_at so config edits retry immediately. Skips are logged and recorded in the debug run's mcp_connect summary with a "skipped" outcome. Fast failures (auth, DNS) are not cached: they are cheap to retry and may be fixed mid-conversation. --- coderd/x/chatd/chatd.go | 2 + coderd/x/chatd/generation_preparer.go | 5 +- coderd/x/chatd/mcp_negative_cache_test.go | 130 +++++++++++++++++ coderd/x/chatd/mcpclient/negativecache.go | 133 ++++++++++++++++++ .../x/chatd/mcpclient/negativecache_test.go | 120 ++++++++++++++++ 5 files changed, 389 insertions(+), 1 deletion(-) create mode 100644 coderd/x/chatd/mcp_negative_cache_test.go create mode 100644 coderd/x/chatd/mcpclient/negativecache.go create mode 100644 coderd/x/chatd/mcpclient/negativecache_test.go diff --git a/coderd/x/chatd/chatd.go b/coderd/x/chatd/chatd.go index ae0c5069673b8..befda635f88c4 100644 --- a/coderd/x/chatd/chatd.go +++ b/coderd/x/chatd/chatd.go @@ -189,6 +189,7 @@ type Server struct { providerAPIKeys chatprovider.ProviderAPIKeys allowBYOK bool oidcTokenSource mcpclient.UserOIDCTokenSource + mcpNegativeCache *mcpclient.NegativeCache debugSvc *chatdebug.Service debugSvcFactory func() *chatdebug.Service debugSvcReady atomic.Bool @@ -3161,6 +3162,7 @@ func New(ps pubsub.Pubsub, cfg Config) *Server { providerAPIKeys: cfg.ProviderAPIKeys, allowBYOK: allowBYOK, oidcTokenSource: cfg.OIDCTokenSource, + mcpNegativeCache: mcpclient.NewNegativeCache(clk), debugSvcFactory: func() *chatdebug.Service { debugSvc := chatdebug.NewService( cfg.Database, diff --git a/coderd/x/chatd/generation_preparer.go b/coderd/x/chatd/generation_preparer.go index 27964fdbf5a9c..18ebc5a66abbb 100644 --- a/coderd/x/chatd/generation_preparer.go +++ b/coderd/x/chatd/generation_preparer.go @@ -356,15 +356,18 @@ func (server *Server) prepareGeneration( logger.Warn(ctx, "failed to load MCP user tokens", slog.Error(tokenErr)) } mcpTokens = server.refreshExpiredMCPTokens(ctx, logger, mcpConnectConfigs, mcpTokens) + connectable, skipped := server.mcpNegativeCache.Filter(ctx, logger, mcpConnectConfigs) mcpTools, mcpSummaries, mcpCleanup = mcpclient.ConnectAll( ctx, logger, - mcpConnectConfigs, + connectable, mcpTokens, chat.OwnerID, server.oidcTokenSource, chatprovider.CoderHeaders(chat), ) + server.mcpNegativeCache.Record(connectable, mcpSummaries) + mcpSummaries = append(mcpSummaries, skipped...) return nil }) } diff --git a/coderd/x/chatd/mcp_negative_cache_test.go b/coderd/x/chatd/mcp_negative_cache_test.go new file mode 100644 index 0000000000000..14546210be06d --- /dev/null +++ b/coderd/x/chatd/mcp_negative_cache_test.go @@ -0,0 +1,130 @@ +package chatd_test + +import ( + "encoding/json" + "net" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + + "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/database/dbgen" + "github.com/coder/coder/v2/coderd/database/dbtestutil" + "github.com/coder/coder/v2/coderd/x/chatd" + "github.com/coder/coder/v2/coderd/x/chatd/chattest" + "github.com/coder/coder/v2/codersdk" + "github.com/coder/coder/v2/testutil" +) + +// TestGeneration_MCPNegativeCacheSkipsTimedOutServer proves the +// negative-cache wiring end to end: the first turn pays the full +// connect budget against a black-holed MCP server and records a +// timeout, and the next turn skips the server entirely, visible as +// a "skipped" outcome in the debug run's mcp_connect summary. +func TestGeneration_MCPNegativeCacheSkipsTimedOutServer(t *testing.T) { + t.Parallel() + + db, ps := dbtestutil.NewDB(t) + ctx := testutil.Context(t, testutil.WaitSuperLong) + + // Accepts TCP connections and never responds, so the MCP + // connect burns its whole budget and classifies as a timeout. + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = ln.Close() }) + go func() { + for { + conn, acceptErr := ln.Accept() + if acceptErr != nil { + return + } + defer conn.Close() + } + }() + + openAIURL, _ := newToolRecordingOpenAI(t) + user, org, model := seedChatDependenciesWithProvider(t, db, "openai-compat", openAIURL) + + server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) { + withoutMCPToolSearch(cfg) + cfg.AlwaysEnableDebugLogs = true + cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(chattest.NewMockAIBridgeTransport(t, openAIURL)) + }) + + dbgen.MCPServerConfig(t, db, database.MCPServerConfig{ + OrganizationID: org.ID, + DisplayName: "Black Hole", + Slug: "blackhole", + Url: "http://" + ln.Addr().String(), + Availability: "force_on", + CreatedBy: uuid.NullUUID{UUID: user.ID, Valid: true}, + UpdatedBy: uuid.NullUUID{UUID: user.ID, Valid: true}, + }) + + chat, err := server.CreateChat(ctx, chatd.CreateOptions{ + OrganizationID: org.ID, + OwnerID: user.ID, + Title: "negative-cache", + ModelConfigID: model.ID, + InitialUserContent: []codersdk.ChatMessagePart{ + codersdk.ChatMessageText("hello"), + }, + }) + require.NoError(t, err) + // The first turn pays the ~10s connect budget, so wait longer + // than waitForChatProcessed's WaitShort allows. + require.Eventually(t, func() bool { + c, getErr := db.GetChatByID(ctx, chat.ID) + return getErr == nil && c.Status != database.ChatStatusRunning + }, testutil.WaitSuperLong, testutil.IntervalMedium) + chatd.WaitUntilIdleForTest(server) + + _, err = server.SendMessage(ctx, chatd.SendMessageOptions{ + ChatID: chat.ID, + CreatedBy: user.ID, + Content: []codersdk.ChatMessagePart{codersdk.ChatMessageText("again")}, + }) + require.NoError(t, err) + // The second turn skips the black-holed server, so it settles + // fast. + require.Eventually(t, func() bool { + c, getErr := db.GetChatByID(ctx, chat.ID) + return getErr == nil && c.Status != database.ChatStatusRunning + }, testutil.WaitLong, testutil.IntervalMedium) + chatd.WaitUntilIdleForTest(server) + + // Collect per-turn mcp_connect outcomes for the blackhole + // server from the chat_turn debug runs, oldest turn first. + runs, err := db.GetChatDebugRunsByChatID(ctx, database.GetChatDebugRunsByChatIDParams{ + ChatID: chat.ID, + LimitVal: 50, + }) + require.NoError(t, err) + type connectEntry struct { + Slug string `json:"slug"` + Outcome string `json:"outcome"` + } + var outcomes []string + for i := len(runs) - 1; i >= 0; i-- { + run := runs[i] + if run.Kind != string(codersdk.ChatDebugRunKindChatTurn) { + continue + } + var summary struct { + MCPConnect []connectEntry `json:"mcp_connect"` + } + require.NoError(t, json.Unmarshal(run.Summary, &summary)) + for _, entry := range summary.MCPConnect { + if entry.Slug == "blackhole" { + outcomes = append(outcomes, entry.Outcome) + } + } + } + require.GreaterOrEqual(t, len(outcomes), 2, + "expected mcp_connect summaries for both turns, got %v", outcomes) + require.Equal(t, "timeout", outcomes[0], + "first turn must record the connect timeout") + require.Equal(t, "skipped", outcomes[1], + "second turn must skip the server via the negative cache") +} diff --git a/coderd/x/chatd/mcpclient/negativecache.go b/coderd/x/chatd/mcpclient/negativecache.go new file mode 100644 index 0000000000000..b7198a3c931c1 --- /dev/null +++ b/coderd/x/chatd/mcpclient/negativecache.go @@ -0,0 +1,133 @@ +package mcpclient + +import ( + "context" + "sync" + "time" + + "github.com/google/uuid" + + "cdr.dev/slog/v3" + "github.com/coder/coder/v2/coderd/database" + "github.com/coder/quartz" +) + +// NegativeCacheTTL is how long a server that timed out during +// connect is skipped before being retried. Connects run on every +// generation step, so without this cache a black-holed server +// costs the full connect budget on every step of every chat on +// the pod. +const NegativeCacheTTL = 60 * time.Second + +// ConnectOutcomeSkipped means the server was not dialed because a +// recent connect attempt timed out and the failure is cached. +const ConnectOutcomeSkipped ConnectOutcome = "skipped" + +// NegativeCache is a per-process cache of MCP servers whose +// connect attempts recently timed out. Entries are keyed by config +// ID and bound to the config's UpdatedAt, so editing a server +// config busts its entry immediately. +type NegativeCache struct { + clock quartz.Clock + + mu sync.Mutex + entries map[uuid.UUID]negativeCacheEntry +} + +type negativeCacheEntry struct { + configUpdatedAt time.Time + expiresAt time.Time +} + +// NewNegativeCache creates a NegativeCache. A nil clock uses the +// real clock. +func NewNegativeCache(clock quartz.Clock) *NegativeCache { + if clock == nil { + clock = quartz.NewReal() + } + return &NegativeCache{ + clock: clock, + entries: make(map[uuid.UUID]negativeCacheEntry), + } +} + +// Filter partitions configs into those that should be dialed and +// ConnectSummary values for those skipped due to a cached recent +// timeout. Expired and stale (config edited) entries are evicted. +func (c *NegativeCache) Filter( + ctx context.Context, + logger slog.Logger, + configs []database.MCPServerConfig, +) ([]database.MCPServerConfig, []ConnectSummary) { + if c == nil { + return configs, nil + } + + c.mu.Lock() + defer c.mu.Unlock() + + now := c.clock.Now() + connectable := make([]database.MCPServerConfig, 0, len(configs)) + var skipped []ConnectSummary + for _, cfg := range configs { + entry, ok := c.entries[cfg.ID] + if !ok { + connectable = append(connectable, cfg) + continue + } + if now.After(entry.expiresAt) || !entry.configUpdatedAt.Equal(cfg.UpdatedAt) { + delete(c.entries, cfg.ID) + connectable = append(connectable, cfg) + continue + } + logger.Warn(ctx, + "skipping MCP server due to recent connect timeout", + slog.F("server_slug", cfg.Slug), + slog.F("server_url", RedactURL(cfg.Url)), + slog.F("retry_after", entry.expiresAt), + ) + skipped = append(skipped, ConnectSummary{ + ConfigID: cfg.ID, + Slug: cfg.Slug, + Outcome: ConnectOutcomeSkipped, + Error: "recent connect timeout; retrying after " + entry.expiresAt.UTC().Format(time.RFC3339), + }) + } + return connectable, skipped +} + +// Record caches connect timeouts from summaries. Only timeouts are +// cached: fast failures (auth, DNS) are cheap to retry every step +// and may be fixed mid-conversation, while a timeout costs the +// whole connect budget on every step until it recovers. +func (c *NegativeCache) Record( + configs []database.MCPServerConfig, + summaries []ConnectSummary, +) { + if c == nil { + return + } + + updatedAtByID := make(map[uuid.UUID]time.Time, len(configs)) + for _, cfg := range configs { + updatedAtByID[cfg.ID] = cfg.UpdatedAt + } + + c.mu.Lock() + defer c.mu.Unlock() + + now := c.clock.Now() + for _, summary := range summaries { + if summary.Outcome != ConnectOutcomeTimeout { + continue + } + updatedAt, ok := updatedAtByID[summary.ConfigID] + if !ok { + continue + } + c.entries[summary.ConfigID] = negativeCacheEntry{ + configUpdatedAt: updatedAt, + expiresAt: now.Add(NegativeCacheTTL), + } + } +} diff --git a/coderd/x/chatd/mcpclient/negativecache_test.go b/coderd/x/chatd/mcpclient/negativecache_test.go new file mode 100644 index 0000000000000..21c3221ac8fe9 --- /dev/null +++ b/coderd/x/chatd/mcpclient/negativecache_test.go @@ -0,0 +1,120 @@ +package mcpclient_test + +import ( + "context" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + + "cdr.dev/slog/v3/sloggers/slogtest" + "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/x/chatd/mcpclient" + "github.com/coder/quartz" +) + +func TestNegativeCache(t *testing.T) { + t.Parallel() + + newConfig := func(slug string) database.MCPServerConfig { + cfg := makeConfig(slug, "http://127.0.0.1:1") + cfg.UpdatedAt = time.Date(2026, 8, 20, 12, 0, 0, 0, time.UTC) + return cfg + } + timeoutSummary := func(cfg database.MCPServerConfig) mcpclient.ConnectSummary { + return mcpclient.ConnectSummary{ + ConfigID: cfg.ID, + Slug: cfg.Slug, + Outcome: mcpclient.ConnectOutcomeTimeout, + Error: "connect: context deadline exceeded", + } + } + + t.Run("SkipsRecentTimeoutUntilTTL", func(t *testing.T) { + t.Parallel() + ctx := context.Background() + logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}) + clock := quartz.NewMock(t) + cache := mcpclient.NewNegativeCache(clock) + cfg := newConfig("srv") + + // Not cached: config passes through. + connectable, skipped := cache.Filter(ctx, logger, []database.MCPServerConfig{cfg}) + require.Len(t, connectable, 1) + require.Empty(t, skipped) + + cache.Record(connectable, []mcpclient.ConnectSummary{timeoutSummary(cfg)}) + + // Within TTL: skipped with a summary for the debug run. + connectable, skipped = cache.Filter(ctx, logger, []database.MCPServerConfig{cfg}) + require.Empty(t, connectable) + require.Len(t, skipped, 1) + require.Equal(t, mcpclient.ConnectOutcomeSkipped, skipped[0].Outcome) + require.Equal(t, cfg.ID, skipped[0].ConfigID) + require.NotEmpty(t, skipped[0].Error) + + // After TTL: retried. + clock.Advance(mcpclient.NegativeCacheTTL + time.Second) + connectable, skipped = cache.Filter(ctx, logger, []database.MCPServerConfig{cfg}) + require.Len(t, connectable, 1) + require.Empty(t, skipped) + }) + + t.Run("ConfigEditBustsEntry", func(t *testing.T) { + t.Parallel() + ctx := context.Background() + logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}) + clock := quartz.NewMock(t) + cache := mcpclient.NewNegativeCache(clock) + cfg := newConfig("srv") + + cache.Record([]database.MCPServerConfig{cfg}, []mcpclient.ConnectSummary{timeoutSummary(cfg)}) + + edited := cfg + edited.UpdatedAt = cfg.UpdatedAt.Add(time.Minute) + connectable, skipped := cache.Filter(ctx, logger, []database.MCPServerConfig{edited}) + require.Len(t, connectable, 1) + require.Empty(t, skipped) + }) + + t.Run("OnlyTimeoutsAreCached", func(t *testing.T) { + t.Parallel() + ctx := context.Background() + logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}) + clock := quartz.NewMock(t) + cache := mcpclient.NewNegativeCache(clock) + cfg := newConfig("srv") + + cache.Record([]database.MCPServerConfig{cfg}, []mcpclient.ConnectSummary{ + { + ConfigID: cfg.ID, + Slug: cfg.Slug, + Outcome: mcpclient.ConnectOutcomeError, + Error: "401 unauthorized", + }, + { + ConfigID: uuid.New(), + Slug: "unknown", + Outcome: mcpclient.ConnectOutcomeTimeout, + }, + }) + + connectable, skipped := cache.Filter(ctx, logger, []database.MCPServerConfig{cfg}) + require.Len(t, connectable, 1) + require.Empty(t, skipped) + }) + + t.Run("NilCacheIsNoop", func(t *testing.T) { + t.Parallel() + ctx := context.Background() + logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}) + var cache *mcpclient.NegativeCache + cfg := newConfig("srv") + + connectable, skipped := cache.Filter(ctx, logger, []database.MCPServerConfig{cfg}) + require.Len(t, connectable, 1) + require.Empty(t, skipped) + cache.Record(connectable, []mcpclient.ConnectSummary{timeoutSummary(cfg)}) + }) +}