coder · ibetitsmike · Aug 20, 2026
diff --git a/coderd/x/chatd/chatd.go b/coderd/x/chatd/chatd.go
@@ -189,6 +189,7 @@ type Server struct {
 	providerAPIKeys                chatprovider.ProviderAPIKeys
 	allowBYOK                      bool
 	oidcTokenSource                mcpclient.UserOIDCTokenSource
+	mcpNegativeCache               *mcpclient.NegativeCache
 	debugSvc                       *chatdebug.Service
 	debugSvcFactory                func() *chatdebug.Service
 	debugSvcReady                  atomic.Bool
@@ -3161,6 +3162,7 @@ func New(ps pubsub.Pubsub, cfg Config) *Server {
 		providerAPIKeys:                cfg.ProviderAPIKeys,
 		allowBYOK:                      allowBYOK,
 		oidcTokenSource:                cfg.OIDCTokenSource,
+		mcpNegativeCache:               mcpclient.NewNegativeCache(clk),
 		debugSvcFactory: func() *chatdebug.Service {
 			debugSvc := chatdebug.NewService(
 				cfg.Database,

diff --git a/coderd/x/chatd/generation_preparer.go b/coderd/x/chatd/generation_preparer.go
@@ -356,15 +356,18 @@ func (server *Server) prepareGeneration(
 				logger.Warn(ctx, "failed to load MCP user tokens", slog.Error(tokenErr))
 			}
 			mcpTokens = server.refreshExpiredMCPTokens(ctx, logger, mcpConnectConfigs, mcpTokens)
+			connectable, skipped := server.mcpNegativeCache.Filter(ctx, logger, mcpConnectConfigs)
 			mcpTools, mcpSummaries, mcpCleanup = mcpclient.ConnectAll(
 				ctx,
 				logger,
-				mcpConnectConfigs,
+				connectable,
 				mcpTokens,
 				chat.OwnerID,
 				server.oidcTokenSource,
 				chatprovider.CoderHeaders(chat),
 			)
+			server.mcpNegativeCache.Record(connectable, mcpSummaries)
+			mcpSummaries = append(mcpSummaries, skipped...)
 			return nil
 		})
 	}

diff --git a/coderd/x/chatd/mcp_negative_cache_test.go b/coderd/x/chatd/mcp_negative_cache_test.go
@@ -0,0 +1,130 @@
+package chatd_test
+
+import (
+	"encoding/json"
+	"net"
+	"testing"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/require"
+
+	"github.com/coder/coder/v2/coderd/database"
+	"github.com/coder/coder/v2/coderd/database/dbgen"
+	"github.com/coder/coder/v2/coderd/database/dbtestutil"
+	"github.com/coder/coder/v2/coderd/x/chatd"
+	"github.com/coder/coder/v2/coderd/x/chatd/chattest"
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/testutil"
+)
+
+// TestGeneration_MCPNegativeCacheSkipsTimedOutServer proves the
+// negative-cache wiring end to end: the first turn pays the full
+// connect budget against a black-holed MCP server and records a
+// timeout, and the next turn skips the server entirely, visible as
+// a "skipped" outcome in the debug run's mcp_connect summary.
+func TestGeneration_MCPNegativeCacheSkipsTimedOutServer(t *testing.T) {
+	t.Parallel()
+
+	db, ps := dbtestutil.NewDB(t)
+	ctx := testutil.Context(t, testutil.WaitSuperLong)
+
+	// Accepts TCP connections and never responds, so the MCP
+	// connect burns its whole budget and classifies as a timeout.
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = ln.Close() })
+	go func() {
+		for {
+			conn, acceptErr := ln.Accept()
+			if acceptErr != nil {
+				return
+			}
+			defer conn.Close()
+		}
+	}()
+
+	openAIURL, _ := newToolRecordingOpenAI(t)
+	user, org, model := seedChatDependenciesWithProvider(t, db, "openai-compat", openAIURL)
+
+	server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) {
+		withoutMCPToolSearch(cfg)
+		cfg.AlwaysEnableDebugLogs = true
+		cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(chattest.NewMockAIBridgeTransport(t, openAIURL))
+	})
+
+	dbgen.MCPServerConfig(t, db, database.MCPServerConfig{
+		OrganizationID: org.ID,
+		DisplayName:    "Black Hole",
+		Slug:           "blackhole",
+		Url:            "http://" + ln.Addr().String(),
+		Availability:   "force_on",
+		CreatedBy:      uuid.NullUUID{UUID: user.ID, Valid: true},
+		UpdatedBy:      uuid.NullUUID{UUID: user.ID, Valid: true},
+	})
+
+	chat, err := server.CreateChat(ctx, chatd.CreateOptions{
+		OrganizationID: org.ID,
+		OwnerID:        user.ID,
+		Title:          "negative-cache",
+		ModelConfigID:  model.ID,
+		InitialUserContent: []codersdk.ChatMessagePart{
+			codersdk.ChatMessageText("hello"),
+		},
+	})
+	require.NoError(t, err)
+	// The first turn pays the ~10s connect budget, so wait longer
+	// than waitForChatProcessed's WaitShort allows.
+	require.Eventually(t, func() bool {
+		c, getErr := db.GetChatByID(ctx, chat.ID)
+		return getErr == nil && c.Status != database.ChatStatusRunning
+	}, testutil.WaitSuperLong, testutil.IntervalMedium)
+	chatd.WaitUntilIdleForTest(server)
+
+	_, err = server.SendMessage(ctx, chatd.SendMessageOptions{
+		ChatID:    chat.ID,
+		CreatedBy: user.ID,
+		Content:   []codersdk.ChatMessagePart{codersdk.ChatMessageText("again")},
+	})
+	require.NoError(t, err)
+	// The second turn skips the black-holed server, so it settles
+	// fast.
+	require.Eventually(t, func() bool {
+		c, getErr := db.GetChatByID(ctx, chat.ID)
+		return getErr == nil && c.Status != database.ChatStatusRunning
+	}, testutil.WaitLong, testutil.IntervalMedium)
+	chatd.WaitUntilIdleForTest(server)
+
+	// Collect per-turn mcp_connect outcomes for the blackhole
+	// server from the chat_turn debug runs, oldest turn first.
+	runs, err := db.GetChatDebugRunsByChatID(ctx, database.GetChatDebugRunsByChatIDParams{
+		ChatID:   chat.ID,
+		LimitVal: 50,
+	})
+	require.NoError(t, err)
+	type connectEntry struct {
+		Slug    string `json:"slug"`
+		Outcome string `json:"outcome"`
+	}
+	var outcomes []string
+	for i := len(runs) - 1; i >= 0; i-- {
+		run := runs[i]
+		if run.Kind != string(codersdk.ChatDebugRunKindChatTurn) {
+			continue
+		}
+		var summary struct {
+			MCPConnect []connectEntry `json:"mcp_connect"`
+		}
+		require.NoError(t, json.Unmarshal(run.Summary, &summary))
+		for _, entry := range summary.MCPConnect {
+			if entry.Slug == "blackhole" {
+				outcomes = append(outcomes, entry.Outcome)
+			}
+		}
+	}
+	require.GreaterOrEqual(t, len(outcomes), 2,
+		"expected mcp_connect summaries for both turns, got %v", outcomes)
+	require.Equal(t, "timeout", outcomes[0],
+		"first turn must record the connect timeout")
+	require.Equal(t, "skipped", outcomes[1],
+		"second turn must skip the server via the negative cache")
+}
diff --git a/coderd/x/chatd/mcpclient/negativecache.go b/coderd/x/chatd/mcpclient/negativecache.go
@@ -0,0 +1,133 @@
+package mcpclient
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"github.com/google/uuid"
+
+	"cdr.dev/slog/v3"
+	"github.com/coder/coder/v2/coderd/database"
+	"github.com/coder/quartz"
+)
+
+// NegativeCacheTTL is how long a server that timed out during
+// connect is skipped before being retried. Connects run on every
+// generation step, so without this cache a black-holed server
+// costs the full connect budget on every step of every chat on
+// the pod.
+const NegativeCacheTTL = 60 * time.Second
+
+// ConnectOutcomeSkipped means the server was not dialed because a
+// recent connect attempt timed out and the failure is cached.
+const ConnectOutcomeSkipped ConnectOutcome = "skipped"
+
+// NegativeCache is a per-process cache of MCP servers whose
+// connect attempts recently timed out. Entries are keyed by config
+// ID and bound to the config's UpdatedAt, so editing a server
+// config busts its entry immediately.
+type NegativeCache struct {
+	clock quartz.Clock
+
+	mu      sync.Mutex
+	entries map[uuid.UUID]negativeCacheEntry
+}
+
+type negativeCacheEntry struct {
+	configUpdatedAt time.Time
+	expiresAt       time.Time
+}
+
+// NewNegativeCache creates a NegativeCache. A nil clock uses the
+// real clock.
+func NewNegativeCache(clock quartz.Clock) *NegativeCache {
+	if clock == nil {
+		clock = quartz.NewReal()
+	}
+	return &NegativeCache{
+		clock:   clock,
+		entries: make(map[uuid.UUID]negativeCacheEntry),
+	}
+}
+
+// Filter partitions configs into those that should be dialed and
+// ConnectSummary values for those skipped due to a cached recent
+// timeout. Expired and stale (config edited) entries are evicted.
+func (c *NegativeCache) Filter(
+	ctx context.Context,
+	logger slog.Logger,
+	configs []database.MCPServerConfig,
+) ([]database.MCPServerConfig, []ConnectSummary) {
+	if c == nil {
+		return configs, nil
+	}
+
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	now := c.clock.Now()
+	connectable := make([]database.MCPServerConfig, 0, len(configs))
+	var skipped []ConnectSummary
+	for _, cfg := range configs {
+		entry, ok := c.entries[cfg.ID]
+		if !ok {
+			connectable = append(connectable, cfg)
+			continue
+		}
+		if now.After(entry.expiresAt) || !entry.configUpdatedAt.Equal(cfg.UpdatedAt) {
+			delete(c.entries, cfg.ID)
+			connectable = append(connectable, cfg)
+			continue
+		}
+		logger.Warn(ctx,
+			"skipping MCP server due to recent connect timeout",
+			slog.F("server_slug", cfg.Slug),
+			slog.F("server_url", RedactURL(cfg.Url)),
+			slog.F("retry_after", entry.expiresAt),
+		)
+		skipped = append(skipped, ConnectSummary{
+			ConfigID: cfg.ID,
+			Slug:     cfg.Slug,
+			Outcome:  ConnectOutcomeSkipped,
+			Error:    "recent connect timeout; retrying after " + entry.expiresAt.UTC().Format(time.RFC3339),
+		})
+	}
+	return connectable, skipped
+}
+
+// Record caches connect timeouts from summaries. Only timeouts are
+// cached: fast failures (auth, DNS) are cheap to retry every step
+// and may be fixed mid-conversation, while a timeout costs the
+// whole connect budget on every step until it recovers.
+func (c *NegativeCache) Record(
+	configs []database.MCPServerConfig,
+	summaries []ConnectSummary,
+) {
+	if c == nil {
+		return
+	}
+
+	updatedAtByID := make(map[uuid.UUID]time.Time, len(configs))
+	for _, cfg := range configs {
+		updatedAtByID[cfg.ID] = cfg.UpdatedAt
+	}
+
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	now := c.clock.Now()
+	for _, summary := range summaries {
+		if summary.Outcome != ConnectOutcomeTimeout {
+			continue
+		}
+		updatedAt, ok := updatedAtByID[summary.ConfigID]
+		if !ok {
+			continue
+		}
+		c.entries[summary.ConfigID] = negativeCacheEntry{
+			configUpdatedAt: updatedAt,
+			expiresAt:       now.Add(NegativeCacheTTL),
+		}
+	}
+}