From 0d076b448b3fdfa81709b3df8d82e266b2b3c871 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 9 Jul 2026 17:40:45 +0000 Subject: [PATCH 01/12] feat(coderd): allow admins to override the chat compaction model Adds a deployment-wide 'compaction' context to the chat model override framework. When set, chatd generates compaction summaries with the override model instead of the chat model. Thresholds use the stricter of the two context limits, the prompt is re-sanitized for the override provider, and metrics plus debug runs record the model that actually generated the summary. --- coderd/database/dbauthz/dbauthz.go | 14 ++ coderd/database/dbauthz/dbauthz_test.go | 8 + coderd/database/dbmetrics/querymetrics.go | 16 ++ coderd/database/dbmock/dbmock.go | 29 +++ coderd/database/querier.go | 2 + coderd/database/queries.sql.go | 22 ++ coderd/database/queries/siteconfig.sql | 8 + coderd/exp_chats.go | 6 + coderd/exp_chats_test.go | 10 + coderd/x/chatd/ARCHITECTURE.md | 13 ++ coderd/x/chatd/chatd_test.go | 200 ++++++++++++++++++ coderd/x/chatd/chatloop/chatloop.go | 8 + coderd/x/chatd/chatloop/compaction.go | 37 +++- coderd/x/chatd/compaction_override.go | 117 ++++++++++ .../compaction_override_internal_test.go | 167 +++++++++++++++ coderd/x/chatd/compaction_prompt.go | 145 +++++++++++++ .../chatd/compaction_prompt_internal_test.go | 185 ++++++++++++++++ coderd/x/chatd/generation.go | 6 +- coderd/x/chatd/generation_preparer.go | 47 +++- codersdk/chats.go | 5 +- site/src/api/typesGenerated.ts | 2 + .../CoderAgentsPage/CoderAgentsPage.tsx | 13 ++ .../CoderAgentsPageView.stories.tsx | 82 ++++++- .../CoderAgentsPage/CoderAgentsPageView.tsx | 22 ++ .../AgentsPage/AgentsPageView.stories.tsx | 3 + 25 files changed, 1152 insertions(+), 15 deletions(-) create mode 100644 coderd/x/chatd/compaction_override.go create mode 100644 coderd/x/chatd/compaction_override_internal_test.go create mode 100644 coderd/x/chatd/compaction_prompt.go create mode 100644 coderd/x/chatd/compaction_prompt_internal_test.go diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index 10f6bcd855a..0b1baaddec9 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -3036,6 +3036,13 @@ func (q *querier) GetChatByIDForUpdate(ctx context.Context, id uuid.UUID) (datab return fetch(q.log, q.auth, q.db.GetChatByIDForUpdate)(ctx, id) } +func (q *querier) GetChatCompactionModelOverride(ctx context.Context) (string, error) { + if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceDeploymentConfig); err != nil { + return "", err + } + return q.db.GetChatCompactionModelOverride(ctx) +} + func (q *querier) GetChatComputerUseProvider(ctx context.Context) (string, error) { // The computer-use provider is a deployment-wide runtime chat setting // read by authenticated chat users and chatd. Feature and experiment @@ -8707,6 +8714,13 @@ func (q *querier) UpsertChatAutoArchiveDays(ctx context.Context, autoArchiveDays return q.db.UpsertChatAutoArchiveDays(ctx, autoArchiveDays) } +func (q *querier) UpsertChatCompactionModelOverride(ctx context.Context, value string) error { + if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceDeploymentConfig); err != nil { + return err + } + return q.db.UpsertChatCompactionModelOverride(ctx, value) +} + func (q *querier) UpsertChatComputerUseProvider(ctx context.Context, provider string) error { if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceDeploymentConfig); err != nil { return err diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index 67c7cee24c6..88ad172a452 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -1209,6 +1209,10 @@ func (s *MethodTestSuite) TestChats() { dbm.EXPECT().GetChatTitleGenerationModelOverride(gomock.Any()).Return("", nil).AnyTimes() check.Args().Asserts(rbac.ResourceDeploymentConfig, policy.ActionRead) })) + s.Run("GetChatCompactionModelOverride", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) { + dbm.EXPECT().GetChatCompactionModelOverride(gomock.Any()).Return("", nil).AnyTimes() + check.Args().Asserts(rbac.ResourceDeploymentConfig, policy.ActionRead) + })) s.Run("GetChatPlanModeInstructions", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) { dbm.EXPECT().GetChatPlanModeInstructions(gomock.Any()).Return("", nil).AnyTimes() check.Args().Asserts(rbac.ResourceDeploymentConfig, policy.ActionUpdate) @@ -1640,6 +1644,10 @@ func (s *MethodTestSuite) TestChats() { dbm.EXPECT().UpsertChatTitleGenerationModelOverride(gomock.Any(), "").Return(nil).AnyTimes() check.Args("").Asserts(rbac.ResourceDeploymentConfig, policy.ActionUpdate) })) + s.Run("UpsertChatCompactionModelOverride", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) { + dbm.EXPECT().UpsertChatCompactionModelOverride(gomock.Any(), "").Return(nil).AnyTimes() + check.Args("").Asserts(rbac.ResourceDeploymentConfig, policy.ActionUpdate) + })) s.Run("UpsertChatPlanModeInstructions", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) { dbm.EXPECT().UpsertChatPlanModeInstructions(gomock.Any(), "").Return(nil).AnyTimes() check.Args("").Asserts(rbac.ResourceDeploymentConfig, policy.ActionUpdate) diff --git a/coderd/database/dbmetrics/querymetrics.go b/coderd/database/dbmetrics/querymetrics.go index a2592c270ec..19efe41292b 100644 --- a/coderd/database/dbmetrics/querymetrics.go +++ b/coderd/database/dbmetrics/querymetrics.go @@ -1425,6 +1425,14 @@ func (m queryMetricsStore) GetChatByIDForUpdate(ctx context.Context, id uuid.UUI return r0, r1 } +func (m queryMetricsStore) GetChatCompactionModelOverride(ctx context.Context) (string, error) { + start := time.Now() + r0, r1 := m.s.GetChatCompactionModelOverride(ctx) + m.queryLatencies.WithLabelValues("GetChatCompactionModelOverride").Observe(time.Since(start).Seconds()) + m.queryCounts.WithLabelValues(httpmw.ExtractHTTPRoute(ctx), httpmw.ExtractHTTPMethod(ctx), "GetChatCompactionModelOverride").Inc() + return r0, r1 +} + func (m queryMetricsStore) GetChatComputerUseProvider(ctx context.Context) (string, error) { start := time.Now() r0, r1 := m.s.GetChatComputerUseProvider(ctx) @@ -6145,6 +6153,14 @@ func (m queryMetricsStore) UpsertChatAutoArchiveDays(ctx context.Context, autoAr return r0 } +func (m queryMetricsStore) UpsertChatCompactionModelOverride(ctx context.Context, value string) error { + start := time.Now() + r0 := m.s.UpsertChatCompactionModelOverride(ctx, value) + m.queryLatencies.WithLabelValues("UpsertChatCompactionModelOverride").Observe(time.Since(start).Seconds()) + m.queryCounts.WithLabelValues(httpmw.ExtractHTTPRoute(ctx), httpmw.ExtractHTTPMethod(ctx), "UpsertChatCompactionModelOverride").Inc() + return r0 +} + func (m queryMetricsStore) UpsertChatComputerUseProvider(ctx context.Context, provider string) error { start := time.Now() r0 := m.s.UpsertChatComputerUseProvider(ctx, provider) diff --git a/coderd/database/dbmock/dbmock.go b/coderd/database/dbmock/dbmock.go index ba2884a0d44..649a4dfd256 100644 --- a/coderd/database/dbmock/dbmock.go +++ b/coderd/database/dbmock/dbmock.go @@ -2623,6 +2623,21 @@ func (mr *MockStoreMockRecorder) GetChatByIDForUpdate(ctx, id any) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetChatByIDForUpdate", reflect.TypeOf((*MockStore)(nil).GetChatByIDForUpdate), ctx, id) } +// GetChatCompactionModelOverride mocks base method. +func (m *MockStore) GetChatCompactionModelOverride(ctx context.Context) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetChatCompactionModelOverride", ctx) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetChatCompactionModelOverride indicates an expected call of GetChatCompactionModelOverride. +func (mr *MockStoreMockRecorder) GetChatCompactionModelOverride(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetChatCompactionModelOverride", reflect.TypeOf((*MockStore)(nil).GetChatCompactionModelOverride), ctx) +} + // GetChatComputerUseProvider mocks base method. func (m *MockStore) GetChatComputerUseProvider(ctx context.Context) (string, error) { m.ctrl.T.Helper() @@ -11514,6 +11529,20 @@ func (mr *MockStoreMockRecorder) UpsertChatAutoArchiveDays(ctx, autoArchiveDays return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertChatAutoArchiveDays", reflect.TypeOf((*MockStore)(nil).UpsertChatAutoArchiveDays), ctx, autoArchiveDays) } +// UpsertChatCompactionModelOverride mocks base method. +func (m *MockStore) UpsertChatCompactionModelOverride(ctx context.Context, value string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpsertChatCompactionModelOverride", ctx, value) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpsertChatCompactionModelOverride indicates an expected call of UpsertChatCompactionModelOverride. +func (mr *MockStoreMockRecorder) UpsertChatCompactionModelOverride(ctx, value any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertChatCompactionModelOverride", reflect.TypeOf((*MockStore)(nil).UpsertChatCompactionModelOverride), ctx, value) +} + // UpsertChatComputerUseProvider mocks base method. func (m *MockStore) UpsertChatComputerUseProvider(ctx context.Context, provider string) error { m.ctrl.T.Helper() diff --git a/coderd/database/querier.go b/coderd/database/querier.go index 218688f7b40..d8a170a1186 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -376,6 +376,7 @@ type sqlcQuerier interface { GetChatByID(ctx context.Context, id uuid.UUID) (Chat, error) GetChatByIDForShare(ctx context.Context, id uuid.UUID) (Chat, error) GetChatByIDForUpdate(ctx context.Context, id uuid.UUID) (Chat, error) + GetChatCompactionModelOverride(ctx context.Context) (string, error) GetChatComputerUseProvider(ctx context.Context) (string, error) // Per-root-chat cost breakdown for a single user within a date range. // Groups by root_chat_id so forked chats roll up under their root. @@ -1529,6 +1530,7 @@ type sqlcQuerier interface { // to JSON before invoking this query. UpsertChatAdvisorConfig(ctx context.Context, value string) error UpsertChatAutoArchiveDays(ctx context.Context, autoArchiveDays int32) error + UpsertChatCompactionModelOverride(ctx context.Context, value string) error UpsertChatComputerUseProvider(ctx context.Context, provider string) error // UpsertChatDebugLoggingAllowUsers updates the runtime admin setting that // allows users to opt into chat debug logging. diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 451a233abcf..884ab6c7a1c 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -24159,6 +24159,18 @@ func (q *sqlQuerier) GetChatAutoArchiveDays(ctx context.Context, defaultAutoArch return auto_archive_days, err } +const getChatCompactionModelOverride = `-- name: GetChatCompactionModelOverride :one +SELECT + COALESCE((SELECT value FROM site_configs WHERE key = 'agents_chat_compaction_model_override'), '') :: text AS model_config_id +` + +func (q *sqlQuerier) GetChatCompactionModelOverride(ctx context.Context) (string, error) { + row := q.db.QueryRowContext(ctx, getChatCompactionModelOverride) + var model_config_id string + err := row.Scan(&model_config_id) + return model_config_id, err +} + const getChatComputerUseProvider = `-- name: GetChatComputerUseProvider :one SELECT COALESCE((SELECT value FROM site_configs WHERE key = 'agents_computer_use_provider'), '') :: text AS provider @@ -24600,6 +24612,16 @@ func (q *sqlQuerier) UpsertChatAutoArchiveDays(ctx context.Context, autoArchiveD return err } +const upsertChatCompactionModelOverride = `-- name: UpsertChatCompactionModelOverride :exec +INSERT INTO site_configs (key, value) VALUES ('agents_chat_compaction_model_override', $1) +ON CONFLICT (key) DO UPDATE SET value = $1 WHERE site_configs.key = 'agents_chat_compaction_model_override' +` + +func (q *sqlQuerier) UpsertChatCompactionModelOverride(ctx context.Context, value string) error { + _, err := q.db.ExecContext(ctx, upsertChatCompactionModelOverride, value) + return err +} + const upsertChatComputerUseProvider = `-- name: UpsertChatComputerUseProvider :exec INSERT INTO site_configs (key, value) VALUES ('agents_computer_use_provider', $1) ON CONFLICT (key) DO UPDATE SET value = $1 WHERE site_configs.key = 'agents_computer_use_provider' diff --git a/coderd/database/queries/siteconfig.sql b/coderd/database/queries/siteconfig.sql index 709cd287ca6..3eb3aacaf02 100644 --- a/coderd/database/queries/siteconfig.sql +++ b/coderd/database/queries/siteconfig.sql @@ -191,6 +191,14 @@ SELECT INSERT INTO site_configs (key, value) VALUES ('agents_chat_title_generation_model_override', $1) ON CONFLICT (key) DO UPDATE SET value = $1 WHERE site_configs.key = 'agents_chat_title_generation_model_override'; +-- name: GetChatCompactionModelOverride :one +SELECT + COALESCE((SELECT value FROM site_configs WHERE key = 'agents_chat_compaction_model_override'), '') :: text AS model_config_id; + +-- name: UpsertChatCompactionModelOverride :exec +INSERT INTO site_configs (key, value) VALUES ('agents_chat_compaction_model_override', $1) +ON CONFLICT (key) DO UPDATE SET value = $1 WHERE site_configs.key = 'agents_chat_compaction_model_override'; + -- name: GetChatDesktopEnabled :one SELECT COALESCE((SELECT value = 'true' FROM site_configs WHERE key = 'agents_desktop_enabled'), false) :: boolean AS enable_desktop; diff --git a/coderd/exp_chats.go b/coderd/exp_chats.go index 1c35f334573..401b966e781 100644 --- a/coderd/exp_chats.go +++ b/coderd/exp_chats.go @@ -638,6 +638,12 @@ func (api *API) chatModelOverrideSiteConfig( getter: api.Database.GetChatTitleGenerationModelOverride, upsert: api.Database.UpsertChatTitleGenerationModelOverride, }, nil + case codersdk.ChatModelOverrideContextCompaction: + return chatModelOverrideSiteConfig{ + label: "compaction", + getter: api.Database.GetChatCompactionModelOverride, + upsert: api.Database.UpsertChatCompactionModelOverride, + }, nil default: return chatModelOverrideSiteConfig{}, xerrors.Errorf( "unknown chat model override context %q", diff --git a/coderd/exp_chats_test.go b/coderd/exp_chats_test.go index 69c29d71e76..dfeb190b86b 100644 --- a/coderd/exp_chats_test.go +++ b/coderd/exp_chats_test.go @@ -11867,6 +11867,16 @@ func TestChatModelOverrides(t *testing.T) { return db.UpsertChatTitleGenerationModelOverride(dbauthz.AsSystemRestricted(ctx), value) }, }, + { + name: "Compaction", + context: codersdk.ChatModelOverrideContextCompaction, + dbGet: func(ctx context.Context, db database.Store) (string, error) { + return db.GetChatCompactionModelOverride(dbauthz.AsSystemRestricted(ctx)) + }, + dbUpsert: func(ctx context.Context, db database.Store, value string) error { + return db.UpsertChatCompactionModelOverride(dbauthz.AsSystemRestricted(ctx), value) + }, + }, } for _, setting := range settings { diff --git a/coderd/x/chatd/ARCHITECTURE.md b/coderd/x/chatd/ARCHITECTURE.md index bc588ccd436..b18bd69b306 100644 --- a/coderd/x/chatd/ARCHITECTURE.md +++ b/coderd/x/chatd/ARCHITECTURE.md @@ -825,6 +825,19 @@ The generation goroutine supports: - turn limit after a user message (the LLM shouldn't be able to spin forever in loop) - and other things +#### Compaction model selection + +Compaction is an auxiliary LLM call: when the conversation approaches the context limit, the generation goroutine asks a model to summarize the history, commits the summary as a compressed boundary, and continues the turn on the chat model. + +By default the summary is generated with the chat model. Admins can override the compaction model deployment-wide via the `compaction` context of the chat model override API (`/api/experimental/chats/config/model-override/{context}`, stored in the `agents_chat_compaction_model_override` site config). The override affects only the summary call; thresholds, compressed-message storage, and the post-compaction assistant generation keep using the chat model. + +Details that follow from the override: + +- Context limits: the compaction trigger uses the stricter of the chat model's and the compaction model's context limits, because the history must also fit the summarizer's window. The post-compaction "still over limit" check stays against the chat model's limit, since continuation runs on the chat model. +- Failure semantics: an unset override uses the chat model. Stale or malformed stored references (deleted or disabled config or provider, missing credentials, non-UUID value) fall back to the chat model with a log. A usable override that fails at use (route or client construction, provider call failure) fails the generation visibly through the normal error path; there is no silent fallback. +- Prompt safety: the prompt is built and sanitized for the chat model, so when the override points at a different provider the compaction copy of the prompt is re-sanitized: provider-executed tool history is stripped, file parts the compaction model rejects are replaced with text placeholders, and Anthropic provider-tool sanitization is re-run for the compaction provider. The assistant generation prompt is never mutated. +- Observability: compaction metrics and chat debug runs record the provider and model that actually generated the summary. + #### Interrupt goroutine The interrupt goroutine is responsible for handling interrupts. It is spawned when the event indicates the core state machine is in `I0` or `I1` (status is `interrupting`). diff --git a/coderd/x/chatd/chatd_test.go b/coderd/x/chatd/chatd_test.go index b89fed6068b..10c76ff7fe1 100644 --- a/coderd/x/chatd/chatd_test.go +++ b/coderd/x/chatd/chatd_test.go @@ -50,6 +50,7 @@ import ( "github.com/coder/coder/v2/coderd/workspacestats" "github.com/coder/coder/v2/coderd/x/chatd" "github.com/coder/coder/v2/coderd/x/chatd/chatadvisor" + "github.com/coder/coder/v2/coderd/x/chatd/chatdebug" "github.com/coder/coder/v2/coderd/x/chatd/chatprompt" "github.com/coder/coder/v2/coderd/x/chatd/chatsanitize" "github.com/coder/coder/v2/coderd/x/chatd/chatstate" @@ -5860,6 +5861,205 @@ func singlePartOfType(t *testing.T, msg database.ChatMessage, typ codersdk.ChatM return matches[0] } +func TestActiveServer_CompactionModelOverride(t *testing.T) { + t.Parallel() + + const ( + compactionSummary = "summary text for compaction" + chatModelName = "claude-sonnet-4-20250514" + overrideModelName = "claude-3-5-haiku-latest" + thresholdPercent = int32(70) + ) + + seedOverrideModel := func(ctx context.Context, t *testing.T, db database.Store, chatModel database.ChatModelConfig, contextLimit int64) database.ChatModelConfig { + t.Helper() + overrideModel := dbgen.ChatModelConfig(t, db, database.ChatModelConfig{ + Model: overrideModelName, + AIProviderID: chatModel.AIProviderID, + ContextLimit: contextLimit, + }) + require.NoError(t, db.UpsertChatCompactionModelOverride(ctx, overrideModel.ID.String())) + return overrideModel + } + + t.Run("summary routes to the override model and continuation stays on the chat model", func(t *testing.T) { + t.Parallel() + + ctx := testutil.Context(t, testutil.WaitLong) + db, ps := dbtestutil.NewDB(t) + reg := prometheus.NewRegistry() + var streamCount atomic.Int32 + anthropicURL := chattest.NewAnthropic(t, func(req *chattest.AnthropicRequest) chattest.AnthropicResponse { + body := anthropicRequestBody(t, *req) + if !req.Stream { + if strings.Contains(body, "You are performing a context compaction") { + require.Equal(t, overrideModelName, req.Model) + return anthropicCompactionResponse(compactionSummary) + } + return chattest.AnthropicNonStreamingResponse("title") + } + require.Equal(t, chatModelName, req.Model) + switch streamCount.Add(1) { + case 1: + return highUsageReadFileResponse("/tmp/a.txt") + default: + require.Contains(t, body, compactionSummary) + return chattest.AnthropicStreamingResponse(chattest.AnthropicTextChunksWithCacheUsage(chattest.AnthropicUsage{ + InputTokens: 20, + OutputTokens: 5, + }, "continued after compaction")...) + } + }) + user, org, model := seedAnthropicChatDependencies(t, db, anthropicURL) + model = updateChatModelCompressionThreshold(t, db, model, 100, thresholdPercent) + overrideModel := seedOverrideModel(ctx, t, db, model, 1_000_000) + ws, dbAgent := seedWorkspaceWithAgent(t, db, user.ID) + + ctrl := gomock.NewController(t) + mockConn := agentconnmock.NewMockAgentConn(ctrl) + setupToolExecutionAgentConn(t, mockConn) + mockConn.EXPECT().ReadFileLines(gomock.Any(), "/tmp/a.txt", int64(1), int64(0), gomock.Any()). + Return(workspacesdk.ReadFileLinesResponse{Success: true, FileSize: 12, TotalLines: 1, LinesRead: 1, Content: "1\tpackage main"}, nil). + Times(1) + + server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) { + cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(chattest.NewMockAIBridgeTransport(t, anthropicURL, chattest.WithPreservePath())) + cfg.PrometheusRegistry = reg + cfg.AlwaysEnableDebugLogs = true + cfg.AgentConn = func(_ context.Context, agentID uuid.UUID) (workspacesdk.AgentConn, func(), error) { + require.Equal(t, dbAgent.ID, agentID) + return mockConn, func() {}, nil + } + }) + chat, err := server.CreateChat(ctx, chatd.CreateOptions{ + OrganizationID: org.ID, + OwnerID: user.ID, + APIKeyID: testAPIKeyID(t, db, user.ID), + WorkspaceID: uuid.NullUUID{UUID: ws.ID, Valid: true}, + AgentID: uuid.NullUUID{UUID: dbAgent.ID, Valid: true}, + Title: "compaction-override", + ModelConfigID: model.ID, + InitialUserContent: []codersdk.ChatMessagePart{ + codersdk.ChatMessageText("read the file and continue"), + }, + }) + require.NoError(t, err) + waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusWaiting) + + messages := chatMessages(ctx, t, db, chat.ID) + promptMessages, err := db.GetChatMessagesForPromptByChatID(ctx, chat.ID) + require.NoError(t, err) + compressed := compressedChatSummarizedMessages(t, append(promptMessages, messages...)) + require.Len(t, compressed.summaries, 1) + require.Contains(t, messageText(t, compressed.summaries[0]), compactionSummary) + requireTextPart(t, messages[len(messages)-1], "continued after compaction") + + requireChatdMetricCounter(t, reg, "coderd_chatd_compaction_total", 1, map[string]string{ + "provider": "anthropic", + "model": overrideModelName, + "result": "success", + }) + + require.NoError(t, server.Close()) + debugCtx := testutil.Context(t, testutil.WaitLong) + var compactionRun database.ChatDebugRun + testutil.Eventually(debugCtx, t, func(ctx context.Context) bool { + runs, err := db.GetChatDebugRunsByChatID(ctx, database.GetChatDebugRunsByChatIDParams{ + ChatID: chat.ID, + LimitVal: 100, + }) + if err != nil { + return false + } + for _, run := range runs { + if run.Kind == string(chatdebug.KindCompaction) { + compactionRun = run + return true + } + } + return false + }, testutil.IntervalMedium) + require.True(t, compactionRun.Provider.Valid) + require.Equal(t, "anthropic", compactionRun.Provider.String) + require.True(t, compactionRun.Model.Valid) + require.Equal(t, overrideModelName, compactionRun.Model.String) + require.True(t, compactionRun.ModelConfigID.Valid) + require.Equal(t, overrideModel.ID, compactionRun.ModelConfigID.UUID) + }) + + t.Run("compaction triggers at the stricter override context limit", func(t *testing.T) { + t.Parallel() + + ctx := testutil.Context(t, testutil.WaitLong) + db, ps := dbtestutil.NewDB(t) + var streamCount atomic.Int32 + anthropicURL := chattest.NewAnthropic(t, func(req *chattest.AnthropicRequest) chattest.AnthropicResponse { + body := anthropicRequestBody(t, *req) + if !req.Stream { + if strings.Contains(body, "You are performing a context compaction") { + require.Equal(t, overrideModelName, req.Model) + return anthropicCompactionResponse(compactionSummary) + } + return chattest.AnthropicNonStreamingResponse("title") + } + switch streamCount.Add(1) { + case 1: + return highUsageReadFileResponse("/tmp/a.txt") + default: + require.Contains(t, body, compactionSummary) + return chattest.AnthropicStreamingResponse(chattest.AnthropicTextChunksWithCacheUsage(chattest.AnthropicUsage{ + InputTokens: 20, + OutputTokens: 5, + }, "continued after compaction")...) + } + }) + user, org, model := seedAnthropicChatDependencies(t, db, anthropicURL) + // The chat model alone would not compact: 80 tokens of usage is + // 8% of its 1000-token limit. The override model's 100-token + // limit makes the effective threshold 70 tokens, so compaction + // must trigger. + model = updateChatModelCompressionThreshold(t, db, model, 1_000, thresholdPercent) + seedOverrideModel(ctx, t, db, model, 100) + ws, dbAgent := seedWorkspaceWithAgent(t, db, user.ID) + + ctrl := gomock.NewController(t) + mockConn := agentconnmock.NewMockAgentConn(ctrl) + setupToolExecutionAgentConn(t, mockConn) + mockConn.EXPECT().ReadFileLines(gomock.Any(), "/tmp/a.txt", int64(1), int64(0), gomock.Any()). + Return(workspacesdk.ReadFileLinesResponse{Success: true, FileSize: 12, TotalLines: 1, LinesRead: 1, Content: "1\tpackage main"}, nil). + Times(1) + + server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) { + cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(chattest.NewMockAIBridgeTransport(t, anthropicURL, chattest.WithPreservePath())) + cfg.AgentConn = func(_ context.Context, agentID uuid.UUID) (workspacesdk.AgentConn, func(), error) { + require.Equal(t, dbAgent.ID, agentID) + return mockConn, func() {}, nil + } + }) + chat, err := server.CreateChat(ctx, chatd.CreateOptions{ + OrganizationID: org.ID, + OwnerID: user.ID, + APIKeyID: testAPIKeyID(t, db, user.ID), + WorkspaceID: uuid.NullUUID{UUID: ws.ID, Valid: true}, + AgentID: uuid.NullUUID{UUID: dbAgent.ID, Valid: true}, + Title: "compaction-override-limit", + ModelConfigID: model.ID, + InitialUserContent: []codersdk.ChatMessagePart{ + codersdk.ChatMessageText("read the file and continue"), + }, + }) + require.NoError(t, err) + waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusWaiting) + + messages := chatMessages(ctx, t, db, chat.ID) + promptMessages, err := db.GetChatMessagesForPromptByChatID(ctx, chat.ID) + require.NoError(t, err) + compressed := compressedChatSummarizedMessages(t, append(promptMessages, messages...)) + require.Len(t, compressed.summaries, 1) + requireTextPart(t, messages[len(messages)-1], "continued after compaction") + }) +} + func TestActiveServer_BasicAssistantGenerationAndPromptPreparation(t *testing.T) { t.Parallel() diff --git a/coderd/x/chatd/chatloop/chatloop.go b/coderd/x/chatd/chatloop/chatloop.go index 33505aa593b..4d55fa8ed82 100644 --- a/coderd/x/chatd/chatloop/chatloop.go +++ b/coderd/x/chatd/chatloop/chatloop.go @@ -280,6 +280,14 @@ type GenerateCompactionOptions struct { ToolCallID string ToolName string + // ResolvedProvider, ResolvedModel, and ModelConfigID identify the + // model that actually generates the summary, which can differ from + // the chat model when a compaction model override is configured. + // Debug runs record these instead of the parent run's identity. + ResolvedProvider string + ResolvedModel string + ModelConfigID uuid.UUID + PublishMessagePart func(codersdk.ChatMessageRole, codersdk.ChatMessagePart) } diff --git a/coderd/x/chatd/chatloop/compaction.go b/coderd/x/chatd/chatloop/compaction.go index 946023edb28..c4a23512ca2 100644 --- a/coderd/x/chatd/chatloop/compaction.go +++ b/coderd/x/chatd/chatloop/compaction.go @@ -64,6 +64,12 @@ type CompactionOptions struct { ChatID uuid.UUID HistoryTipMessageID int64 + // ResolvedProvider, ResolvedModel, and ModelConfigID identify the + // model generating the summary; see GenerateCompactionOptions. + ResolvedProvider string + ResolvedModel string + ModelConfigID uuid.UUID + // ToolCallID and ToolName identify the synthetic tool call // used to represent compaction in the message stream. ToolCallID string @@ -169,6 +175,9 @@ func normalizedCompactionGenerateConfig(opts GenerateCompactionOptions) (Compact DebugSvc: opts.DebugSvc, ChatID: opts.ChatID, HistoryTipMessageID: opts.HistoryTipMessageID, + ResolvedProvider: opts.ResolvedProvider, + ResolvedModel: opts.ResolvedModel, + ModelConfigID: opts.ModelConfigID, ToolCallID: opts.ToolCallID, ToolName: opts.ToolName, PublishMessagePart: opts.PublishMessagePart, @@ -276,6 +285,22 @@ func startCompactionDebugRun( historyTipMessageID = parentRun.HistoryTipMessageID } + // The compaction model can differ from the chat model when a + // deployment override is configured, so record the summary model's + // identity when the caller supplies it. + provider := parentRun.Provider + if options.ResolvedProvider != "" { + provider = options.ResolvedProvider + } + model := parentRun.Model + if options.ResolvedModel != "" { + model = options.ResolvedModel + } + modelConfigID := parentRun.ModelConfigID + if options.ModelConfigID != uuid.Nil { + modelConfigID = options.ModelConfigID + } + // Use a separate short-lived context for the debug insert so a // slow or locked DB cannot block the model call. Detached from // the parent so cancellation of the compaction run still lets @@ -288,13 +313,13 @@ func startCompactionDebugRun( ChatID: options.ChatID, RootChatID: parentRun.RootChatID, ParentChatID: parentRun.ParentChatID, - ModelConfigID: parentRun.ModelConfigID, + ModelConfigID: modelConfigID, TriggerMessageID: parentRun.TriggerMessageID, HistoryTipMessageID: historyTipMessageID, Kind: chatdebug.KindCompaction, Status: chatdebug.StatusInProgress, - Provider: parentRun.Provider, - Model: parentRun.Model, + Provider: provider, + Model: model, }) createRunCancel() if err != nil { @@ -307,12 +332,12 @@ func startCompactionDebugRun( ChatID: options.ChatID, RootChatID: parentRun.RootChatID, ParentChatID: parentRun.ParentChatID, - ModelConfigID: parentRun.ModelConfigID, + ModelConfigID: modelConfigID, TriggerMessageID: parentRun.TriggerMessageID, HistoryTipMessageID: historyTipMessageID, Kind: chatdebug.KindCompaction, - Provider: parentRun.Provider, - Model: parentRun.Model, + Provider: provider, + Model: model, }) return compactionCtx, func(runErr error) { diff --git a/coderd/x/chatd/compaction_override.go b/coderd/x/chatd/compaction_override.go new file mode 100644 index 00000000000..c006ec6201d --- /dev/null +++ b/coderd/x/chatd/compaction_override.go @@ -0,0 +1,117 @@ +package chatd + +import ( + "context" + + "charm.land/fantasy" + "github.com/google/uuid" + "golang.org/x/xerrors" + + "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/database/dbauthz" + "github.com/coder/coder/v2/coderd/x/chatd/chatprovider" +) + +const compactionOverrideContext = "compaction" + +func readCompactionModelOverride( + ctx context.Context, + db database.Store, +) (string, error) { + //nolint:gocritic // Chatd is internal, not a user, so this read uses AsChatd. + chatdCtx := dbauthz.AsChatd(ctx) + raw, err := db.GetChatCompactionModelOverride(chatdCtx) + if err != nil { + return "", xerrors.Errorf( + "get chat compaction model override: %w", + err, + ) + } + return raw, nil +} + +// compactionModelOverride carries the resolved deployment-wide compaction +// model override: the model to run compaction summaries with plus the +// identity metadata debug runs and prompt sanitization need. +type compactionModelOverride struct { + modelConfig database.ChatModelConfig + model fantasy.LanguageModel + resolvedProvider string + resolvedModel string +} + +// resolveCompactionModelOverride resolves the deployment-wide compaction +// model override. Unset, malformed, stale (deleted or disabled config or +// provider), and credential-less overrides fall back to the chat model +// (overrideSet is false; the shared resolver logs the reason). Errors are +// hard failures: a configured, usable override that cannot be routed or +// constructed must fail the generation visibly instead of silently +// compacting with the chat model. +func (p *Server) resolveCompactionModelOverride( + ctx context.Context, + chat database.Chat, + modelOpts modelBuildOptions, +) (compactionModelOverride, bool, error) { + raw, err := readCompactionModelOverride(ctx, p.db) + if err != nil { + return compactionModelOverride{}, false, xerrors.Errorf( + "read compaction model override: %w", + err, + ) + } + + modelConfig, overrideSet, err := p.resolveConfiguredModelOverride( + ctx, + compactionOverrideContext, + raw, + chat.OwnerID, + p.resolveModelConfigAndNormalizedProvider, + func(ctx context.Context, ownerID uuid.UUID, aiProviderID uuid.UUID) (chatprovider.ProviderAPIKeys, error) { + return p.resolveUserProviderAPIKeys(ctx, ownerID, aiProviderID) + }, + modelOverrideFailureModeSoft, + ) + if err != nil { + return compactionModelOverride{}, false, err + } + if !overrideSet { + return compactionModelOverride{}, false, nil + } + + //nolint:gocritic // Compaction overrides need chatd-scoped provider reads for user-owned chats. + route, err := p.resolveModelRouteForConfig(dbauthz.AsChatd(ctx), chat.OwnerID, modelConfig) + if err != nil { + return compactionModelOverride{}, true, xerrors.Errorf( + "resolve compaction model override route: %w", + err, + ) + } + resolvedProvider, resolvedModel, err := chatprovider.ResolveModelWithProviderHint( + modelConfig.Model, + route.ModelProviderHint, + ) + if err != nil { + return compactionModelOverride{}, true, xerrors.Errorf( + "resolve compaction model override metadata: %w", + err, + ) + } + model, _, err := p.newDebugAwareModel(ctx, modelClientRequest{ + Chat: chat, + ModelName: modelConfig.Model, + UserAgent: chatprovider.UserAgent(), + ExtraHeaders: chatprovider.CoderHeaders(chat), + }, route, modelOpts) + if err != nil { + return compactionModelOverride{}, true, xerrors.Errorf( + "create compaction model override: %w", + err, + ) + } + return compactionModelOverride{ + modelConfig: modelConfig, + model: model, + resolvedProvider: resolvedProvider, + resolvedModel: resolvedModel, + }, true, nil +} diff --git a/coderd/x/chatd/compaction_override_internal_test.go b/coderd/x/chatd/compaction_override_internal_test.go new file mode 100644 index 00000000000..e6ddea32498 --- /dev/null +++ b/coderd/x/chatd/compaction_override_internal_test.go @@ -0,0 +1,167 @@ +package chatd + +import ( + "database/sql" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "cdr.dev/slog/v3/sloggers/slogtest" + "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/database/dbmock" + "github.com/coder/coder/v2/testutil" +) + +func TestResolveCompactionModelOverride_Unset(t *testing.T) { + t.Parallel() + + ctx := testutil.Context(t, testutil.WaitShort) + ctrl := gomock.NewController(t) + db := dbmock.NewMockStore(ctrl) + logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}) + chat, _ := titleOverrideTestChatAndMessages(t) + + db.EXPECT().GetChatCompactionModelOverride(gomock.Any()).Return("", nil) + + server := titleOverrideTestServer(db, logger) + override, overrideSet, err := server.resolveCompactionModelOverride(ctx, chat, modelBuildOptions{}) + require.NoError(t, err) + require.False(t, overrideSet) + require.Nil(t, override.model) +} + +func TestResolveCompactionModelOverride_ReadDBError(t *testing.T) { + t.Parallel() + + ctx := testutil.Context(t, testutil.WaitShort) + ctrl := gomock.NewController(t) + db := dbmock.NewMockStore(ctrl) + logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}) + chat, _ := titleOverrideTestChatAndMessages(t) + + db.EXPECT().GetChatCompactionModelOverride(gomock.Any()).Return("", sql.ErrConnDone) + + server := titleOverrideTestServer(db, logger) + _, overrideSet, err := server.resolveCompactionModelOverride(ctx, chat, modelBuildOptions{}) + require.Error(t, err) + require.ErrorContains(t, err, "read compaction model override") + require.False(t, overrideSet) +} + +func TestResolveCompactionModelOverride_MalformedFallsBack(t *testing.T) { + t.Parallel() + + ctx := testutil.Context(t, testutil.WaitShort) + ctrl := gomock.NewController(t) + db := dbmock.NewMockStore(ctrl) + logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}) + chat, _ := titleOverrideTestChatAndMessages(t) + + db.EXPECT().GetChatCompactionModelOverride(gomock.Any()).Return("not-a-uuid", nil) + + server := titleOverrideTestServer(db, logger) + _, overrideSet, err := server.resolveCompactionModelOverride(ctx, chat, modelBuildOptions{}) + require.NoError(t, err) + require.False(t, overrideSet) +} + +func TestResolveCompactionModelOverride_DeletedConfigFallsBack(t *testing.T) { + t.Parallel() + + ctx := testutil.Context(t, testutil.WaitShort) + ctrl := gomock.NewController(t) + db := dbmock.NewMockStore(ctrl) + logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}) + chat, _ := titleOverrideTestChatAndMessages(t) + missingID := uuid.New() + + db.EXPECT().GetChatCompactionModelOverride(gomock.Any()).Return(missingID.String(), nil) + db.EXPECT().GetChatModelConfigByID(gomock.Any(), missingID).Return(database.ChatModelConfig{}, sql.ErrNoRows) + + server := titleOverrideTestServer(db, logger) + _, overrideSet, err := server.resolveCompactionModelOverride(ctx, chat, modelBuildOptions{}) + require.NoError(t, err) + require.False(t, overrideSet) +} + +func TestResolveCompactionModelOverride_DisabledConfigFallsBack(t *testing.T) { + t.Parallel() + + ctx := testutil.Context(t, testutil.WaitShort) + ctrl := gomock.NewController(t) + db := dbmock.NewMockStore(ctrl) + logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}) + chat, _ := titleOverrideTestChatAndMessages(t) + overrideConfig := titleOverrideModelConfig("gpt-4.1", false) + + db.EXPECT().GetChatCompactionModelOverride(gomock.Any()).Return(overrideConfig.ID.String(), nil) + db.EXPECT().GetChatModelConfigByID(gomock.Any(), overrideConfig.ID).Return(overrideConfig, nil) + + server := titleOverrideTestServer(db, logger) + _, overrideSet, err := server.resolveCompactionModelOverride(ctx, chat, modelBuildOptions{}) + require.NoError(t, err) + require.False(t, overrideSet) +} + +func TestResolveCompactionModelOverride_MissingCredentialsFallsBack(t *testing.T) { + t.Parallel() + + ctx := testutil.Context(t, testutil.WaitShort) + ctrl := gomock.NewController(t) + db := dbmock.NewMockStore(ctrl) + logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}) + chat, _ := titleOverrideTestChatAndMessages(t) + overrideConfig := titleOverrideModelConfig("gpt-4.1", true) + providerID := uuid.New() + overrideConfig.AIProviderID = uuid.NullUUID{UUID: providerID, Valid: true} + + db.EXPECT().GetChatCompactionModelOverride(gomock.Any()).Return(overrideConfig.ID.String(), nil) + db.EXPECT().GetChatModelConfigByID(gomock.Any(), overrideConfig.ID).Return(overrideConfig, nil) + db.EXPECT().GetAIProviderByID(gomock.Any(), providerID).Return(database.AIProvider{ + ID: providerID, + Type: database.AIProviderTypeOpenai, + Enabled: true, + }, nil).AnyTimes() + db.EXPECT().GetAIProviderKeysByProviderID(gomock.Any(), providerID).Return(nil, nil).AnyTimes() + + server := titleOverrideTestServer(db, logger) + _, overrideSet, err := server.resolveCompactionModelOverride(ctx, chat, modelBuildOptions{}) + require.NoError(t, err) + require.False(t, overrideSet) +} + +func TestResolveCompactionModelOverride_SetUsable(t *testing.T) { + t.Parallel() + + ctx := testutil.Context(t, testutil.WaitShort) + ctrl := gomock.NewController(t) + db := dbmock.NewMockStore(ctrl) + logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}) + chat, _ := titleOverrideTestChatAndMessages(t) + overrideConfig := titleOverrideModelConfig("gpt-4.1", true) + providerID := uuid.New() + overrideConfig.AIProviderID = uuid.NullUUID{UUID: providerID, Valid: true} + + db.EXPECT().GetChatCompactionModelOverride(gomock.Any()).Return(overrideConfig.ID.String(), nil) + db.EXPECT().GetChatModelConfigByID(gomock.Any(), overrideConfig.ID).Return(overrideConfig, nil) + db.EXPECT().GetAIProviderByID(gomock.Any(), providerID).Return(aibridgeTestAIProvider(providerID, "primary-openai", database.AIProviderTypeOpenai), nil).AnyTimes() + db.EXPECT().GetAIProviderKeysByProviderID(gomock.Any(), providerID).Return([]database.AIProviderKey{{ + ProviderID: providerID, + APIKey: "test-key", + }}, nil).AnyTimes() + + server := titleOverrideTestServer(db, logger) + override, overrideSet, err := server.resolveCompactionModelOverride( + ctx, + chat, + modelBuildOptions{ActiveAPIKeyID: uuid.NewString()}, + ) + require.NoError(t, err) + require.True(t, overrideSet) + require.NotNil(t, override.model) + require.Equal(t, overrideConfig.ID, override.modelConfig.ID) + require.Equal(t, "openai", override.resolvedProvider) + require.Equal(t, "gpt-4.1", override.resolvedModel) +} diff --git a/coderd/x/chatd/compaction_prompt.go b/coderd/x/chatd/compaction_prompt.go new file mode 100644 index 00000000000..66a27014a4c --- /dev/null +++ b/coderd/x/chatd/compaction_prompt.go @@ -0,0 +1,145 @@ +package chatd + +import ( + "context" + "fmt" + + "charm.land/fantasy" + + "cdr.dev/slog/v3" + "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/x/chatd/chatprovider" + "github.com/coder/coder/v2/coderd/x/chatd/chatsanitize" +) + +// sameCompactionProviderIdentity reports whether the chat model and the +// compaction override model are backed by the same provider instance. +// Legacy configs without an AIProviderID compare as different (fail +// closed), so cross-provider stripping applies. +func sameCompactionProviderIdentity(chatConfig, overrideConfig database.ChatModelConfig) bool { + return chatConfig.AIProviderID.Valid && overrideConfig.AIProviderID.Valid && + chatConfig.AIProviderID.UUID == overrideConfig.AIProviderID.UUID +} + +// sanitizeCompactionPrompt adapts the chat prompt for a compaction model +// that differs from the chat model. The prompt was built for the chat +// model, so provider-executed tool history and file parts the compaction +// provider rejects must not replay to it. The input messages are never +// mutated; the assistant generation keeps using the original prompt. +func sanitizeCompactionPrompt( + ctx context.Context, + logger slog.Logger, + prompt []fantasy.Message, + compactionModel fantasy.LanguageModel, + chatConfig database.ChatModelConfig, + overrideConfig database.ChatModelConfig, +) []fantasy.Message { + messages := prompt + if !sameCompactionProviderIdentity(chatConfig, overrideConfig) { + messages = stripProviderExecutedToolParts(ctx, logger, messages) + } + messages = replaceUnsupportedFileParts(ctx, logger, messages, func(mediaType string) bool { + return chatprovider.AcceptsFilePartMediaType( + compactionModel.Provider(), + compactionModel.Model(), + mediaType, + ) + }) + sanitized, stats := chatsanitize.SanitizeAnthropicProviderToolHistory( + compactionModel.Provider(), + messages, + ) + chatsanitize.LogAnthropicProviderToolSanitization( + ctx, + logger, + "compaction_prompt", + compactionModel.Provider(), + compactionModel.Model(), + stats, + ) + return sanitized +} + +// stripProviderExecutedToolParts removes provider-executed tool calls and +// results from a copy of messages. Provider-executed blocks produced by +// one provider can be rejected when replayed to another, and compaction +// only needs the conversation text. Messages emptied by stripping are +// dropped. The input is never mutated. +func stripProviderExecutedToolParts( + ctx context.Context, + logger slog.Logger, + messages []fantasy.Message, +) []fantasy.Message { + removed := 0 + out := make([]fantasy.Message, 0, len(messages)) + for _, msg := range messages { + parts := make([]fantasy.MessagePart, 0, len(msg.Content)) + for _, part := range msg.Content { + switch typed := part.(type) { + case fantasy.ToolCallPart: + if typed.ProviderExecuted { + removed++ + continue + } + case fantasy.ToolResultPart: + if typed.ProviderExecuted { + removed++ + continue + } + } + parts = append(parts, part) + } + if len(parts) == 0 && len(msg.Content) > 0 { + continue + } + msg.Content = parts + out = append(out, msg) + } + if removed > 0 { + logger.Debug(ctx, "stripped provider-executed tool history from compaction prompt", + slog.F("removed_parts", removed), + ) + } + return out +} + +// replaceUnsupportedFileParts swaps file parts the compaction model does +// not accept for short text placeholders in a copy of messages, so the +// summary notes the attachment existed instead of silently losing it. +// The input is never mutated. +func replaceUnsupportedFileParts( + ctx context.Context, + logger slog.Logger, + messages []fantasy.Message, + acceptsFilePart func(mediaType string) bool, +) []fantasy.Message { + replaced := 0 + out := make([]fantasy.Message, 0, len(messages)) + for _, msg := range messages { + parts := make([]fantasy.MessagePart, 0, len(msg.Content)) + for _, part := range msg.Content { + filePart, ok := part.(fantasy.FilePart) + if !ok || acceptsFilePart(filePart.MediaType) { + parts = append(parts, part) + continue + } + replaced++ + parts = append(parts, fantasy.TextPart{ + Text: fmt.Sprintf( + "[Attachment %q (%s) omitted: not supported by the compaction model]", + filePart.Filename, + filePart.MediaType, + ), + ProviderOptions: filePart.ProviderOptions, + }) + } + msg.Content = parts + out = append(out, msg) + } + if replaced > 0 { + logger.Debug(ctx, "replaced unsupported file parts in compaction prompt", + slog.F("replaced_parts", replaced), + ) + } + return out +} diff --git a/coderd/x/chatd/compaction_prompt_internal_test.go b/coderd/x/chatd/compaction_prompt_internal_test.go new file mode 100644 index 00000000000..8c01640c0f9 --- /dev/null +++ b/coderd/x/chatd/compaction_prompt_internal_test.go @@ -0,0 +1,185 @@ +package chatd + +import ( + "testing" + + "charm.land/fantasy" + "github.com/google/uuid" + "github.com/stretchr/testify/require" + + "cdr.dev/slog/v3/sloggers/slogtest" + "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/x/chatd/chattest" + "github.com/coder/coder/v2/testutil" +) + +func TestSameCompactionProviderIdentity(t *testing.T) { + t.Parallel() + + providerID := uuid.New() + + require.True(t, sameCompactionProviderIdentity(configWithProvider(providerID), configWithProvider(providerID))) + require.False(t, sameCompactionProviderIdentity(configWithProvider(providerID), configWithProvider(uuid.New()))) + // Legacy configs without a provider FK compare as different (fail closed). + require.False(t, sameCompactionProviderIdentity(database.ChatModelConfig{}, configWithProvider(providerID))) + require.False(t, sameCompactionProviderIdentity(database.ChatModelConfig{}, database.ChatModelConfig{})) +} + +func configWithProvider(id uuid.UUID) database.ChatModelConfig { + return database.ChatModelConfig{AIProviderID: uuid.NullUUID{UUID: id, Valid: true}} +} + +func TestSanitizeCompactionPrompt_StripsForeignProviderExecutedToolParts(t *testing.T) { + t.Parallel() + + ctx := testutil.Context(t, testutil.WaitShort) + logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}) + prompt := []fantasy.Message{ + { + Role: fantasy.MessageRoleUser, + Content: []fantasy.MessagePart{ + fantasy.TextPart{Text: "search the web"}, + }, + }, + { + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + fantasy.TextPart{Text: "searching"}, + fantasy.ToolCallPart{ + ToolCallID: "ws-1", + ToolName: "web_search", + Input: `{"query":"coder"}`, + ProviderExecuted: true, + }, + fantasy.ToolResultPart{ + ToolCallID: "ws-1", + Output: fantasy.ToolResultOutputContentText{Text: "results"}, + ProviderExecuted: true, + }, + }, + }, + { + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + fantasy.ToolCallPart{ + ToolCallID: "local-1", + ToolName: "read_file", + Input: `{"path":"/tmp/a.txt"}`, + }, + }, + }, + } + + compactionModel := &chattest.FakeModel{ProviderName: "openai", ModelName: "gpt-4.1-mini"} + sanitized := sanitizeCompactionPrompt(ctx, logger, prompt, compactionModel, configWithProvider(uuid.New()), configWithProvider(uuid.New())) + + require.Len(t, sanitized, 3) + require.Len(t, sanitized[1].Content, 1) + require.Equal(t, fantasy.TextPart{Text: "searching"}, sanitized[1].Content[0]) + // Local tool calls replay fine across providers and must survive. + require.Len(t, sanitized[2].Content, 1) + + // The original prompt used for assistant generation is untouched. + require.Len(t, prompt[1].Content, 3) +} + +func TestSanitizeCompactionPrompt_DropsEmptiedMessages(t *testing.T) { + t.Parallel() + + ctx := testutil.Context(t, testutil.WaitShort) + logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}) + prompt := []fantasy.Message{ + { + Role: fantasy.MessageRoleTool, + Content: []fantasy.MessagePart{ + fantasy.ToolResultPart{ + ToolCallID: "ws-1", + Output: fantasy.ToolResultOutputContentText{Text: "results"}, + ProviderExecuted: true, + }, + }, + }, + { + Role: fantasy.MessageRoleUser, + Content: []fantasy.MessagePart{ + fantasy.TextPart{Text: "hello"}, + }, + }, + } + + compactionModel := &chattest.FakeModel{ProviderName: "openai", ModelName: "gpt-4.1-mini"} + sanitized := sanitizeCompactionPrompt(ctx, logger, prompt, compactionModel, configWithProvider(uuid.New()), configWithProvider(uuid.New())) + + require.Len(t, sanitized, 1) + require.Equal(t, fantasy.MessageRoleUser, sanitized[0].Role) + require.Len(t, prompt, 2) +} + +func TestSanitizeCompactionPrompt_ReplacesUnsupportedFileParts(t *testing.T) { + t.Parallel() + + ctx := testutil.Context(t, testutil.WaitShort) + logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}) + prompt := []fantasy.Message{ + { + Role: fantasy.MessageRoleUser, + Content: []fantasy.MessagePart{ + fantasy.TextPart{Text: "look at this"}, + fantasy.FilePart{ + Filename: "diagram.pdf", + Data: []byte("%PDF-"), + MediaType: "application/pdf", + }, + }, + }, + } + + // Mistral accepts images but not PDFs, so the PDF part must become a + // placeholder while the prompt stays otherwise intact. + compactionModel := &chattest.FakeModel{ProviderName: "mistral", ModelName: "mistral-large"} + sharedProviderID := uuid.New() + sanitized := sanitizeCompactionPrompt(ctx, logger, prompt, compactionModel, configWithProvider(sharedProviderID), configWithProvider(sharedProviderID)) + + require.Len(t, sanitized, 1) + require.Len(t, sanitized[0].Content, 2) + textPart, ok := sanitized[0].Content[1].(fantasy.TextPart) + require.True(t, ok) + require.Contains(t, textPart.Text, "diagram.pdf") + require.Contains(t, textPart.Text, "not supported by the compaction model") + + // The original prompt keeps its file part. + _, ok = prompt[0].Content[1].(fantasy.FilePart) + require.True(t, ok) +} + +func TestSanitizeCompactionPrompt_SameProviderKeepsProviderExecutedParts(t *testing.T) { + t.Parallel() + + ctx := testutil.Context(t, testutil.WaitShort) + logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}) + prompt := []fantasy.Message{ + { + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + fantasy.ToolCallPart{ + ToolCallID: "ws-1", + ToolName: "web_search", + Input: `{"query":"coder"}`, + ProviderExecuted: true, + }, + fantasy.ToolResultPart{ + ToolCallID: "ws-1", + Output: fantasy.ToolResultOutputContentText{Text: "results"}, + ProviderExecuted: true, + }, + }, + }, + } + + compactionModel := &chattest.FakeModel{ProviderName: "openai", ModelName: "gpt-4.1-mini"} + sharedProviderID := uuid.New() + sanitized := sanitizeCompactionPrompt(ctx, logger, prompt, compactionModel, configWithProvider(sharedProviderID), configWithProvider(sharedProviderID)) + + require.Len(t, sanitized, 1) + require.Len(t, sanitized[0].Content, 2) +} diff --git a/coderd/x/chatd/generation.go b/coderd/x/chatd/generation.go index ac3bcd9a128..06e8bf645fe 100644 --- a/coderd/x/chatd/generation.go +++ b/coderd/x/chatd/generation.go @@ -684,7 +684,11 @@ func (s *taskStarter) generateCompaction( } compactionOpts := prepared.Compaction.Options compactionOpts.PublishMessagePart = attempt.publish - outcome, err := chatloop.GenerateCompaction(ctx, compactionOpts) + // Attach the turn debug run so the compaction call records a child + // debug run; without it startCompactionDebugRun finds no parent and + // skips debug instrumentation entirely. + runCtx := input.DebugTurn.Ensure(ctx, prepared.Chat, prepared.Debug) + outcome, err := chatloop.GenerateCompaction(runCtx, compactionOpts) if err != nil { s.server.metrics.RecordCompaction(compactionProvider(compactionOpts), compactionModel(compactionOpts), false, err) return xerrors.Errorf("generate compaction: %w", err) diff --git a/coderd/x/chatd/generation_preparer.go b/coderd/x/chatd/generation_preparer.go index 0a80f2de16a..ef33083e06e 100644 --- a/coderd/x/chatd/generation_preparer.go +++ b/coderd/x/chatd/generation_preparer.go @@ -571,20 +571,57 @@ func (server *Server) prepareGeneration( if override, ok := server.resolveUserCompactionThreshold(ctx, chat.OwnerID, modelConfig.ID); ok { effectiveThreshold = override } + compactionModel := model + compactionPrompt := prompt + compactionResolvedProvider := resolvedProvider + compactionResolvedModel := debugModel + compactionModelConfigID := modelConfig.ID + // The effective compaction limit is the stricter of the chat model's + // and the compaction model's context limits: the history must also fit + // the summarizer's window. Post-compaction continuation checks keep + // using the chat model's limit (ContextLimitFallback below), since the + // follow-up assistant generation runs on the chat model. + compactionContextLimit := modelConfig.ContextLimit + compactionOverride, compactionOverrideSet, err := server.resolveCompactionModelOverride(ctx, chat, modelOpts) + if err != nil { + cleanup() + return generationPrepared{}, err + } + if compactionOverrideSet { + compactionModel = compactionOverride.model + compactionResolvedProvider = compactionOverride.resolvedProvider + compactionResolvedModel = compactionOverride.resolvedModel + compactionModelConfigID = compactionOverride.modelConfig.ID + if overrideLimit := compactionOverride.modelConfig.ContextLimit; overrideLimit > 0 && + (compactionContextLimit <= 0 || overrideLimit < compactionContextLimit) { + compactionContextLimit = overrideLimit + } + compactionPrompt = sanitizeCompactionPrompt( + ctx, + logger, + prompt, + compactionOverride.model, + modelConfig, + compactionOverride.modelConfig, + ) + } compactionOptions := chatloop.GenerateCompactionOptions{ - Model: model, - Messages: prompt, + Model: compactionModel, + Messages: compactionPrompt, ThresholdPercent: effectiveThreshold, - ContextLimit: modelConfig.ContextLimit, - ContextLimitFallback: modelConfig.ContextLimit, + ContextLimit: compactionContextLimit, + ContextLimitFallback: compactionContextLimit, ToolCallID: compactionToolCallID, ToolName: "chat_summarized", DebugSvc: debugSvc, ChatID: chat.ID, HistoryTipMessageID: historyTipMessageID, + ResolvedProvider: compactionResolvedProvider, + ResolvedModel: compactionResolvedModel, + ModelConfigID: compactionModelConfigID, } compactionOptions.StepUsage = latestPromptUsage(promptRows) - compactionNeeded := shouldCompactPromptUsage(compactionOptions.StepUsage, modelConfig.ContextLimit, effectiveThreshold) + compactionNeeded := shouldCompactPromptUsage(compactionOptions.StepUsage, compactionContextLimit, effectiveThreshold) // workspaceCtx.currentChatSnapshot may carry a freshly persisted // AgentID/BuildID binding from the getWorkspaceAgent call above. diff --git a/codersdk/chats.go b/codersdk/chats.go index d9330a2f26d..d07c60b0241 100644 --- a/codersdk/chats.go +++ b/codersdk/chats.go @@ -749,6 +749,7 @@ const ( ChatModelOverrideContextGeneral ChatModelOverrideContext = "general" ChatModelOverrideContextExplore ChatModelOverrideContext = "explore" ChatModelOverrideContextTitleGeneration ChatModelOverrideContext = "title_generation" + ChatModelOverrideContextCompaction ChatModelOverrideContext = "compaction" ) // Valid reports whether the override context is one of the supported values. @@ -756,7 +757,8 @@ func (c ChatModelOverrideContext) Valid() bool { switch c { case ChatModelOverrideContextGeneral, ChatModelOverrideContextExplore, - ChatModelOverrideContextTitleGeneration: + ChatModelOverrideContextTitleGeneration, + ChatModelOverrideContextCompaction: return true default: return false @@ -769,6 +771,7 @@ func AllChatModelOverrideContexts() []ChatModelOverrideContext { ChatModelOverrideContextGeneral, ChatModelOverrideContextExplore, ChatModelOverrideContextTitleGeneration, + ChatModelOverrideContextCompaction, } } diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts index ee8d0d44b46..ad143af02f7 100644 --- a/site/src/api/typesGenerated.ts +++ b/site/src/api/typesGenerated.ts @@ -2632,11 +2632,13 @@ export interface ChatModelOpenRouterProviderOptions { // From codersdk/chats.go export type ChatModelOverrideContext = + | "compaction" | "explore" | "general" | "title_generation"; export const ChatModelOverrideContexts: ChatModelOverrideContext[] = [ + "compaction", "explore", "general", "title_generation", diff --git a/site/src/pages/AISettingsPage/CoderAgentsPage/CoderAgentsPage.tsx b/site/src/pages/AISettingsPage/CoderAgentsPage/CoderAgentsPage.tsx index 4d549028c54..d794dd74f3b 100644 --- a/site/src/pages/AISettingsPage/CoderAgentsPage/CoderAgentsPage.tsx +++ b/site/src/pages/AISettingsPage/CoderAgentsPage/CoderAgentsPage.tsx @@ -28,6 +28,8 @@ const generalOverrideContext: TypesGen.ChatModelOverrideContext = "general"; const exploreOverrideContext: TypesGen.ChatModelOverrideContext = "explore"; const titleGenerationOverrideContext: TypesGen.ChatModelOverrideContext = "title_generation"; +const compactionOverrideContext: TypesGen.ChatModelOverrideContext = + "compaction"; const chatModelOverrideKey = (context: TypesGen.ChatModelOverrideContext) => ["chat-model-override", context] as const; @@ -79,6 +81,10 @@ const CoderAgentsPage: FC = () => { ...chatModelOverrideQuery(titleGenerationOverrideContext), enabled: canEditDeploymentConfig, }); + const compactionModelQuery = useQuery({ + ...chatModelOverrideQuery(compactionOverrideContext), + enabled: canEditDeploymentConfig, + }); const modelConfigsQuery = useQuery(chatModelConfigs()); const advisorConfigQuery = useQuery({ ...chatAdvisorConfig(), @@ -104,6 +110,9 @@ const CoderAgentsPage: FC = () => { titleGenerationOverrideContext, ), ); + const saveCompactionModelMutation = useMutation( + updateChatModelOverrideMutation(queryClient, compactionOverrideContext), + ); const saveExploreModelOverrideMutation = useMutation( updateChatModelOverrideMutation(queryClient, exploreOverrideContext), ); @@ -141,6 +150,7 @@ const CoderAgentsPage: FC = () => { } generalModelOverrideData={generalModelOverrideQuery.data} titleGenerationModelOverrideData={titleGenerationModelQuery.data} + compactionModelOverrideData={compactionModelQuery.data} exploreModelOverrideData={exploreModelOverrideQuery.data} modelConfigsData={modelConfigsQuery.data} providerInfoByID={providerInfoByID} @@ -161,6 +171,9 @@ const CoderAgentsPage: FC = () => { isSaveTitleGenerationModelError={ saveTitleGenerationModelMutation.isError } + onSaveCompactionModel={saveCompactionModelMutation.mutate} + isSavingCompactionModel={saveCompactionModelMutation.isPending} + isSaveCompactionModelError={saveCompactionModelMutation.isError} onSaveExploreModelOverride={saveExploreModelOverrideMutation.mutate} isSavingExploreModelOverride={ saveExploreModelOverrideMutation.isPending diff --git a/site/src/pages/AISettingsPage/CoderAgentsPage/CoderAgentsPageView.stories.tsx b/site/src/pages/AISettingsPage/CoderAgentsPage/CoderAgentsPageView.stories.tsx index d004de666a8..63976e9b599 100644 --- a/site/src/pages/AISettingsPage/CoderAgentsPage/CoderAgentsPageView.stories.tsx +++ b/site/src/pages/AISettingsPage/CoderAgentsPage/CoderAgentsPageView.stories.tsx @@ -94,6 +94,14 @@ const exploreDisabledModelConfig = buildModelConfig({ context_limit: 200_000, }); +const compactionDisabledModelConfig = buildModelConfig({ + id: "model-compaction-disabled", + model: "gpt-4.1-nano-legacy", + display_name: "GPT 4.1 Nano Legacy", + enabled: false, + context_limit: 128_000, +}); + const allModelConfigs: TypesGen.ChatModelConfig[] = [ generalModelConfig, claudeSonnetModelConfig, @@ -102,6 +110,7 @@ const allModelConfigs: TypesGen.ChatModelConfig[] = [ generalDisabledModelConfig, titleDisabledModelConfig, exploreDisabledModelConfig, + compactionDisabledModelConfig, ]; const providerInfoByID = new Map([ @@ -124,6 +133,7 @@ const buildArgs = ( isSaveAdminOverridesError: false, generalModelOverrideData: buildOverrideData("general"), titleGenerationModelOverrideData: buildTitleGenerationModelOverrideData(), + compactionModelOverrideData: buildOverrideData("compaction"), exploreModelOverrideData: buildOverrideData("explore"), modelConfigsData: allModelConfigs, providerInfoByID, @@ -136,6 +146,9 @@ const buildArgs = ( onSaveTitleGenerationModel: fn(), isSavingTitleGenerationModel: false, isSaveTitleGenerationModelError: false, + onSaveCompactionModel: fn(), + isSavingCompactionModel: false, + isSaveCompactionModelError: false, onSaveExploreModelOverride: fn(), isSavingExploreModelOverride: false, isSaveExploreModelOverrideError: false, @@ -211,6 +224,7 @@ export const AllOverridesUnset: Story = { expect(headings.map((heading) => heading.textContent?.trim())).toEqual([ "General model", "Title generation model", + "Compaction model", "Explore subagent model", ]); await canvas.findByText( @@ -223,6 +237,10 @@ export const AllOverridesUnset: Story = { headingName: "Title generation model", placeholder: "Use title default", }, + { + headingName: "Compaction model", + placeholder: "Use chat model", + }, { headingName: "Explore subagent model", placeholder: "Use chat default", @@ -293,6 +311,9 @@ export const EachOverrideSetToEnabledModel: Story = { titleGenerationModelOverrideData: buildTitleGenerationModelOverrideData({ model_config_id: titleModelConfig.id, }), + compactionModelOverrideData: buildOverrideData("compaction", { + model_config_id: claudeSonnetModelConfig.id, + }), exploreModelOverrideData: buildOverrideData("explore", { model_config_id: exploreFallbackModelConfig.id, }), @@ -303,6 +324,10 @@ export const EachOverrideSetToEnabledModel: Story = { canvasElement, "Title generation model", ); + const compactionSection = await getSection( + canvasElement, + "Compaction model", + ); const exploreSection = await getSection( canvasElement, "Explore subagent model", @@ -360,6 +385,26 @@ export const EachOverrideSetToEnabledModel: Story = { ); }); + await selectModelInSection( + compactionSection, + canvasElement, + /claude sonnet 4/i, + "GPT 4o Mini", + ); + const compactionSaveButton = within(compactionSection).getByRole("button", { + name: "Save", + }); + await waitFor(() => { + expect(compactionSaveButton).toBeEnabled(); + }); + await userEvent.click(compactionSaveButton); + await waitFor(() => { + expect(args.onSaveCompactionModel).toHaveBeenCalledWith( + { model_config_id: titleModelConfig.id }, + expect.anything(), + ); + }); + const exploreClearButton = within(exploreSection).getByRole("button", { name: "Clear", }); @@ -388,6 +433,9 @@ export const MalformedOverridesRemainClearableAndSaveable: Story = { titleGenerationModelOverrideData: buildTitleGenerationModelOverrideData({ is_malformed: true, }), + compactionModelOverrideData: buildOverrideData("compaction", { + is_malformed: true, + }), exploreModelOverrideData: buildOverrideData("explore", { is_malformed: true, }), @@ -398,12 +446,21 @@ export const MalformedOverridesRemainClearableAndSaveable: Story = { canvasElement, "Title generation model", ); + const compactionSection = await getSection( + canvasElement, + "Compaction model", + ); const exploreSection = await getSection( canvasElement, "Explore subagent model", ); - for (const section of [generalSection, titleSection, exploreSection]) { + for (const section of [ + generalSection, + titleSection, + compactionSection, + exploreSection, + ]) { await within(section).findByText(OVERRIDE_MALFORMED_WARNING); } @@ -448,6 +505,20 @@ export const MalformedOverridesRemainClearableAndSaveable: Story = { expect.anything(), ); }); + + const compactionSaveButton = within(compactionSection).getByRole("button", { + name: "Save", + }); + await waitFor(() => { + expect(compactionSaveButton).toBeEnabled(); + }); + await userEvent.click(compactionSaveButton); + await waitFor(() => { + expect(args.onSaveCompactionModel).toHaveBeenCalledWith( + { model_config_id: "" }, + expect.anything(), + ); + }); }, }; @@ -459,6 +530,9 @@ export const UnavailableSavedModels: Story = { titleGenerationModelOverrideData: buildTitleGenerationModelOverrideData({ model_config_id: titleDisabledModelConfig.id, }), + compactionModelOverrideData: buildOverrideData("compaction", { + model_config_id: compactionDisabledModelConfig.id, + }), exploreModelOverrideData: buildOverrideData("explore", { model_config_id: exploreDisabledModelConfig.id, }), @@ -469,12 +543,16 @@ export const UnavailableSavedModels: Story = { canvasElement, "Title generation model", ); + const compactionSection = await getSection( + canvasElement, + "Compaction model", + ); const exploreSection = await getSection( canvasElement, "Explore subagent model", ); - for (const section of [generalSection, exploreSection]) { + for (const section of [generalSection, compactionSection, exploreSection]) { await within(section).findByText(UNAVAILABLE_SAVED_MODEL_WARNING); expect( within(section).getByRole("combobox", { name: "Unavailable model" }), diff --git a/site/src/pages/AISettingsPage/CoderAgentsPage/CoderAgentsPageView.tsx b/site/src/pages/AISettingsPage/CoderAgentsPage/CoderAgentsPageView.tsx index 656d3498578..b8fccf0fe46 100644 --- a/site/src/pages/AISettingsPage/CoderAgentsPage/CoderAgentsPageView.tsx +++ b/site/src/pages/AISettingsPage/CoderAgentsPage/CoderAgentsPageView.tsx @@ -33,6 +33,7 @@ export interface CoderAgentsPageViewProps { isSaveAdminOverridesError: boolean; generalModelOverrideData?: TypesGen.ChatModelOverrideResponse; titleGenerationModelOverrideData?: TypesGen.ChatModelOverrideResponse; + compactionModelOverrideData?: TypesGen.ChatModelOverrideResponse; exploreModelOverrideData?: TypesGen.ChatModelOverrideResponse; modelConfigsData: TypesGen.ChatModelConfig[] | undefined; providerInfoByID: ReadonlyMap; @@ -45,6 +46,9 @@ export interface CoderAgentsPageViewProps { onSaveTitleGenerationModel: SaveModelOverride; isSavingTitleGenerationModel: boolean; isSaveTitleGenerationModelError: boolean; + onSaveCompactionModel: SaveModelOverride; + isSavingCompactionModel: boolean; + isSaveCompactionModelError: boolean; onSaveExploreModelOverride: SaveModelOverride; isSavingExploreModelOverride: boolean; isSaveExploreModelOverrideError: boolean; @@ -83,6 +87,7 @@ export const CoderAgentsPageView: FC = ({ isSaveAdminOverridesError, generalModelOverrideData, titleGenerationModelOverrideData, + compactionModelOverrideData, exploreModelOverrideData, modelConfigsData, providerInfoByID, @@ -95,6 +100,9 @@ export const CoderAgentsPageView: FC = ({ onSaveTitleGenerationModel, isSavingTitleGenerationModel, isSaveTitleGenerationModelError, + onSaveCompactionModel, + isSavingCompactionModel, + isSaveCompactionModelError, onSaveExploreModelOverride, isSavingExploreModelOverride, isSaveExploreModelOverrideError, @@ -172,6 +180,20 @@ export const CoderAgentsPageView: FC = ({ unsetPlaceholder="Use title default" unavailableModelWarning="The selected model is currently unavailable. Title generation will be skipped until you choose another model or clear this setting." /> + ( onSaveTitleGenerationModel={fn()} isSavingTitleGenerationModel={false} isSaveTitleGenerationModelError={false} + onSaveCompactionModel={fn()} + isSavingCompactionModel={false} + isSaveCompactionModelError={false} onSaveExploreModelOverride={fn()} isSavingExploreModelOverride={false} isSaveExploreModelOverrideError={false} From f09d3957820c765374241c07f6e0b485fe101012 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 9 Jul 2026 17:53:45 +0000 Subject: [PATCH 02/12] refactor(coderd/x/chatd): sanitize compaction prompt only when compaction runs --- coderd/x/chatd/generation_preparer.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/coderd/x/chatd/generation_preparer.go b/coderd/x/chatd/generation_preparer.go index ef33083e06e..b83b599150d 100644 --- a/coderd/x/chatd/generation_preparer.go +++ b/coderd/x/chatd/generation_preparer.go @@ -596,6 +596,14 @@ func (server *Server) prepareGeneration( (compactionContextLimit <= 0 || overrideLimit < compactionContextLimit) { compactionContextLimit = overrideLimit } + } + compactionStepUsage := latestPromptUsage(promptRows) + compactionNeeded := shouldCompactPromptUsage(compactionStepUsage, compactionContextLimit, effectiveThreshold) + // Sanitizing rewrites the full prompt, so skip it unless this turn + // actually compacts. The sanitized prompt is only consumed by the + // compaction generation, which only runs when compactionNeeded is + // true at prepare time. + if compactionOverrideSet && compactionNeeded { compactionPrompt = sanitizeCompactionPrompt( ctx, logger, @@ -619,9 +627,8 @@ func (server *Server) prepareGeneration( ResolvedProvider: compactionResolvedProvider, ResolvedModel: compactionResolvedModel, ModelConfigID: compactionModelConfigID, + StepUsage: compactionStepUsage, } - compactionOptions.StepUsage = latestPromptUsage(promptRows) - compactionNeeded := shouldCompactPromptUsage(compactionOptions.StepUsage, compactionContextLimit, effectiveThreshold) // workspaceCtx.currentChatSnapshot may carry a freshly persisted // AgentID/BuildID binding from the getWorkspaceAgent call above. From 89f78c73694ec27606efd858a56607dd51f6fb69 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Thu, 9 Jul 2026 17:58:54 +0000 Subject: [PATCH 03/12] chore(coderd/x/chatd): trim redundant non-mutation comment sentences --- coderd/x/chatd/compaction_prompt.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/coderd/x/chatd/compaction_prompt.go b/coderd/x/chatd/compaction_prompt.go index 66a27014a4c..a49e9900ae0 100644 --- a/coderd/x/chatd/compaction_prompt.go +++ b/coderd/x/chatd/compaction_prompt.go @@ -64,7 +64,7 @@ func sanitizeCompactionPrompt( // results from a copy of messages. Provider-executed blocks produced by // one provider can be rejected when replayed to another, and compaction // only needs the conversation text. Messages emptied by stripping are -// dropped. The input is never mutated. +// dropped. func stripProviderExecutedToolParts( ctx context.Context, logger slog.Logger, @@ -106,7 +106,6 @@ func stripProviderExecutedToolParts( // replaceUnsupportedFileParts swaps file parts the compaction model does // not accept for short text placeholders in a copy of messages, so the // summary notes the attachment existed instead of silently losing it. -// The input is never mutated. func replaceUnsupportedFileParts( ctx context.Context, logger slog.Logger, From c488f6b6f57f8412098807d958c810a8315a4d09 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 10 Jul 2026 10:19:21 +0000 Subject: [PATCH 04/12] fix(coderd/x/chatd): defer compaction override model construction Resolving the compaction model override previously built the route and model client on every generation prepare, so a configured but temporarily broken override could fail ordinary turns that never compact. Split the resolver: the stored config (whose context limit feeds the compaction trigger) is resolved on every prepare with soft fallback, while route and client construction happen only once compaction is known to run. --- coderd/x/chatd/ARCHITECTURE.md | 2 +- coderd/x/chatd/compaction_override.go | 52 +++++++++++-------- .../compaction_override_internal_test.go | 36 +++++++------ coderd/x/chatd/generation_preparer.go | 28 ++++++---- 4 files changed, 69 insertions(+), 49 deletions(-) diff --git a/coderd/x/chatd/ARCHITECTURE.md b/coderd/x/chatd/ARCHITECTURE.md index b18bd69b306..5b54a3ff857 100644 --- a/coderd/x/chatd/ARCHITECTURE.md +++ b/coderd/x/chatd/ARCHITECTURE.md @@ -834,7 +834,7 @@ By default the summary is generated with the chat model. Admins can override the Details that follow from the override: - Context limits: the compaction trigger uses the stricter of the chat model's and the compaction model's context limits, because the history must also fit the summarizer's window. The post-compaction "still over limit" check stays against the chat model's limit, since continuation runs on the chat model. -- Failure semantics: an unset override uses the chat model. Stale or malformed stored references (deleted or disabled config or provider, missing credentials, non-UUID value) fall back to the chat model with a log. A usable override that fails at use (route or client construction, provider call failure) fails the generation visibly through the normal error path; there is no silent fallback. +- Failure semantics: an unset override uses the chat model. Stale or malformed stored references (deleted or disabled config or provider, missing credentials, non-UUID value) fall back to the chat model with a log. A usable override that fails at use (route or client construction, provider call failure) fails the generation visibly through the normal error path; there is no silent fallback. The override model client is constructed only on turns that actually compact, so a broken override cannot fail turns that stay under the compaction threshold. - Prompt safety: the prompt is built and sanitized for the chat model, so when the override points at a different provider the compaction copy of the prompt is re-sanitized: provider-executed tool history is stripped, file parts the compaction model rejects are replaced with text placeholders, and Anthropic provider-tool sanitization is re-run for the compaction provider. The assistant generation prompt is never mutated. - Observability: compaction metrics and chat debug runs record the provider and model that actually generated the summary. diff --git a/coderd/x/chatd/compaction_override.go b/coderd/x/chatd/compaction_override.go index c006ec6201d..561c70f670a 100644 --- a/coderd/x/chatd/compaction_override.go +++ b/coderd/x/chatd/compaction_override.go @@ -40,27 +40,27 @@ type compactionModelOverride struct { resolvedModel string } -// resolveCompactionModelOverride resolves the deployment-wide compaction -// model override. Unset, malformed, stale (deleted or disabled config or -// provider), and credential-less overrides fall back to the chat model -// (overrideSet is false; the shared resolver logs the reason). Errors are -// hard failures: a configured, usable override that cannot be routed or -// constructed must fail the generation visibly instead of silently -// compacting with the chat model. -func (p *Server) resolveCompactionModelOverride( +// resolveCompactionOverrideConfig resolves the stored deployment-wide +// compaction model override to its model config. Unset, malformed, stale +// (deleted or disabled config or provider), and credential-less overrides +// fall back to the chat model (overrideSet is false; the shared resolver +// logs the reason). This runs on every generation prepare because the +// override's context limit feeds the compaction trigger; the model client +// is built separately by buildCompactionOverrideModel only when compaction +// actually runs. +func (p *Server) resolveCompactionOverrideConfig( ctx context.Context, chat database.Chat, - modelOpts modelBuildOptions, -) (compactionModelOverride, bool, error) { +) (database.ChatModelConfig, bool, error) { raw, err := readCompactionModelOverride(ctx, p.db) if err != nil { - return compactionModelOverride{}, false, xerrors.Errorf( + return database.ChatModelConfig{}, false, xerrors.Errorf( "read compaction model override: %w", err, ) } - modelConfig, overrideSet, err := p.resolveConfiguredModelOverride( + return p.resolveConfiguredModelOverride( ctx, compactionOverrideContext, raw, @@ -71,17 +71,25 @@ func (p *Server) resolveCompactionModelOverride( }, modelOverrideFailureModeSoft, ) - if err != nil { - return compactionModelOverride{}, false, err - } - if !overrideSet { - return compactionModelOverride{}, false, nil - } +} +// buildCompactionOverrideModel resolves the route and constructs the model +// client for a usable compaction override config. Errors are hard failures: +// a configured, usable override that cannot be routed or constructed must +// fail the generation visibly instead of silently compacting with the chat +// model. Callers invoke this only when compaction is about to run, so a +// broken override cannot fail turns that stay under the compaction +// threshold. +func (p *Server) buildCompactionOverrideModel( + ctx context.Context, + chat database.Chat, + modelConfig database.ChatModelConfig, + modelOpts modelBuildOptions, +) (compactionModelOverride, error) { //nolint:gocritic // Compaction overrides need chatd-scoped provider reads for user-owned chats. route, err := p.resolveModelRouteForConfig(dbauthz.AsChatd(ctx), chat.OwnerID, modelConfig) if err != nil { - return compactionModelOverride{}, true, xerrors.Errorf( + return compactionModelOverride{}, xerrors.Errorf( "resolve compaction model override route: %w", err, ) @@ -91,7 +99,7 @@ func (p *Server) resolveCompactionModelOverride( route.ModelProviderHint, ) if err != nil { - return compactionModelOverride{}, true, xerrors.Errorf( + return compactionModelOverride{}, xerrors.Errorf( "resolve compaction model override metadata: %w", err, ) @@ -103,7 +111,7 @@ func (p *Server) resolveCompactionModelOverride( ExtraHeaders: chatprovider.CoderHeaders(chat), }, route, modelOpts) if err != nil { - return compactionModelOverride{}, true, xerrors.Errorf( + return compactionModelOverride{}, xerrors.Errorf( "create compaction model override: %w", err, ) @@ -113,5 +121,5 @@ func (p *Server) resolveCompactionModelOverride( model: model, resolvedProvider: resolvedProvider, resolvedModel: resolvedModel, - }, true, nil + }, nil } diff --git a/coderd/x/chatd/compaction_override_internal_test.go b/coderd/x/chatd/compaction_override_internal_test.go index e6ddea32498..6f1567af9d0 100644 --- a/coderd/x/chatd/compaction_override_internal_test.go +++ b/coderd/x/chatd/compaction_override_internal_test.go @@ -14,7 +14,7 @@ import ( "github.com/coder/coder/v2/testutil" ) -func TestResolveCompactionModelOverride_Unset(t *testing.T) { +func TestResolveCompactionOverrideConfig_Unset(t *testing.T) { t.Parallel() ctx := testutil.Context(t, testutil.WaitShort) @@ -26,13 +26,12 @@ func TestResolveCompactionModelOverride_Unset(t *testing.T) { db.EXPECT().GetChatCompactionModelOverride(gomock.Any()).Return("", nil) server := titleOverrideTestServer(db, logger) - override, overrideSet, err := server.resolveCompactionModelOverride(ctx, chat, modelBuildOptions{}) + _, overrideSet, err := server.resolveCompactionOverrideConfig(ctx, chat) require.NoError(t, err) require.False(t, overrideSet) - require.Nil(t, override.model) } -func TestResolveCompactionModelOverride_ReadDBError(t *testing.T) { +func TestResolveCompactionOverrideConfig_ReadDBError(t *testing.T) { t.Parallel() ctx := testutil.Context(t, testutil.WaitShort) @@ -44,13 +43,13 @@ func TestResolveCompactionModelOverride_ReadDBError(t *testing.T) { db.EXPECT().GetChatCompactionModelOverride(gomock.Any()).Return("", sql.ErrConnDone) server := titleOverrideTestServer(db, logger) - _, overrideSet, err := server.resolveCompactionModelOverride(ctx, chat, modelBuildOptions{}) + _, overrideSet, err := server.resolveCompactionOverrideConfig(ctx, chat) require.Error(t, err) require.ErrorContains(t, err, "read compaction model override") require.False(t, overrideSet) } -func TestResolveCompactionModelOverride_MalformedFallsBack(t *testing.T) { +func TestResolveCompactionOverrideConfig_MalformedFallsBack(t *testing.T) { t.Parallel() ctx := testutil.Context(t, testutil.WaitShort) @@ -62,12 +61,12 @@ func TestResolveCompactionModelOverride_MalformedFallsBack(t *testing.T) { db.EXPECT().GetChatCompactionModelOverride(gomock.Any()).Return("not-a-uuid", nil) server := titleOverrideTestServer(db, logger) - _, overrideSet, err := server.resolveCompactionModelOverride(ctx, chat, modelBuildOptions{}) + _, overrideSet, err := server.resolveCompactionOverrideConfig(ctx, chat) require.NoError(t, err) require.False(t, overrideSet) } -func TestResolveCompactionModelOverride_DeletedConfigFallsBack(t *testing.T) { +func TestResolveCompactionOverrideConfig_DeletedConfigFallsBack(t *testing.T) { t.Parallel() ctx := testutil.Context(t, testutil.WaitShort) @@ -81,12 +80,12 @@ func TestResolveCompactionModelOverride_DeletedConfigFallsBack(t *testing.T) { db.EXPECT().GetChatModelConfigByID(gomock.Any(), missingID).Return(database.ChatModelConfig{}, sql.ErrNoRows) server := titleOverrideTestServer(db, logger) - _, overrideSet, err := server.resolveCompactionModelOverride(ctx, chat, modelBuildOptions{}) + _, overrideSet, err := server.resolveCompactionOverrideConfig(ctx, chat) require.NoError(t, err) require.False(t, overrideSet) } -func TestResolveCompactionModelOverride_DisabledConfigFallsBack(t *testing.T) { +func TestResolveCompactionOverrideConfig_DisabledConfigFallsBack(t *testing.T) { t.Parallel() ctx := testutil.Context(t, testutil.WaitShort) @@ -100,12 +99,12 @@ func TestResolveCompactionModelOverride_DisabledConfigFallsBack(t *testing.T) { db.EXPECT().GetChatModelConfigByID(gomock.Any(), overrideConfig.ID).Return(overrideConfig, nil) server := titleOverrideTestServer(db, logger) - _, overrideSet, err := server.resolveCompactionModelOverride(ctx, chat, modelBuildOptions{}) + _, overrideSet, err := server.resolveCompactionOverrideConfig(ctx, chat) require.NoError(t, err) require.False(t, overrideSet) } -func TestResolveCompactionModelOverride_MissingCredentialsFallsBack(t *testing.T) { +func TestResolveCompactionOverrideConfig_MissingCredentialsFallsBack(t *testing.T) { t.Parallel() ctx := testutil.Context(t, testutil.WaitShort) @@ -127,12 +126,12 @@ func TestResolveCompactionModelOverride_MissingCredentialsFallsBack(t *testing.T db.EXPECT().GetAIProviderKeysByProviderID(gomock.Any(), providerID).Return(nil, nil).AnyTimes() server := titleOverrideTestServer(db, logger) - _, overrideSet, err := server.resolveCompactionModelOverride(ctx, chat, modelBuildOptions{}) + _, overrideSet, err := server.resolveCompactionOverrideConfig(ctx, chat) require.NoError(t, err) require.False(t, overrideSet) } -func TestResolveCompactionModelOverride_SetUsable(t *testing.T) { +func TestCompactionOverride_SetUsable(t *testing.T) { t.Parallel() ctx := testutil.Context(t, testutil.WaitShort) @@ -153,13 +152,18 @@ func TestResolveCompactionModelOverride_SetUsable(t *testing.T) { }}, nil).AnyTimes() server := titleOverrideTestServer(db, logger) - override, overrideSet, err := server.resolveCompactionModelOverride( + resolvedConfig, overrideSet, err := server.resolveCompactionOverrideConfig(ctx, chat) + require.NoError(t, err) + require.True(t, overrideSet) + require.Equal(t, overrideConfig.ID, resolvedConfig.ID) + + override, err := server.buildCompactionOverrideModel( ctx, chat, + resolvedConfig, modelBuildOptions{ActiveAPIKeyID: uuid.NewString()}, ) require.NoError(t, err) - require.True(t, overrideSet) require.NotNil(t, override.model) require.Equal(t, overrideConfig.ID, override.modelConfig.ID) require.Equal(t, "openai", override.resolvedProvider) diff --git a/coderd/x/chatd/generation_preparer.go b/coderd/x/chatd/generation_preparer.go index b83b599150d..7c3c8e4cfc6 100644 --- a/coderd/x/chatd/generation_preparer.go +++ b/coderd/x/chatd/generation_preparer.go @@ -582,28 +582,36 @@ func (server *Server) prepareGeneration( // using the chat model's limit (ContextLimitFallback below), since the // follow-up assistant generation runs on the chat model. compactionContextLimit := modelConfig.ContextLimit - compactionOverride, compactionOverrideSet, err := server.resolveCompactionModelOverride(ctx, chat, modelOpts) + compactionOverrideConfig, compactionOverrideSet, err := server.resolveCompactionOverrideConfig(ctx, chat) if err != nil { cleanup() return generationPrepared{}, err } if compactionOverrideSet { - compactionModel = compactionOverride.model - compactionResolvedProvider = compactionOverride.resolvedProvider - compactionResolvedModel = compactionOverride.resolvedModel - compactionModelConfigID = compactionOverride.modelConfig.ID - if overrideLimit := compactionOverride.modelConfig.ContextLimit; overrideLimit > 0 && + if overrideLimit := compactionOverrideConfig.ContextLimit; overrideLimit > 0 && (compactionContextLimit <= 0 || overrideLimit < compactionContextLimit) { compactionContextLimit = overrideLimit } } compactionStepUsage := latestPromptUsage(promptRows) compactionNeeded := shouldCompactPromptUsage(compactionStepUsage, compactionContextLimit, effectiveThreshold) - // Sanitizing rewrites the full prompt, so skip it unless this turn - // actually compacts. The sanitized prompt is only consumed by the - // compaction generation, which only runs when compactionNeeded is - // true at prepare time. + // The override model client is built (and the prompt sanitized for it) + // only when this turn actually compacts: route or client construction + // failures are hard, and they must not fail ordinary turns that stay + // under the compaction threshold. Sanitizing also rewrites the full + // prompt, and the sanitized prompt is only consumed by the compaction + // generation, which only runs when compactionNeeded is true at prepare + // time. if compactionOverrideSet && compactionNeeded { + compactionOverride, err := server.buildCompactionOverrideModel(ctx, chat, compactionOverrideConfig, modelOpts) + if err != nil { + cleanup() + return generationPrepared{}, err + } + compactionModel = compactionOverride.model + compactionResolvedProvider = compactionOverride.resolvedProvider + compactionResolvedModel = compactionOverride.resolvedModel + compactionModelConfigID = compactionOverride.modelConfig.ID compactionPrompt = sanitizeCompactionPrompt( ctx, logger, From 358739808c0c779e541ac7ddf6255b373e5c69d6 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 10 Jul 2026 10:28:11 +0000 Subject: [PATCH 05/12] test(coderd): include compaction in invalid-context detail assertion Adding the compaction context to AllChatModelOverrideContexts extends the invalid-context error detail, so the UnknownContextReturns400 expectations must list it. --- coderd/exp_chats_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/coderd/exp_chats_test.go b/coderd/exp_chats_test.go index dfeb190b86b..03918f7db25 100644 --- a/coderd/exp_chats_test.go +++ b/coderd/exp_chats_test.go @@ -12022,7 +12022,7 @@ func TestChatModelOverrides(t *testing.T) { require.Equal(t, "Invalid chat model override context.", sdkErr.Message) require.Equal( t, - `Expected one of general, explore, title_generation. Got "not-a-context".`, + `Expected one of general, explore, title_generation, compaction. Got "not-a-context".`, sdkErr.Detail, ) @@ -12031,7 +12031,7 @@ func TestChatModelOverrides(t *testing.T) { require.Equal(t, "Invalid chat model override context.", sdkErr.Message) require.Equal( t, - `Expected one of general, explore, title_generation. Got "not-a-context".`, + `Expected one of general, explore, title_generation, compaction. Got "not-a-context".`, sdkErr.Detail, ) }) From b866d4fde3d5c367ae73794cbdb608ba53a4a5c2 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 10 Jul 2026 10:40:47 +0000 Subject: [PATCH 06/12] fix(coderd/x/chatd): build compaction override in the compact action compactionNeeded at prepare time does not guarantee compaction runs: decideGenerationAction can return FinishTurn first (completed step, stop-after tool, max steps), and building the override client during prepare failed such turns on hard construction errors. Carry the resolved override config through generationCompaction and build the client, swap identity, and sanitize the prompt inside generateCompaction, which only runs for the compact action. --- coderd/x/chatd/ARCHITECTURE.md | 2 +- coderd/x/chatd/compaction_override.go | 5 +-- coderd/x/chatd/generation.go | 31 +++++++++++++++ coderd/x/chatd/generation_preparer.go | 54 ++++++++------------------- 4 files changed, 50 insertions(+), 42 deletions(-) diff --git a/coderd/x/chatd/ARCHITECTURE.md b/coderd/x/chatd/ARCHITECTURE.md index 5b54a3ff857..f6e3ece2810 100644 --- a/coderd/x/chatd/ARCHITECTURE.md +++ b/coderd/x/chatd/ARCHITECTURE.md @@ -834,7 +834,7 @@ By default the summary is generated with the chat model. Admins can override the Details that follow from the override: - Context limits: the compaction trigger uses the stricter of the chat model's and the compaction model's context limits, because the history must also fit the summarizer's window. The post-compaction "still over limit" check stays against the chat model's limit, since continuation runs on the chat model. -- Failure semantics: an unset override uses the chat model. Stale or malformed stored references (deleted or disabled config or provider, missing credentials, non-UUID value) fall back to the chat model with a log. A usable override that fails at use (route or client construction, provider call failure) fails the generation visibly through the normal error path; there is no silent fallback. The override model client is constructed only on turns that actually compact, so a broken override cannot fail turns that stay under the compaction threshold. +- Failure semantics: an unset override uses the chat model. Stale or malformed stored references (deleted or disabled config or provider, missing credentials, non-UUID value) fall back to the chat model with a log. A usable override that fails at use (route or client construction, provider call failure) fails the generation visibly through the normal error path; there is no silent fallback. The override model client is constructed inside the compact generation action, not at prepare time, so a broken override cannot fail turns that finish without compacting (including turns over the threshold whose last assistant step already completed). - Prompt safety: the prompt is built and sanitized for the chat model, so when the override points at a different provider the compaction copy of the prompt is re-sanitized: provider-executed tool history is stripped, file parts the compaction model rejects are replaced with text placeholders, and Anthropic provider-tool sanitization is re-run for the compaction provider. The assistant generation prompt is never mutated. - Observability: compaction metrics and chat debug runs record the provider and model that actually generated the summary. diff --git a/coderd/x/chatd/compaction_override.go b/coderd/x/chatd/compaction_override.go index 561c70f670a..3762bbfe8cc 100644 --- a/coderd/x/chatd/compaction_override.go +++ b/coderd/x/chatd/compaction_override.go @@ -77,9 +77,8 @@ func (p *Server) resolveCompactionOverrideConfig( // client for a usable compaction override config. Errors are hard failures: // a configured, usable override that cannot be routed or constructed must // fail the generation visibly instead of silently compacting with the chat -// model. Callers invoke this only when compaction is about to run, so a -// broken override cannot fail turns that stay under the compaction -// threshold. +// model. Callers invoke this from the compact generation action, so a +// broken override cannot fail turns that finish without compacting. func (p *Server) buildCompactionOverrideModel( ctx context.Context, chat database.Chat, diff --git a/coderd/x/chatd/generation.go b/coderd/x/chatd/generation.go index 06e8bf645fe..91f233fabb1 100644 --- a/coderd/x/chatd/generation.go +++ b/coderd/x/chatd/generation.go @@ -69,6 +69,15 @@ type generationPrepared struct { // generationCompaction contains compaction inputs prepared for generation. type generationCompaction struct { + // OverrideConfig, when non-nil, is the resolved compaction model + // override config. The override model client is built (and the prompt + // sanitized for it) in the compact action path, not at prepare time, + // so hard construction failures cannot fail turns that finish without + // compacting. ChatModelConfig is the chat model's config, needed to + // sanitize the prompt across provider boundaries. + OverrideConfig *database.ChatModelConfig + ChatModelConfig database.ChatModelConfig + Required bool Options chatloop.GenerateCompactionOptions } @@ -683,6 +692,28 @@ func (s *taskStarter) generateCompaction( return s.finishGenerationError(ctx, machine, input, xerrors.New("compaction action missing options"), requireGenerationAttempt(attempt.number)) } compactionOpts := prepared.Compaction.Options + if overrideConfig := prepared.Compaction.OverrideConfig; overrideConfig != nil { + override, err := s.server.buildCompactionOverrideModel(ctx, prepared.Chat, *overrideConfig, prepared.ModelBuildOptions) + if err != nil { + return xerrors.Errorf("build compaction model override: %w", err) + } + logger := s.server.logger.With( + slog.F("chat_id", prepared.Chat.ID), + slog.F("owner_id", prepared.Chat.OwnerID), + ) + compactionOpts.Model = override.model + compactionOpts.ResolvedProvider = override.resolvedProvider + compactionOpts.ResolvedModel = override.resolvedModel + compactionOpts.ModelConfigID = override.modelConfig.ID + compactionOpts.Messages = sanitizeCompactionPrompt( + ctx, + logger, + compactionOpts.Messages, + override.model, + prepared.Compaction.ChatModelConfig, + override.modelConfig, + ) + } compactionOpts.PublishMessagePart = attempt.publish // Attach the turn debug run so the compaction call records a child // debug run; without it startCompactionDebugRun finds no parent and diff --git a/coderd/x/chatd/generation_preparer.go b/coderd/x/chatd/generation_preparer.go index 7c3c8e4cfc6..0c3e97c18b5 100644 --- a/coderd/x/chatd/generation_preparer.go +++ b/coderd/x/chatd/generation_preparer.go @@ -571,11 +571,6 @@ func (server *Server) prepareGeneration( if override, ok := server.resolveUserCompactionThreshold(ctx, chat.OwnerID, modelConfig.ID); ok { effectiveThreshold = override } - compactionModel := model - compactionPrompt := prompt - compactionResolvedProvider := resolvedProvider - compactionResolvedModel := debugModel - compactionModelConfigID := modelConfig.ID // The effective compaction limit is the stricter of the chat model's // and the compaction model's context limits: the history must also fit // the summarizer's window. Post-compaction continuation checks keep @@ -587,7 +582,9 @@ func (server *Server) prepareGeneration( cleanup() return generationPrepared{}, err } + var compactionOverride *database.ChatModelConfig if compactionOverrideSet { + compactionOverride = &compactionOverrideConfig if overrideLimit := compactionOverrideConfig.ContextLimit; overrideLimit > 0 && (compactionContextLimit <= 0 || overrideLimit < compactionContextLimit) { compactionContextLimit = overrideLimit @@ -595,35 +592,14 @@ func (server *Server) prepareGeneration( } compactionStepUsage := latestPromptUsage(promptRows) compactionNeeded := shouldCompactPromptUsage(compactionStepUsage, compactionContextLimit, effectiveThreshold) - // The override model client is built (and the prompt sanitized for it) - // only when this turn actually compacts: route or client construction - // failures are hard, and they must not fail ordinary turns that stay - // under the compaction threshold. Sanitizing also rewrites the full - // prompt, and the sanitized prompt is only consumed by the compaction - // generation, which only runs when compactionNeeded is true at prepare - // time. - if compactionOverrideSet && compactionNeeded { - compactionOverride, err := server.buildCompactionOverrideModel(ctx, chat, compactionOverrideConfig, modelOpts) - if err != nil { - cleanup() - return generationPrepared{}, err - } - compactionModel = compactionOverride.model - compactionResolvedProvider = compactionOverride.resolvedProvider - compactionResolvedModel = compactionOverride.resolvedModel - compactionModelConfigID = compactionOverride.modelConfig.ID - compactionPrompt = sanitizeCompactionPrompt( - ctx, - logger, - prompt, - compactionOverride.model, - modelConfig, - compactionOverride.modelConfig, - ) - } + // The options carry the chat model; when an override is configured, + // generateCompaction builds the override client (a hard failure) and + // swaps model, identity, and sanitized prompt in the compact action + // path, so a broken override cannot fail turns that finish without + // compacting. compactionOptions := chatloop.GenerateCompactionOptions{ - Model: compactionModel, - Messages: compactionPrompt, + Model: model, + Messages: prompt, ThresholdPercent: effectiveThreshold, ContextLimit: compactionContextLimit, ContextLimitFallback: compactionContextLimit, @@ -632,9 +608,9 @@ func (server *Server) prepareGeneration( DebugSvc: debugSvc, ChatID: chat.ID, HistoryTipMessageID: historyTipMessageID, - ResolvedProvider: compactionResolvedProvider, - ResolvedModel: compactionResolvedModel, - ModelConfigID: compactionModelConfigID, + ResolvedProvider: resolvedProvider, + ResolvedModel: debugModel, + ModelConfigID: modelConfig.ID, StepUsage: compactionStepUsage, } @@ -669,8 +645,10 @@ func (server *Server) prepareGeneration( ToolNameToConfigID: toolNameToConfigID, MaxSteps: maxChatSteps, Compaction: &generationCompaction{ - Required: compactionNeeded, - Options: compactionOptions, + OverrideConfig: compactionOverride, + ChatModelConfig: modelConfig, + Required: compactionNeeded, + Options: compactionOptions, }, Cleanup: cleanup, Debug: debug, From f9bbc7b1818d2bbcdea26551c2f5692c98070016 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 10 Jul 2026 19:49:13 +0000 Subject: [PATCH 07/12] fix(coderd/x/chatd): flatten provider-executed tool history for compaction Align the cross-provider compaction prompt pass with coder/mux's flattenProviderExecutedToolParts: rewrite provider-executed tool calls and results into plain text parts instead of dropping them, so the summary keeps server-side tool content (for example web_search results) while still shedding the provider-specific wire shape other providers reject. Anomalous provider-executed parts outside assistant messages are still dropped. --- coderd/x/chatd/ARCHITECTURE.md | 2 +- coderd/x/chatd/compaction_prompt.go | 80 ++++++++++++++++--- .../chatd/compaction_prompt_internal_test.go | 20 ++++- 3 files changed, 84 insertions(+), 18 deletions(-) diff --git a/coderd/x/chatd/ARCHITECTURE.md b/coderd/x/chatd/ARCHITECTURE.md index ba7a18ed0c0..62040cd6e03 100644 --- a/coderd/x/chatd/ARCHITECTURE.md +++ b/coderd/x/chatd/ARCHITECTURE.md @@ -841,7 +841,7 @@ Details that follow from the override: - Context limits: the compaction trigger uses the stricter of the chat model's and the compaction model's context limits, because the history must also fit the summarizer's window. The post-compaction "still over limit" check stays against the chat model's limit, since continuation runs on the chat model. - Failure semantics: an unset override uses the chat model. Stale or malformed stored references (deleted or disabled config or provider, missing credentials, non-UUID value) fall back to the chat model with a log. A usable override that fails at use (route or client construction, provider call failure) fails the generation visibly through the normal error path; there is no silent fallback. The override model client is constructed inside the compact generation action, not at prepare time, so a broken override cannot fail turns that finish without compacting (including turns over the threshold whose last assistant step already completed). -- Prompt safety: the prompt is built and sanitized for the chat model, so when the override points at a different provider the compaction copy of the prompt is re-sanitized: provider-executed tool history is stripped, file parts the compaction model rejects are replaced with text placeholders, and Anthropic provider-tool sanitization is re-run for the compaction provider. The assistant generation prompt is never mutated. +- Prompt safety: the prompt is built and sanitized for the chat model, so when the override points at a different provider the compaction copy of the prompt is re-sanitized: provider-executed tool history is flattened into plain text parts (keeping its content while dropping the provider-specific wire shape), file parts the compaction model rejects are replaced with text placeholders, and Anthropic provider-tool sanitization is re-run for the compaction provider. The assistant generation prompt is never mutated. - Observability: compaction metrics and chat debug runs record the provider and model that actually generated the summary. #### Interrupt goroutine diff --git a/coderd/x/chatd/compaction_prompt.go b/coderd/x/chatd/compaction_prompt.go index a49e9900ae0..17014efe5e8 100644 --- a/coderd/x/chatd/compaction_prompt.go +++ b/coderd/x/chatd/compaction_prompt.go @@ -36,7 +36,7 @@ func sanitizeCompactionPrompt( ) []fantasy.Message { messages := prompt if !sameCompactionProviderIdentity(chatConfig, overrideConfig) { - messages = stripProviderExecutedToolParts(ctx, logger, messages) + messages = flattenProviderExecutedToolParts(ctx, logger, messages) } messages = replaceUnsupportedFileParts(ctx, logger, messages, func(mediaType string) bool { return chatprovider.AcceptsFilePartMediaType( @@ -60,30 +60,61 @@ func sanitizeCompactionPrompt( return sanitized } -// stripProviderExecutedToolParts removes provider-executed tool calls and -// results from a copy of messages. Provider-executed blocks produced by -// one provider can be rejected when replayed to another, and compaction -// only needs the conversation text. Messages emptied by stripping are -// dropped. -func stripProviderExecutedToolParts( +// flattenProviderExecutedToolParts rewrites provider-executed tool calls +// and results in assistant messages into plain text parts on a copy of +// messages. Provider-executed blocks carry provider-specific wire shapes and +// ids (for example Anthropic srvtoolu_... ids) that other providers reject +// on replay, but their content is still useful to the summary, so it is kept +// as text instead of dropped. Mirrors flattenProviderExecutedToolParts in +// coder/mux. +// +// Assistant messages are the only legitimate home for provider-executed +// parts (chatsanitize flags other roles as violations). Anomalous parts in +// other roles are dropped rather than flattened, because a text part is not +// valid tool-message content for every provider; messages emptied by that +// drop are removed. +func flattenProviderExecutedToolParts( ctx context.Context, logger slog.Logger, messages []fantasy.Message, ) []fantasy.Message { - removed := 0 + flattened := 0 + dropped := 0 + // Tool names live on the call part only; results reference the call by ID. + toolNamesByCallID := make(map[string]string) out := make([]fantasy.Message, 0, len(messages)) for _, msg := range messages { + flattenToText := msg.Role == fantasy.MessageRoleAssistant parts := make([]fantasy.MessagePart, 0, len(msg.Content)) for _, part := range msg.Content { switch typed := part.(type) { case fantasy.ToolCallPart: if typed.ProviderExecuted { - removed++ + if !flattenToText { + dropped++ + continue + } + toolNamesByCallID[typed.ToolCallID] = typed.ToolName + flattened++ + parts = append(parts, fantasy.TextPart{ + Text: fmt.Sprintf("[Server tool call: %s] %s", typed.ToolName, typed.Input), + }) continue } case fantasy.ToolResultPart: if typed.ProviderExecuted { - removed++ + if !flattenToText { + dropped++ + continue + } + flattened++ + parts = append(parts, fantasy.TextPart{ + Text: fmt.Sprintf( + "[Server tool result: %s] %s", + toolNamesByCallID[typed.ToolCallID], + stringifyToolResultOutput(typed.Output), + ), + }) continue } } @@ -95,14 +126,37 @@ func stripProviderExecutedToolParts( msg.Content = parts out = append(out, msg) } - if removed > 0 { - logger.Debug(ctx, "stripped provider-executed tool history from compaction prompt", - slog.F("removed_parts", removed), + if flattened > 0 || dropped > 0 { + logger.Debug(ctx, "flattened provider-executed tool history in compaction prompt", + slog.F("flattened_parts", flattened), + slog.F("dropped_parts", dropped), ) } return out } +// stringifyToolResultOutput renders a tool result output as prompt text. +// Media payloads are summarized instead of inlined so base64 data does not +// enter the compaction prompt as text. +func stringifyToolResultOutput(output fantasy.ToolResultOutputContent) string { + switch typed := output.(type) { + case fantasy.ToolResultOutputContentText: + return typed.Text + case fantasy.ToolResultOutputContentError: + if typed.Error == nil { + return "error" + } + return typed.Error.Error() + case fantasy.ToolResultOutputContentMedia: + if typed.Text != "" { + return fmt.Sprintf("%s [media %s omitted]", typed.Text, typed.MediaType) + } + return fmt.Sprintf("[media %s omitted]", typed.MediaType) + default: + return "[unserializable tool output]" + } +} + // replaceUnsupportedFileParts swaps file parts the compaction model does // not accept for short text placeholders in a copy of messages, so the // summary notes the attachment existed instead of silently losing it. diff --git a/coderd/x/chatd/compaction_prompt_internal_test.go b/coderd/x/chatd/compaction_prompt_internal_test.go index 8c01640c0f9..f3b21f7bc10 100644 --- a/coderd/x/chatd/compaction_prompt_internal_test.go +++ b/coderd/x/chatd/compaction_prompt_internal_test.go @@ -29,7 +29,7 @@ func configWithProvider(id uuid.UUID) database.ChatModelConfig { return database.ChatModelConfig{AIProviderID: uuid.NullUUID{UUID: id, Valid: true}} } -func TestSanitizeCompactionPrompt_StripsForeignProviderExecutedToolParts(t *testing.T) { +func TestSanitizeCompactionPrompt_FlattensForeignProviderExecutedToolParts(t *testing.T) { t.Parallel() ctx := testutil.Context(t, testutil.WaitShort) @@ -74,20 +74,32 @@ func TestSanitizeCompactionPrompt_StripsForeignProviderExecutedToolParts(t *test sanitized := sanitizeCompactionPrompt(ctx, logger, prompt, compactionModel, configWithProvider(uuid.New()), configWithProvider(uuid.New())) require.Len(t, sanitized, 3) - require.Len(t, sanitized[1].Content, 1) + // Provider-executed parts are flattened to text so the summary keeps + // their content without the provider-specific wire shape. + require.Len(t, sanitized[1].Content, 3) require.Equal(t, fantasy.TextPart{Text: "searching"}, sanitized[1].Content[0]) + require.Equal(t, fantasy.TextPart{Text: `[Server tool call: web_search] {"query":"coder"}`}, sanitized[1].Content[1]) + require.Equal(t, fantasy.TextPart{Text: "[Server tool result: web_search] results"}, sanitized[1].Content[2]) // Local tool calls replay fine across providers and must survive. require.Len(t, sanitized[2].Content, 1) + require.Equal(t, "read_file", sanitized[2].Content[0].(fantasy.ToolCallPart).ToolName) // The original prompt used for assistant generation is untouched. - require.Len(t, prompt[1].Content, 3) + require.Equal(t, fantasy.ToolCallPart{ + ToolCallID: "ws-1", + ToolName: "web_search", + Input: `{"query":"coder"}`, + ProviderExecuted: true, + }, prompt[1].Content[1]) } -func TestSanitizeCompactionPrompt_DropsEmptiedMessages(t *testing.T) { +func TestSanitizeCompactionPrompt_DropsNonAssistantProviderExecutedParts(t *testing.T) { t.Parallel() ctx := testutil.Context(t, testutil.WaitShort) logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}) + // Provider-executed parts outside assistant messages are anomalous; a + // flattened text part is not valid tool-message content, so they drop. prompt := []fantasy.Message{ { Role: fantasy.MessageRoleTool, From b32de95342d928d881e73346898a03585ce9013a Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Fri, 10 Jul 2026 20:14:35 +0000 Subject: [PATCH 08/12] fix(coderd/x/chatd): honor compaction override reasoning effort The compaction override resolver folds the admin-configured reasoning effort into the override config's call options, but the summary call never consumed them. Convert the override's provider options and resolved reasoning effort when building the override client and pass them on the summary fantasy.Call, so a compaction model's configured effort actually applies. --- coderd/x/chatd/chatd_test.go | 13 ++++++ coderd/x/chatd/chatloop/chatloop.go | 4 ++ coderd/x/chatd/chatloop/compaction.go | 12 ++++-- coderd/x/chatd/chattest/anthropic.go | 1 + coderd/x/chatd/compaction_override.go | 38 +++++++++++++++++ .../compaction_override_internal_test.go | 41 +++++++++++++++++++ coderd/x/chatd/generation.go | 1 + 7 files changed, 106 insertions(+), 4 deletions(-) diff --git a/coderd/x/chatd/chatd_test.go b/coderd/x/chatd/chatd_test.go index af91e740047..b7c7de1f96b 100644 --- a/coderd/x/chatd/chatd_test.go +++ b/coderd/x/chatd/chatd_test.go @@ -5879,6 +5879,13 @@ func TestActiveServer_CompactionModelOverride(t *testing.T) { AIProviderID: chatModel.AIProviderID, ContextLimit: contextLimit, }) + lowEffort := "low" + overrideModel = updateChatModelCallConfig(t, db, overrideModel, codersdk.ChatModelCallConfig{ + ReasoningEffort: &codersdk.ChatModelReasoningEffortConfig{ + Default: &lowEffort, + Max: &lowEffort, + }, + }) require.NoError(t, db.UpsertChatCompactionModelOverride(ctx, overrideModel.ID.String())) return overrideModel } @@ -5895,6 +5902,10 @@ func TestActiveServer_CompactionModelOverride(t *testing.T) { if !req.Stream { if strings.Contains(body, "You are performing a context compaction") { require.Equal(t, overrideModelName, req.Model) + // The override config's reasoning effort must reach the + // summary request (Anthropic serializes it as + // output_config effort). + require.Contains(t, string(req.OutputConfig), `"effort":"low"`) return anthropicCompactionResponse(compactionSummary) } return chattest.AnthropicNonStreamingResponse("title") @@ -5905,6 +5916,8 @@ func TestActiveServer_CompactionModelOverride(t *testing.T) { return highUsageReadFileResponse("/tmp/a.txt") default: require.Contains(t, body, compactionSummary) + require.Empty(t, string(req.OutputConfig), + "the override reasoning effort must not leak into chat model generations") return chattest.AnthropicStreamingResponse(chattest.AnthropicTextChunksWithCacheUsage(chattest.AnthropicUsage{ InputTokens: 20, OutputTokens: 5, diff --git a/coderd/x/chatd/chatloop/chatloop.go b/coderd/x/chatd/chatloop/chatloop.go index 4d55fa8ed82..24988c357c1 100644 --- a/coderd/x/chatd/chatloop/chatloop.go +++ b/coderd/x/chatd/chatloop/chatloop.go @@ -288,6 +288,10 @@ type GenerateCompactionOptions struct { ResolvedModel string ModelConfigID uuid.UUID + // ProviderOptions are provider-specific call options for the summary + // model, such as a compaction override's configured reasoning effort. + ProviderOptions fantasy.ProviderOptions + PublishMessagePart func(codersdk.ChatMessageRole, codersdk.ChatMessagePart) } diff --git a/coderd/x/chatd/chatloop/compaction.go b/coderd/x/chatd/chatloop/compaction.go index c4a23512ca2..e0e2f89d44b 100644 --- a/coderd/x/chatd/chatloop/compaction.go +++ b/coderd/x/chatd/chatloop/compaction.go @@ -64,11 +64,13 @@ type CompactionOptions struct { ChatID uuid.UUID HistoryTipMessageID int64 - // ResolvedProvider, ResolvedModel, and ModelConfigID identify the - // model generating the summary; see GenerateCompactionOptions. + // ResolvedProvider, ResolvedModel, ModelConfigID, and + // ProviderOptions describe the model generating the summary; see + // GenerateCompactionOptions. ResolvedProvider string ResolvedModel string ModelConfigID uuid.UUID + ProviderOptions fantasy.ProviderOptions // ToolCallID and ToolName identify the synthetic tool call // used to represent compaction in the message stream. @@ -178,6 +180,7 @@ func normalizedCompactionGenerateConfig(opts GenerateCompactionOptions) (Compact ResolvedProvider: opts.ResolvedProvider, ResolvedModel: opts.ResolvedModel, ModelConfigID: opts.ModelConfigID, + ProviderOptions: opts.ProviderOptions, ToolCallID: opts.ToolCallID, ToolName: opts.ToolName, PublishMessagePart: opts.PublishMessagePart, @@ -392,8 +395,9 @@ func generateCompactionSummary( }() response, err := model.Generate(summaryCtx, fantasy.Call{ - Prompt: summaryPrompt, - ToolChoice: &toolChoice, + Prompt: summaryPrompt, + ToolChoice: &toolChoice, + ProviderOptions: options.ProviderOptions, }) if err != nil { return "", xerrors.Errorf("generate summary text: %w", err) diff --git a/coderd/x/chatd/chattest/anthropic.go b/coderd/x/chatd/chattest/anthropic.go index ba23571b8db..c88352b8ba8 100644 --- a/coderd/x/chatd/chattest/anthropic.go +++ b/coderd/x/chatd/chattest/anthropic.go @@ -31,6 +31,7 @@ type AnthropicRequest struct { Tools []AnthropicRequestTool `json:"tools,omitempty"` Stream bool `json:"stream,omitempty"` MaxTokens int `json:"max_tokens,omitempty"` + OutputConfig json.RawMessage `json:"output_config,omitempty"` // TODO: encoding/json ignores inline tags. Add custom UnmarshalJSON to capture unknown keys. Options map[string]interface{} `json:",inline"` //nolint:revive } diff --git a/coderd/x/chatd/compaction_override.go b/coderd/x/chatd/compaction_override.go index 9fc1b0b10f0..952e367d888 100644 --- a/coderd/x/chatd/compaction_override.go +++ b/coderd/x/chatd/compaction_override.go @@ -2,6 +2,7 @@ package chatd import ( "context" + "encoding/json" "charm.land/fantasy" "github.com/google/uuid" @@ -10,6 +11,7 @@ import ( "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/database/dbauthz" "github.com/coder/coder/v2/coderd/x/chatd/chatprovider" + "github.com/coder/coder/v2/codersdk" ) const compactionOverrideContext = "compaction" @@ -38,6 +40,9 @@ type compactionModelOverride struct { model fantasy.LanguageModel resolvedProvider string resolvedModel string + // providerOptions carry the override config's provider options and + // resolved reasoning effort for the summary call. + providerOptions fantasy.ProviderOptions } // resolveCompactionOverrideConfig resolves the stored deployment-wide @@ -119,10 +124,43 @@ func (p *Server) buildCompactionOverrideModel( err, ) } + providerOptions, err := compactionOverrideProviderOptions(model, modelConfig) + if err != nil { + return compactionModelOverride{}, err + } return compactionModelOverride{ modelConfig: modelConfig, model: model, resolvedProvider: resolvedProvider, resolvedModel: resolvedModel, + providerOptions: providerOptions, }, nil } + +// compactionOverrideProviderOptions converts the override config's call +// options (provider options plus the admin-resolved reasoning effort folded +// in by resolveCompactionOverrideConfig) into provider options for the +// summary call. +func compactionOverrideProviderOptions( + model fantasy.LanguageModel, + modelConfig database.ChatModelConfig, +) (fantasy.ProviderOptions, error) { + callConfig := codersdk.ChatModelCallConfig{} + if len(modelConfig.Options) > 0 { + if err := json.Unmarshal(modelConfig.Options, &callConfig); err != nil { + return nil, xerrors.Errorf( + "parse compaction model override call config: %w", + err, + ) + } + } + providerOptions := chatprovider.ProviderOptionsFromChatModelConfig( + model, + callConfig.ProviderOptions, + ) + reasoningEffort := chatprovider.ResolveReasoningEffort( + nil, + callConfig.ReasoningEffort, + ) + return chatprovider.ApplyReasoningEffort(model, providerOptions, reasoningEffort), nil +} diff --git a/coderd/x/chatd/compaction_override_internal_test.go b/coderd/x/chatd/compaction_override_internal_test.go index 6f1567af9d0..c61f4b8c43c 100644 --- a/coderd/x/chatd/compaction_override_internal_test.go +++ b/coderd/x/chatd/compaction_override_internal_test.go @@ -2,8 +2,10 @@ package chatd import ( "database/sql" + "encoding/json" "testing" + fantasyanthropic "charm.land/fantasy/providers/anthropic" "github.com/google/uuid" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" @@ -11,9 +13,48 @@ import ( "cdr.dev/slog/v3/sloggers/slogtest" "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/database/dbmock" + "github.com/coder/coder/v2/coderd/x/chatd/chattest" + "github.com/coder/coder/v2/codersdk" "github.com/coder/coder/v2/testutil" ) +func TestCompactionOverrideProviderOptions(t *testing.T) { + t.Parallel() + + model := &chattest.FakeModel{ProviderName: "anthropic", ModelName: "claude-3-5-haiku"} + + t.Run("NoOptions", func(t *testing.T) { + t.Parallel() + opts, err := compactionOverrideProviderOptions(model, database.ChatModelConfig{}) + require.NoError(t, err) + require.Nil(t, opts) + }) + + t.Run("ReasoningEffort", func(t *testing.T) { + t.Parallel() + effort := "low" + options, err := json.Marshal(codersdk.ChatModelCallConfig{ + ReasoningEffort: &codersdk.ChatModelReasoningEffortConfig{ + Default: &effort, + Max: &effort, + }, + }) + require.NoError(t, err) + opts, err := compactionOverrideProviderOptions(model, database.ChatModelConfig{Options: options}) + require.NoError(t, err) + anthropicOpts, ok := opts[fantasyanthropic.Name].(*fantasyanthropic.ProviderOptions) + require.True(t, ok) + require.NotNil(t, anthropicOpts.Effort) + require.Equal(t, fantasyanthropic.Effort("low"), *anthropicOpts.Effort) + }) + + t.Run("MalformedOptions", func(t *testing.T) { + t.Parallel() + _, err := compactionOverrideProviderOptions(model, database.ChatModelConfig{Options: []byte("{")}) + require.ErrorContains(t, err, "parse compaction model override call config") + }) +} + func TestResolveCompactionOverrideConfig_Unset(t *testing.T) { t.Parallel() diff --git a/coderd/x/chatd/generation.go b/coderd/x/chatd/generation.go index 91f233fabb1..917099e4955 100644 --- a/coderd/x/chatd/generation.go +++ b/coderd/x/chatd/generation.go @@ -705,6 +705,7 @@ func (s *taskStarter) generateCompaction( compactionOpts.ResolvedProvider = override.resolvedProvider compactionOpts.ResolvedModel = override.resolvedModel compactionOpts.ModelConfigID = override.modelConfig.ID + compactionOpts.ProviderOptions = override.providerOptions compactionOpts.Messages = sanitizeCompactionPrompt( ctx, logger, From 8e705c1717b69b91346c7f9e91170ae46313ee21 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Sun, 12 Jul 2026 13:29:11 +0000 Subject: [PATCH 09/12] refactor(coderd/x/chatd): rename compaction_prompt.go to compaction_sanitize.go The file never defined the compaction prompt; the summary prompt text lives in chatloop/compaction.go. It sanitizes the already-built prompt messages for the compaction override model, so name it after its entry point sanitizeCompactionPrompt and the chatsanitize package it wraps. --- coderd/x/chatd/{compaction_prompt.go => compaction_sanitize.go} | 0 ...ompt_internal_test.go => compaction_sanitize_internal_test.go} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename coderd/x/chatd/{compaction_prompt.go => compaction_sanitize.go} (100%) rename coderd/x/chatd/{compaction_prompt_internal_test.go => compaction_sanitize_internal_test.go} (100%) diff --git a/coderd/x/chatd/compaction_prompt.go b/coderd/x/chatd/compaction_sanitize.go similarity index 100% rename from coderd/x/chatd/compaction_prompt.go rename to coderd/x/chatd/compaction_sanitize.go diff --git a/coderd/x/chatd/compaction_prompt_internal_test.go b/coderd/x/chatd/compaction_sanitize_internal_test.go similarity index 100% rename from coderd/x/chatd/compaction_prompt_internal_test.go rename to coderd/x/chatd/compaction_sanitize_internal_test.go From 458669c5704c8229fdd515bf0a7153ab6ba1b50b Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Mon, 13 Jul 2026 18:50:40 +0000 Subject: [PATCH 10/12] fix(coderd/x/chatd): attribute still-over-limit compaction metric to the override model The still-over-limit terminal error is recorded at decision time, before the compact action builds the override client, so it used the chat model labels from the prepared options while the compact action's own events used the built override client's labels. A failing override compaction then split across two metric series: success on the override model, error on the chat model. Prepare-time override resolution already computes the normalized provider and validates ResolveModelWithProviderHint, so keep that identity in resolvedCompactionOverride and source every compaction metric label from one compactionMetricIdentity helper. --- coderd/x/chatd/ARCHITECTURE.md | 2 +- coderd/x/chatd/compaction_override.go | 53 +++++++++++++++---- .../compaction_override_internal_test.go | 37 +++++++------ coderd/x/chatd/generation.go | 50 ++++++++++------- coderd/x/chatd/generation_internal_test.go | 27 ++++++++++ coderd/x/chatd/generation_preparer.go | 10 ++-- coderd/x/chatd/subagent.go | 26 ++++----- coderd/x/chatd/subagent_internal_test.go | 2 +- coderd/x/chatd/title_override.go | 2 +- 9 files changed, 144 insertions(+), 65 deletions(-) diff --git a/coderd/x/chatd/ARCHITECTURE.md b/coderd/x/chatd/ARCHITECTURE.md index 62040cd6e03..9cea5ce8475 100644 --- a/coderd/x/chatd/ARCHITECTURE.md +++ b/coderd/x/chatd/ARCHITECTURE.md @@ -842,7 +842,7 @@ Details that follow from the override: - Context limits: the compaction trigger uses the stricter of the chat model's and the compaction model's context limits, because the history must also fit the summarizer's window. The post-compaction "still over limit" check stays against the chat model's limit, since continuation runs on the chat model. - Failure semantics: an unset override uses the chat model. Stale or malformed stored references (deleted or disabled config or provider, missing credentials, non-UUID value) fall back to the chat model with a log. A usable override that fails at use (route or client construction, provider call failure) fails the generation visibly through the normal error path; there is no silent fallback. The override model client is constructed inside the compact generation action, not at prepare time, so a broken override cannot fail turns that finish without compacting (including turns over the threshold whose last assistant step already completed). - Prompt safety: the prompt is built and sanitized for the chat model, so when the override points at a different provider the compaction copy of the prompt is re-sanitized: provider-executed tool history is flattened into plain text parts (keeping its content while dropping the provider-specific wire shape), file parts the compaction model rejects are replaced with text placeholders, and Anthropic provider-tool sanitization is re-run for the compaction provider. The assistant generation prompt is never mutated. -- Observability: compaction metrics and chat debug runs record the provider and model that actually generated the summary. +- Observability: compaction metrics and chat debug runs record the provider and model that actually generated the summary. This includes the "still over limit" terminal error, which is recorded before the override client is built: prepare-time resolution keeps the override's provider/model identity so that error lands on the same metric series as the compact action's own events. #### Interrupt goroutine diff --git a/coderd/x/chatd/compaction_override.go b/coderd/x/chatd/compaction_override.go index 952e367d888..1e62faf0dde 100644 --- a/coderd/x/chatd/compaction_override.go +++ b/coderd/x/chatd/compaction_override.go @@ -45,27 +45,44 @@ type compactionModelOverride struct { providerOptions fantasy.ProviderOptions } +// resolvedCompactionOverride is the outcome of resolving the compaction +// model override at prepare time: the override config plus the +// provider/model identity resolved without building the model client. The +// identity lets error paths that never build the client (for example the +// still-over-limit terminal error) attribute metrics to the same model as +// events recorded by the compact action after the client is built. +type resolvedCompactionOverride struct { + Config database.ChatModelConfig + // ResolvedProvider and ResolvedModel match the identity + // buildCompactionOverrideModel resolves for the built client: + // ResolveModelWithProviderHint normalizes its hint, so the normalized + // provider name and the route's provider-type hint yield the same + // result. + ResolvedProvider string + ResolvedModel string +} + // resolveCompactionOverrideConfig resolves the stored deployment-wide // compaction model override to its model config. Unset, malformed, stale // (deleted or disabled config or provider), and credential-less overrides -// fall back to the chat model (overrideSet is false; the shared resolver -// logs the reason). This runs on every generation prepare because the -// override's context limit feeds the compaction trigger; the model client -// is built separately by buildCompactionOverrideModel only when compaction -// actually runs. +// fall back to the chat model (nil override; the shared resolver logs the +// reason). This runs on every generation prepare because the override's +// context limit feeds the compaction trigger; the model client is built +// separately by buildCompactionOverrideModel only when compaction actually +// runs. func (p *Server) resolveCompactionOverrideConfig( ctx context.Context, chat database.Chat, -) (database.ChatModelConfig, bool, error) { +) (*resolvedCompactionOverride, error) { raw, err := readCompactionModelOverride(ctx, p.db) if err != nil { - return database.ChatModelConfig{}, false, xerrors.Errorf( + return nil, xerrors.Errorf( "read compaction model override: %w", err, ) } - modelConfig, overrideEffort, overrideSet, err := p.resolveConfiguredModelOverride( + modelConfig, providerName, overrideEffort, overrideSet, err := p.resolveConfiguredModelOverride( ctx, compactionOverrideContext, raw, @@ -77,9 +94,25 @@ func (p *Server) resolveCompactionOverrideConfig( modelOverrideFailureModeSoft, ) if err != nil || !overrideSet { - return database.ChatModelConfig{}, overrideSet, err + return nil, err + } + // The shared resolver already validated this resolution, so an error + // here is unreachable with the same inputs. + resolvedProvider, resolvedModel, err := chatprovider.ResolveModelWithProviderHint( + modelConfig.Model, + providerName, + ) + if err != nil { + return nil, xerrors.Errorf( + "resolve compaction model override identity: %w", + err, + ) } - return withResolvedReasoningEffort(modelConfig, overrideEffort), true, nil + return &resolvedCompactionOverride{ + Config: withResolvedReasoningEffort(modelConfig, overrideEffort), + ResolvedProvider: resolvedProvider, + ResolvedModel: resolvedModel, + }, nil } // buildCompactionOverrideModel resolves the route and constructs the model diff --git a/coderd/x/chatd/compaction_override_internal_test.go b/coderd/x/chatd/compaction_override_internal_test.go index c61f4b8c43c..6e1338ff79d 100644 --- a/coderd/x/chatd/compaction_override_internal_test.go +++ b/coderd/x/chatd/compaction_override_internal_test.go @@ -67,9 +67,9 @@ func TestResolveCompactionOverrideConfig_Unset(t *testing.T) { db.EXPECT().GetChatCompactionModelOverride(gomock.Any()).Return("", nil) server := titleOverrideTestServer(db, logger) - _, overrideSet, err := server.resolveCompactionOverrideConfig(ctx, chat) + override, err := server.resolveCompactionOverrideConfig(ctx, chat) require.NoError(t, err) - require.False(t, overrideSet) + require.Nil(t, override) } func TestResolveCompactionOverrideConfig_ReadDBError(t *testing.T) { @@ -84,10 +84,10 @@ func TestResolveCompactionOverrideConfig_ReadDBError(t *testing.T) { db.EXPECT().GetChatCompactionModelOverride(gomock.Any()).Return("", sql.ErrConnDone) server := titleOverrideTestServer(db, logger) - _, overrideSet, err := server.resolveCompactionOverrideConfig(ctx, chat) + override, err := server.resolveCompactionOverrideConfig(ctx, chat) require.Error(t, err) require.ErrorContains(t, err, "read compaction model override") - require.False(t, overrideSet) + require.Nil(t, override) } func TestResolveCompactionOverrideConfig_MalformedFallsBack(t *testing.T) { @@ -102,9 +102,9 @@ func TestResolveCompactionOverrideConfig_MalformedFallsBack(t *testing.T) { db.EXPECT().GetChatCompactionModelOverride(gomock.Any()).Return("not-a-uuid", nil) server := titleOverrideTestServer(db, logger) - _, overrideSet, err := server.resolveCompactionOverrideConfig(ctx, chat) + override, err := server.resolveCompactionOverrideConfig(ctx, chat) require.NoError(t, err) - require.False(t, overrideSet) + require.Nil(t, override) } func TestResolveCompactionOverrideConfig_DeletedConfigFallsBack(t *testing.T) { @@ -121,9 +121,9 @@ func TestResolveCompactionOverrideConfig_DeletedConfigFallsBack(t *testing.T) { db.EXPECT().GetChatModelConfigByID(gomock.Any(), missingID).Return(database.ChatModelConfig{}, sql.ErrNoRows) server := titleOverrideTestServer(db, logger) - _, overrideSet, err := server.resolveCompactionOverrideConfig(ctx, chat) + override, err := server.resolveCompactionOverrideConfig(ctx, chat) require.NoError(t, err) - require.False(t, overrideSet) + require.Nil(t, override) } func TestResolveCompactionOverrideConfig_DisabledConfigFallsBack(t *testing.T) { @@ -140,9 +140,9 @@ func TestResolveCompactionOverrideConfig_DisabledConfigFallsBack(t *testing.T) { db.EXPECT().GetChatModelConfigByID(gomock.Any(), overrideConfig.ID).Return(overrideConfig, nil) server := titleOverrideTestServer(db, logger) - _, overrideSet, err := server.resolveCompactionOverrideConfig(ctx, chat) + override, err := server.resolveCompactionOverrideConfig(ctx, chat) require.NoError(t, err) - require.False(t, overrideSet) + require.Nil(t, override) } func TestResolveCompactionOverrideConfig_MissingCredentialsFallsBack(t *testing.T) { @@ -167,9 +167,9 @@ func TestResolveCompactionOverrideConfig_MissingCredentialsFallsBack(t *testing. db.EXPECT().GetAIProviderKeysByProviderID(gomock.Any(), providerID).Return(nil, nil).AnyTimes() server := titleOverrideTestServer(db, logger) - _, overrideSet, err := server.resolveCompactionOverrideConfig(ctx, chat) + override, err := server.resolveCompactionOverrideConfig(ctx, chat) require.NoError(t, err) - require.False(t, overrideSet) + require.Nil(t, override) } func TestCompactionOverride_SetUsable(t *testing.T) { @@ -193,15 +193,15 @@ func TestCompactionOverride_SetUsable(t *testing.T) { }}, nil).AnyTimes() server := titleOverrideTestServer(db, logger) - resolvedConfig, overrideSet, err := server.resolveCompactionOverrideConfig(ctx, chat) + resolved, err := server.resolveCompactionOverrideConfig(ctx, chat) require.NoError(t, err) - require.True(t, overrideSet) - require.Equal(t, overrideConfig.ID, resolvedConfig.ID) + require.NotNil(t, resolved) + require.Equal(t, overrideConfig.ID, resolved.Config.ID) override, err := server.buildCompactionOverrideModel( ctx, chat, - resolvedConfig, + resolved.Config, modelBuildOptions{ActiveAPIKeyID: uuid.NewString()}, ) require.NoError(t, err) @@ -209,4 +209,9 @@ func TestCompactionOverride_SetUsable(t *testing.T) { require.Equal(t, overrideConfig.ID, override.modelConfig.ID) require.Equal(t, "openai", override.resolvedProvider) require.Equal(t, "gpt-4.1", override.resolvedModel) + // The prepare-time identity must match the built client's identity so + // metrics recorded before the client is built (still-over-limit) land + // on the same series as metrics recorded by the compact action. + require.Equal(t, override.resolvedProvider, resolved.ResolvedProvider) + require.Equal(t, override.resolvedModel, resolved.ResolvedModel) } diff --git a/coderd/x/chatd/generation.go b/coderd/x/chatd/generation.go index 917099e4955..1e4a97dc77c 100644 --- a/coderd/x/chatd/generation.go +++ b/coderd/x/chatd/generation.go @@ -69,13 +69,13 @@ type generationPrepared struct { // generationCompaction contains compaction inputs prepared for generation. type generationCompaction struct { - // OverrideConfig, when non-nil, is the resolved compaction model - // override config. The override model client is built (and the prompt + // Override, when non-nil, is the compaction model override resolved at + // prepare time. The override model client is built (and the prompt // sanitized for it) in the compact action path, not at prepare time, // so hard construction failures cannot fail turns that finish without // compacting. ChatModelConfig is the chat model's config, needed to // sanitize the prompt across provider boundaries. - OverrideConfig *database.ChatModelConfig + Override *resolvedCompactionOverride ChatModelConfig database.ChatModelConfig Required bool @@ -342,9 +342,10 @@ func (s *taskStarter) StartGeneration(ctx context.Context, input chatWorkerTaskS return xerrors.Errorf("decide generation: %w", err) } if errors.Is(err, errCompactionStillOverLimit) && prepared.Compaction != nil { + metricProvider, metricModel := compactionMetricIdentity(prepared.Compaction) s.server.metrics.RecordCompaction( - compactionProvider(prepared.Compaction.Options), - compactionModel(prepared.Compaction.Options), + metricProvider, + metricModel, false, errCompactionStillOverLimit, ) @@ -692,8 +693,9 @@ func (s *taskStarter) generateCompaction( return s.finishGenerationError(ctx, machine, input, xerrors.New("compaction action missing options"), requireGenerationAttempt(attempt.number)) } compactionOpts := prepared.Compaction.Options - if overrideConfig := prepared.Compaction.OverrideConfig; overrideConfig != nil { - override, err := s.server.buildCompactionOverrideModel(ctx, prepared.Chat, *overrideConfig, prepared.ModelBuildOptions) + metricProvider, metricModel := compactionMetricIdentity(prepared.Compaction) + if override := prepared.Compaction.Override; override != nil { + overrideModel, err := s.server.buildCompactionOverrideModel(ctx, prepared.Chat, override.Config, prepared.ModelBuildOptions) if err != nil { return xerrors.Errorf("build compaction model override: %w", err) } @@ -701,18 +703,18 @@ func (s *taskStarter) generateCompaction( slog.F("chat_id", prepared.Chat.ID), slog.F("owner_id", prepared.Chat.OwnerID), ) - compactionOpts.Model = override.model - compactionOpts.ResolvedProvider = override.resolvedProvider - compactionOpts.ResolvedModel = override.resolvedModel - compactionOpts.ModelConfigID = override.modelConfig.ID - compactionOpts.ProviderOptions = override.providerOptions + compactionOpts.Model = overrideModel.model + compactionOpts.ResolvedProvider = overrideModel.resolvedProvider + compactionOpts.ResolvedModel = overrideModel.resolvedModel + compactionOpts.ModelConfigID = overrideModel.modelConfig.ID + compactionOpts.ProviderOptions = overrideModel.providerOptions compactionOpts.Messages = sanitizeCompactionPrompt( ctx, logger, compactionOpts.Messages, - override.model, + overrideModel.model, prepared.Compaction.ChatModelConfig, - override.modelConfig, + overrideModel.modelConfig, ) } compactionOpts.PublishMessagePart = attempt.publish @@ -722,12 +724,12 @@ func (s *taskStarter) generateCompaction( runCtx := input.DebugTurn.Ensure(ctx, prepared.Chat, prepared.Debug) outcome, err := chatloop.GenerateCompaction(runCtx, compactionOpts) if err != nil { - s.server.metrics.RecordCompaction(compactionProvider(compactionOpts), compactionModel(compactionOpts), false, err) + s.server.metrics.RecordCompaction(metricProvider, metricModel, false, err) return xerrors.Errorf("generate compaction: %w", err) } if strings.TrimSpace(outcome.SystemSummary) == "" || strings.TrimSpace(outcome.SummaryReport) == "" { err := xerrors.New("compaction produced no summary") - s.server.metrics.RecordCompaction(compactionProvider(compactionOpts), compactionModel(compactionOpts), false, err) + s.server.metrics.RecordCompaction(metricProvider, metricModel, false, err) return s.finishGenerationError(ctx, machine, input, err, requireGenerationAttempt(attempt.number)) } messages, err := buildCompactionMessages(buildCompactionMessagesInput{ @@ -739,20 +741,32 @@ func (s *taskStarter) generateCompaction( contentVersion: chatprompt.CurrentContentVersion, }) if err != nil { - s.server.metrics.RecordCompaction(compactionProvider(compactionOpts), compactionModel(compactionOpts), false, err) + s.server.metrics.RecordCompaction(metricProvider, metricModel, false, err) return s.finishGenerationError(ctx, machine, input, err, requireGenerationAttempt(attempt.number)) } err = s.commitGenerationStep(ctx, machine, input, attempt.number, generationActionCompact, stepMessagesForCommit{ Messages: messages.Messages, VisibleIndexes: visibleMessageIndexes(messages.Messages), }) - s.server.metrics.RecordCompaction(compactionProvider(compactionOpts), compactionModel(compactionOpts), err == nil, err) + s.server.metrics.RecordCompaction(metricProvider, metricModel, err == nil, err) if err != nil { return xerrors.Errorf("commit generation step: %w", err) } return nil } +// compactionMetricIdentity returns the provider/model labels for compaction +// metrics. When an override is configured the labels come from the identity +// resolved at prepare time, so events recorded before the override client +// is built (the still-over-limit terminal error) carry the same labels as +// events recorded by the compact action after it swaps in the built client. +func compactionMetricIdentity(compaction *generationCompaction) (provider, model string) { + if compaction.Override != nil { + return compaction.Override.ResolvedProvider, compaction.Override.ResolvedModel + } + return compactionProvider(compaction.Options), compactionModel(compaction.Options) +} + func compactionProvider(opts chatloop.GenerateCompactionOptions) string { if opts.Model == nil { return "" diff --git a/coderd/x/chatd/generation_internal_test.go b/coderd/x/chatd/generation_internal_test.go index 9090506b671..12b196620d1 100644 --- a/coderd/x/chatd/generation_internal_test.go +++ b/coderd/x/chatd/generation_internal_test.go @@ -7,10 +7,37 @@ import ( "golang.org/x/xerrors" "github.com/coder/coder/v2/coderd/x/chatd/chatdebug" + "github.com/coder/coder/v2/coderd/x/chatd/chatloop" "github.com/coder/coder/v2/coderd/x/chatd/chatstate" + "github.com/coder/coder/v2/coderd/x/chatd/chattest" "github.com/coder/coder/v2/testutil" ) +func TestCompactionMetricIdentity(t *testing.T) { + t.Parallel() + + compaction := &generationCompaction{ + Options: chatloop.GenerateCompactionOptions{ + Model: &chattest.FakeModel{ProviderName: "anthropic", ModelName: "claude-sonnet-4-5"}, + }, + } + + provider, model := compactionMetricIdentity(compaction) + require.Equal(t, "anthropic", provider) + require.Equal(t, "claude-sonnet-4-5", model) + + // With an override configured, metrics use the identity resolved at + // prepare time instead of the chat model carried by the options, even + // before the override client is built. + compaction.Override = &resolvedCompactionOverride{ + ResolvedProvider: "openai", + ResolvedModel: "gpt-4.1-mini", + } + provider, model = compactionMetricIdentity(compaction) + require.Equal(t, "openai", provider) + require.Equal(t, "gpt-4.1-mini", model) +} + func TestRecordGenerationFinishFailure(t *testing.T) { t.Parallel() diff --git a/coderd/x/chatd/generation_preparer.go b/coderd/x/chatd/generation_preparer.go index 112a6806099..96010540606 100644 --- a/coderd/x/chatd/generation_preparer.go +++ b/coderd/x/chatd/generation_preparer.go @@ -586,15 +586,13 @@ func (server *Server) prepareGeneration( // using the chat model's limit (ContextLimitFallback below), since the // follow-up assistant generation runs on the chat model. compactionContextLimit := modelConfig.ContextLimit - compactionOverrideConfig, compactionOverrideSet, err := server.resolveCompactionOverrideConfig(ctx, chat) + compactionOverride, err := server.resolveCompactionOverrideConfig(ctx, chat) if err != nil { cleanup() return generationPrepared{}, err } - var compactionOverride *database.ChatModelConfig - if compactionOverrideSet { - compactionOverride = &compactionOverrideConfig - if overrideLimit := compactionOverrideConfig.ContextLimit; overrideLimit > 0 && + if compactionOverride != nil { + if overrideLimit := compactionOverride.Config.ContextLimit; overrideLimit > 0 && (compactionContextLimit <= 0 || overrideLimit < compactionContextLimit) { compactionContextLimit = overrideLimit } @@ -654,7 +652,7 @@ func (server *Server) prepareGeneration( ToolNameToConfigID: toolNameToConfigID, MaxSteps: maxChatSteps, Compaction: &generationCompaction{ - OverrideConfig: compactionOverride, + Override: compactionOverride, ChatModelConfig: modelConfig, Required: compactionNeeded, Options: compactionOptions, diff --git a/coderd/x/chatd/subagent.go b/coderd/x/chatd/subagent.go index ccb7e1a26d3..852dab02166 100644 --- a/coderd/x/chatd/subagent.go +++ b/coderd/x/chatd/subagent.go @@ -184,6 +184,8 @@ func modelOverrideErrorLabel(overrideContext string) string { // resolveConfiguredModelOverride returns ok when a usable override is // resolved. In hard failure mode, ok is also true for configured but unusable // overrides so callers can distinguish them from unset or malformed values. +// providerName is the normalized provider name of the resolved config; it is +// only meaningful on the usable-override return. func (p *Server) resolveConfiguredModelOverride( ctx context.Context, overrideContext string, @@ -192,7 +194,7 @@ func (p *Server) resolveConfiguredModelOverride( resolveModelConfig modelOverrideConfigResolver, resolveProviderKeys modelOverrideProviderKeysResolver, failureMode modelOverrideFailureMode, -) (database.ChatModelConfig, *string, bool, error) { +) (database.ChatModelConfig, string, *string, bool, error) { parsed, ok := parseModelOverride(raw) if !ok { p.logger.Info(ctx, @@ -200,10 +202,10 @@ func (p *Server) resolveConfiguredModelOverride( slog.F("override_context", overrideContext), slog.F("raw_model_config_id", strings.TrimSpace(raw)), ) - return database.ChatModelConfig{}, nil, false, nil + return database.ChatModelConfig{}, "", nil, false, nil } if parsed.modelConfigID == uuid.Nil { - return database.ChatModelConfig{}, nil, false, nil + return database.ChatModelConfig{}, "", nil, false, nil } modelConfig, providerName, err := resolveModelConfig( @@ -215,20 +217,20 @@ func (p *Server) resolveConfiguredModelOverride( label := modelOverrideErrorLabel(overrideContext) switch { case errors.Is(err, sql.ErrNoRows): - return database.ChatModelConfig{}, parsed.reasoningEffort, true, xerrors.Errorf( + return database.ChatModelConfig{}, "", parsed.reasoningEffort, true, xerrors.Errorf( "%s model override is unavailable: %s", label, parsed.modelConfigID, ) case errors.Is(err, errInvalidModelOverrideMetadata): - return database.ChatModelConfig{}, parsed.reasoningEffort, true, xerrors.Errorf( + return database.ChatModelConfig{}, "", parsed.reasoningEffort, true, xerrors.Errorf( "%s model override metadata is invalid for %s: %w", label, parsed.modelConfigID, err, ) default: - return database.ChatModelConfig{}, parsed.reasoningEffort, true, xerrors.Errorf( + return database.ChatModelConfig{}, "", parsed.reasoningEffort, true, xerrors.Errorf( "resolve %s model override %s: %w", label, parsed.modelConfigID, @@ -259,19 +261,19 @@ func (p *Server) resolveConfiguredModelOverride( slog.Error(err), ) } - return database.ChatModelConfig{}, nil, false, nil + return database.ChatModelConfig{}, "", nil, false, nil } providerKeys, err := resolveProviderKeys(ctx, ownerID, modelConfigAIProviderID(modelConfig)) if err != nil { - return database.ChatModelConfig{}, nil, false, xerrors.Errorf( + return database.ChatModelConfig{}, "", nil, false, xerrors.Errorf( "resolve provider API keys: %w", err, ) } if !userCanUseProviderKeys(providerKeys, providerName) { if failureMode == modelOverrideFailureModeHard { - return database.ChatModelConfig{}, parsed.reasoningEffort, true, xerrors.Errorf( + return database.ChatModelConfig{}, "", parsed.reasoningEffort, true, xerrors.Errorf( "%s model override credentials are unavailable for provider %q", modelOverrideErrorLabel(overrideContext), providerName, @@ -284,9 +286,9 @@ func (p *Server) resolveConfiguredModelOverride( slog.F("model_config_id", parsed.modelConfigID), slog.F("provider", providerName), ) - return database.ChatModelConfig{}, nil, false, nil + return database.ChatModelConfig{}, "", nil, false, nil } - return modelConfig, parsed.reasoningEffort, true, nil + return modelConfig, providerName, parsed.reasoningEffort, true, nil } func (p *Server) resolvePersonalSubagentModelConfigID( @@ -483,7 +485,7 @@ func (p *Server) resolveSubagentModelConfigID( err, ) } - modelConfig, reasoningEffort, ok, err := p.resolveConfiguredModelOverride( + modelConfig, _, reasoningEffort, ok, err := p.resolveConfiguredModelOverride( chatdCtx, string(overrideContext), raw, diff --git a/coderd/x/chatd/subagent_internal_test.go b/coderd/x/chatd/subagent_internal_test.go index 9ea4d057ad7..ada9354c3da 100644 --- a/coderd/x/chatd/subagent_internal_test.go +++ b/coderd/x/chatd/subagent_internal_test.go @@ -1490,7 +1490,7 @@ func TestResolveConfiguredModelOverride_AcceptsAmbientCredentialsProvider( Enabled: true, } - resolvedModelConfig, reasoningEffort, ok, err := server.resolveConfiguredModelOverride( + resolvedModelConfig, _, reasoningEffort, ok, err := server.resolveConfiguredModelOverride( ctx, "plan", modelConfig.ID.String(), diff --git a/coderd/x/chatd/title_override.go b/coderd/x/chatd/title_override.go index 0072587498a..a4cfb4de8f7 100644 --- a/coderd/x/chatd/title_override.go +++ b/coderd/x/chatd/title_override.go @@ -70,7 +70,7 @@ func (p *Server) resolveTitleGenerationModelOverride( ) } - modelConfig, overrideEffort, overrideSet, err := p.resolveConfiguredModelOverride( + modelConfig, _, overrideEffort, overrideSet, err := p.resolveConfiguredModelOverride( ctx, titleGenerationOverrideContext, raw, From e4a684d89b26ee44dbdbb5a00bf2aaf684e38982 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Tue, 14 Jul 2026 08:27:50 +0000 Subject: [PATCH 11/12] refactor(coderd/x/chatd): tighten compaction override comments --- coderd/x/chatd/chatloop/chatloop.go | 9 ++- coderd/x/chatd/chatloop/compaction.go | 8 +-- coderd/x/chatd/compaction_override.go | 55 ++++++++----------- .../compaction_override_internal_test.go | 5 +- coderd/x/chatd/compaction_sanitize.go | 41 +++++--------- coderd/x/chatd/generation.go | 18 +++--- coderd/x/chatd/generation_internal_test.go | 5 +- coderd/x/chatd/generation_preparer.go | 15 ++--- coderd/x/chatd/subagent.go | 3 +- 9 files changed, 62 insertions(+), 97 deletions(-) diff --git a/coderd/x/chatd/chatloop/chatloop.go b/coderd/x/chatd/chatloop/chatloop.go index 24988c357c1..d1166b0fc96 100644 --- a/coderd/x/chatd/chatloop/chatloop.go +++ b/coderd/x/chatd/chatloop/chatloop.go @@ -281,15 +281,14 @@ type GenerateCompactionOptions struct { ToolName string // ResolvedProvider, ResolvedModel, and ModelConfigID identify the - // model that actually generates the summary, which can differ from - // the chat model when a compaction model override is configured. - // Debug runs record these instead of the parent run's identity. + // summary model, which can differ from the chat model when a + // compaction override is configured. Debug runs record these. ResolvedProvider string ResolvedModel string ModelConfigID uuid.UUID - // ProviderOptions are provider-specific call options for the summary - // model, such as a compaction override's configured reasoning effort. + // ProviderOptions carry summary-model call options such as an + // override's reasoning effort. ProviderOptions fantasy.ProviderOptions PublishMessagePart func(codersdk.ChatMessageRole, codersdk.ChatMessagePart) diff --git a/coderd/x/chatd/chatloop/compaction.go b/coderd/x/chatd/chatloop/compaction.go index e0e2f89d44b..5ba9918c44d 100644 --- a/coderd/x/chatd/chatloop/compaction.go +++ b/coderd/x/chatd/chatloop/compaction.go @@ -64,8 +64,7 @@ type CompactionOptions struct { ChatID uuid.UUID HistoryTipMessageID int64 - // ResolvedProvider, ResolvedModel, ModelConfigID, and - // ProviderOptions describe the model generating the summary; see + // Summary model identity and call options; see // GenerateCompactionOptions. ResolvedProvider string ResolvedModel string @@ -288,9 +287,8 @@ func startCompactionDebugRun( historyTipMessageID = parentRun.HistoryTipMessageID } - // The compaction model can differ from the chat model when a - // deployment override is configured, so record the summary model's - // identity when the caller supplies it. + // Prefer the caller-supplied summary model identity; it can differ + // from the parent run's chat model under a compaction override. provider := parentRun.Provider if options.ResolvedProvider != "" { provider = options.ResolvedProvider diff --git a/coderd/x/chatd/compaction_override.go b/coderd/x/chatd/compaction_override.go index 1e62faf0dde..0e3d4d48604 100644 --- a/coderd/x/chatd/compaction_override.go +++ b/coderd/x/chatd/compaction_override.go @@ -32,44 +32,37 @@ func readCompactionModelOverride( return raw, nil } -// compactionModelOverride carries the resolved deployment-wide compaction -// model override: the model to run compaction summaries with plus the -// identity metadata debug runs and prompt sanitization need. +// compactionModelOverride carries the built compaction override model plus +// the identity metadata debug runs and prompt sanitization need. type compactionModelOverride struct { modelConfig database.ChatModelConfig model fantasy.LanguageModel resolvedProvider string resolvedModel string - // providerOptions carry the override config's provider options and - // resolved reasoning effort for the summary call. + // providerOptions include the override's reasoning effort for the + // summary call. providerOptions fantasy.ProviderOptions } -// resolvedCompactionOverride is the outcome of resolving the compaction -// model override at prepare time: the override config plus the -// provider/model identity resolved without building the model client. The -// identity lets error paths that never build the client (for example the -// still-over-limit terminal error) attribute metrics to the same model as -// events recorded by the compact action after the client is built. +// resolvedCompactionOverride is the compaction override resolved at +// prepare time. The provider/model identity is resolved without building +// the model client so metrics recorded before the client exists +// (still-over-limit) attribute to the same model as the compact action's. type resolvedCompactionOverride struct { Config database.ChatModelConfig - // ResolvedProvider and ResolvedModel match the identity - // buildCompactionOverrideModel resolves for the built client: - // ResolveModelWithProviderHint normalizes its hint, so the normalized - // provider name and the route's provider-type hint yield the same - // result. + // ResolvedProvider and ResolvedModel match the built client's + // identity: ResolveModelWithProviderHint normalizes its hint, so the + // normalized provider name here and the route's raw provider type in + // buildCompactionOverrideModel yield the same result. ResolvedProvider string ResolvedModel string } // resolveCompactionOverrideConfig resolves the stored deployment-wide -// compaction model override to its model config. Unset, malformed, stale -// (deleted or disabled config or provider), and credential-less overrides -// fall back to the chat model (nil override; the shared resolver logs the -// reason). This runs on every generation prepare because the override's -// context limit feeds the compaction trigger; the model client is built -// separately by buildCompactionOverrideModel only when compaction actually -// runs. +// compaction model override. Unset, malformed, stale, and credential-less +// overrides fall back to the chat model (nil override). This runs on every +// generation prepare because the override's context limit feeds the +// compaction trigger; the model client is built only when compaction runs. func (p *Server) resolveCompactionOverrideConfig( ctx context.Context, chat database.Chat, @@ -96,8 +89,7 @@ func (p *Server) resolveCompactionOverrideConfig( if err != nil || !overrideSet { return nil, err } - // The shared resolver already validated this resolution, so an error - // here is unreachable with the same inputs. + // Already validated by the shared resolver; failure is unreachable. resolvedProvider, resolvedModel, err := chatprovider.ResolveModelWithProviderHint( modelConfig.Model, providerName, @@ -116,11 +108,9 @@ func (p *Server) resolveCompactionOverrideConfig( } // buildCompactionOverrideModel resolves the route and constructs the model -// client for a usable compaction override config. Errors are hard failures: -// a configured, usable override that cannot be routed or constructed must -// fail the generation visibly instead of silently compacting with the chat -// model. Callers invoke this from the compact generation action, so a -// broken override cannot fail turns that finish without compacting. +// client for a usable override config. Errors are hard failures: a usable +// override that cannot be constructed must fail the generation visibly +// instead of silently compacting with the chat model. func (p *Server) buildCompactionOverrideModel( ctx context.Context, chat database.Chat, @@ -171,9 +161,8 @@ func (p *Server) buildCompactionOverrideModel( } // compactionOverrideProviderOptions converts the override config's call -// options (provider options plus the admin-resolved reasoning effort folded -// in by resolveCompactionOverrideConfig) into provider options for the -// summary call. +// options, including the admin-resolved reasoning effort, into provider +// options for the summary call. func compactionOverrideProviderOptions( model fantasy.LanguageModel, modelConfig database.ChatModelConfig, diff --git a/coderd/x/chatd/compaction_override_internal_test.go b/coderd/x/chatd/compaction_override_internal_test.go index 6e1338ff79d..9267c84446d 100644 --- a/coderd/x/chatd/compaction_override_internal_test.go +++ b/coderd/x/chatd/compaction_override_internal_test.go @@ -209,9 +209,8 @@ func TestCompactionOverride_SetUsable(t *testing.T) { require.Equal(t, overrideConfig.ID, override.modelConfig.ID) require.Equal(t, "openai", override.resolvedProvider) require.Equal(t, "gpt-4.1", override.resolvedModel) - // The prepare-time identity must match the built client's identity so - // metrics recorded before the client is built (still-over-limit) land - // on the same series as metrics recorded by the compact action. + // Prepare-time identity must match the built client's so + // still-over-limit metrics land on the same series. require.Equal(t, override.resolvedProvider, resolved.ResolvedProvider) require.Equal(t, override.resolvedModel, resolved.ResolvedModel) } diff --git a/coderd/x/chatd/compaction_sanitize.go b/coderd/x/chatd/compaction_sanitize.go index 17014efe5e8..9f4676b29f5 100644 --- a/coderd/x/chatd/compaction_sanitize.go +++ b/coderd/x/chatd/compaction_sanitize.go @@ -12,20 +12,17 @@ import ( "github.com/coder/coder/v2/coderd/x/chatd/chatsanitize" ) -// sameCompactionProviderIdentity reports whether the chat model and the -// compaction override model are backed by the same provider instance. -// Legacy configs without an AIProviderID compare as different (fail -// closed), so cross-provider stripping applies. +// sameCompactionProviderIdentity reports whether the chat and compaction +// override models share a provider instance. Configs without an +// AIProviderID compare as different (fail closed). func sameCompactionProviderIdentity(chatConfig, overrideConfig database.ChatModelConfig) bool { return chatConfig.AIProviderID.Valid && overrideConfig.AIProviderID.Valid && chatConfig.AIProviderID.UUID == overrideConfig.AIProviderID.UUID } -// sanitizeCompactionPrompt adapts the chat prompt for a compaction model -// that differs from the chat model. The prompt was built for the chat -// model, so provider-executed tool history and file parts the compaction -// provider rejects must not replay to it. The input messages are never -// mutated; the assistant generation keeps using the original prompt. +// sanitizeCompactionPrompt adapts a prompt built for the chat model to a +// differing compaction model. The input messages are never mutated; the +// assistant generation keeps using the original prompt. func sanitizeCompactionPrompt( ctx context.Context, logger slog.Logger, @@ -61,18 +58,11 @@ func sanitizeCompactionPrompt( } // flattenProviderExecutedToolParts rewrites provider-executed tool calls -// and results in assistant messages into plain text parts on a copy of -// messages. Provider-executed blocks carry provider-specific wire shapes and -// ids (for example Anthropic srvtoolu_... ids) that other providers reject -// on replay, but their content is still useful to the summary, so it is kept -// as text instead of dropped. Mirrors flattenProviderExecutedToolParts in -// coder/mux. -// -// Assistant messages are the only legitimate home for provider-executed -// parts (chatsanitize flags other roles as violations). Anomalous parts in -// other roles are dropped rather than flattened, because a text part is not -// valid tool-message content for every provider; messages emptied by that -// drop are removed. +// and results in assistant messages into text parts on a copy of messages, +// keeping their content while shedding provider-specific wire shapes other +// providers reject on replay. Provider-executed parts outside assistant +// messages are anomalous and dropped, since a text part is not valid +// tool-message content everywhere; messages emptied by the drop are removed. func flattenProviderExecutedToolParts( ctx context.Context, logger slog.Logger, @@ -135,9 +125,8 @@ func flattenProviderExecutedToolParts( return out } -// stringifyToolResultOutput renders a tool result output as prompt text. -// Media payloads are summarized instead of inlined so base64 data does not -// enter the compaction prompt as text. +// stringifyToolResultOutput renders a tool result as prompt text. Media +// payloads are summarized so base64 data does not enter the prompt. func stringifyToolResultOutput(output fantasy.ToolResultOutputContent) string { switch typed := output.(type) { case fantasy.ToolResultOutputContentText: @@ -158,8 +147,8 @@ func stringifyToolResultOutput(output fantasy.ToolResultOutputContent) string { } // replaceUnsupportedFileParts swaps file parts the compaction model does -// not accept for short text placeholders in a copy of messages, so the -// summary notes the attachment existed instead of silently losing it. +// not accept for text placeholders in a copy of messages, so the summary +// notes the attachment existed instead of silently losing it. func replaceUnsupportedFileParts( ctx context.Context, logger slog.Logger, diff --git a/coderd/x/chatd/generation.go b/coderd/x/chatd/generation.go index 1e4a97dc77c..ed1ccbc5432 100644 --- a/coderd/x/chatd/generation.go +++ b/coderd/x/chatd/generation.go @@ -70,12 +70,11 @@ type generationPrepared struct { // generationCompaction contains compaction inputs prepared for generation. type generationCompaction struct { // Override, when non-nil, is the compaction model override resolved at - // prepare time. The override model client is built (and the prompt - // sanitized for it) in the compact action path, not at prepare time, - // so hard construction failures cannot fail turns that finish without - // compacting. ChatModelConfig is the chat model's config, needed to - // sanitize the prompt across provider boundaries. - Override *resolvedCompactionOverride + // prepare time. Its model client is built in the compact action path, + // so construction failures cannot fail turns that never compact. + Override *resolvedCompactionOverride + // ChatModelConfig is the chat model's config, used to detect provider + // changes when sanitizing the compaction prompt. ChatModelConfig database.ChatModelConfig Required bool @@ -756,10 +755,9 @@ func (s *taskStarter) generateCompaction( } // compactionMetricIdentity returns the provider/model labels for compaction -// metrics. When an override is configured the labels come from the identity -// resolved at prepare time, so events recorded before the override client -// is built (the still-over-limit terminal error) carry the same labels as -// events recorded by the compact action after it swaps in the built client. +// metrics. Override labels come from prepare-time resolution so events +// recorded before the override client is built (still-over-limit) match +// the compact action's own events. func compactionMetricIdentity(compaction *generationCompaction) (provider, model string) { if compaction.Override != nil { return compaction.Override.ResolvedProvider, compaction.Override.ResolvedModel diff --git a/coderd/x/chatd/generation_internal_test.go b/coderd/x/chatd/generation_internal_test.go index 12b196620d1..e9736f59dc9 100644 --- a/coderd/x/chatd/generation_internal_test.go +++ b/coderd/x/chatd/generation_internal_test.go @@ -26,9 +26,8 @@ func TestCompactionMetricIdentity(t *testing.T) { require.Equal(t, "anthropic", provider) require.Equal(t, "claude-sonnet-4-5", model) - // With an override configured, metrics use the identity resolved at - // prepare time instead of the chat model carried by the options, even - // before the override client is built. + // With an override, metrics use the prepare-time identity, not the + // chat model carried by the options. compaction.Override = &resolvedCompactionOverride{ ResolvedProvider: "openai", ResolvedModel: "gpt-4.1-mini", diff --git a/coderd/x/chatd/generation_preparer.go b/coderd/x/chatd/generation_preparer.go index 96010540606..ffee255d126 100644 --- a/coderd/x/chatd/generation_preparer.go +++ b/coderd/x/chatd/generation_preparer.go @@ -580,11 +580,9 @@ func (server *Server) prepareGeneration( if override, ok := server.resolveUserCompactionThreshold(ctx, chat.OwnerID, modelConfig.ID); ok { effectiveThreshold = override } - // The effective compaction limit is the stricter of the chat model's - // and the compaction model's context limits: the history must also fit - // the summarizer's window. Post-compaction continuation checks keep - // using the chat model's limit (ContextLimitFallback below), since the - // follow-up assistant generation runs on the chat model. + // The compaction trigger uses the stricter of the chat and override + // models' context limits: the history must also fit the summarizer's + // window. compactionContextLimit := modelConfig.ContextLimit compactionOverride, err := server.resolveCompactionOverrideConfig(ctx, chat) if err != nil { @@ -599,11 +597,8 @@ func (server *Server) prepareGeneration( } compactionStepUsage := latestPromptUsage(promptRows) compactionNeeded := shouldCompactPromptUsage(compactionStepUsage, compactionContextLimit, effectiveThreshold) - // The options carry the chat model; when an override is configured, - // generateCompaction builds the override client (a hard failure) and - // swaps model, identity, and sanitized prompt in the compact action - // path, so a broken override cannot fail turns that finish without - // compacting. + // The options carry the chat model; generateCompaction swaps in the + // override client when one is configured. compactionOptions := chatloop.GenerateCompactionOptions{ Model: model, Messages: prompt, diff --git a/coderd/x/chatd/subagent.go b/coderd/x/chatd/subagent.go index 852dab02166..9ad563247c5 100644 --- a/coderd/x/chatd/subagent.go +++ b/coderd/x/chatd/subagent.go @@ -184,8 +184,7 @@ func modelOverrideErrorLabel(overrideContext string) string { // resolveConfiguredModelOverride returns ok when a usable override is // resolved. In hard failure mode, ok is also true for configured but unusable // overrides so callers can distinguish them from unset or malformed values. -// providerName is the normalized provider name of the resolved config; it is -// only meaningful on the usable-override return. +// The normalized provider name is only meaningful for a usable override. func (p *Server) resolveConfiguredModelOverride( ctx context.Context, overrideContext string, From 9dc171de103e67b60d16134931c0dfb38909cc1c Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Tue, 14 Jul 2026 09:08:09 +0000 Subject: [PATCH 12/12] fix(coderd/x/chatd): use compaction limit in still-over-limit decision The compaction trigger uses the stricter of the chat and override models' context limits, but the still-over-limit check compared against the chat model's limit. A stricter override then looped through repeated compactions instead of failing the turn. --- coderd/x/chatd/generation.go | 14 +++++++++++++- coderd/x/chatd/generation_internal_test.go | 14 ++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/coderd/x/chatd/generation.go b/coderd/x/chatd/generation.go index ed1ccbc5432..25d553ef2cd 100644 --- a/coderd/x/chatd/generation.go +++ b/coderd/x/chatd/generation.go @@ -246,6 +246,18 @@ func generationCompactionThreshold(compaction *generationCompaction) int32 { return compaction.Options.ThresholdPercent } +// generationCompactionContextLimit returns the context limit the compaction +// trigger was evaluated against at prepare time (the stricter of the chat and +// override models' limits). The still-over-limit check must compare against +// the same limit, otherwise a stricter override loops through repeated +// compactions instead of surfacing errCompactionStillOverLimit. +func generationCompactionContextLimit(compaction *generationCompaction) int64 { + if compaction == nil { + return 0 + } + return compaction.Options.ContextLimit +} + func unresolvedToolCallsFromHistory( messages []database.ChatMessage, dynamicToolNames map[string]bool, @@ -332,7 +344,7 @@ func (s *taskStarter) StartGeneration(ctx context.Context, input chatWorkerTaskS compactionEnabled: prepared.Compaction != nil, compactionNeeded: prepared.Compaction != nil && prepared.Compaction.Required, compactionThresholdPercent: generationCompactionThreshold(prepared.Compaction), - compactionContextLimit: prepared.ContextLimitFallback, + compactionContextLimit: generationCompactionContextLimit(prepared.Compaction), }) }) if err != nil { diff --git a/coderd/x/chatd/generation_internal_test.go b/coderd/x/chatd/generation_internal_test.go index e9736f59dc9..aa8e93a3d96 100644 --- a/coderd/x/chatd/generation_internal_test.go +++ b/coderd/x/chatd/generation_internal_test.go @@ -37,6 +37,20 @@ func TestCompactionMetricIdentity(t *testing.T) { require.Equal(t, "gpt-4.1-mini", model) } +func TestGenerationCompactionContextLimit(t *testing.T) { + t.Parallel() + + require.EqualValues(t, 0, generationCompactionContextLimit(nil)) + + // The decision path must see the prepare-time compaction limit (the + // stricter of the chat and override models' limits), not the chat + // model's limit. + compaction := &generationCompaction{ + Options: chatloop.GenerateCompactionOptions{ContextLimit: 50_000}, + } + require.EqualValues(t, 50_000, generationCompactionContextLimit(compaction)) +} + func TestRecordGenerationFinishFailure(t *testing.T) { t.Parallel()