diff --git a/coderd/x/chatd/ARCHITECTURE.md b/coderd/x/chatd/ARCHITECTURE.md index dee9660132cd2..8b869367e20d5 100644 --- a/coderd/x/chatd/ARCHITECTURE.md +++ b/coderd/x/chatd/ARCHITECTURE.md @@ -839,7 +839,7 @@ The generation goroutine supports: Model configs may carry a `reasoning_effort` config (`{default, max}`) inside `chat_model_configs.options`. Users select a per-turn effort when sending or editing a message; the value is stored on `chat_messages.reasoning_effort` and on `chat_queued_messages.reasoning_effort` for queued messages. Queued messages carry the value through promotion, and `chats.last_reasoning_effort` tracks the most recent message that set one, mirroring `last_model_config_id`. -During generation preparation, the effective effort is resolved as the chat's `last_reasoning_effort` if set, else the config's `default`; clamped to the config's `max` on the global scale `none < minimal < low < medium < high < xhigh < max`; and passed through to the provider. The provider verifies whether the configured value is valid for that model at runtime. If the model config has no `reasoning_effort`, any user-selected value is ignored. The resolved value is injected into the provider-native options with `chatprovider.ApplyReasoningEffort` after provider option conversion. +During generation preparation, the effective effort is resolved as the chat's `last_reasoning_effort` if set, else the config's `default`; clamped to the config's `max` on the global scale `none < minimal < low < medium < high < xhigh < max`; and passed through to the provider. The provider verifies whether the configured value is valid for that model at runtime. If the model config has no `reasoning_effort`, any user-selected value is ignored. The resolved value is injected into the provider-native options with `chatprovider.ApplyReasoningEffort` after provider option conversion. For Anthropic, the fantasy provider converts effort into enabled budget thinking on models older than Claude 4.6, which reject adaptive thinking. #### Compaction model selection diff --git a/coderd/x/chatd/chatd_test.go b/coderd/x/chatd/chatd_test.go index e0b2c668e5e73..f89dd2404dd60 100644 --- a/coderd/x/chatd/chatd_test.go +++ b/coderd/x/chatd/chatd_test.go @@ -5917,133 +5917,172 @@ func TestActiveServer_CompactionModelOverride(t *testing.T) { thresholdPercent = int32(70) ) - seedOverrideModel := func(ctx context.Context, t *testing.T, db database.Store, chatModel database.ChatModelConfig, contextLimit int64) database.ChatModelConfig { + seedOverrideModel := func(ctx context.Context, t *testing.T, db database.Store, chatModel database.ChatModelConfig, modelName, effort string, contextLimit int64) database.ChatModelConfig { t.Helper() overrideModel := dbgen.ChatModelConfig(t, db, database.ChatModelConfig{ - Model: overrideModelName, + Model: modelName, AIProviderID: chatModel.AIProviderID, ContextLimit: contextLimit, }) - lowEffort := "low" overrideModel = updateChatModelCallConfig(t, db, overrideModel, codersdk.ChatModelCallConfig{ ReasoningEffort: &codersdk.ChatModelReasoningEffortConfig{ - Default: &lowEffort, - Max: &lowEffort, + Default: &effort, + Max: &effort, }, }) require.NoError(t, db.UpsertChatCompactionModelOverride(ctx, overrideModel.ID.String())) return overrideModel } - t.Run("summary routes to the override model and continuation stays on the chat model", func(t *testing.T) { - t.Parallel() - - ctx := testutil.Context(t, testutil.WaitLong) - db, ps := dbtestutil.NewDB(t) - reg := prometheus.NewRegistry() - var streamCount atomic.Int32 - anthropicURL := chattest.NewAnthropic(t, func(req *chattest.AnthropicRequest) chattest.AnthropicResponse { - body := anthropicRequestBody(t, *req) - if !req.Stream { - if strings.Contains(body, "You are performing a context compaction") { - require.Equal(t, overrideModelName, req.Model) - // The override config's reasoning effort must reach the - // summary request (Anthropic serializes it as - // output_config effort). - require.Contains(t, string(req.OutputConfig), `"effort":"low"`) - return anthropicCompactionResponse(compactionSummary) - } - return chattest.AnthropicNonStreamingResponse("title") - } - require.Equal(t, chatModelName, req.Model) - switch streamCount.Add(1) { - case 1: - return highUsageReadFileResponse("/tmp/a.txt") - default: - require.Contains(t, body, compactionSummary) - require.Empty(t, string(req.OutputConfig), - "the override reasoning effort must not leak into chat model generations") - return chattest.AnthropicStreamingResponse(chattest.AnthropicTextChunksWithCacheUsage(chattest.AnthropicUsage{ - InputTokens: 20, - OutputTokens: 5, - }, "continued after compaction")...) - } - }) - user, org, model := seedAnthropicChatDependencies(t, db, anthropicURL) - model = updateChatModelCompressionThreshold(t, db, model, 100, thresholdPercent) - overrideModel := seedOverrideModel(ctx, t, db, model, 1_000_000) - ws, dbAgent := seedWorkspaceWithAgent(t, db, user.ID) + routingCases := []struct { + name string + overrideModel string + effort string + assertSummaryRequest func(t *testing.T, req *chattest.AnthropicRequest) + }{ + { + // Claude 3.5 predates extended thinking, so effort sends + // neither thinking nor output_config. + name: "pre-thinking override model", + overrideModel: overrideModelName, + effort: "high", + assertSummaryRequest: func(t *testing.T, req *chattest.AnthropicRequest) { + require.Empty(t, string(req.OutputConfig)) + require.Empty(t, string(req.Thinking)) + }, + }, + { + // 3276 is 0.8 (high) of the summary call's default 4096 max_tokens. + name: "legacy budget-thinking override model", + overrideModel: "claude-haiku-4-5", + effort: "high", + assertSummaryRequest: func(t *testing.T, req *chattest.AnthropicRequest) { + require.Empty(t, string(req.OutputConfig)) + require.Contains(t, string(req.Thinking), `"type":"enabled"`) + require.Contains(t, string(req.Thinking), `"budget_tokens":3276`) + }, + }, + { + name: "adaptive-capable override model", + overrideModel: "claude-sonnet-4-6", + effort: "low", + assertSummaryRequest: func(t *testing.T, req *chattest.AnthropicRequest) { + require.Contains(t, string(req.OutputConfig), `"effort":"low"`) + require.Contains(t, string(req.Thinking), `"type":"adaptive"`) + }, + }, + } - ctrl := gomock.NewController(t) - mockConn := agentconnmock.NewMockAgentConn(ctrl) - setupToolExecutionAgentConn(t, mockConn) - mockConn.EXPECT().ReadFileLines(gomock.Any(), "/tmp/a.txt", int64(1), int64(0), gomock.Any()). - Return(workspacesdk.ReadFileLinesResponse{Success: true, FileSize: 12, TotalLines: 1, LinesRead: 1, Content: "1\tpackage main"}, nil). - Times(1) + for _, tc := range routingCases { + t.Run("summary routes to the override model and continuation stays on the chat model/"+tc.name, func(t *testing.T) { + t.Parallel() - server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) { - cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(chattest.NewMockAIBridgeTransport(t, anthropicURL, chattest.WithPreservePath())) - cfg.PrometheusRegistry = reg - cfg.AlwaysEnableDebugLogs = true - cfg.AgentConn = func(_ context.Context, agentID uuid.UUID) (workspacesdk.AgentConn, func(), error) { - require.Equal(t, dbAgent.ID, agentID) - return mockConn, func() {}, nil - } - }) - chat, err := server.CreateChat(ctx, chatd.CreateOptions{ - OrganizationID: org.ID, - OwnerID: user.ID, - WorkspaceID: uuid.NullUUID{UUID: ws.ID, Valid: true}, - AgentID: uuid.NullUUID{UUID: dbAgent.ID, Valid: true}, - Title: "compaction-override", - ModelConfigID: model.ID, - InitialUserContent: []codersdk.ChatMessagePart{ - codersdk.ChatMessageText("read the file and continue"), - }, - }) - require.NoError(t, err) - waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusWaiting) + ctx := testutil.Context(t, testutil.WaitLong) + db, ps := dbtestutil.NewDB(t) + reg := prometheus.NewRegistry() + var streamCount atomic.Int32 + anthropicURL := chattest.NewAnthropic(t, func(req *chattest.AnthropicRequest) chattest.AnthropicResponse { + body := anthropicRequestBody(t, *req) + if !req.Stream { + if strings.Contains(body, "You are performing a context compaction") { + require.Equal(t, tc.overrideModel, req.Model) + tc.assertSummaryRequest(t, req) + return anthropicCompactionResponse(compactionSummary) + } + return chattest.AnthropicNonStreamingResponse("title") + } + require.Equal(t, chatModelName, req.Model) + switch streamCount.Add(1) { + case 1: + return highUsageReadFileResponse("/tmp/a.txt") + default: + require.Contains(t, body, compactionSummary) + require.Empty(t, string(req.OutputConfig), + "the override reasoning effort must not leak into chat model generations") + require.Empty(t, string(req.Thinking), + "the override reasoning effort must not leak into chat model generations") + return chattest.AnthropicStreamingResponse(chattest.AnthropicTextChunksWithCacheUsage(chattest.AnthropicUsage{ + InputTokens: 20, + OutputTokens: 5, + }, "continued after compaction")...) + } + }) + user, org, model := seedAnthropicChatDependencies(t, db, anthropicURL) + model = updateChatModelCompressionThreshold(t, db, model, 100, thresholdPercent) + overrideModel := seedOverrideModel(ctx, t, db, model, tc.overrideModel, tc.effort, 1_000_000) + ws, dbAgent := seedWorkspaceWithAgent(t, db, user.ID) - messages := chatMessages(ctx, t, db, chat.ID) - promptMessages, err := db.GetChatMessagesForPromptByChatID(ctx, chat.ID) - require.NoError(t, err) - compressed := compressedChatSummarizedMessages(t, append(promptMessages, messages...)) - require.Len(t, compressed.summaries, 1) - require.Contains(t, messageText(t, compressed.summaries[0]), compactionSummary) - requireTextPart(t, messages[len(messages)-1], "continued after compaction") + ctrl := gomock.NewController(t) + mockConn := agentconnmock.NewMockAgentConn(ctrl) + setupToolExecutionAgentConn(t, mockConn) + mockConn.EXPECT().ReadFileLines(gomock.Any(), "/tmp/a.txt", int64(1), int64(0), gomock.Any()). + Return(workspacesdk.ReadFileLinesResponse{Success: true, FileSize: 12, TotalLines: 1, LinesRead: 1, Content: "1\tpackage main"}, nil). + Times(1) - requireChatdMetricCounter(t, reg, "coderd_chatd_compaction_total", 1, map[string]string{ - "provider": "anthropic", - "model": overrideModelName, - "result": "success", - }) + server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) { + cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(chattest.NewMockAIBridgeTransport(t, anthropicURL, chattest.WithPreservePath())) + cfg.PrometheusRegistry = reg + cfg.AlwaysEnableDebugLogs = true + cfg.AgentConn = func(_ context.Context, agentID uuid.UUID) (workspacesdk.AgentConn, func(), error) { + require.Equal(t, dbAgent.ID, agentID) + return mockConn, func() {}, nil + } + }) + chat, err := server.CreateChat(ctx, chatd.CreateOptions{ + OrganizationID: org.ID, + OwnerID: user.ID, + WorkspaceID: uuid.NullUUID{UUID: ws.ID, Valid: true}, + AgentID: uuid.NullUUID{UUID: dbAgent.ID, Valid: true}, + Title: "compaction-override", + ModelConfigID: model.ID, + InitialUserContent: []codersdk.ChatMessagePart{ + codersdk.ChatMessageText("read the file and continue"), + }, + }) + require.NoError(t, err) + waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusWaiting) - require.NoError(t, server.Close()) - debugCtx := testutil.Context(t, testutil.WaitLong) - var compactionRun database.ChatDebugRun - testutil.Eventually(debugCtx, t, func(ctx context.Context) bool { - runs, err := db.GetChatDebugRunsByChatID(ctx, database.GetChatDebugRunsByChatIDParams{ - ChatID: chat.ID, - LimitVal: 100, + messages := chatMessages(ctx, t, db, chat.ID) + promptMessages, err := db.GetChatMessagesForPromptByChatID(ctx, chat.ID) + require.NoError(t, err) + compressed := compressedChatSummarizedMessages(t, append(promptMessages, messages...)) + require.Len(t, compressed.summaries, 1) + require.Contains(t, messageText(t, compressed.summaries[0]), compactionSummary) + requireTextPart(t, messages[len(messages)-1], "continued after compaction") + + requireChatdMetricCounter(t, reg, "coderd_chatd_compaction_total", 1, map[string]string{ + "provider": "anthropic", + "model": tc.overrideModel, + "result": "success", }) - if err != nil { - return false - } - for _, run := range runs { - if run.Kind == string(chatdebug.KindCompaction) { - compactionRun = run - return true + + require.NoError(t, server.Close()) + debugCtx := testutil.Context(t, testutil.WaitLong) + var compactionRun database.ChatDebugRun + testutil.Eventually(debugCtx, t, func(ctx context.Context) bool { + runs, err := db.GetChatDebugRunsByChatID(ctx, database.GetChatDebugRunsByChatIDParams{ + ChatID: chat.ID, + LimitVal: 100, + }) + if err != nil { + return false } - } - return false - }, testutil.IntervalMedium) - require.True(t, compactionRun.Provider.Valid) - require.Equal(t, "anthropic", compactionRun.Provider.String) - require.True(t, compactionRun.Model.Valid) - require.Equal(t, overrideModelName, compactionRun.Model.String) - require.True(t, compactionRun.ModelConfigID.Valid) - require.Equal(t, overrideModel.ID, compactionRun.ModelConfigID.UUID) - }) + for _, run := range runs { + if run.Kind == string(chatdebug.KindCompaction) { + compactionRun = run + return true + } + } + return false + }, testutil.IntervalMedium) + require.True(t, compactionRun.Provider.Valid) + require.Equal(t, "anthropic", compactionRun.Provider.String) + require.True(t, compactionRun.Model.Valid) + require.Equal(t, tc.overrideModel, compactionRun.Model.String) + require.True(t, compactionRun.ModelConfigID.Valid) + require.Equal(t, overrideModel.ID, compactionRun.ModelConfigID.UUID) + }) + } t.Run("compaction triggers at the stricter override context limit", func(t *testing.T) { t.Parallel() @@ -6077,7 +6116,7 @@ func TestActiveServer_CompactionModelOverride(t *testing.T) { // limit makes the effective threshold 70 tokens, so compaction // must trigger. model = updateChatModelCompressionThreshold(t, db, model, 1_000, thresholdPercent) - seedOverrideModel(ctx, t, db, model, 100) + seedOverrideModel(ctx, t, db, model, overrideModelName, "high", 100) ws, dbAgent := seedWorkspaceWithAgent(t, db, user.ID) ctrl := gomock.NewController(t) @@ -6117,6 +6156,83 @@ func TestActiveServer_CompactionModelOverride(t *testing.T) { }) } +func TestActiveServer_AnthropicModelReasoningEffort(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + model string + effort string + checkThinking func(t *testing.T, thinking string) + }{ + { + // Pre-4.6 models reject adaptive thinking, so effort becomes an + // enabled-thinking budget derived from max_tokens. + name: "LegacyModelEffortHigh", + model: "claude-haiku-4-5", + effort: "high", + checkThinking: func(t *testing.T, thinking string) { + require.Contains(t, thinking, `"type":"enabled"`) + require.Contains(t, thinking, `"budget_tokens":3276`) + }, + }, + { + // Claude 5+ models run adaptive thinking when the request omits + // the thinking field, so effort none must send an explicit + // disable. + name: "AdaptiveDefaultModelEffortNone", + model: "claude-sonnet-5", + effort: "none", + checkThinking: func(t *testing.T, thinking string) { + require.Contains(t, thinking, `"type":"disabled"`) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + ctx := testutil.Context(t, testutil.WaitLong) + db, ps := dbtestutil.NewDB(t) + requests := newAnthropicRequestRecorder() + anthropicURL := chattest.NewAnthropic(t, func(req *chattest.AnthropicRequest) chattest.AnthropicResponse { + requests.record(req) + if !req.Stream { + return chattest.AnthropicNonStreamingResponse("title") + } + return chattest.AnthropicStreamingResponse(chattest.AnthropicTextChunks("done")...) + }) + user, org, model := seedAnthropicChatDependencies(t, db, anthropicURL) + model.Model = tt.model + model = updateChatModelContextLimit(t, db, model) + model = updateChatModelCallConfig(t, db, model, codersdk.ChatModelCallConfig{ + MaxOutputTokens: ptr.Ref(int64(4096)), + ReasoningEffort: &codersdk.ChatModelReasoningEffortConfig{ + Default: &tt.effort, + Max: &tt.effort, + }, + }) + + server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) { + cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(chattest.NewMockAIBridgeTransport(t, anthropicURL, chattest.WithPreservePath())) + }) + chat := createChatThroughServer(ctx, t, db, server, org.ID, user.ID, model.ID, "hello") + waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusWaiting) + + messages := chatMessages(ctx, t, db, chat.ID) + requireTextPart(t, messages[len(messages)-1], "done") + + generationRequests := filterAnthropicStreamingRequests(requests.all()) + require.Len(t, generationRequests, 1) + req := generationRequests[0] + require.Equal(t, tt.model, req.Model) + require.Empty(t, string(req.OutputConfig)) + tt.checkThinking(t, string(req.Thinking)) + }) + } +} + func TestActiveServer_BasicAssistantGenerationAndPromptPreparation(t *testing.T) { t.Parallel() diff --git a/coderd/x/chatd/chattest/anthropic.go b/coderd/x/chatd/chattest/anthropic.go index c88352b8ba823..c8193a98c6c68 100644 --- a/coderd/x/chatd/chattest/anthropic.go +++ b/coderd/x/chatd/chattest/anthropic.go @@ -32,6 +32,7 @@ type AnthropicRequest struct { Stream bool `json:"stream,omitempty"` MaxTokens int `json:"max_tokens,omitempty"` OutputConfig json.RawMessage `json:"output_config,omitempty"` + Thinking json.RawMessage `json:"thinking,omitempty"` // TODO: encoding/json ignores inline tags. Add custom UnmarshalJSON to capture unknown keys. Options map[string]interface{} `json:",inline"` //nolint:revive } diff --git a/go.mod b/go.mod index 374c884580f73..d50e007cc8bdf 100644 --- a/go.mod +++ b/go.mod @@ -108,8 +108,13 @@ replace github.com/spf13/afero => github.com/aslilac/afero v0.0.0-20250403163713 // content-filter. // 16) coder/fantasy#46, route the gpt-5.6 family (sol, terra, luna) // through the OpenAI Responses API. -// See: https://github.com/coder/fantasy/commits/6da0c3b10237 -replace charm.land/fantasy => github.com/coder/fantasy v0.0.0-20260709180403-6da0c3b10237 +// 17) coder/fantasy#47, convert Anthropic reasoning effort to enabled +// budget thinking on pre-4.6 models instead of sending adaptive +// thinking, which those models reject with an HTTP 400, and send +// an explicit thinking disable for effort none on Claude 5+ models +// that otherwise run adaptive thinking by default. +// See: https://github.com/coder/fantasy/commits/a63de4b40315 +replace charm.land/fantasy => github.com/coder/fantasy v0.0.0-20260718195754-a63de4b40315 // coder/coder uses a fork of charmbracelet's fork of the Anthropic Go SDK // with performance improvements and Bedrock header cleanup. diff --git a/go.sum b/go.sum index 1ab4b93cc3993..1d59eba828222 100644 --- a/go.sum +++ b/go.sum @@ -333,8 +333,8 @@ github.com/coder/bubbletea v1.2.2-0.20241212190825-007a1cdb2c41 h1:SBN/DA63+ZHwu github.com/coder/bubbletea v1.2.2-0.20241212190825-007a1cdb2c41/go.mod h1:I9ULxr64UaOSUv7hcb3nX4kowodJCVS7vt7VVJk/kW4= github.com/coder/clistat v1.2.1 h1:P9/10njXMyj5cWzIU5wkRsSy5LVQH49+tcGMsAgWX0w= github.com/coder/clistat v1.2.1/go.mod h1:m7SC0uj88eEERgvF8Kn6+w6XF21BeSr+15f7GoLAw0A= -github.com/coder/fantasy v0.0.0-20260709180403-6da0c3b10237 h1:DyIn89F5RErX8Z6BKtzQBg5PvDTasjX8+PXkdJUxK3Q= -github.com/coder/fantasy v0.0.0-20260709180403-6da0c3b10237/go.mod h1:/1tM8tL1viuF1COpWPwbcUl4s1kAnjSruzo9wAdg4z0= +github.com/coder/fantasy v0.0.0-20260718195754-a63de4b40315 h1:5kd2/5JGhWJCLYyT4Vl1ujB+j9E9y8CJQF21V5vMyuE= +github.com/coder/fantasy v0.0.0-20260718195754-a63de4b40315/go.mod h1:ErEfF4rbdLcZmXG00xPZ4p3y0t+FGYmqCP5hRPG+U9A= github.com/coder/flog v1.1.0 h1:kbAes1ai8fIS5OeV+QAnKBQE22ty1jRF/mcAwHpLBa4= github.com/coder/flog v1.1.0/go.mod h1:UQlQvrkJBvnRGo69Le8E24Tcl5SJleAAR7gYEHzAmdQ= github.com/coder/go-httpstat v0.0.0-20230801153223-321c88088322 h1:m0lPZjlQ7vdVpRBPKfYIFlmgevoTkBxB10wv6l2gOaU= @@ -1415,8 +1415,8 @@ golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= -golang.org/x/image v0.42.0 h1:1gSs6ehNWXLbkHBIPcWztk3D/6aIA/8hauiAYtlodVY= -golang.org/x/image v0.42.0/go.mod h1:rrpelvGFt+kLPAjPM4HeWPgrl0FtafueU//e5N0qk/Q= +golang.org/x/image v0.43.0 h1:FLxcP4ec2350nTfOC8ysKtqYSIFbk/QGjw1ZHNP4tsY= +golang.org/x/image v0.43.0/go.mod h1:rrpelvGFt+kLPAjPM4HeWPgrl0FtafueU//e5N0qk/Q= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=