diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index 2fbb0cd65342b..e78055c42a3a0 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -1854,17 +1854,6 @@ func (q *querier) CleanupDeletedMCPServerIDsFromChats(ctx context.Context) error return q.db.CleanupDeletedMCPServerIDsFromChats(ctx) } -func (q *querier) ClearChatMessageProviderResponseIDsByChatID(ctx context.Context, chatID uuid.UUID) error { - chat, err := q.db.GetChatByID(ctx, chatID) - if err != nil { - return err - } - if err := q.authorizeContext(ctx, policy.ActionUpdate, chat); err != nil { - return err - } - return q.db.ClearChatMessageProviderResponseIDsByChatID(ctx, chatID) -} - func (q *querier) CountAIBridgeSessions(ctx context.Context, arg database.CountAIBridgeSessionsParams) (int64, error) { prep, err := prepareSQLFilter(ctx, q.auth, policy.ActionRead, rbac.ResourceAibridgeInterception.Type) if err != nil { diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index 550ce623aaf90..615c8ce92f2bb 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -848,12 +848,6 @@ func (s *MethodTestSuite) TestChats() { dbm.EXPECT().SoftDeleteContextFileMessages(gomock.Any(), chat.ID).Return(nil).AnyTimes() check.Args(chat.ID).Asserts(chat, policy.ActionUpdate).Returns() })) - s.Run("ClearChatMessageProviderResponseIDsByChatID", s.Mocked(func(dbm *dbmock.MockStore, faker *gofakeit.Faker, check *expects) { - chat := testutil.Fake(s.T(), faker, database.Chat{}) - dbm.EXPECT().GetChatByID(gomock.Any(), chat.ID).Return(chat, nil).AnyTimes() - dbm.EXPECT().ClearChatMessageProviderResponseIDsByChatID(gomock.Any(), chat.ID).Return(nil).AnyTimes() - check.Args(chat.ID).Asserts(chat, policy.ActionUpdate).Returns() - })) s.Run("GetChatCostPerChat", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) { arg := database.GetChatCostPerChatParams{ OwnerID: uuid.New(), diff --git a/coderd/database/dbgen/dbgen.go b/coderd/database/dbgen/dbgen.go index b857ffe2a5da6..97d3094ea06c1 100644 --- a/coderd/database/dbgen/dbgen.go +++ b/coderd/database/dbgen/dbgen.go @@ -145,7 +145,6 @@ func ChatMessage(t testing.TB, db database.Store, seed database.ChatMessage) dat Compressed: []bool{seed.Compressed}, TotalCostMicros: []int64{seed.TotalCostMicros.Int64}, RuntimeMs: []int64{seed.RuntimeMs.Int64}, - ProviderResponseID: []string{seed.ProviderResponseID.String}, }) require.NoError(t, err, "insert chat message") require.Len(t, msgs, 1) diff --git a/coderd/database/dbgen/dbgen_test.go b/coderd/database/dbgen/dbgen_test.go index 776531484711a..35ba905e90995 100644 --- a/coderd/database/dbgen/dbgen_test.go +++ b/coderd/database/dbgen/dbgen_test.go @@ -398,7 +398,6 @@ func TestGenerator(t *testing.T) { ContextLimit: sql.NullInt64{Int64: 77, Valid: true}, Compressed: true, TotalCostMicros: sql.NullInt64{Int64: 88, Valid: true}, - ProviderResponseID: sql.NullString{String: "resp-123", Valid: true}, }) require.Equal(t, database.ChatMessageRoleAssistant, msg2.Role) require.True(t, msg2.Content.Valid) @@ -412,7 +411,6 @@ func TestGenerator(t *testing.T) { require.Equal(t, sql.NullInt64{Int64: 77, Valid: true}, msg2.ContextLimit) require.True(t, msg2.Compressed) require.Equal(t, sql.NullInt64{Int64: 88, Valid: true}, msg2.TotalCostMicros) - require.Equal(t, sql.NullString{String: "resp-123", Valid: true}, msg2.ProviderResponseID) }) t.Run("MCPServerConfig", func(t *testing.T) { diff --git a/coderd/database/dbmetrics/querymetrics.go b/coderd/database/dbmetrics/querymetrics.go index 5afa5fcdc1289..d31796e73bf90 100644 --- a/coderd/database/dbmetrics/querymetrics.go +++ b/coderd/database/dbmetrics/querymetrics.go @@ -305,14 +305,6 @@ func (m queryMetricsStore) CleanupDeletedMCPServerIDsFromChats(ctx context.Conte return r0 } -func (m queryMetricsStore) ClearChatMessageProviderResponseIDsByChatID(ctx context.Context, chatID uuid.UUID) error { - start := time.Now() - r0 := m.s.ClearChatMessageProviderResponseIDsByChatID(ctx, chatID) - m.queryLatencies.WithLabelValues("ClearChatMessageProviderResponseIDsByChatID").Observe(time.Since(start).Seconds()) - m.queryCounts.WithLabelValues(httpmw.ExtractHTTPRoute(ctx), httpmw.ExtractHTTPMethod(ctx), "ClearChatMessageProviderResponseIDsByChatID").Inc() - return r0 -} - func (m queryMetricsStore) CountAIBridgeSessions(ctx context.Context, arg database.CountAIBridgeSessionsParams) (int64, error) { start := time.Now() r0, r1 := m.s.CountAIBridgeSessions(ctx, arg) diff --git a/coderd/database/dbmock/dbmock.go b/coderd/database/dbmock/dbmock.go index 97eacb11fa3bf..466c5c36f4abc 100644 --- a/coderd/database/dbmock/dbmock.go +++ b/coderd/database/dbmock/dbmock.go @@ -408,20 +408,6 @@ func (mr *MockStoreMockRecorder) CleanupDeletedMCPServerIDsFromChats(ctx any) *g return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CleanupDeletedMCPServerIDsFromChats", reflect.TypeOf((*MockStore)(nil).CleanupDeletedMCPServerIDsFromChats), ctx) } -// ClearChatMessageProviderResponseIDsByChatID mocks base method. -func (m *MockStore) ClearChatMessageProviderResponseIDsByChatID(ctx context.Context, chatID uuid.UUID) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ClearChatMessageProviderResponseIDsByChatID", ctx, chatID) - ret0, _ := ret[0].(error) - return ret0 -} - -// ClearChatMessageProviderResponseIDsByChatID indicates an expected call of ClearChatMessageProviderResponseIDsByChatID. -func (mr *MockStoreMockRecorder) ClearChatMessageProviderResponseIDsByChatID(ctx, chatID any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClearChatMessageProviderResponseIDsByChatID", reflect.TypeOf((*MockStore)(nil).ClearChatMessageProviderResponseIDsByChatID), ctx, chatID) -} - // CountAIBridgeSessions mocks base method. func (m *MockStore) CountAIBridgeSessions(ctx context.Context, arg database.CountAIBridgeSessionsParams) (int64, error) { m.ctrl.T.Helper() diff --git a/coderd/database/querier.go b/coderd/database/querier.go index 0576f50e65b73..4aab48dde14d7 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -88,7 +88,6 @@ type sqlcQuerier interface { CleanTailnetLostPeers(ctx context.Context) error CleanTailnetTunnels(ctx context.Context) error CleanupDeletedMCPServerIDsFromChats(ctx context.Context) error - ClearChatMessageProviderResponseIDsByChatID(ctx context.Context, chatID uuid.UUID) error CountAIBridgeSessions(ctx context.Context, arg CountAIBridgeSessionsParams) (int64, error) CountAuditLogs(ctx context.Context, arg CountAuditLogsParams) (int64, error) // Cheap queue-length check used by ChatMachine.Update when deciding diff --git a/coderd/database/querier_test.go b/coderd/database/querier_test.go index 629e633c0c740..587aaafcfc93b 100644 --- a/coderd/database/querier_test.go +++ b/coderd/database/querier_test.go @@ -12803,7 +12803,6 @@ func TestUpdateChatLastTurnSummary(t *testing.T) { Compressed: []bool{false}, TotalCostMicros: []int64{0}, RuntimeMs: []int64{0}, - ProviderResponseID: []string{""}, }) require.NoError(t, err) @@ -14497,7 +14496,6 @@ func TestGetChatsFilter(t *testing.T) { Compressed: []bool{false}, TotalCostMicros: []int64{0}, RuntimeMs: []int64{0}, - ProviderResponseID: []string{""}, }) require.NoError(t, err) } @@ -14742,7 +14740,6 @@ func TestChatHasUnread(t *testing.T) { Compressed: []bool{false}, TotalCostMicros: []int64{0}, RuntimeMs: []int64{0}, - ProviderResponseID: []string{""}, }) require.NoError(t, err) } diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index d070615d2448a..d23291dabe150 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -6164,19 +6164,6 @@ func (q *sqlQuerier) BatchUpsertChatHeartbeats(ctx context.Context, arg BatchUps return err } -const clearChatMessageProviderResponseIDsByChatID = `-- name: ClearChatMessageProviderResponseIDsByChatID :exec -UPDATE chat_messages -SET provider_response_id = NULL -WHERE chat_id = $1::uuid - AND deleted = false - AND provider_response_id IS NOT NULL -` - -func (q *sqlQuerier) ClearChatMessageProviderResponseIDsByChatID(ctx context.Context, chatID uuid.UUID) error { - _, err := q.db.ExecContext(ctx, clearChatMessageProviderResponseIDsByChatID, chatID) - return err -} - const countChatQueuedMessages = `-- name: CountChatQueuedMessages :one SELECT COUNT(*)::bigint AS count FROM chat_queued_messages @@ -9728,8 +9715,7 @@ INSERT INTO chat_messages ( context_limit, compressed, total_cost_micros, - runtime_ms, - provider_response_id + runtime_ms ) SELECT $1::uuid, @@ -9749,8 +9735,7 @@ SELECT NULLIF(UNNEST($15::bigint[]), 0), UNNEST($16::boolean[]), NULLIF(UNNEST($17::bigint[]), 0), - NULLIF(UNNEST($18::bigint[]), 0), - NULLIF(UNNEST($19::text[]), '') + NULLIF(UNNEST($18::bigint[]), 0) RETURNING id, chat_id, model_config_id, created_at, role, content, visibility, input_tokens, output_tokens, total_tokens, reasoning_tokens, cache_creation_tokens, cache_read_tokens, context_limit, compressed, created_by, content_version, total_cost_micros, runtime_ms, deleted, provider_response_id, api_key_id, revision ` @@ -9774,7 +9759,6 @@ type InsertChatMessagesParams struct { Compressed []bool `db:"compressed" json:"compressed"` TotalCostMicros []int64 `db:"total_cost_micros" json:"total_cost_micros"` RuntimeMs []int64 `db:"runtime_ms" json:"runtime_ms"` - ProviderResponseID []string `db:"provider_response_id" json:"provider_response_id"` } func (q *sqlQuerier) InsertChatMessages(ctx context.Context, arg InsertChatMessagesParams) ([]ChatMessage, error) { @@ -9797,7 +9781,6 @@ func (q *sqlQuerier) InsertChatMessages(ctx context.Context, arg InsertChatMessa pq.Array(arg.Compressed), pq.Array(arg.TotalCostMicros), pq.Array(arg.RuntimeMs), - pq.Array(arg.ProviderResponseID), ) if err != nil { return nil, err diff --git a/coderd/database/queries/chats.sql b/coderd/database/queries/chats.sql index b7bbe8ecb38d7..8d4feb773b215 100644 --- a/coderd/database/queries/chats.sql +++ b/coderd/database/queries/chats.sql @@ -836,8 +836,7 @@ INSERT INTO chat_messages ( context_limit, compressed, total_cost_micros, - runtime_ms, - provider_response_id + runtime_ms ) SELECT @chat_id::uuid, @@ -857,8 +856,7 @@ SELECT NULLIF(UNNEST(@context_limit::bigint[]), 0), UNNEST(@compressed::boolean[]), NULLIF(UNNEST(@total_cost_micros::bigint[]), 0), - NULLIF(UNNEST(@runtime_ms::bigint[]), 0), - NULLIF(UNNEST(@provider_response_id::text[]), '') + NULLIF(UNNEST(@runtime_ms::bigint[]), 0) RETURNING *; @@ -2598,13 +2596,6 @@ WHERE agent_id = @agent_id::uuid AND status IN ('waiting', 'running', 'paused', 'pending', 'requires_action') ORDER BY updated_at DESC; --- name: ClearChatMessageProviderResponseIDsByChatID :exec -UPDATE chat_messages -SET provider_response_id = NULL -WHERE chat_id = @chat_id::uuid - AND deleted = false - AND provider_response_id IS NOT NULL; - -- name: SoftDeleteContextFileMessages :exec UPDATE chat_messages SET deleted = true WHERE chat_id = @chat_id::uuid diff --git a/coderd/x/chatd/attempt.go b/coderd/x/chatd/attempt.go index 0b803e8586306..5283cbe6479aa 100644 --- a/coderd/x/chatd/attempt.go +++ b/coderd/x/chatd/attempt.go @@ -25,11 +25,10 @@ const ( // stepData is the durable content produced by one provider attempt. type stepData struct { - Content []fantasy.Content - Usage fantasy.Usage - ContextLimit sql.NullInt64 - ProviderResponseID string - Runtime time.Duration + Content []fantasy.Content + Usage fantasy.Usage + ContextLimit sql.NullInt64 + Runtime time.Duration ToolCallCreatedAt map[string]time.Time ToolResultCreatedAt map[string]time.Time diff --git a/coderd/x/chatd/chatadvisor/runner.go b/coderd/x/chatd/chatadvisor/runner.go index fe31afbc5c2e1..4247f385dd7e7 100644 --- a/coderd/x/chatd/chatadvisor/runner.go +++ b/coderd/x/chatd/chatadvisor/runner.go @@ -43,10 +43,8 @@ func (rt *Runtime) RunAdvisor( }, nil } - // Clone per invocation and reset inherited state so chatloop cannot - // mutate the Runtime's stored options across calls, and so the nested - // call never runs as a chain-mode continuation against stale parent - // state or persists an orphan stored response on the provider side. + // resetProviderOptionsForNestedCall mutates its argument; give it a + // clone so the Runtime's stored options stay unchanged across calls. nestedProviderOptions := cloneProviderOptions(rt.cfg.ProviderOptions) resetProviderOptionsForNestedCall(nestedProviderOptions) diff --git a/coderd/x/chatd/chatadvisor/runner_test.go b/coderd/x/chatd/chatadvisor/runner_test.go index 42cb7e16b3e40..c4a4ff96e54e5 100644 --- a/coderd/x/chatd/chatadvisor/runner_test.go +++ b/coderd/x/chatd/chatadvisor/runner_test.go @@ -371,9 +371,9 @@ func TestNewRuntimeValidation(t *testing.T) { func TestNewRuntimeDeepClonesOpenAIResponsesProviderOptions(t *testing.T) { t.Parallel() - parentPrevID := "resp_parent_abc123" + parentStore := true parentOpts := &fantasyopenai.ResponsesProviderOptions{ - PreviousResponseID: &parentPrevID, + Store: &parentStore, } parentProviderOpts := fantasy.ProviderOptions{ fantasyopenai.Name: parentOpts, @@ -402,30 +402,29 @@ func TestNewRuntimeDeepClonesOpenAIResponsesProviderOptions(t *testing.T) { require.NoError(t, err) require.Equal(t, chatadvisor.ResultTypeAdvice, result.Type) - // Parent's OpenAI Responses entry must still carry its PreviousResponseID; - // the advisor's nested chatloop run must not have mutated the shared pointer. - require.NotNil(t, parentOpts.PreviousResponseID) - require.Equal(t, parentPrevID, *parentOpts.PreviousResponseID) + // Parent's OpenAI Responses entry must still carry its Store setting; + // the advisor must have mutated only its per-call clone, never the + // shared pointer. + require.NotNil(t, parentOpts.Store) + require.True(t, *parentOpts.Store) } -func TestAdvisorRunStripsChainStateAndIsConsistentAcrossCalls(t *testing.T) { +func TestAdvisorRunDisablesStoreAndIsConsistentAcrossCalls(t *testing.T) { t.Parallel() - parentPrevID := "resp_parent_xyz" + parentStore := true parentOpts := &fantasyopenai.ResponsesProviderOptions{ - PreviousResponseID: &parentPrevID, + Store: &parentStore, } parentProviderOpts := fantasy.ProviderOptions{ fantasyopenai.Name: parentOpts, } - // Snapshot PreviousResponseID and Store at stream time, before chatloop - // has any chance to clear them on the shared map. Comparing across calls - // proves the advisor observes consistent (non-chained, non-persisted) - // options each invocation. + // Snapshot Store at stream time to capture exactly what each call sent. + // Comparing across calls proves the advisor observes consistent + // (non-persisted) options each invocation. type observedOpts struct { - prevID *string - store *bool + store *bool } var observed []observedOpts runtime, err := chatadvisor.NewRuntime(chatadvisor.RuntimeConfig{ @@ -438,10 +437,6 @@ func TestAdvisorRunStripsChainStateAndIsConsistentAcrossCalls(t *testing.T) { observed = append(observed, observedOpts{}) } else { snap := observedOpts{} - if openaiOpts.PreviousResponseID != nil { - copied := *openaiOpts.PreviousResponseID - snap.prevID = &copied - } if openaiOpts.Store != nil { copied := *openaiOpts.Store snap.store = &copied @@ -470,19 +465,15 @@ func TestAdvisorRunStripsChainStateAndIsConsistentAcrossCalls(t *testing.T) { require.Len(t, observed, 2) for i, snap := range observed { - // Each nested call must run without chain mode so prompts built - // from full history by BuildAdvisorMessages are accepted. - require.Nil(t, snap.prevID, "call %d unexpectedly ran in chain mode", i) - // Store must be explicitly disabled so the provider does not - // persist an orphan response that later chain-mode calls would - // fail to resume. + // Store must be explicitly disabled so the advisor call leaves no + // stored response behind on the provider. require.NotNil(t, snap.store, "call %d did not disable Store", i) require.False(t, *snap.store, "call %d ran with Store enabled", i) } // The parent's pointer must be untouched across repeated advisor runs. - require.NotNil(t, parentOpts.PreviousResponseID) - require.Equal(t, parentPrevID, *parentOpts.PreviousResponseID) + require.NotNil(t, parentOpts.Store) + require.True(t, *parentOpts.Store) } func TestBuildAdvisorMessagesTruncatesToRecentMessageLimit(t *testing.T) { diff --git a/coderd/x/chatd/chatadvisor/runtime.go b/coderd/x/chatd/chatadvisor/runtime.go index f50514b8f6878..d7282e9706dda 100644 --- a/coderd/x/chatd/chatadvisor/runtime.go +++ b/coderd/x/chatd/chatadvisor/runtime.go @@ -61,14 +61,12 @@ func NewRuntime(cfg RuntimeConfig) (*Runtime, error) { return &Runtime{cfg: normalized}, nil } -// cloneProviderOptions returns a copy of opts with pointer entries for known, -// in-place mutated provider option types replaced by a shallow struct copy. -// chatloop mutates the OpenAI Responses entry (PreviousResponseID) on -// chain-mode exit, so sharing the pointer with the parent run would let an -// advisor call corrupt the parent's chain state. Value fields such as -// Metadata and Include are still shared with the parent; nothing in this -// package mutates them, but callers that need true deep-copy semantics must -// handle those fields explicitly. +// cloneProviderOptions returns a copy of opts with pointer entries for +// known, in-place mutated provider option types replaced by shallow struct +// copies, so a nested advisor call that disables Store does so on its own +// copy rather than the parent run's entry. Value fields such as Metadata +// and Include remain shared; callers that need true deep-copy semantics +// must handle those fields explicitly. func cloneProviderOptions(opts fantasy.ProviderOptions) fantasy.ProviderOptions { if opts == nil { return nil @@ -90,18 +88,14 @@ func cloneProviderOptions(opts fantasy.ProviderOptions) fantasy.ProviderOptions return cloned } -// resetProviderOptionsForNestedCall strips inherited state from opts that -// does not apply to an ephemeral advisor call. PreviousResponseID is -// cleared so the nested call is not sent as a chain-mode continuation -// (BuildAdvisorMessages sends the full history, not an incremental turn). -// Store is forced off so the advisor call does not persist an orphan -// response on the provider side. Must be called on a cloned map to avoid -// mutating shared parent state. +// resetProviderOptionsForNestedCall forces Store off so ephemeral advisor +// calls leave no stored response behind on the provider. It mutates opts +// in place, so it must be called on a cloned map, never on options shared +// with the parent run. func resetProviderOptionsForNestedCall(opts fantasy.ProviderOptions) { for _, value := range opts { if typed, ok := value.(*fantasyopenai.ResponsesProviderOptions); ok && typed != nil { storeDisabled := false - typed.PreviousResponseID = nil typed.Store = &storeDisabled } } diff --git a/coderd/x/chatd/chatd.go b/coderd/x/chatd/chatd.go index c337f7df82d85..29eabaad24f2b 100644 --- a/coderd/x/chatd/chatd.go +++ b/coderd/x/chatd/chatd.go @@ -2928,7 +2928,6 @@ func recordManualTitleUsage( Compressed: []bool{false}, TotalCostMicros: []int64{ptr.NilToDefault(totalCostMicros, 0)}, RuntimeMs: []int64{0}, - ProviderResponseID: []string{""}, }) if err != nil { return xerrors.Errorf("insert manual title usage message: %w", err) @@ -2982,7 +2981,6 @@ type chatMessage struct { contextLimit int64 totalCostMicros int64 runtimeMs int64 - providerResponseID string } type userChatMessage struct { @@ -3057,7 +3055,6 @@ func appendMessageFields( params.Compressed = append(params.Compressed, msg.compressed) params.TotalCostMicros = append(params.TotalCostMicros, msg.totalCostMicros) params.RuntimeMs = append(params.RuntimeMs, msg.runtimeMs) - params.ProviderResponseID = append(params.ProviderResponseID, msg.providerResponseID) } func appendChatMessage(params *database.InsertChatMessagesParams, msg chatMessage) { @@ -3786,9 +3783,9 @@ func mergeTurnSkills( ) } -// buildSystemPrompt applies system-level prompt injections in the -// canonical order. It is used by both the initial prompt assembly -// and the ReloadMessages callback to keep them in sync. +// buildSystemPrompt applies system-level prompt injections in a fixed +// order: subagent instruction, chat instruction, skill index, user prompt, +// then mode overlay prompts. func buildSystemPrompt( prompt []fantasy.Message, subagentInstruction string, diff --git a/coderd/x/chatd/chatd_chainmode_test.go b/coderd/x/chatd/chatd_chainmode_test.go deleted file mode 100644 index 417dfedfa8163..0000000000000 --- a/coderd/x/chatd/chatd_chainmode_test.go +++ /dev/null @@ -1,589 +0,0 @@ -package chatd_test - -import ( - "context" - "encoding/json" - "net/http" - "strings" - "sync" - "sync/atomic" - "testing" - - "charm.land/fantasy" - fantasyanthropic "charm.land/fantasy/providers/anthropic" - "github.com/google/uuid" - "github.com/stretchr/testify/require" - - "github.com/coder/coder/v2/coderd/database" - "github.com/coder/coder/v2/coderd/database/dbgen" - "github.com/coder/coder/v2/coderd/database/dbtestutil" - "github.com/coder/coder/v2/coderd/x/chatd" - "github.com/coder/coder/v2/coderd/x/chatd/chatprompt" - "github.com/coder/coder/v2/coderd/x/chatd/chattest" - "github.com/coder/coder/v2/codersdk" - "github.com/coder/coder/v2/testutil" -) - -func TestActiveServer_ChainBrokenRecovery(t *testing.T) { - t.Parallel() - - const ( - previousResponseID = "resp_poisoned" - recoveredAnswer = "recovered answer" - ) - ctx := testutil.Context(t, testutil.WaitLong) - db, ps := dbtestutil.NewDB(t) - requests := newOpenAIRequestRecorder() - openAIURL := chattest.NewOpenAI(t, func(req *chattest.OpenAIRequest) chattest.OpenAIResponse { - requests.record(req) - if req.PreviousResponseID != nil { - return chattest.OpenAIErrorResponse(http.StatusNotFound, "invalid_request_error", chainBrokenProviderErrorMessage) - } - return chattest.OpenAIStreamingResponse(chattest.OpenAITextChunks(recoveredAnswer)...) - }) - user, org, model := seedChatDependenciesWithProvider(t, db, "openai", openAIURL) - model = updateModelForChainMode(t, db, model) - - factory := chattest.NewMockAIBridgeTransport(t, openAIURL) - server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) { - cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(factory) - }) - chat := createChatThroughServer(ctx, t, db, server, org.ID, user.ID, model.ID, "first user") - waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusWaiting) - insertProviderResponseID(ctx, t, db, chat.ID, "first assistant", model.ID, previousResponseID) - _, err := server.SendMessage(ctx, chatd.SendMessageOptions{ - ChatID: chat.ID, - CreatedBy: user.ID, - APIKeyID: testAPIKeyID(t, db, user.ID), - ModelConfigID: model.ID, - Content: []codersdk.ChatMessagePart{codersdk.ChatMessageText("follow up")}, - BusyBehavior: chatd.SendMessageBusyBehaviorQueue, - }) - require.NoError(t, err) - - waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusWaiting) - - got := requests.all() - require.GreaterOrEqual(t, len(got), 3) - generationRequests := filterStreamingRequests(got) - require.Len(t, generationRequests, 3) - require.Nil(t, generationRequests[0].PreviousResponseID) - require.Equal(t, previousResponseID, requirePreviousResponseID(t, generationRequests[1])) - require.Nil(t, generationRequests[2].PreviousResponseID) - requireRawPromptContains(t, generationRequests[2], "first user") - requireRawPromptContains(t, generationRequests[2], "first assistant") - requireRawPromptContains(t, generationRequests[2], "follow up") - - messages := chatMessages(ctx, t, db, chat.ID) - requireTextPart(t, messages[len(messages)-1], recoveredAnswer) -} - -func TestActiveServer_ChainBrokenRecoveryAppliesProviderPromptPrep(t *testing.T) { - t.Parallel() - - const previousResponseID = "resp_anthropic_chain" - ctx := testutil.Context(t, testutil.WaitLong) - db, ps := dbtestutil.NewDB(t) - requests := newAnthropicRequestRecorder() - var streamCalls atomic.Int32 - anthropicURL := chattest.NewAnthropic(t, func(req *chattest.AnthropicRequest) chattest.AnthropicResponse { - requests.record(req) - if streamCalls.Add(1) == 2 { - return chattest.AnthropicErrorResponse(http.StatusInternalServerError, "server_error", chainBrokenProviderErrorMessage) - } - return chattest.AnthropicStreamingResponse(chattest.AnthropicTextChunks("anthropic answer")...) - }) - user, org, model := seedAnthropicChatDependencies(t, db, anthropicURL) - model = updateModelForChainMode(t, db, model) - - factory := chattest.NewMockAIBridgeTransport(t, anthropicURL, chattest.WithPreservePath()) - server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) { - cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(factory) - }) - chat := createChatThroughServer(ctx, t, db, server, org.ID, user.ID, model.ID, "hello") - waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusWaiting) - insertSystemTextMessage(ctx, t, db, chat.ID, "sys-1", model.ID) - insertProviderResponseID(ctx, t, db, chat.ID, "hi", model.ID, previousResponseID) - _, err := server.SendMessage(ctx, chatd.SendMessageOptions{ - ChatID: chat.ID, - CreatedBy: user.ID, - APIKeyID: testAPIKeyID(t, db, user.ID), - ModelConfigID: model.ID, - Content: []codersdk.ChatMessagePart{codersdk.ChatMessageText("follow up")}, - BusyBehavior: chatd.SendMessageBusyBehaviorQueue, - }) - require.NoError(t, err) - - waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusWaiting) - - generationRequests := filterAnthropicStreamingRequests(requests.all()) - require.Len(t, generationRequests, 2) - recovered := generationRequests[1] - require.Len(t, recovered.Messages, 4) - require.True(t, anthropicSystemHasEphemeralCacheControl(t, recovered)) - require.False(t, anthropicMessageHasEphemeralCacheControl(t, recovered.Messages[0])) - require.False(t, anthropicMessageHasEphemeralCacheControl(t, recovered.Messages[1])) - require.True(t, anthropicMessageHasEphemeralCacheControl(t, recovered.Messages[2])) - require.True(t, anthropicMessageHasEphemeralCacheControl(t, recovered.Messages[3])) -} - -func TestActiveServer_NonChainBrokenRetryPreservesChainMode(t *testing.T) { - t.Parallel() - - const previousResponseID = "resp_still_valid" - ctx := testutil.Context(t, testutil.WaitLong) - db, ps := dbtestutil.NewDB(t) - requests := newOpenAIRequestRecorder() - var streamCalls atomic.Int32 - openAIURL := chattest.NewOpenAI(t, func(req *chattest.OpenAIRequest) chattest.OpenAIResponse { - requests.record(req) - if req.Stream && streamCalls.Add(1) == 2 { - return chattest.OpenAIServerErrorResponse() - } - return chattest.OpenAIStreamingResponse(chattest.OpenAITextChunks("answer")...) - }) - user, org, model := seedChatDependenciesWithProvider(t, db, "openai", openAIURL) - model = updateModelForChainMode(t, db, model) - - factory := chattest.NewMockAIBridgeTransport(t, openAIURL) - server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) { - cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(factory) - }) - chat := createChatThroughServer(ctx, t, db, server, org.ID, user.ID, model.ID, "first user") - waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusWaiting) - insertProviderResponseID(ctx, t, db, chat.ID, "first assistant", model.ID, previousResponseID) - _, err := server.SendMessage(ctx, chatd.SendMessageOptions{ - ChatID: chat.ID, - CreatedBy: user.ID, - APIKeyID: testAPIKeyID(t, db, user.ID), - ModelConfigID: model.ID, - Content: []codersdk.ChatMessagePart{codersdk.ChatMessageText("follow up")}, - BusyBehavior: chatd.SendMessageBusyBehaviorQueue, - }) - require.NoError(t, err) - - waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusWaiting) - - generationRequests := filterStreamingRequests(requests.all()) - require.Len(t, generationRequests, 3) - require.Equal(t, previousResponseID, requirePreviousResponseID(t, generationRequests[1])) - require.Equal(t, previousResponseID, requirePreviousResponseID(t, generationRequests[2])) - requireRawPromptNotContains(t, generationRequests[2], "first user") - requireRawPromptContains(t, generationRequests[2], "follow up") -} - -func TestActiveServer_ChainBrokenRecoveryPersistsAcrossGenerationActions(t *testing.T) { - t.Parallel() - - const previousResponseID = "resp_tool_poisoned" - ctx := testutil.Context(t, testutil.WaitLong) - db, ps := dbtestutil.NewDB(t) - requests := newOpenAIRequestRecorder() - var streamCalls atomic.Int32 - openAIURL := chattest.NewOpenAI(t, func(req *chattest.OpenAIRequest) chattest.OpenAIResponse { - requests.record(req) - if !req.Stream { - return chattest.OpenAINonStreamingResponse(`{"title":"test"}`) - } - switch streamCalls.Add(1) { - case 1: - return chattest.OpenAIStreamingResponse(chattest.OpenAITextChunks("first answer")...) - case 2: - return chattest.OpenAIErrorResponse(http.StatusNotFound, "invalid_request_error", chainBrokenProviderErrorMessage) - case 3: - return chattest.OpenAIStreamingResponse(chattest.OpenAIToolCallChunk("read_skill", `{"name":"x"}`)) - default: - return chattest.OpenAIStreamingResponse(chattest.OpenAITextChunks("final answer")...) - } - }) - user, org, model := seedChatDependenciesWithProvider(t, db, "openai", openAIURL) - model = updateModelForChainMode(t, db, model) - - factory := chattest.NewMockAIBridgeTransport(t, openAIURL) - server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) { - cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(factory) - }) - chat := createChatThroughServer(ctx, t, db, server, org.ID, user.ID, model.ID, "first user") - waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusWaiting) - insertProviderResponseID(ctx, t, db, chat.ID, "first assistant", model.ID, previousResponseID) - _, err := server.SendMessage(ctx, chatd.SendMessageOptions{ - ChatID: chat.ID, - CreatedBy: user.ID, - APIKeyID: testAPIKeyID(t, db, user.ID), - ModelConfigID: model.ID, - Content: []codersdk.ChatMessagePart{codersdk.ChatMessageText("follow up")}, - BusyBehavior: chatd.SendMessageBusyBehaviorQueue, - }) - require.NoError(t, err) - - waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusWaiting) - - generationRequests := filterStreamingRequests(requests.all()) - require.Len(t, generationRequests, 4) - require.Equal(t, previousResponseID, requirePreviousResponseID(t, generationRequests[1])) - require.Nil(t, generationRequests[2].PreviousResponseID) - require.Nil(t, generationRequests[3].PreviousResponseID) -} - -func TestActiveServer_ChainBrokenWithoutChainModeIsSafe(t *testing.T) { - t.Parallel() - - ctx := testutil.Context(t, testutil.WaitLong) - db, ps := dbtestutil.NewDB(t) - requests := newOpenAIRequestRecorder() - var streamCalls atomic.Int32 - openAIURL := chattest.NewOpenAI(t, func(req *chattest.OpenAIRequest) chattest.OpenAIResponse { - requests.record(req) - if req.Stream && streamCalls.Add(1) == 1 { - return chattest.OpenAIErrorResponse(http.StatusNotFound, "invalid_request_error", chainBrokenProviderErrorMessage) - } - return chattest.OpenAIStreamingResponse(chattest.OpenAITextChunks("recovered")...) - }) - user, org, model := seedChatDependenciesWithProvider(t, db, "openai", openAIURL) - model = updateModelForChainMode(t, db, model) - - factory := chattest.NewMockAIBridgeTransport(t, openAIURL) - server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) { - cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(factory) - }) - chat := createChatThroughServer(ctx, t, db, server, org.ID, user.ID, model.ID, "only user") - waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusWaiting) - - generationRequests := filterStreamingRequests(requests.all()) - require.Len(t, generationRequests, 2) - require.Nil(t, generationRequests[0].PreviousResponseID) - require.Nil(t, generationRequests[1].PreviousResponseID) -} - -func TestActiveServer_ChainBrokenRecoveryDropsOrphanProviderToolCall(t *testing.T) { - t.Parallel() - - const previousResponseID = "resp_orphan_provider_tool" - ctx := testutil.Context(t, testutil.WaitLong) - db, ps := dbtestutil.NewDB(t) - requests := newAnthropicRequestRecorder() - var streamCalls atomic.Int32 - anthropicURL := chattest.NewAnthropic(t, func(req *chattest.AnthropicRequest) chattest.AnthropicResponse { - requests.record(req) - if streamCalls.Add(1) == 2 { - return chattest.AnthropicErrorResponse(http.StatusInternalServerError, "server_error", chainBrokenProviderErrorMessage) - } - return chattest.AnthropicStreamingResponse(chattest.AnthropicTextChunks("cleaned")...) - }) - user, org, model := seedAnthropicChatDependencies(t, db, anthropicURL) - model = updateModelForChainMode(t, db, model) - - factory := chattest.NewMockAIBridgeTransport(t, anthropicURL, chattest.WithPreservePath()) - server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) { - cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(factory) - }) - chat := createChatThroughServer(ctx, t, db, server, org.ID, user.ID, model.ID, "first user") - waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusWaiting) - insertProviderResponseID(ctx, t, db, chat.ID, "first assistant", model.ID, previousResponseID) - insertOrphanProviderToolCall(ctx, t, db, chat.ID, model.ID) - _, err := server.SendMessage(ctx, chatd.SendMessageOptions{ - ChatID: chat.ID, - CreatedBy: user.ID, - APIKeyID: testAPIKeyID(t, db, user.ID), - ModelConfigID: model.ID, - Content: []codersdk.ChatMessagePart{codersdk.ChatMessageText("continue")}, - BusyBehavior: chatd.SendMessageBusyBehaviorQueue, - }) - require.NoError(t, err) - - waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusWaiting) - - generationRequests := filterAnthropicStreamingRequests(requests.all()) - require.Len(t, generationRequests, 2) - recoveredBody := anthropicRequestBody(t, generationRequests[1]) - require.NotContains(t, recoveredBody, "web_search") - require.Contains(t, recoveredBody, "partial") - require.Contains(t, recoveredBody, "continue") - requireAnthropicRequestRedactedReasoning(t, generationRequests[1], "redacted-payload") -} - -type anthropicRequestRecorder struct { - mu sync.Mutex - requests []chattest.AnthropicRequest -} - -func newAnthropicRequestRecorder() *anthropicRequestRecorder { - return &anthropicRequestRecorder{} -} - -func (r *anthropicRequestRecorder) record(req *chattest.AnthropicRequest) { - r.mu.Lock() - defer r.mu.Unlock() - r.requests = append(r.requests, *req) -} - -func (r *anthropicRequestRecorder) all() []chattest.AnthropicRequest { - r.mu.Lock() - defer r.mu.Unlock() - return append([]chattest.AnthropicRequest(nil), r.requests...) -} - -func filterAnthropicStreamingRequests(requests []chattest.AnthropicRequest) []chattest.AnthropicRequest { - out := make([]chattest.AnthropicRequest, 0, len(requests)) - for _, req := range requests { - if req.Stream { - out = append(out, req) - } - } - return out -} - -func seedAnthropicChatDependencies(t *testing.T, db database.Store, baseURL string) (database.User, database.Organization, database.ChatModelConfig) { - t.Helper() - user := dbgen.User(t, db, database.User{}) - _ = testAPIKeyID(t, db, user.ID) - org := dbgen.Organization(t, db, database.Organization{}) - dbgen.OrganizationMember(t, db, database.OrganizationMember{UserID: user.ID, OrganizationID: org.ID}) - provider := dbgen.AIProvider(t, db, database.AIProvider{Type: database.AIProviderTypeAnthropic}, func(params *database.InsertAIProviderParams) { - params.BaseUrl = baseURL - }) - dbgen.AIProviderKey(t, db, database.AIProviderKey{ProviderID: provider.ID}) - model := dbgen.ChatModelConfig(t, db, database.ChatModelConfig{ - Model: "claude-sonnet-4-20250514", - IsDefault: true, - AIProviderID: uuid.NullUUID{UUID: provider.ID, Valid: true}, - }) - return user, org, model -} - -func anthropicSystemHasEphemeralCacheControl(t *testing.T, req chattest.AnthropicRequest) bool { - t.Helper() - return strings.Contains(string(req.System), `"cache_control":{"type":"ephemeral"}`) -} - -func anthropicMessageHasEphemeralCacheControl(t *testing.T, message chattest.AnthropicRequestMessage) bool { - t.Helper() - return strings.Contains(string(message.Content), `"cache_control":{"type":"ephemeral"}`) -} - -func anthropicRequestBody(t *testing.T, req chattest.AnthropicRequest) string { - t.Helper() - data, err := json.Marshal(req.Messages) - require.NoError(t, err) - return string(data) -} - -func insertSystemTextMessage( - ctx context.Context, - t *testing.T, - db database.Store, - chatID uuid.UUID, - text string, - modelID uuid.UUID, -) { - t.Helper() - content, err := chatprompt.MarshalParts([]codersdk.ChatMessagePart{codersdk.ChatMessageText(text)}) - require.NoError(t, err) - params := chatd.BuildSingleChatMessageInsertParams( - chatID, - database.ChatMessageRoleSystem, - content, - database.ChatMessageVisibilityBoth, - modelID, - chatprompt.CurrentContentVersion, - uuid.Nil, - ) - _, err = db.InsertChatMessages(ctx, params) - require.NoError(t, err) -} - -func requireAnthropicRequestRedactedReasoning(t *testing.T, req chattest.AnthropicRequest, redactedData string) { - t.Helper() - body := anthropicRequestBody(t, req) - require.Contains(t, body, "redacted-payload") - require.Contains(t, body, redactedData) -} - -func insertOrphanProviderToolCall(ctx context.Context, t *testing.T, db database.Store, chatID uuid.UUID, modelID uuid.UUID) { - t.Helper() - reasoningMetadata, err := json.Marshal(fantasy.ProviderMetadata{ - fantasyanthropic.Name: &fantasyanthropic.ReasoningOptionMetadata{RedactedData: "redacted-payload"}, - }) - require.NoError(t, err) - parts := []codersdk.ChatMessagePart{ - { - Type: codersdk.ChatMessagePartTypeReasoning, - ProviderMetadata: reasoningMetadata, - }, - { - Type: codersdk.ChatMessagePartTypeToolCall, - ToolCallID: "ws-orphan", - ToolName: "web_search", - Args: json.RawMessage(`{"query":"coder"}`), - ProviderExecuted: true, - }, - codersdk.ChatMessageText("partial"), - } - content, err := chatprompt.MarshalParts(parts) - require.NoError(t, err) - params := chatd.BuildSingleChatMessageInsertParams( - chatID, - database.ChatMessageRoleAssistant, - content, - database.ChatMessageVisibilityBoth, - modelID, - chatprompt.CurrentContentVersion, - uuid.Nil, - ) - _, err = db.InsertChatMessages(ctx, params) - require.NoError(t, err) -} - -const chainBrokenProviderErrorMessage = "Previous response with id 'resp_abc' not found." - -type openAIRequestRecorder struct { - mu sync.Mutex - requests []chattest.OpenAIRequest -} - -func newOpenAIRequestRecorder() *openAIRequestRecorder { - return &openAIRequestRecorder{} -} - -func (r *openAIRequestRecorder) record(req *chattest.OpenAIRequest) { - r.mu.Lock() - defer r.mu.Unlock() - r.requests = append(r.requests, *req) -} - -func (r *openAIRequestRecorder) all() []chattest.OpenAIRequest { - r.mu.Lock() - defer r.mu.Unlock() - return append([]chattest.OpenAIRequest(nil), r.requests...) -} - -func updateModelForChainMode(t *testing.T, db database.Store, model database.ChatModelConfig) database.ChatModelConfig { - t.Helper() - store := true - options, err := json.Marshal(codersdk.ChatModelCallConfig{ - ProviderOptions: &codersdk.ChatModelProviderOptions{ - OpenAI: &codersdk.ChatModelOpenAIProviderOptions{Store: &store}, - }, - }) - require.NoError(t, err) - updated, err := db.UpdateChatModelConfig(context.Background(), database.UpdateChatModelConfigParams{ - ID: model.ID, - DisplayName: model.DisplayName, - Model: model.Model, - Enabled: model.Enabled, - ContextLimit: model.ContextLimit, - CompressionThreshold: model.CompressionThreshold, - Options: options, - AIProviderID: model.AIProviderID, - }) - require.NoError(t, err) - return updated -} - -func createChatThroughServer( - ctx context.Context, - t *testing.T, - db database.Store, - server *chatd.Server, - orgID uuid.UUID, - userID uuid.UUID, - modelID uuid.UUID, - text string, -) database.Chat { - t.Helper() - chat, err := server.CreateChat(ctx, chatd.CreateOptions{ - OrganizationID: orgID, - OwnerID: userID, - APIKeyID: testAPIKeyID(t, db, userID), - Title: "chain mode test", - InitialUserContent: []codersdk.ChatMessagePart{codersdk.ChatMessageText(text)}, - ModelConfigID: modelID, - }) - require.NoError(t, err) - return chat -} - -func waitForChatStatus(ctx context.Context, t *testing.T, db database.Store, chatID uuid.UUID, status database.ChatStatus) database.Chat { - t.Helper() - var chat database.Chat - testutil.Eventually(ctx, t, func(ctx context.Context) bool { - latest, err := db.GetChatByID(ctx, chatID) - if err != nil { - return false - } - chat = latest - return latest.Status == status && !latest.WorkerID.Valid && !latest.RunnerID.Valid - }, testutil.IntervalFast) - return chat -} - -func insertProviderResponseID( - ctx context.Context, - t *testing.T, - db database.Store, - chatID uuid.UUID, - text string, - modelID uuid.UUID, - providerResponseID string, -) { - t.Helper() - content, err := chatprompt.MarshalParts([]codersdk.ChatMessagePart{codersdk.ChatMessageText(text)}) - require.NoError(t, err) - params := chatd.BuildSingleChatMessageInsertParams( - chatID, - database.ChatMessageRoleAssistant, - content, - database.ChatMessageVisibilityBoth, - modelID, - chatprompt.CurrentContentVersion, - uuid.Nil, - ) - params.ProviderResponseID[0] = providerResponseID - _, err = db.InsertChatMessages(ctx, params) - require.NoError(t, err) -} - -func chatMessages(ctx context.Context, t *testing.T, db database.Store, chatID uuid.UUID) []database.ChatMessage { - t.Helper() - messages, err := db.GetChatMessagesByChatID(ctx, database.GetChatMessagesByChatIDParams{ChatID: chatID}) - require.NoError(t, err) - return messages -} - -func filterStreamingRequests(requests []chattest.OpenAIRequest) []chattest.OpenAIRequest { - out := make([]chattest.OpenAIRequest, 0, len(requests)) - for _, req := range requests { - if req.Stream { - out = append(out, req) - } - } - return out -} - -func requirePreviousResponseID(t *testing.T, req chattest.OpenAIRequest) string { - t.Helper() - require.NotNil(t, req.PreviousResponseID) - return *req.PreviousResponseID -} - -func requireRawPromptContains(t *testing.T, req chattest.OpenAIRequest, text string) { - t.Helper() - require.Contains(t, string(req.RawBody), text) -} - -func requireRawPromptNotContains(t *testing.T, req chattest.OpenAIRequest, text string) { - t.Helper() - require.NotContains(t, string(req.RawBody), text) -} - -func requireTextPart(t *testing.T, msg database.ChatMessage, text string) { - t.Helper() - parts, err := chatprompt.ParseContent(msg) - require.NoError(t, err) - for _, part := range parts { - if part.Type == codersdk.ChatMessagePartTypeText && part.Text == text { - return - } - } - t.Fatalf("missing text part %q in message %d", text, msg.ID) -} diff --git a/coderd/x/chatd/chatd_helpers_test.go b/coderd/x/chatd/chatd_helpers_test.go new file mode 100644 index 0000000000000..b51d7c55c6d77 --- /dev/null +++ b/coderd/x/chatd/chatd_helpers_test.go @@ -0,0 +1,200 @@ +package chatd_test + +// Shared helpers for chatd active-server tests. + +import ( + "context" + "encoding/json" + "strings" + "sync" + "testing" + + "charm.land/fantasy" + fantasyanthropic "charm.land/fantasy/providers/anthropic" + "github.com/google/uuid" + "github.com/stretchr/testify/require" + + "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/database/dbgen" + "github.com/coder/coder/v2/coderd/x/chatd" + "github.com/coder/coder/v2/coderd/x/chatd/chatprompt" + "github.com/coder/coder/v2/coderd/x/chatd/chattest" + "github.com/coder/coder/v2/codersdk" + "github.com/coder/coder/v2/testutil" +) + +type anthropicRequestRecorder struct { + mu sync.Mutex + requests []chattest.AnthropicRequest +} + +func newAnthropicRequestRecorder() *anthropicRequestRecorder { + return &anthropicRequestRecorder{} +} + +func (r *anthropicRequestRecorder) record(req *chattest.AnthropicRequest) { + r.mu.Lock() + defer r.mu.Unlock() + r.requests = append(r.requests, *req) +} + +func (r *anthropicRequestRecorder) all() []chattest.AnthropicRequest { + r.mu.Lock() + defer r.mu.Unlock() + return append([]chattest.AnthropicRequest(nil), r.requests...) +} + +func filterAnthropicStreamingRequests(requests []chattest.AnthropicRequest) []chattest.AnthropicRequest { + out := make([]chattest.AnthropicRequest, 0, len(requests)) + for _, req := range requests { + if req.Stream { + out = append(out, req) + } + } + return out +} + +func seedAnthropicChatDependencies(t *testing.T, db database.Store, baseURL string) (database.User, database.Organization, database.ChatModelConfig) { + t.Helper() + user := dbgen.User(t, db, database.User{}) + _ = testAPIKeyID(t, db, user.ID) + org := dbgen.Organization(t, db, database.Organization{}) + dbgen.OrganizationMember(t, db, database.OrganizationMember{UserID: user.ID, OrganizationID: org.ID}) + provider := dbgen.AIProvider(t, db, database.AIProvider{Type: database.AIProviderTypeAnthropic}, func(params *database.InsertAIProviderParams) { + params.BaseUrl = baseURL + }) + dbgen.AIProviderKey(t, db, database.AIProviderKey{ProviderID: provider.ID}) + model := dbgen.ChatModelConfig(t, db, database.ChatModelConfig{ + Model: "claude-sonnet-4-20250514", + IsDefault: true, + AIProviderID: uuid.NullUUID{UUID: provider.ID, Valid: true}, + }) + return user, org, model +} + +func anthropicMessageHasEphemeralCacheControl(t *testing.T, message chattest.AnthropicRequestMessage) bool { + t.Helper() + return strings.Contains(string(message.Content), `"cache_control":{"type":"ephemeral"}`) +} + +func anthropicRequestBody(t *testing.T, req chattest.AnthropicRequest) string { + t.Helper() + data, err := json.Marshal(req.Messages) + require.NoError(t, err) + return string(data) +} + +func insertSystemTextMessage( + ctx context.Context, + t *testing.T, + db database.Store, + chatID uuid.UUID, + text string, + modelID uuid.UUID, +) { + t.Helper() + content, err := chatprompt.MarshalParts([]codersdk.ChatMessagePart{codersdk.ChatMessageText(text)}) + require.NoError(t, err) + params := chatd.BuildSingleChatMessageInsertParams( + chatID, + database.ChatMessageRoleSystem, + content, + database.ChatMessageVisibilityBoth, + modelID, + chatprompt.CurrentContentVersion, + uuid.Nil, + ) + _, err = db.InsertChatMessages(ctx, params) + require.NoError(t, err) +} + +func insertOrphanProviderToolCall(ctx context.Context, t *testing.T, db database.Store, chatID uuid.UUID, modelID uuid.UUID) { + t.Helper() + reasoningMetadata, err := json.Marshal(fantasy.ProviderMetadata{ + fantasyanthropic.Name: &fantasyanthropic.ReasoningOptionMetadata{RedactedData: "redacted-payload"}, + }) + require.NoError(t, err) + parts := []codersdk.ChatMessagePart{ + { + Type: codersdk.ChatMessagePartTypeReasoning, + ProviderMetadata: reasoningMetadata, + }, + { + Type: codersdk.ChatMessagePartTypeToolCall, + ToolCallID: "ws-orphan", + ToolName: "web_search", + Args: json.RawMessage(`{"query":"coder"}`), + ProviderExecuted: true, + }, + codersdk.ChatMessageText("partial"), + } + content, err := chatprompt.MarshalParts(parts) + require.NoError(t, err) + params := chatd.BuildSingleChatMessageInsertParams( + chatID, + database.ChatMessageRoleAssistant, + content, + database.ChatMessageVisibilityBoth, + modelID, + chatprompt.CurrentContentVersion, + uuid.Nil, + ) + _, err = db.InsertChatMessages(ctx, params) + require.NoError(t, err) +} + +func createChatThroughServer( + ctx context.Context, + t *testing.T, + db database.Store, + server *chatd.Server, + orgID uuid.UUID, + userID uuid.UUID, + modelID uuid.UUID, + text string, +) database.Chat { + t.Helper() + chat, err := server.CreateChat(ctx, chatd.CreateOptions{ + OrganizationID: orgID, + OwnerID: userID, + APIKeyID: testAPIKeyID(t, db, userID), + Title: "test chat", + InitialUserContent: []codersdk.ChatMessagePart{codersdk.ChatMessageText(text)}, + ModelConfigID: modelID, + }) + require.NoError(t, err) + return chat +} + +func waitForChatStatus(ctx context.Context, t *testing.T, db database.Store, chatID uuid.UUID, status database.ChatStatus) database.Chat { + t.Helper() + var chat database.Chat + testutil.Eventually(ctx, t, func(ctx context.Context) bool { + latest, err := db.GetChatByID(ctx, chatID) + if err != nil { + return false + } + chat = latest + return latest.Status == status && !latest.WorkerID.Valid && !latest.RunnerID.Valid + }, testutil.IntervalFast) + return chat +} + +func chatMessages(ctx context.Context, t *testing.T, db database.Store, chatID uuid.UUID) []database.ChatMessage { + t.Helper() + messages, err := db.GetChatMessagesByChatID(ctx, database.GetChatMessagesByChatIDParams{ChatID: chatID}) + require.NoError(t, err) + return messages +} + +func requireTextPart(t *testing.T, msg database.ChatMessage, text string) { + t.Helper() + parts, err := chatprompt.ParseContent(msg) + require.NoError(t, err) + for _, part := range parts { + if part.Type == codersdk.ChatMessagePartTypeText && part.Text == text { + return + } + } + t.Fatalf("missing text part %q in message %d", text, msg.ID) +} diff --git a/coderd/x/chatd/chatd_retry_test.go b/coderd/x/chatd/chatd_retry_test.go index b1f162a75cfab..51eb8930970c6 100644 --- a/coderd/x/chatd/chatd_retry_test.go +++ b/coderd/x/chatd/chatd_retry_test.go @@ -77,7 +77,6 @@ func TestActiveServer_RetryStatePersistedDuringBackoff(t *testing.T) { require.Equal(t, "generate_assistant", retrySinkFieldValue(t, entries[0].Fields, "action")) require.Equal(t, "openai", retrySinkFieldValue(t, entries[0].Fields, "provider")) require.Equal(t, "429", retrySinkFieldValue(t, entries[0].Fields, "status_code")) - require.Equal(t, "false", retrySinkFieldValue(t, entries[0].Fields, "chain_broken")) require.Greater(t, latest.RetryStateVersion, withRetry.RetryStateVersion) messages, err := db.GetChatMessagesByChatID(ctx, database.GetChatMessagesByChatIDParams{ChatID: chat.ID}) require.NoError(t, err) @@ -123,10 +122,9 @@ func TestActiveServer_RetryStreamSilenceTimeoutAndClassification(t *testing.T) { require.NoError(t, err) requireTextPart(t, messages[len(messages)-1], "recovered") requireRetryCounter(t, reg, "coderd_chatd_stream_retries_total", 1, map[string]string{ - "provider": "openai", - "model": "gpt-4o", - "kind": string(codersdk.ChatErrorKindRateLimit), - "chain_broken": "false", + "provider": "openai", + "model": "gpt-4o", + "kind": string(codersdk.ChatErrorKindRateLimit), }) }) @@ -187,10 +185,9 @@ func TestActiveServer_RetryStreamSilenceTimeoutAndClassification(t *testing.T) { require.Equal(t, string(codersdk.ChatErrorKindStreamSilenceTimeout), retrySinkFieldValue(t, entries[0].Fields, "error_kind")) require.Equal(t, "openai", retrySinkFieldValue(t, entries[0].Fields, "provider")) requireRetryCounter(t, reg, "coderd_chatd_stream_retries_total", 1, map[string]string{ - "provider": "openai", - "model": model.Model, - "kind": string(codersdk.ChatErrorKindStreamSilenceTimeout), - "chain_broken": "false", + "provider": "openai", + "model": model.Model, + "kind": string(codersdk.ChatErrorKindStreamSilenceTimeout), }) }) } diff --git a/coderd/x/chatd/chatd_test.go b/coderd/x/chatd/chatd_test.go index d83543399c2d5..954399308c6a3 100644 --- a/coderd/x/chatd/chatd_test.go +++ b/coderd/x/chatd/chatd_test.go @@ -65,11 +65,10 @@ import ( ) type recordedOpenAIRequest struct { - Messages []chattest.OpenAIMessage - Tools []string - Store *bool - PreviousResponseID *string - ContentLength int64 + Messages []chattest.OpenAIMessage + Tools []string + Store *bool + ContentLength int64 } func testAPIKeyID(t testing.TB, db database.Store, userID uuid.UUID) string { @@ -130,23 +129,16 @@ func recordOpenAIRequest(req *chattest.OpenAIRequest) recordedOpenAIRequest { store = &value } - var previousResponseID *string - if req.PreviousResponseID != nil { - value := *req.PreviousResponseID - previousResponseID = &value - } - var contentLength int64 if req.Request != nil { contentLength = req.Request.ContentLength } return recordedOpenAIRequest{ - Messages: messages, - Tools: tools, - Store: store, - PreviousResponseID: previousResponseID, - ContentLength: contentLength, + Messages: messages, + Tools: tools, + Store: store, + ContentLength: contentLength, } } @@ -5904,7 +5896,7 @@ func TestActiveServer_BasicAssistantGenerationAndPromptPreparation(t *testing.T) generationRequests := filterAnthropicStreamingRequests(requests.all()) require.Len(t, generationRequests, 2) recovered := generationRequests[1] - require.True(t, anthropicSystemHasEphemeralCacheControl(t, recovered)) + require.Contains(t, string(recovered.System), `"cache_control":{"type":"ephemeral"}`) require.Len(t, recovered.Messages, 4) require.False(t, anthropicMessageHasEphemeralCacheControl(t, recovered.Messages[0])) require.False(t, anthropicMessageHasEphemeralCacheControl(t, recovered.Messages[1])) @@ -6915,7 +6907,7 @@ func TestActiveServer_AnthropicSanitizesProviderToolBeforeRequest(t *testing.T) require.NotContains(t, body, "web_search") require.Contains(t, body, "partial") require.Contains(t, body, "continue") - requireAnthropicRequestRedactedReasoning(t, generationRequests[1], "redacted-payload") + require.Contains(t, body, "redacted-payload") } func TestActiveServer_AnthropicProviderToolPreRequestGuard(t *testing.T) { @@ -12500,198 +12492,6 @@ func TestAdvisorGating_ExploreSubagent(t *testing.T) { } } -// TestAdvisorChainMode_SnapshotKeepsFullHistory exercises the advisor -// runtime together with chain mode and asserts the snapshot captured for -// the nested advisor call retains the full pre-chain prompt. Chain mode -// otherwise strips assistant and tool turns from the prompt the outer -// loop sees, so a regression that captures the advisor snapshot after -// filterPromptForChainMode, or removes the chain-mode guard around -// advisor snapshotting, would leak the filtered view into the advisor's -// nested call. The advisor would then only see the trailing user -// message, losing the context the outer model had been building on. -func TestAdvisorChainMode_SnapshotKeepsFullHistory(t *testing.T) { - t.Parallel() - - db, ps := dbtestutil.NewDB(t) - ctx := testutil.Context(t, testutil.WaitLong) - - const ( - turn1User = "help me refactor this module" - turn1Reply = "happy to help, tell me more" - turn1RespID = "resp_turn1_advisor_chain" - turn2User = "follow up question" - advisorReply = "narrow the scope to one module" - finalReply = "acknowledged" - ) - - var ( - requestsMu sync.Mutex - requests []recordedOpenAIRequest - advisorRequestRaw []byte - advisorCallSeen atomic.Bool - ) - - openAIURL := chattest.NewOpenAI(t, func(req *chattest.OpenAIRequest) chattest.OpenAIResponse { - if !req.Stream { - return chattest.OpenAINonStreamingResponse("title") - } - - // The advisor's nested call runs with no tools (MaxSteps=1, - // empty tool set). Parent calls always carry the chat's tool - // set, which includes the advisor tool. - isAdvisorNested := len(req.Tools) == 0 - - requestsMu.Lock() - requests = append(requests, recordOpenAIRequest(req)) - if isAdvisorNested { - advisorRequestRaw = append([]byte(nil), req.RawBody...) - advisorCallSeen.Store(true) - } - requestsMu.Unlock() - - if isAdvisorNested { - return chattest.OpenAIStreamingResponse( - chattest.OpenAITextChunks(advisorReply)..., - ) - } - - // Turn 1 parent request: no previous_response_id yet, so chain - // mode cannot activate. Respond with a plain text reply and - // tag the stored response id so turn 2 can chain off it. - if req.PreviousResponseID == nil { - resp := chattest.OpenAIStreamingResponse( - chattest.OpenAITextChunks(turn1Reply)..., - ) - resp.ResponseID = turn1RespID - return resp - } - - // Turn 2 parent: chain mode is active. On the first pass call - // advisor; on the continuation after the tool result arrives, - // close out with a final text reply. - var hasAdvisorResult bool - for _, m := range req.Messages { - if m.Role == "tool" && strings.Contains(m.Content, advisorReply) { - hasAdvisorResult = true - break - } - } - if !hasAdvisorResult { - return chattest.OpenAIStreamingResponse(chattest.OpenAIToolCallChunk( - "advisor", - `{"question":"should I keep going?"}`, - )) - } - return chattest.OpenAIStreamingResponse( - chattest.OpenAITextChunks(finalReply)..., - ) - }) - - user, org, _ := seedChatDependenciesWithProvider(t, db, "openai", openAIURL) - storeEnabled := true - // The OpenAI Responses API is the only provider code path where - // chain mode activates. Store=true is the switch that routes this - // provider/model through the Responses API and lets - // IsResponsesStoreEnabled return true. - responsesModel := insertChatModelConfigWithCallConfig( - t, db, user.ID, "openai", "gpt-4o", - codersdk.ChatModelCallConfig{ - ProviderOptions: &codersdk.ChatModelProviderOptions{ - OpenAI: &codersdk.ChatModelOpenAIProviderOptions{ - Store: &storeEnabled, - }, - }, - }, - ) - seedAdvisorConfig(ctx, t, db, codersdk.AdvisorConfig{ - Enabled: true, - MaxUsesPerRun: 3, - MaxOutputTokens: 16384, - }) - server := newOpenAIResponsesTestServer(t, db, ps, func(cfg *chatd.Config) { - cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer( - chattest.NewMockAIBridgeTransport(t, openAIURL), - ) - }) - - chat, err := server.CreateChat(ctx, chatd.CreateOptions{ - OrganizationID: org.ID, - OwnerID: user.ID, - APIKeyID: testAPIKeyID(t, db, user.ID), - Title: "advisor-chain-mode", - ModelConfigID: responsesModel.ID, - InitialUserContent: []codersdk.ChatMessagePart{ - codersdk.ChatMessageText(turn1User), - }, - }) - require.NoError(t, err) - - // Turn 1 must settle before turn 2 starts so the assistant row - // with ProviderResponseID is visible to resolveChainMode. - waitForChatProcessed(ctx, t, db, chat.ID, server) - turn1Chat, err := db.GetChatByID(ctx, chat.ID) - require.NoError(t, err) - require.Equal(t, database.ChatStatusWaiting, turn1Chat.Status, - "turn 1 must complete before turn 2 can be sent; last_error=%q", chatLastErrorMessage(turn1Chat.LastError)) - - _, err = server.SendMessage(ctx, chatd.SendMessageOptions{ - ChatID: chat.ID, - CreatedBy: user.ID, - APIKeyID: testAPIKeyID(t, db, user.ID), - Content: []codersdk.ChatMessagePart{ - codersdk.ChatMessageText(turn2User), - }, - }) - require.NoError(t, err) - - require.Eventually(t, func() bool { - if !advisorCallSeen.Load() { - return false - } - got, getErr := db.GetChatByID(ctx, chat.ID) - if getErr != nil { - return false - } - return got.Status == database.ChatStatusWaiting || - got.Status == database.ChatStatusError - }, testutil.WaitLong, testutil.IntervalFast) - - requestsMu.Lock() - gotAdvisorBody := append([]byte(nil), advisorRequestRaw...) - gotRequests := append([]recordedOpenAIRequest(nil), requests...) - requestsMu.Unlock() - - // Chain mode must have actually fired on turn 2, otherwise this - // test degenerates to TestAdvisorHappyPath_RootChat. - var chainModeActivated bool - for _, r := range gotRequests { - if r.PreviousResponseID != nil && *r.PreviousResponseID == turn1RespID { - chainModeActivated = true - break - } - } - require.True(t, chainModeActivated, - "turn 2 parent request must carry previous_response_id; without it this test does not exercise chain mode") - - require.True(t, advisorCallSeen.Load(), - "the nested advisor call must execute under chain mode") - require.NotEmpty(t, gotAdvisorBody, - "advisor call must receive a non-empty request body") - - // The core assertion: the advisor snapshot must retain turn 1 - // context. Chain mode filtering strips assistant and tool turns - // from the prompt the outer loop sees, so if that filtered view - // leaked into the snapshot the advisor would only see turn 2's - // trailing user message. The advisor's nested call goes through - // the OpenAI Responses API, which encodes its prompt in the - // "input" field rather than "messages", so we inspect the raw - // request body for both turn-1 substrings. - require.Contains(t, string(gotAdvisorBody), turn1User, - "advisor snapshot must retain the turn 1 user message even when chain mode is active") - require.Contains(t, string(gotAdvisorBody), turn1Reply, - "advisor snapshot must retain the turn 1 assistant message even when chain mode is active") -} - // TestProviderSwitchSanitizesAndRestoresPEToolHistory verifies the A→B→A // provider-switch contract: // diff --git a/coderd/x/chatd/chatdebug/service_test.go b/coderd/x/chatd/chatdebug/service_test.go index df39abb300086..0c8176d6aa7b2 100644 --- a/coderd/x/chatd/chatdebug/service_test.go +++ b/coderd/x/chatd/chatdebug/service_test.go @@ -1143,13 +1143,12 @@ func insertMessage( require.NoError(t, err) msg := dbgen.ChatMessage(t, db, database.ChatMessage{ - ChatID: chatID, - CreatedBy: uuid.NullUUID{UUID: createdBy, Valid: true}, - ModelConfigID: uuid.NullUUID{UUID: modelID, Valid: true}, - Role: role, - Content: parts, - ContentVersion: chatprompt.CurrentContentVersion, - ProviderResponseID: sql.NullString{}, + ChatID: chatID, + CreatedBy: uuid.NullUUID{UUID: createdBy, Valid: true}, + ModelConfigID: uuid.NullUUID{UUID: modelID, Valid: true}, + Role: role, + Content: parts, + ContentVersion: chatprompt.CurrentContentVersion, }) return msg } diff --git a/coderd/x/chatd/chaterror/classify.go b/coderd/x/chatd/chaterror/classify.go index 0147248e2fc6a..aeadc723cb6b5 100644 --- a/coderd/x/chatd/chaterror/classify.go +++ b/coderd/x/chatd/chaterror/classify.go @@ -29,15 +29,6 @@ type ClassifiedError struct { // RetryAfter is a normalized minimum retry delay derived from // provider response metadata when available. RetryAfter time.Duration - - // ChainBroken is true when the provider reported that the - // previous_response_id (or analogous chain anchor) is no longer - // retrievable. The chatloop retry path uses this signal to exit - // chain mode and replay full history before the next attempt. - // This is an internal signal; it is not surfaced as a separate - // codersdk.ChatErrorKind so the user-visible kind set stays - // stable. - ChainBroken bool } // http2PeerResetCause mirrors golang.org/x/net/http2's unexported @@ -186,20 +177,6 @@ func Classify(err error) ClassifiedError { return classified } - // Chain-broken detection runs before the generic rule table so a - // 404 carrying a chain anchor failure is not classified as a - // generic non-retryable error. The chatloop retry callback uses - // the ChainBroken flag to exit chain mode and replay full - // history. - if classified, ok := chainBrokenClassification( - lower, - provider, - statusCode, - structured, - ); ok { - return classified - } - retryableHTTP2StreamReset, hasHTTP2StreamReset := classifyHTTP2StreamReset(err) providerDisabledMatch := containsAny(lower, providerDisabledPatterns...) deadline := errors.Is(err, context.DeadlineExceeded) || strings.Contains(lower, "context deadline exceeded") @@ -396,35 +373,6 @@ func streamIncompleteMessage(provider string) string { return providerSubject(provider) + " stream closed unexpectedly before the response completed." } -// chainBrokenClassification recognizes the OpenAI error -// "Previous response with id ... not found" returned when a -// chained turn references a previous_response_id the provider no -// longer recognizes. -func chainBrokenClassification( - lowerMessage string, - provider string, - statusCode int, - structured providerErrorDetails, -) (ClassifiedError, bool) { - if !(strings.Contains(lowerMessage, "previous response with id") && - strings.Contains(lowerMessage, "not found")) { - return ClassifiedError{}, false - } - // This class of error has so far only been observed with OpenAI. - if provider == "" { - provider = "openai" - } - return normalizeClassification(ClassifiedError{ - Detail: structured.detail, - Kind: codersdk.ChatErrorKindGeneric, - Provider: provider, - Retryable: true, - StatusCode: statusCode, - RetryAfter: structured.retryAfter, - ChainBroken: true, - }), true -} - func responsesAPIDiagnostic(lowerMessage, detail string) (string, bool) { lowerDetail := strings.ToLower(detail) for _, match := range responsesAPIDiagnosticMatches { diff --git a/coderd/x/chatd/chaterror/classify_test.go b/coderd/x/chatd/chaterror/classify_test.go index 1fb0a46e5b021..2ab7549e59e39 100644 --- a/coderd/x/chatd/chaterror/classify_test.go +++ b/coderd/x/chatd/chaterror/classify_test.go @@ -1465,123 +1465,6 @@ func TestClassify_TruncatesProviderDetail(t *testing.T) { require.True(t, strings.HasSuffix(classified.Detail, "…")) } -func TestClassify_ChainBroken(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - err error - wantChainBroken bool - wantRetryable bool - wantProvider string - wantStatusCode int - }{ - { - name: "OpenAIPreviousResponseNotFoundBareString", - err: xerrors.New( - "Previous response with id 'resp_abc' not found.", - ), - wantChainBroken: true, - wantRetryable: true, - wantProvider: "openai", - wantStatusCode: 0, - }, - { - name: "OpenAIPreviousResponseNotFoundProviderError", - err: testProviderError( - "Previous response with id 'resp_096c70c5bb8d52bc0069fa11e0630c81a3ba210cddfa75bae9' not found.", - 404, - nil, - ), - wantChainBroken: true, - wantRetryable: true, - wantProvider: "openai", - wantStatusCode: 404, - }, - { - name: "OpenAIPreviousResponseCaseInsensitive", - err: testProviderError( - "PREVIOUS RESPONSE WITH ID 'resp_abc' NOT FOUND.", - 404, - nil, - ), - wantChainBroken: true, - wantRetryable: true, - wantProvider: "openai", - wantStatusCode: 404, - }, - { - name: "PreviousResponseWithoutNotFoundIsNotChainBroken", - err: testProviderError( - "Previous response with id 'resp_abc' is invalid.", - 400, - nil, - ), - wantChainBroken: false, - }, - { - name: "UnrelatedNotFoundIsNotChainBroken", - err: testProviderError( - "resource not found", - 404, - nil, - ), - wantChainBroken: false, - }, - { - name: "UnrelatedInvalidRequestIsNotChainBroken", - err: testProviderError( - "", - 400, - nil, - testProviderResponseDump(`{"error":{"type":"invalid_request_error","message":"Image exceeds 5 MB maximum."}}`), - ), - wantChainBroken: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - classified := chaterror.Classify(tt.err) - require.Equal(t, tt.wantChainBroken, classified.ChainBroken, - "chain broken flag mismatch") - if !tt.wantChainBroken { - return - } - require.Equal(t, tt.wantRetryable, classified.Retryable, - "chain-broken errors must be retryable so the loop"+ - " can self-heal") - require.Equal(t, tt.wantProvider, classified.Provider) - require.Equal(t, tt.wantStatusCode, classified.StatusCode) - require.Equal(t, codersdk.ChatErrorKindGeneric, classified.Kind, - "chain-broken keeps the user-visible kind unchanged"+ - " so we don't add a new codersdk surface") - }) - } -} - -func TestClassify_ChainBrokenSurvivesWithClassification(t *testing.T) { - t.Parallel() - - original := chaterror.Classify(testProviderError( - "Previous response with id 'resp_abc' not found.", - 404, - nil, - )) - require.True(t, original.ChainBroken) - - wrapped := chaterror.WithClassification( - xerrors.New("transport blew up"), - original, - ) - round := chaterror.Classify(wrapped) - require.True(t, round.ChainBroken, - "WithClassification round-trips ChainBroken so the retry path"+ - " can detect it after re-classification") -} - func TestClassify_MissingKeyPreClassified(t *testing.T) { t.Parallel() diff --git a/coderd/x/chatd/chatloop/chatloop.go b/coderd/x/chatd/chatloop/chatloop.go index 0560cfd21e1ed..2086868d9e943 100644 --- a/coderd/x/chatd/chatloop/chatloop.go +++ b/coderd/x/chatd/chatloop/chatloop.go @@ -24,7 +24,6 @@ import ( "github.com/coder/coder/v2/coderd/database/dbtime" "github.com/coder/coder/v2/coderd/x/chatd/chatdebug" "github.com/coder/coder/v2/coderd/x/chatd/chaterror" - "github.com/coder/coder/v2/coderd/x/chatd/chatopenai" "github.com/coder/coder/v2/coderd/x/chatd/chatprompt" "github.com/coder/coder/v2/coderd/x/chatd/chatretry" "github.com/coder/coder/v2/coderd/x/chatd/chatsanitize" @@ -63,16 +62,13 @@ type PendingToolCall struct { Args string } -// PersistedStep contains the full content of a completed or -// interrupted agent step. Content includes both assistant blocks -// (text, reasoning, tool calls) and tool result blocks. The -// persistence layer is responsible for splitting these into -// separate database messages by role. +// PersistedStep is the unit the persistence layer splits into role-separated +// database messages. Content mixes assistant blocks (text, reasoning, tool +// calls) and tool result blocks from one completed or interrupted agent step. type PersistedStep struct { - Content []fantasy.Content - Usage fantasy.Usage - ContextLimit sql.NullInt64 - ProviderResponseID string + Content []fantasy.Content + Usage fantasy.Usage + ContextLimit sql.NullInt64 // Runtime is the wall-clock duration of this step, // covering LLM streaming, tool execution, and retries. // Zero indicates the duration was not measured (e.g. @@ -162,19 +158,8 @@ type RunOptions struct { ) // Callers should attach correlation fields (chat_id, owner_id, etc.) // using Logger.With before passing the logger in. - Logger slog.Logger - Compaction *CompactionOptions - ReloadMessages func(context.Context) ([]fantasy.Message, error) - DisableChainMode func() - // PrepareMessages is called at least once before each LLM step - // with the current message history. If it returns non-nil, the - // returned slice replaces messages for this and all subsequent - // steps. - // Used to inject system context that becomes available mid-loop - // (e.g. AGENTS.md after create_workspace). - // NOTE: It may be called more than once per step in case of a - // retry, so callbacks should avoid duplicating messages. - PrepareMessages func([]fantasy.Message) []fantasy.Message + Logger slog.Logger + Compaction *CompactionOptions // PrepareTools is called once before each LLM step with the // current tool list. If it returns non-nil, the returned slice @@ -436,7 +421,6 @@ func GenerateAssistant(ctx context.Context, opts GenerateAssistantOptions) (Assi Content: result.content, Usage: result.usage, ContextLimit: contextLimit, - ProviderResponseID: chatopenai.ExtractResponseIDIfStored(opts.ProviderOptions, result.providerMetadata), Runtime: opts.Clock.Since(stepStart), ToolCallCreatedAt: result.toolCallCreatedAt, ToolResultCreatedAt: result.toolResultCreatedAt, @@ -581,11 +565,6 @@ func prepareMessagesForRequest( totalSteps int, ) (canonical []fantasy.Message, prompt []fantasy.Message, err error) { canonical = messages - if opts.PrepareMessages != nil { - if updated := opts.PrepareMessages(canonical); updated != nil { - canonical = updated - } - } // Copy messages so provider-specific caching mutations don't leak // back to the canonical message slice. prompt = slices.Clone(canonical) diff --git a/coderd/x/chatd/chatloop/metrics.go b/coderd/x/chatd/chatloop/metrics.go index 6beddf9a54613..3263f01d87c59 100644 --- a/coderd/x/chatd/chatloop/metrics.go +++ b/coderd/x/chatd/chatloop/metrics.go @@ -3,7 +3,6 @@ package chatloop import ( "context" "errors" - "strconv" "charm.land/fantasy" "github.com/prometheus/client_golang/prometheus" @@ -109,7 +108,7 @@ func NewMetrics(reg prometheus.Registerer) *Metrics { Subsystem: metricsSubsystem, Name: "stream_retries_total", Help: "Total LLM stream retries.", - }, []string{"provider", "model", "kind", "chain_broken"}), + }, []string{"provider", "model", "kind"}), StreamBufferDroppedTotal: factory.NewCounter(prometheus.CounterOpts{ Namespace: metricsNamespace, Subsystem: metricsSubsystem, @@ -148,9 +147,7 @@ func (m *Metrics) RecordCompaction(provider, model string, compacted bool, err e // RecordStreamRetry increments stream_retries_total. The caller // must obtain classified via chaterror.Classify (non-empty Kind). -// No-op when m is nil. The chain_broken label is "true" for chain -// anchor failures (e.g. OpenAI previous_response_id 404) recovered -// by the chatloop, and "false" otherwise. +// No-op when m is nil. func (m *Metrics) RecordStreamRetry(provider, model string, classified chaterror.ClassifiedError) { if m == nil { return @@ -159,7 +156,6 @@ func (m *Metrics) RecordStreamRetry(provider, model string, classified chaterror provider, model, string(classified.Kind), - strconv.FormatBool(classified.ChainBroken), ).Inc() } diff --git a/coderd/x/chatd/chatloop/metrics_test.go b/coderd/x/chatd/chatloop/metrics_test.go index 1f48fa4de0c62..0094867fe1c54 100644 --- a/coderd/x/chatd/chatloop/metrics_test.go +++ b/coderd/x/chatd/chatloop/metrics_test.go @@ -2,7 +2,6 @@ package chatloop_test import ( "context" - "strconv" "testing" "charm.land/fantasy" @@ -33,7 +32,7 @@ func TestNewMetrics_RegistersAllMetrics(t *testing.T) { m.PromptSizeBytes.WithLabelValues("anthropic", "claude-sonnet-4-5") m.TTFTSeconds.WithLabelValues("anthropic", "claude-sonnet-4-5") m.StepsTotal.WithLabelValues("anthropic", "claude-sonnet-4-5") - m.StreamRetriesTotal.WithLabelValues("anthropic", "claude-sonnet-4-5", string(codersdk.ChatErrorKindTimeout), "false") + m.StreamRetriesTotal.WithLabelValues("anthropic", "claude-sonnet-4-5", string(codersdk.ChatErrorKindTimeout)) // StreamBufferDroppedTotal is a plain Counter, so it's always present // in Gather output once registered; no exerciser call is // needed. @@ -87,7 +86,7 @@ func TestNopMetrics_DoesNotPanic(t *testing.T) { m.CompactionTotal.WithLabelValues("openai", "gpt-5", "error").Inc() m.CompactionTotal.WithLabelValues("google", "gemini-2.5-pro", "timeout").Inc() m.StepsTotal.WithLabelValues("anthropic", "claude-sonnet-4-5").Inc() - m.StreamRetriesTotal.WithLabelValues("anthropic", "claude-sonnet-4-5", string(codersdk.ChatErrorKindTimeout), "false").Inc() + m.StreamRetriesTotal.WithLabelValues("anthropic", "claude-sonnet-4-5", string(codersdk.ChatErrorKindTimeout)).Inc() m.StreamBufferDroppedTotal.Inc() // Nil-receiver guard for RecordStreamRetry and @@ -284,9 +283,8 @@ func TestRecordStreamRetry(t *testing.T) { // guarantees Kind is non-empty, so no empty-string case is // needed. tests := []struct { - name string - kind codersdk.ChatErrorKind - chainBroken bool + name string + kind codersdk.ChatErrorKind }{ {name: "overloaded", kind: codersdk.ChatErrorKindOverloaded}, {name: "rate_limit", kind: codersdk.ChatErrorKindRateLimit}, @@ -296,7 +294,6 @@ func TestRecordStreamRetry(t *testing.T) { {name: "config", kind: codersdk.ChatErrorKindConfig}, {name: "missing_key", kind: codersdk.ChatErrorKindMissingKey}, {name: "generic", kind: codersdk.ChatErrorKindGeneric}, - {name: "chain_broken", kind: codersdk.ChatErrorKindGeneric, chainBroken: true}, } for _, tt := range tests { @@ -306,15 +303,13 @@ func TestRecordStreamRetry(t *testing.T) { reg := prometheus.NewRegistry() m := chatloop.NewMetrics(reg) m.RecordStreamRetry("test-provider", "test-model", chaterror.ClassifiedError{ - Kind: tt.kind, - ChainBroken: tt.chainBroken, + Kind: tt.kind, }) requireCounter(t, reg, "coderd_chatd_stream_retries_total", 1, map[string]string{ - "provider": "test-provider", - "model": "test-model", - "kind": string(tt.kind), - "chain_broken": strconv.FormatBool(tt.chainBroken), + "provider": "test-provider", + "model": "test-model", + "kind": string(tt.kind), }) }) } @@ -453,10 +448,9 @@ func TestGenerateAssistant_StreamRetryRecordsMetric(t *testing.T) { require.Equal(t, "bedrock", chaterror.Classify(err).Provider) // Retry metric keeps the transport provider label, not "bedrock". requireCounter(t, reg, "coderd_chatd_stream_retries_total", 1, map[string]string{ - "provider": "test-provider", - "model": "test-model", - "kind": string(codersdk.ChatErrorKindRateLimit), - "chain_broken": "false", + "provider": "test-provider", + "model": "test-model", + "kind": string(codersdk.ChatErrorKindRateLimit), }) } @@ -487,9 +481,8 @@ func TestGenerateAssistant_StreamRetry_ContextCanceledTransportResetIncrements(t require.Equal(t, 1, attempts) requireCounter(t, reg, "coderd_chatd_stream_retries_total", 1, map[string]string{ - "provider": "test-provider", - "model": "test-model", - "kind": string(codersdk.ChatErrorKindTimeout), - "chain_broken": "false", + "provider": "test-provider", + "model": "test-model", + "kind": string(codersdk.ChatErrorKindTimeout), }) } diff --git a/coderd/x/chatd/chatopenai/responses.go b/coderd/x/chatd/chatopenai/responses.go deleted file mode 100644 index 134ce31590df6..0000000000000 --- a/coderd/x/chatd/chatopenai/responses.go +++ /dev/null @@ -1,370 +0,0 @@ -package chatopenai - -import ( - "maps" - "slices" - "strings" - - "charm.land/fantasy" - fantasyopenai "charm.land/fantasy/providers/openai" - "github.com/google/uuid" - - "github.com/coder/coder/v2/coderd/database" - "github.com/coder/coder/v2/coderd/x/chatd/chatprompt" - "github.com/coder/coder/v2/codersdk" -) - -// ChainModeInfo holds the information needed to determine whether a follow-up turn -// can use OpenAI's previous_response_id chaining instead of replaying full -// conversation history. -type ChainModeInfo struct { - // previousResponseID is the provider response ID from the last assistant - // message, if any. - previousResponseID string - // modelConfigID is the model configuration used to produce the assistant - // message referenced by previousResponseID. - modelConfigID uuid.UUID - // contributingTrailingUserCount counts the trailing user messages that - // materially change the provider input. - contributingTrailingUserCount int - // hasUnresolvedLocalToolCalls is true when previousResponseID points at an - // assistant message with pending local tool calls. - hasUnresolvedLocalToolCalls bool - // providerMissingToolResults is true when the assistant message has local - // tool calls with local results, but no follow-up assistant message exists to - // confirm the results were sent back to the provider. This happens when - // StopAfterTool terminates a turn before the results are round-tripped. - providerMissingToolResults bool -} - -// PreviousResponseID returns the provider response ID from the last assistant -// message, if any. -func (c ChainModeInfo) PreviousResponseID() string { - return c.previousResponseID -} - -// ModelConfigID returns the model configuration used to produce the assistant -// message referenced by PreviousResponseID. -func (c ChainModeInfo) ModelConfigID() uuid.UUID { - return c.modelConfigID -} - -// ContributingTrailingUserCount returns the number of trailing user messages -// that materially change the provider input. -func (c ChainModeInfo) ContributingTrailingUserCount() int { - return c.contributingTrailingUserCount -} - -// HasUnresolvedLocalToolCalls reports whether PreviousResponseID points at an -// assistant message with pending local tool calls. -func (c ChainModeInfo) HasUnresolvedLocalToolCalls() bool { - return c.hasUnresolvedLocalToolCalls -} - -// ProviderMissingToolResults reports whether PreviousResponseID points at an -// assistant message with local tool results, but no follow-up assistant message -// confirms those tool results were sent to the provider (not just persisted -// locally). -func (c ChainModeInfo) ProviderMissingToolResults() bool { - return c.providerMissingToolResults -} - -// IsResponsesStoreEnabled checks if the OpenAI Responses provider options are -// present and have Store set to true. When true, the provider stores -// conversation history server-side, enabling follow-up chaining via -// PreviousResponseID. -func IsResponsesStoreEnabled(opts fantasy.ProviderOptions) bool { - if opts == nil { - return false - } - raw, ok := opts[fantasyopenai.Name] - if !ok { - return false - } - respOpts, ok := raw.(*fantasyopenai.ResponsesProviderOptions) - if !ok || respOpts == nil { - return false - } - return respOpts.Store != nil && *respOpts.Store -} - -// WithPreviousResponseID shallow-clones the provider options map and the OpenAI -// Responses entry, setting PreviousResponseID on the clone. The original map -// and entry are not mutated. -func WithPreviousResponseID( - opts fantasy.ProviderOptions, - previousResponseID string, -) fantasy.ProviderOptions { - cloned := maps.Clone(opts) - if cloned == nil { - cloned = fantasy.ProviderOptions{} - } - if raw, ok := cloned[fantasyopenai.Name]; ok { - if respOpts, ok := raw.(*fantasyopenai.ResponsesProviderOptions); ok && respOpts != nil { - clone := *respOpts - clone.PreviousResponseID = &previousResponseID - cloned[fantasyopenai.Name] = &clone - } - } - return cloned -} - -// extractResponseID extracts the OpenAI Responses API response ID from provider -// metadata. Returns an empty string if no OpenAI Responses metadata is present. -func extractResponseID(metadata fantasy.ProviderMetadata) string { - if len(metadata) == 0 { - return "" - } - - entry, ok := metadata[fantasyopenai.Name] - if !ok { - return "" - } - providerMetadata, ok := entry.(*fantasyopenai.ResponsesProviderMetadata) - if !ok || providerMetadata == nil { - return "" - } - return providerMetadata.ResponseID -} - -// ExtractResponseIDIfStored returns the OpenAI response ID only when the -// provider options indicate store=true. Response IDs from store=false turns are -// not persisted server-side and cannot be used for chaining. -func ExtractResponseIDIfStored( - providerOptions fantasy.ProviderOptions, - metadata fantasy.ProviderMetadata, -) string { - if !IsResponsesStoreEnabled(providerOptions) { - return "" - } - - return extractResponseID(metadata) -} - -// ShouldActivateChainMode reports whether a follow-up turn can use -// previous_response_id instead of replaying history. It requires store=true, a -// matching model config, meaningful trailing user input, non-plan mode, -// complete local tool state, and confirmation that tool results were sent to -// the provider. -func ShouldActivateChainMode( - providerOptions fantasy.ProviderOptions, - info ChainModeInfo, - modelConfigID uuid.UUID, - isPlanModeTurn bool, -) bool { - return IsResponsesStoreEnabled(providerOptions) && - info.previousResponseID != "" && - info.contributingTrailingUserCount > 0 && - info.modelConfigID == modelConfigID && - !isPlanModeTurn && - !info.hasUnresolvedLocalToolCalls && - !info.providerMissingToolResults -} - -// ResolveChainMode scans DB messages from the end to inspect the current -// trailing user turn and detect whether the immediately preceding assistant/tool -// block can chain from a provider response ID. -func ResolveChainMode(messages []database.ChatMessage) ChainModeInfo { - var info ChainModeInfo - i := len(messages) - 1 - for ; i >= 0; i-- { - if messages[i].Role != database.ChatMessageRoleUser { - break - } - if userMessageContributesToChainMode(messages[i]) { - info.contributingTrailingUserCount++ - } - } - for ; i >= 0; i-- { - switch messages[i].Role { - case database.ChatMessageRoleAssistant: - if messages[i].ProviderResponseID.Valid && - messages[i].ProviderResponseID.String != "" { - info.previousResponseID = messages[i].ProviderResponseID.String - if messages[i].ModelConfigID.Valid { - info.modelConfigID = messages[i].ModelConfigID.UUID - } - info.hasUnresolvedLocalToolCalls = assistantHasUnresolvedLocalToolCalls(messages, i) - if !info.hasUnresolvedLocalToolCalls { - info.providerMissingToolResults = providerHasMissingToolResults(messages, i) - } - return info - } - return info - case database.ChatMessageRoleTool: - continue - default: - return info - } - } - return info -} - -// FilterPromptForChainMode keeps only system messages and the trailing user -// messages that still contribute model-visible content to the current turn. -// Assistant and tool messages are dropped because the provider already has -// them via the previous_response_id chain. -func FilterPromptForChainMode( - prompt []fantasy.Message, - info ChainModeInfo, -) []fantasy.Message { - if info.contributingTrailingUserCount <= 0 { - return prompt - } - - totalUsers := 0 - for _, msg := range prompt { - if msg.Role == "user" { - totalUsers++ - } - } - - // Prompt construction already drops user turns with no model-visible - // content, such as skill-only sentinel messages. That means the user - // count here stays aligned with contributingTrailingUserCount even - // when non-contributing DB turns are interleaved in the trailing - // block. - usersToSkip := totalUsers - info.contributingTrailingUserCount - if usersToSkip < 0 { - usersToSkip = 0 - } - - filtered := make([]fantasy.Message, 0, len(prompt)) - usersSeen := 0 - for _, msg := range prompt { - switch msg.Role { - case "system": - filtered = append(filtered, msg) - case "user": - usersSeen++ - if usersSeen > usersToSkip { - filtered = append(filtered, msg) - } - } - } - - return filtered -} - -func userMessageContributesToChainMode(msg database.ChatMessage) bool { - parts, err := chatprompt.ParseContent(msg) - if err != nil { - return false - } - for _, part := range parts { - switch part.Type { - case codersdk.ChatMessagePartTypeText, - codersdk.ChatMessagePartTypeReasoning: - if strings.TrimSpace(part.Text) != "" { - return true - } - case codersdk.ChatMessagePartTypeFile, - codersdk.ChatMessagePartTypeFileReference: - return true - case codersdk.ChatMessagePartTypeContextFile: - if part.ContextFileContent != "" { - return true - } - } - } - return false -} - -// assistantHasUnresolvedLocalToolCalls reports whether the assistant message -// at assistantIdx contains local tool calls that lack matching tool results. It -// returns true when content parsing fails because full-history replay is safer -// than chaining from state that cannot be inspected. -func assistantHasUnresolvedLocalToolCalls( - messages []database.ChatMessage, - assistantIdx int, -) bool { - if assistantIdx < 0 || assistantIdx >= len(messages) { - return false - } - - parts, err := chatprompt.ParseContent(messages[assistantIdx]) - if err != nil { - // Use full replay when persisted assistant content cannot be parsed. - return true - } - - localCallIDs := make(map[string]struct{}) - for _, part := range parts { - if part.Type != codersdk.ChatMessagePartTypeToolCall || - part.ProviderExecuted { - continue - } - localCallIDs[part.ToolCallID] = struct{}{} - } - if len(localCallIDs) == 0 { - return false - } - - resolvedCallIDs := make(map[string]struct{}) - for i := assistantIdx + 1; i < len(messages); i++ { - if messages[i].Role != database.ChatMessageRoleTool { - break - } - parts, err := chatprompt.ParseContent(messages[i]) - if err != nil { - // Use full replay when persisted tool content cannot be parsed. - return true - } - for _, part := range parts { - if part.Type != codersdk.ChatMessagePartTypeToolResult { - continue - } - if _, ok := localCallIDs[part.ToolCallID]; ok { - resolvedCallIDs[part.ToolCallID] = struct{}{} - } - } - } - - return len(resolvedCallIDs) != len(localCallIDs) -} - -// providerHasMissingToolResults reports whether the assistant message at -// assistantIdx has local tool calls whose results exist in the database but -// were never sent back to the provider. This is detected by the absence of a -// follow-up assistant message after the tool results. In normal flow the LLM -// processes tool results and produces a follow-up response, but StopAfterTool -// skips that round-trip. -func providerHasMissingToolResults( - messages []database.ChatMessage, - assistantIdx int, -) bool { - if assistantIdx < 0 || assistantIdx >= len(messages) { - return false - } - - parts, err := chatprompt.ParseContent(messages[assistantIdx]) - if err != nil { - // Parsing errors are already handled by - // assistantHasUnresolvedLocalToolCalls. - return false - } - - if !slices.ContainsFunc(parts, func(p codersdk.ChatMessagePart) bool { - return p.Type == codersdk.ChatMessagePartTypeToolCall && !p.ProviderExecuted - }) { - return false - } - - // Scan forward past tool messages. If the first non-tool message is not an - // assistant, the tool results were never round-tripped to the provider. - for i := assistantIdx + 1; i < len(messages); i++ { - switch messages[i].Role { - case database.ChatMessageRoleTool: - continue - case database.ChatMessageRoleAssistant: - // A follow-up assistant exists, so results were sent. - return false - default: - // User or system message with no follow-up assistant. - return true - } - } - - // Reached end of messages without a follow-up assistant. - return true -} diff --git a/coderd/x/chatd/chatopenai/responses_test.go b/coderd/x/chatd/chatopenai/responses_test.go deleted file mode 100644 index 59c5cdb44f6eb..0000000000000 --- a/coderd/x/chatd/chatopenai/responses_test.go +++ /dev/null @@ -1,913 +0,0 @@ -package chatopenai_test - -import ( - "database/sql" - "encoding/json" - "testing" - - "charm.land/fantasy" - fantasyopenai "charm.land/fantasy/providers/openai" - "github.com/google/uuid" - "github.com/sqlc-dev/pqtype" - "github.com/stretchr/testify/require" - - "github.com/coder/coder/v2/coderd/database" - "github.com/coder/coder/v2/coderd/x/chatd/chatopenai" - "github.com/coder/coder/v2/coderd/x/chatd/chatprompt" - "github.com/coder/coder/v2/coderd/x/chatd/chattest" - "github.com/coder/coder/v2/codersdk" -) - -func TestIsResponsesStoreEnabled(t *testing.T) { - t.Parallel() - - storeTrue := true - storeFalse := false - - tests := []struct { - name string - opts fantasy.ProviderOptions - want bool - }{ - { - name: "NilOptions", - }, - { - name: "NonOpenAIKeysOnly", - opts: fantasy.ProviderOptions{ - "other": &fantasyopenai.ProviderOptions{}, - }, - }, - { - name: "OpenAIKeyWithNonResponsesOptions", - opts: fantasy.ProviderOptions{ - fantasyopenai.Name: &fantasyopenai.ProviderOptions{}, - }, - }, - { - name: "OpenAIKeyWithNilStore", - opts: fantasy.ProviderOptions{ - fantasyopenai.Name: &fantasyopenai.ResponsesProviderOptions{}, - }, - }, - { - name: "OpenAIKeyWithFalseStore", - opts: fantasy.ProviderOptions{ - fantasyopenai.Name: &fantasyopenai.ResponsesProviderOptions{Store: &storeFalse}, - }, - }, - { - name: "OpenAIKeyWithTrueStore", - opts: fantasy.ProviderOptions{ - fantasyopenai.Name: &fantasyopenai.ResponsesProviderOptions{Store: &storeTrue}, - }, - want: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - got := chatopenai.IsResponsesStoreEnabled(tt.opts) - require.Equal(t, tt.want, got) - }) - } -} - -func TestIsResponsesStoreEnabledIgnoresMalformedNonOpenAIKey(t *testing.T) { - t.Parallel() - - store := true - // This intentionally documents the only synthetic mismatch from the old - // chatloop value scan: a malformed map with OpenAI Responses options under a - // non-OpenAI key is not treated as enabled. - opts := fantasy.ProviderOptions{ - "not-openai": &fantasyopenai.ResponsesProviderOptions{Store: &store}, - } - - require.False(t, chatopenai.IsResponsesStoreEnabled(opts)) -} - -func TestShouldActivateChainMode(t *testing.T) { - t.Parallel() - - modelConfigID := uuid.New() - baseInfo := chatopenai.ResolveChainMode([]database.ChatMessage{ - chainModeSystemMessage(), - chainModeUserMessage("prior user message"), - chainModeAssistantMessage(modelConfigID, nil), - chainModeUserMessage("latest user message"), - }) - - localCall := codersdk.ChatMessageToolCall( - "call-local", - "read_file", - json.RawMessage(`{"path":"main.go"}`), - ) - unresolvedLocalInfo := chatopenai.ResolveChainMode([]database.ChatMessage{ - chainModeSystemMessage(), - chainModeUserMessage("prior user message"), - chainModeAssistantMessage(modelConfigID, []codersdk.ChatMessagePart{localCall}), - chainModeUserMessage("latest user message"), - }) - localResult := codersdk.ChatMessageToolResult( - "call-local", - "read_file", - json.RawMessage(`{"ok":true}`), - false, - false, - ) - missingToolResultsInfo := chatopenai.ResolveChainMode([]database.ChatMessage{ - chainModeSystemMessage(), - chainModeUserMessage("prior user message"), - chainModeAssistantMessage(modelConfigID, []codersdk.ChatMessagePart{localCall}), - chainModeToolMessage([]codersdk.ChatMessagePart{localResult}), - chainModeUserMessage("latest user message"), - }) - skillOnlyInfo := chatopenai.ResolveChainMode([]database.ChatMessage{ - chainModeSystemMessage(), - chainModeUserMessage("prior user message"), - chainModeAssistantMessage(modelConfigID, nil), - chainModeSkillOnlyUserMessage(), - }) - missingResponseInfo := chatopenai.ResolveChainMode([]database.ChatMessage{ - chainModeSystemMessage(), - chainModeUserMessage("prior user message"), - chainModeAssistantMessageWithoutResponse(modelConfigID), - chainModeUserMessage("latest user message"), - }) - - tests := []struct { - name string - providerOpts fantasy.ProviderOptions - info chatopenai.ChainModeInfo - modelConfigID uuid.UUID - isPlanModeTurn bool - want bool - }{ - { - name: "StoreDisabled", - providerOpts: chainModeProviderOptions(false), - info: baseInfo, - modelConfigID: modelConfigID, - }, - { - name: "MissingPreviousResponseID", - providerOpts: chainModeProviderOptions(true), - info: missingResponseInfo, - modelConfigID: modelConfigID, - }, - { - name: "MismatchedModelConfigID", - providerOpts: chainModeProviderOptions(true), - info: baseInfo, - modelConfigID: uuid.New(), - }, - { - name: "PlanMode", - providerOpts: chainModeProviderOptions(true), - info: baseInfo, - modelConfigID: modelConfigID, - isPlanModeTurn: true, - }, - { - name: "NoContributingTrailingUser", - providerOpts: chainModeProviderOptions(true), - info: skillOnlyInfo, - modelConfigID: modelConfigID, - }, - { - name: "UnresolvedLocalToolCalls", - providerOpts: chainModeProviderOptions(true), - info: unresolvedLocalInfo, - modelConfigID: modelConfigID, - }, - { - name: "ProviderMissingToolResults", - providerOpts: chainModeProviderOptions(true), - info: missingToolResultsInfo, - modelConfigID: modelConfigID, - }, - { - name: "AllConditionsMet", - providerOpts: chainModeProviderOptions(true), - info: baseInfo, - modelConfigID: modelConfigID, - want: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - got := chatopenai.ShouldActivateChainMode( - tt.providerOpts, - tt.info, - tt.modelConfigID, - tt.isPlanModeTurn, - ) - require.Equal(t, tt.want, got) - }) - } -} - -func TestWithPreviousResponseID(t *testing.T) { - t.Parallel() - - store := true - originalResponses := &fantasyopenai.ResponsesProviderOptions{Store: &store} - otherOptions := &fantasyopenai.ProviderOptions{} - opts := fantasy.ProviderOptions{ - fantasyopenai.Name: originalResponses, - "other": otherOptions, - } - - got := chatopenai.WithPreviousResponseID(opts, "resp-next") - - gotOtherOptions, ok := got["other"].(*fantasyopenai.ProviderOptions) - require.True(t, ok) - require.True(t, otherOptions == gotOtherOptions) - gotOriginalResponses, ok := opts[fantasyopenai.Name].(*fantasyopenai.ResponsesProviderOptions) - require.True(t, ok) - require.True(t, originalResponses == gotOriginalResponses) - require.Nil(t, originalResponses.PreviousResponseID) - - clonedResponses, ok := got[fantasyopenai.Name].(*fantasyopenai.ResponsesProviderOptions) - require.True(t, ok) - require.NotSame(t, originalResponses, clonedResponses) - require.NotNil(t, clonedResponses.PreviousResponseID) - require.Equal(t, "resp-next", *clonedResponses.PreviousResponseID) - require.True(t, originalResponses.Store == clonedResponses.Store) - - got["new"] = otherOptions - require.NotContains(t, opts, "new") -} - -func TestWithPreviousResponseIDNilInput(t *testing.T) { - t.Parallel() - - got := chatopenai.WithPreviousResponseID(nil, "resp-next") - - require.NotNil(t, got) - require.Empty(t, got) -} - -func TestExtractResponseIDIfStoredMetadata(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - metadata fantasy.ProviderMetadata - want string - }{ - { - name: "NilMetadata", - }, - { - name: "NoResponsesMetadata", - metadata: fantasy.ProviderMetadata{ - "other": &fantasyopenai.ProviderOptions{}, - }, - }, - { - name: "ResponsesMetadataUnderNonOpenAIKey", - metadata: fantasy.ProviderMetadata{ - "other": &fantasyopenai.ResponsesProviderMetadata{ - ResponseID: "resp-123", - }, - }, - }, - { - name: "ResponsesMetadata", - metadata: fantasy.ProviderMetadata{ - fantasyopenai.Name: &fantasyopenai.ResponsesProviderMetadata{ - ResponseID: "resp-123", - }, - }, - want: "resp-123", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - got := chatopenai.ExtractResponseIDIfStored( - chainModeProviderOptions(true), - tt.metadata, - ) - require.Equal(t, tt.want, got) - }) - } -} - -func TestExtractResponseIDIfStored(t *testing.T) { - t.Parallel() - - metadata := fantasy.ProviderMetadata{ - fantasyopenai.Name: &fantasyopenai.ResponsesProviderMetadata{ - ResponseID: "resp-123", - }, - } - - require.Empty(t, chatopenai.ExtractResponseIDIfStored( - chainModeProviderOptions(false), - metadata, - )) - require.Equal(t, "resp-123", chatopenai.ExtractResponseIDIfStored( - chainModeProviderOptions(true), - metadata, - )) -} - -func TestResolveChainModeIgnoresSkillOnlySentinelMessages(t *testing.T) { - t.Parallel() - - modelConfigID := uuid.New() - assistant := database.ChatMessage{ - Role: database.ChatMessageRoleAssistant, - ProviderResponseID: sql.NullString{String: "resp-123", Valid: true}, - ModelConfigID: uuid.NullUUID{UUID: modelConfigID, Valid: true}, - } - skillOnly := chainModeSkillOnlyUserMessage() - user := chattest.ChatMessageWithParts([]codersdk.ChatMessagePart{{ - Type: codersdk.ChatMessagePartTypeText, - Text: "latest user message", - }}) - user.Role = database.ChatMessageRoleUser - - got := chatopenai.ResolveChainMode([]database.ChatMessage{assistant, skillOnly, user}) - require.Equal(t, "resp-123", got.PreviousResponseID()) - require.Equal(t, modelConfigID, got.ModelConfigID()) - require.Equal(t, 1, got.ContributingTrailingUserCount()) -} - -func TestResolveChainMode_BlocksOnUnresolvedLocalToolCall(t *testing.T) { - t.Parallel() - - modelConfigID := uuid.New() - toolCall := codersdk.ChatMessageToolCall( - "call-local", - "read_file", - json.RawMessage(`{"path":"main.go"}`), - ) - - chainInfo := chatopenai.ResolveChainMode([]database.ChatMessage{ - chainModeSystemMessage(), - chainModeUserMessage("prior user message"), - chainModeAssistantMessage(modelConfigID, []codersdk.ChatMessagePart{toolCall}), - chainModeUserMessage("latest user message"), - }) - - require.Equal(t, "resp-123", chainInfo.PreviousResponseID()) - require.True(t, chainInfo.HasUnresolvedLocalToolCalls()) - require.False(t, chatopenai.ShouldActivateChainMode( - chainModeProviderOptions(true), - chainInfo, - modelConfigID, - false, - )) -} - -func TestResolveChainMode_BlocksWhenAssistantContentCannotParse(t *testing.T) { - t.Parallel() - - modelConfigID := uuid.New() - chainInfo := chatopenai.ResolveChainMode([]database.ChatMessage{ - chainModeSystemMessage(), - chainModeUserMessage("prior user message"), - chainModeCorruptAssistantMessage(modelConfigID), - chainModeUserMessage("latest user message"), - }) - - require.Equal(t, "resp-123", chainInfo.PreviousResponseID()) - require.True(t, chainInfo.HasUnresolvedLocalToolCalls()) - require.False(t, chatopenai.ShouldActivateChainMode( - chainModeProviderOptions(true), - chainInfo, - modelConfigID, - false, - )) -} - -func TestResolveChainMode_BlocksWhenToolContentCannotParse(t *testing.T) { - t.Parallel() - - modelConfigID := uuid.New() - toolCall := codersdk.ChatMessageToolCall( - "call-local", - "read_file", - json.RawMessage(`{"path":"main.go"}`), - ) - - chainInfo := chatopenai.ResolveChainMode([]database.ChatMessage{ - chainModeSystemMessage(), - chainModeUserMessage("prior user message"), - chainModeAssistantMessage(modelConfigID, []codersdk.ChatMessagePart{toolCall}), - chainModeCorruptToolMessage(), - chainModeUserMessage("latest user message"), - }) - - require.Equal(t, "resp-123", chainInfo.PreviousResponseID()) - require.True(t, chainInfo.HasUnresolvedLocalToolCalls()) - require.False(t, chatopenai.ShouldActivateChainMode( - chainModeProviderOptions(true), - chainInfo, - modelConfigID, - false, - )) -} - -func TestResolveChainMode_AllowsProviderExecutedOnly(t *testing.T) { - t.Parallel() - - modelConfigID := uuid.New() - toolCall := codersdk.ChatMessageToolCall( - "call-web-search", - "web_search", - json.RawMessage(`{"query":"coder docs"}`), - ) - toolCall.ProviderExecuted = true - - chainInfo := chatopenai.ResolveChainMode([]database.ChatMessage{ - chainModeSystemMessage(), - chainModeUserMessage("prior user message"), - chainModeAssistantMessage(modelConfigID, []codersdk.ChatMessagePart{toolCall}), - chainModeUserMessage("latest user message"), - }) - - require.Equal(t, "resp-123", chainInfo.PreviousResponseID()) - require.False(t, chainInfo.HasUnresolvedLocalToolCalls()) - require.False(t, chainInfo.ProviderMissingToolResults()) - require.True(t, chatopenai.ShouldActivateChainMode( - chainModeProviderOptions(true), - chainInfo, - modelConfigID, - false, - )) -} - -func TestResolveChainMode_BlocksOnMixedProviderExecutedAndUnresolvedLocalCall(t *testing.T) { - t.Parallel() - - modelConfigID := uuid.New() - providerCall := codersdk.ChatMessageToolCall( - "call-web-search", - "web_search", - json.RawMessage(`{"query":"coder docs"}`), - ) - providerCall.ProviderExecuted = true - localCall := codersdk.ChatMessageToolCall( - "call-local", - "read_file", - json.RawMessage(`{"path":"main.go"}`), - ) - - chainInfo := chatopenai.ResolveChainMode([]database.ChatMessage{ - chainModeSystemMessage(), - chainModeUserMessage("prior user message"), - chainModeAssistantMessage( - modelConfigID, - []codersdk.ChatMessagePart{providerCall, localCall}, - ), - chainModeUserMessage("latest user message"), - }) - - require.Equal(t, "resp-123", chainInfo.PreviousResponseID()) - require.True(t, chainInfo.HasUnresolvedLocalToolCalls()) - require.False(t, chatopenai.ShouldActivateChainMode( - chainModeProviderOptions(true), - chainInfo, - modelConfigID, - false, - )) -} - -func TestResolveChainMode_AllowsResolvedLocalCall(t *testing.T) { - t.Parallel() - - modelConfigID := uuid.New() - toolCall := codersdk.ChatMessageToolCall( - "call-local", - "read_file", - json.RawMessage(`{"path":"main.go"}`), - ) - toolResult := codersdk.ChatMessageToolResult( - "call-local", - "read_file", - json.RawMessage(`{"ok":true}`), - false, - false, - ) - followUp := chainModeAssistantMessage(modelConfigID, nil) - followUp.ProviderResponseID = sql.NullString{String: "resp-follow-up", Valid: true} - - chainInfo := chatopenai.ResolveChainMode([]database.ChatMessage{ - chainModeSystemMessage(), - chainModeUserMessage("prior user message"), - chainModeAssistantMessage(modelConfigID, []codersdk.ChatMessagePart{toolCall}), - chainModeToolMessage([]codersdk.ChatMessagePart{toolResult}), - followUp, - chainModeUserMessage("latest user message"), - }) - - require.Equal(t, "resp-follow-up", chainInfo.PreviousResponseID()) - require.False(t, chainInfo.HasUnresolvedLocalToolCalls()) - require.False(t, chainInfo.ProviderMissingToolResults()) - require.True(t, chatopenai.ShouldActivateChainMode( - chainModeProviderOptions(true), - chainInfo, - modelConfigID, - false, - )) -} - -func TestResolveChainMode_BlocksOnMixedResolvedAndUnresolved(t *testing.T) { - t.Parallel() - - modelConfigID := uuid.New() - firstCall := codersdk.ChatMessageToolCall( - "call-first", - "read_file", - json.RawMessage(`{"path":"main.go"}`), - ) - secondCall := codersdk.ChatMessageToolCall( - "call-second", - "read_file", - json.RawMessage(`{"path":"README.md"}`), - ) - toolResult := codersdk.ChatMessageToolResult( - "call-first", - "read_file", - json.RawMessage(`{"ok":true}`), - false, - false, - ) - - chainInfo := chatopenai.ResolveChainMode([]database.ChatMessage{ - chainModeSystemMessage(), - chainModeUserMessage("prior user message"), - chainModeAssistantMessage( - modelConfigID, - []codersdk.ChatMessagePart{firstCall, secondCall}, - ), - chainModeToolMessage([]codersdk.ChatMessagePart{toolResult}), - chainModeUserMessage("latest user message"), - }) - - require.Equal(t, "resp-123", chainInfo.PreviousResponseID()) - require.True(t, chainInfo.HasUnresolvedLocalToolCalls()) - require.False(t, chatopenai.ShouldActivateChainMode( - chainModeProviderOptions(true), - chainInfo, - modelConfigID, - false, - )) -} - -func TestResolveChainMode_BlocksWhenToolResultNeverSentToProvider(t *testing.T) { - t.Parallel() - - modelConfigID := uuid.New() - toolCall := codersdk.ChatMessageToolCall( - "call-local", - "propose_plan", - json.RawMessage(`{"path":"plan.md"}`), - ) - toolResult := codersdk.ChatMessageToolResult( - "call-local", - "propose_plan", - json.RawMessage(`{"ok":true}`), - false, - false, - ) - - chainInfo := chatopenai.ResolveChainMode([]database.ChatMessage{ - chainModeSystemMessage(), - chainModeUserMessage("make a plan"), - chainModeAssistantMessage(modelConfigID, []codersdk.ChatMessagePart{toolCall}), - chainModeToolMessage([]codersdk.ChatMessagePart{toolResult}), - chainModeUserMessage("implement the plan"), - }) - - require.Equal(t, "resp-123", chainInfo.PreviousResponseID()) - require.False(t, chainInfo.HasUnresolvedLocalToolCalls()) - require.True(t, chainInfo.ProviderMissingToolResults()) - require.False(t, chatopenai.ShouldActivateChainMode( - chainModeProviderOptions(true), - chainInfo, - modelConfigID, - false, - )) -} - -func TestResolveChainMode_BlocksProviderMissingWithMultipleToolCalls(t *testing.T) { - t.Parallel() - - modelConfigID := uuid.New() - call1 := codersdk.ChatMessageToolCall( - "call-1", - "propose_plan", - json.RawMessage(`{"path":"plan.md"}`), - ) - call2 := codersdk.ChatMessageToolCall( - "call-2", - "write_file", - json.RawMessage(`{"path":"foo.go"}`), - ) - result1 := codersdk.ChatMessageToolResult( - "call-1", - "propose_plan", - json.RawMessage(`{"ok":true}`), - false, - false, - ) - result2 := codersdk.ChatMessageToolResult( - "call-2", - "write_file", - json.RawMessage(`{"ok":true}`), - false, - false, - ) - - chainInfo := chatopenai.ResolveChainMode([]database.ChatMessage{ - chainModeSystemMessage(), - chainModeUserMessage("do it"), - chainModeAssistantMessage(modelConfigID, []codersdk.ChatMessagePart{call1, call2}), - chainModeToolMessage([]codersdk.ChatMessagePart{result1, result2}), - chainModeUserMessage("next"), - }) - - require.False(t, chainInfo.HasUnresolvedLocalToolCalls()) - require.True(t, chainInfo.ProviderMissingToolResults()) - require.False(t, chatopenai.ShouldActivateChainMode( - chainModeProviderOptions(true), - chainInfo, - modelConfigID, - false, - )) -} - -func TestResolveChainMode_AllowsWhenNoToolCalls(t *testing.T) { - t.Parallel() - - modelConfigID := uuid.New() - - chainInfo := chatopenai.ResolveChainMode([]database.ChatMessage{ - chainModeSystemMessage(), - chainModeUserMessage("hello"), - chainModeAssistantMessage(modelConfigID, nil), - chainModeUserMessage("thanks"), - }) - - require.Equal(t, "resp-123", chainInfo.PreviousResponseID()) - require.False(t, chainInfo.HasUnresolvedLocalToolCalls()) - require.False(t, chainInfo.ProviderMissingToolResults()) - require.True(t, chatopenai.ShouldActivateChainMode( - chainModeProviderOptions(true), - chainInfo, - modelConfigID, - false, - )) -} - -func TestFilterPromptForChainModeKeepsContributingUsersAcrossSkippedSentinelTurns(t *testing.T) { - t.Parallel() - - modelConfigID := uuid.New() - priorUser := chattest.ChatMessageWithParts([]codersdk.ChatMessagePart{{ - Type: codersdk.ChatMessagePartTypeText, - Text: "prior user message", - }}) - priorUser.Role = database.ChatMessageRoleUser - assistant := database.ChatMessage{ - Role: database.ChatMessageRoleAssistant, - ProviderResponseID: sql.NullString{String: "resp-123", Valid: true}, - ModelConfigID: uuid.NullUUID{UUID: modelConfigID, Valid: true}, - } - firstTrailingUser := chattest.ChatMessageWithParts([]codersdk.ChatMessagePart{{ - Type: codersdk.ChatMessagePartTypeText, - Text: "first trailing user", - }}) - firstTrailingUser.Role = database.ChatMessageRoleUser - skillOnly := chainModeSkillOnlyUserMessage() - lastTrailingUser := chattest.ChatMessageWithParts([]codersdk.ChatMessagePart{{ - Type: codersdk.ChatMessagePartTypeText, - Text: "last trailing user", - }}) - lastTrailingUser.Role = database.ChatMessageRoleUser - - chainInfo := chatopenai.ResolveChainMode([]database.ChatMessage{ - priorUser, - assistant, - firstTrailingUser, - skillOnly, - lastTrailingUser, - }) - require.Equal(t, 2, chainInfo.ContributingTrailingUserCount()) - - prompt := []fantasy.Message{ - { - Role: fantasy.MessageRoleSystem, - Content: []fantasy.MessagePart{ - fantasy.TextPart{Text: "system instruction"}, - }, - }, - { - Role: fantasy.MessageRoleUser, - Content: []fantasy.MessagePart{ - fantasy.TextPart{Text: "prior user message"}, - }, - }, - { - Role: fantasy.MessageRoleAssistant, - Content: []fantasy.MessagePart{ - fantasy.TextPart{Text: "assistant reply"}, - }, - }, - { - Role: fantasy.MessageRoleUser, - Content: []fantasy.MessagePart{ - fantasy.TextPart{Text: "first trailing user"}, - }, - }, - { - Role: fantasy.MessageRoleUser, - Content: []fantasy.MessagePart{ - fantasy.TextPart{Text: "last trailing user"}, - }, - }, - } - - got := chatopenai.FilterPromptForChainMode(prompt, chainInfo) - require.Len(t, got, 3) - require.Equal(t, fantasy.MessageRoleSystem, got[0].Role) - require.Equal(t, fantasy.MessageRoleUser, got[1].Role) - require.Equal(t, fantasy.MessageRoleUser, got[2].Role) - - firstPart, ok := fantasy.AsMessagePart[fantasy.TextPart](got[1].Content[0]) - require.True(t, ok) - require.Equal(t, "first trailing user", firstPart.Text) - lastPart, ok := fantasy.AsMessagePart[fantasy.TextPart](got[2].Content[0]) - require.True(t, ok) - require.Equal(t, "last trailing user", lastPart.Text) -} - -func TestFilterPromptForChainModeUsesContributingTrailingUsers(t *testing.T) { - t.Parallel() - - modelConfigID := uuid.New() - priorUser := chattest.ChatMessageWithParts([]codersdk.ChatMessagePart{{ - Type: codersdk.ChatMessagePartTypeText, - Text: "prior user message", - }}) - priorUser.Role = database.ChatMessageRoleUser - assistant := database.ChatMessage{ - Role: database.ChatMessageRoleAssistant, - ProviderResponseID: sql.NullString{String: "resp-123", Valid: true}, - ModelConfigID: uuid.NullUUID{UUID: modelConfigID, Valid: true}, - } - skillOnly := chainModeSkillOnlyUserMessage() - latestUser := chattest.ChatMessageWithParts([]codersdk.ChatMessagePart{{ - Type: codersdk.ChatMessagePartTypeText, - Text: "latest user message", - }}) - latestUser.Role = database.ChatMessageRoleUser - - chainInfo := chatopenai.ResolveChainMode([]database.ChatMessage{ - priorUser, - assistant, - skillOnly, - latestUser, - }) - require.Equal(t, 1, chainInfo.ContributingTrailingUserCount()) - - prompt := []fantasy.Message{ - { - Role: fantasy.MessageRoleSystem, - Content: []fantasy.MessagePart{ - fantasy.TextPart{Text: "system instruction"}, - }, - }, - { - Role: fantasy.MessageRoleUser, - Content: []fantasy.MessagePart{ - fantasy.TextPart{Text: "prior user message"}, - }, - }, - { - Role: fantasy.MessageRoleAssistant, - Content: []fantasy.MessagePart{ - fantasy.TextPart{Text: "assistant reply"}, - }, - }, - { - Role: fantasy.MessageRoleUser, - Content: []fantasy.MessagePart{ - fantasy.TextPart{Text: "latest user message"}, - }, - }, - } - - got := chatopenai.FilterPromptForChainMode(prompt, chainInfo) - require.Len(t, got, 2) - require.Equal(t, fantasy.MessageRoleSystem, got[0].Role) - require.Equal(t, fantasy.MessageRoleUser, got[1].Role) - - part, ok := fantasy.AsMessagePart[fantasy.TextPart](got[1].Content[0]) - require.True(t, ok) - require.Equal(t, "latest user message", part.Text) -} - -func chainModeProviderOptions(store bool) fantasy.ProviderOptions { - return fantasy.ProviderOptions{ - fantasyopenai.Name: &fantasyopenai.ResponsesProviderOptions{ - Store: &store, - }, - } -} - -func chainModeSystemMessage() database.ChatMessage { - return database.ChatMessage{Role: database.ChatMessageRoleSystem} -} - -func chainModeUserMessage(text string) database.ChatMessage { - msg := chattest.ChatMessageWithParts([]codersdk.ChatMessagePart{ - codersdk.ChatMessageText(text), - }) - msg.Role = database.ChatMessageRoleUser - return msg -} - -func chainModeSkillOnlyUserMessage() database.ChatMessage { - msg := chattest.ChatMessageWithParts([]codersdk.ChatMessagePart{ - { - Type: codersdk.ChatMessagePartTypeContextFile, - // Keep this in sync with chatd.AgentChatContextSentinelPath. - ContextFilePath: ".coder/agent-chat-context-sentinel", - ContextFileAgentID: uuid.NullUUID{ - UUID: uuid.New(), - Valid: true, - }, - }, - { - Type: codersdk.ChatMessagePartTypeSkill, - SkillName: "repo-helper", - SkillDir: "/skills/repo-helper", - }, - }) - msg.Role = database.ChatMessageRoleUser - return msg -} - -func chainModeAssistantMessage( - modelConfigID uuid.UUID, - parts []codersdk.ChatMessagePart, -) database.ChatMessage { - msg := chattest.ChatMessageWithParts(parts) - msg.Role = database.ChatMessageRoleAssistant - msg.ProviderResponseID = sql.NullString{String: "resp-123", Valid: true} - msg.ModelConfigID = uuid.NullUUID{UUID: modelConfigID, Valid: true} - return msg -} - -func chainModeAssistantMessageWithoutResponse( - modelConfigID uuid.UUID, -) database.ChatMessage { - msg := chattest.ChatMessageWithParts(nil) - msg.Role = database.ChatMessageRoleAssistant - msg.ModelConfigID = uuid.NullUUID{UUID: modelConfigID, Valid: true} - return msg -} - -func chainModeCorruptAssistantMessage(modelConfigID uuid.UUID) database.ChatMessage { - return database.ChatMessage{ - Role: database.ChatMessageRoleAssistant, - ProviderResponseID: sql.NullString{String: "resp-123", Valid: true}, - ModelConfigID: uuid.NullUUID{UUID: modelConfigID, Valid: true}, - Content: pqtype.NullRawMessage{ - RawMessage: []byte("not json"), - Valid: true, - }, - ContentVersion: chatprompt.CurrentContentVersion, - } -} - -func chainModeCorruptToolMessage() database.ChatMessage { - return database.ChatMessage{ - Role: database.ChatMessageRoleTool, - Content: pqtype.NullRawMessage{ - RawMessage: []byte("not json"), - Valid: true, - }, - ContentVersion: chatprompt.CurrentContentVersion, - } -} - -func chainModeToolMessage(parts []codersdk.ChatMessagePart) database.ChatMessage { - msg := chattest.ChatMessageWithParts(parts) - msg.Role = database.ChatMessageRoleTool - return msg -} diff --git a/coderd/x/chatd/chatstate/messages.go b/coderd/x/chatd/chatstate/messages.go index ee92e0ea1305e..90b2b9eae970d 100644 --- a/coderd/x/chatd/chatstate/messages.go +++ b/coderd/x/chatd/chatstate/messages.go @@ -34,7 +34,6 @@ type Message struct { ContextLimit sql.NullInt64 TotalCostMicros sql.NullInt64 RuntimeMs sql.NullInt64 - ProviderResponseID sql.NullString APIKeyID sql.NullString } @@ -65,7 +64,6 @@ func toInsertParams(chatID uuid.UUID, messages []Message) database.InsertChatMes Compressed: make([]bool, n), TotalCostMicros: make([]int64, n), RuntimeMs: make([]int64, n), - ProviderResponseID: make([]string, n), } for i, m := range messages { params.CreatedBy[i] = nullUUIDOrNil(m.CreatedBy) @@ -93,9 +91,6 @@ func toInsertParams(chatID uuid.UUID, messages []Message) database.InsertChatMes params.Compressed[i] = m.Compressed params.TotalCostMicros[i] = nullInt64Or(m.TotalCostMicros, 0) params.RuntimeMs[i] = nullInt64Or(m.RuntimeMs, 0) - if m.ProviderResponseID.Valid { - params.ProviderResponseID[i] = m.ProviderResponseID.String - } } return params } diff --git a/coderd/x/chatd/chattest/openai.go b/coderd/x/chatd/chattest/openai.go index 0320d554d940f..74a2a91691cdf 100644 --- a/coderd/x/chatd/chattest/openai.go +++ b/coderd/x/chatd/chattest/openai.go @@ -48,13 +48,12 @@ type OpenAIWebSearchCall struct { // OpenAIRequest represents an OpenAI chat completion request. type OpenAIRequest struct { *http.Request - Model string `json:"model"` - Messages []OpenAIMessage `json:"messages"` - Stream bool `json:"stream,omitempty"` - Tools []OpenAITool `json:"tools,omitempty"` - Prompt []interface{} `json:"prompt,omitempty"` // Responses API input or prompt. - Store *bool `json:"store,omitempty"` - PreviousResponseID *string `json:"previous_response_id,omitempty"` + Model string `json:"model"` + Messages []OpenAIMessage `json:"messages"` + Stream bool `json:"stream,omitempty"` + Tools []OpenAITool `json:"tools,omitempty"` + Prompt []interface{} `json:"prompt,omitempty"` // Responses API input or prompt. + Store *bool `json:"store,omitempty"` // RawBody holds the original request body so callers can inspect // fields the typed struct does not expose, such as the Responses // API "input" payload. It is populated before JSON decoding. diff --git a/coderd/x/chatd/context_prompt.go b/coderd/x/chatd/context_prompt.go index 558a5b791a3af..7dd8fe909116e 100644 --- a/coderd/x/chatd/context_prompt.go +++ b/coderd/x/chatd/context_prompt.go @@ -17,13 +17,6 @@ import ( "github.com/coder/coder/v2/codersdk/workspacesdk" ) -// AgentChatContextSentinelPath is the canonical path of the synthetic empty -// context-file part that legacy chats used to mark skill-only workspace-agent -// context. New turns no longer emit it; it is retained as the canonical value -// so historical-message handling and the chatopenai chain-mode tests stay in -// sync. -const AgentChatContextSentinelPath = ".coder/agent-chat-context-sentinel" - // contextBodyUnmarshalOptions reads the protojson resource bodies written by // the agent context push (coderd/agentapi/context.go). DiscardUnknown keeps // the reader forward compatible as new body fields are added to the proto. diff --git a/coderd/x/chatd/generation.go b/coderd/x/chatd/generation.go index 4a0085cc526c6..da3b1df6baf49 100644 --- a/coderd/x/chatd/generation.go +++ b/coderd/x/chatd/generation.go @@ -28,9 +28,8 @@ import ( // generationPrepareInput contains the committed state used to prepare one // generation action. type generationPrepareInput struct { - Chat database.Chat - Messages []database.ChatMessage - ChainModeDisabled bool + Chat database.Chat + Messages []database.ChatMessage } // generationPrepared contains the side-effect inputs for a generation task. @@ -299,16 +298,14 @@ func (s *taskStarter) StartGeneration(ctx context.Context, input chatWorkerTaskS return xerrors.New("chatworker: server is required") } machine := chatstate.NewChatMachine(s.opts.Store, s.opts.Pubsub, input.ChatID) - chainModeDisabled := false for { locked, messages, err := loadGenerationState(ctx, machine, input) if err != nil { return xerrors.Errorf("load generation state: %w", err) } prepareInput := generationPrepareInput{ - Chat: locked, - Messages: messages, - ChainModeDisabled: chainModeDisabled, + Chat: locked, + Messages: messages, } prepared, err := retryGenerationPhase(ctx, s, "prepare", func() (generationPrepared, error) { return s.server.prepareGeneration(ctx, prepareInput) @@ -398,12 +395,8 @@ func (s *taskStarter) StartGeneration(ctx context.Context, input chatWorkerTaskS slog.F("error_kind", classified.Kind), slog.F("provider", classified.Provider), slog.F("status_code", classified.StatusCode), - slog.F("chain_broken", classified.ChainBroken), slogError(actionErr), ) - if classified.ChainBroken { - chainModeDisabled = true - } if err := s.waitGenerationRetry(ctx, decision.delay); err != nil { return xerrors.Errorf("wait generation retry: %w", err) } @@ -1057,7 +1050,6 @@ func stepDataFromPersisted(step chatloop.PersistedStep) stepData { Content: step.Content, Usage: step.Usage, ContextLimit: step.ContextLimit, - ProviderResponseID: step.ProviderResponseID, Runtime: step.Runtime, ToolCallCreatedAt: step.ToolCallCreatedAt, ToolResultCreatedAt: step.ToolResultCreatedAt, diff --git a/coderd/x/chatd/generation_preparer.go b/coderd/x/chatd/generation_preparer.go index 9289276ebcf94..0a80f2de16a59 100644 --- a/coderd/x/chatd/generation_preparer.go +++ b/coderd/x/chatd/generation_preparer.go @@ -16,7 +16,6 @@ import ( "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/x/chatd/chatadvisor" "github.com/coder/coder/v2/coderd/x/chatd/chatloop" - "github.com/coder/coder/v2/coderd/x/chatd/chatopenai" "github.com/coder/coder/v2/coderd/x/chatd/chatprompt" "github.com/coder/coder/v2/coderd/x/chatd/chatprovider" "github.com/coder/coder/v2/coderd/x/chatd/chatsanitize" @@ -534,16 +533,6 @@ func (server *Server) prepareGeneration( } providerOptions := chatprovider.ProviderOptionsFromChatModelConfig(model, callConfig.ProviderOptions) - chainInfo := chatopenai.ResolveChainMode(promptRows) - if !input.ChainModeDisabled && chatopenai.ShouldActivateChainMode( - providerOptions, - chainInfo, - modelConfig.ID, - isPlanModeTurn, - ) { - providerOptions = chatopenai.WithPreviousResponseID(providerOptions, chainInfo.PreviousResponseID()) - prompt = chatopenai.FilterPromptForChainMode(prompt, chainInfo) - } activeToolNames := activeToolNamesForTurn(tools, currentPlanMode, chat.ParentChatID, approvedPlanMCPConfigIDs) if isExploreSubagent { diff --git a/coderd/x/chatd/integration_responses_test.go b/coderd/x/chatd/integration_responses_test.go index 2c16a10ec7b92..6d27463f0552e 100644 --- a/coderd/x/chatd/integration_responses_test.go +++ b/coderd/x/chatd/integration_responses_test.go @@ -2,7 +2,6 @@ package chatd_test import ( "context" - "encoding/json" "fmt" "strings" "sync" @@ -13,11 +12,9 @@ import ( "github.com/stretchr/testify/require" "github.com/coder/coder/v2/coderd/database" - "github.com/coder/coder/v2/coderd/database/dbgen" "github.com/coder/coder/v2/coderd/database/dbtestutil" dbpubsub "github.com/coder/coder/v2/coderd/database/pubsub" "github.com/coder/coder/v2/coderd/x/chatd" - "github.com/coder/coder/v2/coderd/x/chatd/chatprompt" "github.com/coder/coder/v2/coderd/x/chatd/chattest" "github.com/coder/coder/v2/codersdk" "github.com/coder/coder/v2/testutil" @@ -105,7 +102,6 @@ func TestOpenAIResponsesNoStaleWebSearchReplay(t *testing.T) { followup := requests[1] require.NotNil(t, followup.Store) require.False(t, *followup.Store) - require.Nil(t, followup.PreviousResponseID) require.NotEmpty(t, followup.Prompt) requireNoResponsesProviderItemReplay(t, followup.Prompt, reasoningID, webSearchID) require.NotContains(t, promptItemTypes(followup.Prompt), "web_search_call") @@ -193,174 +189,13 @@ func TestOpenAIResponsesFullReplayPairsReasoningAndWebSearch(t *testing.T) { followup := requests[1] require.NotNil(t, followup.Store) require.True(t, *followup.Store) - require.Nil(t, followup.PreviousResponseID) require.NotEmpty(t, followup.Prompt) requirePromptItemReferenceOrder(t, followup.Prompt, reasoningID, webSearchID) } -func TestOpenAIResponsesChainModeSkipsWhenLocalCallPending(t *testing.T) { - t.Parallel() - - db, ps := dbtestutil.NewDB(t) - ctx := testutil.Context(t, testutil.WaitLong) - - var recorder responsesRequestRecorder - openAIURL := chattest.NewOpenAI(t, func(req *chattest.OpenAIRequest) chattest.OpenAIResponse { - if !req.Stream { - return chattest.OpenAINonStreamingResponse("title") - } - recorder.record(req) - resp := chattest.OpenAIStreamingResponse( - chattest.OpenAITextChunks("resolved after local call")..., - ) - resp.ResponseID = "resp_local_pending_next" - return resp - }) - - user, org, _ := seedChatDependenciesWithProvider(t, db, "openai", openAIURL) - model := insertOpenAIResponsesModelConfig(t, db, user.ID, true, false) - chat := insertOpenAIResponsesChat(t, db, org.ID, user.ID, model.ID, "local-pending") - - callID := fmt.Sprintf("call_local_%d", time.Now().UnixNano()) - localCall := codersdk.ChatMessageToolCall( - callID, - "read_file", - json.RawMessage(`{"path":"README.md"}`), - ) - insertOpenAIResponsesMessages(ctx, t, db, chat.ID, user.ID, model.ID, - persistedResponsesMessage{ - role: database.ChatMessageRoleUser, - parts: []codersdk.ChatMessagePart{ - codersdk.ChatMessageText("please inspect the README"), - }, - }, - persistedResponsesMessage{ - role: database.ChatMessageRoleAssistant, - parts: []codersdk.ChatMessagePart{localCall}, - providerResponseID: "resp_local_pending_prior", - }, - ) - - factory := chattest.NewMockAIBridgeTransport(t, openAIURL) - server := newOpenAIResponsesTestServer(t, db, ps, func(cfg *chatd.Config) { - cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(factory) - }) - _, err := server.SendMessage(ctx, chatd.SendMessageOptions{ - ChatID: chat.ID, - CreatedBy: user.ID, - APIKeyID: testAPIKeyID(t, db, user.ID), - ModelConfigID: model.ID, - Content: []codersdk.ChatMessagePart{ - codersdk.ChatMessageText("continue after that tool call"), - }, - }) - require.NoError(t, err) - waitForChatProcessed(ctx, t, db, chat.ID, server) - requireResponsesChatWaiting(ctx, t, db, chat.ID) - - requests := recorder.all() - require.Len(t, requests, 1) - request := requests[0] - require.NotNil(t, request.Store) - require.True(t, *request.Store) - require.Nil(t, request.PreviousResponseID) - require.NotEmpty(t, request.Prompt) - requirePromptItemWithTypeAndCallID(t, request.Prompt, "function_call", callID) - requirePromptItemWithTypeAndCallID(t, request.Prompt, "function_call_output", callID) -} - -func TestOpenAIResponsesChainModeStillFiresForProviderExecutedOnly(t *testing.T) { - t.Parallel() - - db, ps := dbtestutil.NewDB(t) - ctx := testutil.Context(t, testutil.WaitLong) - - var recorder responsesRequestRecorder - openAIURL := chattest.NewOpenAI(t, func(req *chattest.OpenAIRequest) chattest.OpenAIResponse { - if !req.Stream { - return chattest.OpenAINonStreamingResponse("title") - } - recorder.record(req) - resp := chattest.OpenAIStreamingResponse( - chattest.OpenAITextChunks("chained answer")..., - ) - resp.ResponseID = "resp_provider_only_next" - return resp - }) - - user, org, _ := seedChatDependenciesWithProvider(t, db, "openai", openAIURL) - model := insertOpenAIResponsesModelConfig(t, db, user.ID, true, true) - chat := insertOpenAIResponsesChat(t, db, org.ID, user.ID, model.ID, "provider-only") - - const ( - previousResponseID = "resp_provider_only_prior" - webSearchID = "ws_provider_only_search" - ) - webSearchCall := codersdk.ChatMessageToolCall( - webSearchID, - "web_search", - json.RawMessage(`{"query":"coder docs"}`), - ) - webSearchCall.ProviderExecuted = true - webSearchResult := codersdk.ChatMessageToolResult( - webSearchID, - "web_search", - json.RawMessage(`{"status":"completed"}`), - false, - false, - ) - webSearchResult.ProviderExecuted = true - insertOpenAIResponsesMessages(ctx, t, db, chat.ID, user.ID, model.ID, - persistedResponsesMessage{ - role: database.ChatMessageRoleUser, - parts: []codersdk.ChatMessagePart{ - codersdk.ChatMessageText("look up the docs"), - }, - }, - persistedResponsesMessage{ - role: database.ChatMessageRoleAssistant, - parts: []codersdk.ChatMessagePart{ - webSearchCall, - webSearchResult, - }, - providerResponseID: previousResponseID, - }, - ) - - factory := chattest.NewMockAIBridgeTransport(t, openAIURL) - server := newOpenAIResponsesTestServer(t, db, ps, func(cfg *chatd.Config) { - cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(factory) - }) - _, err := server.SendMessage(ctx, chatd.SendMessageOptions{ - ChatID: chat.ID, - CreatedBy: user.ID, - APIKeyID: testAPIKeyID(t, db, user.ID), - ModelConfigID: model.ID, - Content: []codersdk.ChatMessagePart{ - codersdk.ChatMessageText("what did it find"), - }, - }) - require.NoError(t, err) - waitForChatProcessed(ctx, t, db, chat.ID, server) - requireResponsesChatWaiting(ctx, t, db, chat.ID) - - requests := recorder.all() - require.Len(t, requests, 1) - request := requests[0] - require.NotNil(t, request.Store) - require.True(t, *request.Store) - require.NotNil(t, request.PreviousResponseID) - require.Equal(t, previousResponseID, *request.PreviousResponseID) - require.NotEmpty(t, request.Prompt) - requireNoResponsesProviderItemReplay(t, request.Prompt, webSearchID) - require.NotContains(t, promptItemTypes(request.Prompt), "web_search_call") - require.NotContains(t, promptItemRoles(request.Prompt), "assistant") -} - type recordedResponsesRequest struct { - Prompt []interface{} - Store *bool - PreviousResponseID *string + Prompt []interface{} + Store *bool } type responsesRequestRecorder struct { @@ -377,15 +212,9 @@ func (r *responsesRequestRecorder) record(req *chattest.OpenAIRequest) int { value := *req.Store store = &value } - var previousResponseID *string - if req.PreviousResponseID != nil { - value := *req.PreviousResponseID - previousResponseID = &value - } r.requests = append(r.requests, recordedResponsesRequest{ - Prompt: append([]interface{}(nil), req.Prompt...), - Store: store, - PreviousResponseID: previousResponseID, + Prompt: append([]interface{}(nil), req.Prompt...), + Store: store, }) return len(r.requests) } @@ -396,12 +225,6 @@ func (r *responsesRequestRecorder) all() []recordedResponsesRequest { return append([]recordedResponsesRequest(nil), r.requests...) } -type persistedResponsesMessage struct { - role database.ChatMessageRole - parts []codersdk.ChatMessagePart - providerResponseID string -} - func newOpenAIResponsesTestServer( t *testing.T, db database.Store, @@ -445,64 +268,6 @@ func insertOpenAIResponsesModelConfig( ) } -func insertOpenAIResponsesChat( - t *testing.T, - db database.Store, - organizationID uuid.UUID, - ownerID uuid.UUID, - modelConfigID uuid.UUID, - titlePrefix string, -) database.Chat { - t.Helper() - return dbgen.Chat(t, db, database.Chat{ - OrganizationID: organizationID, - OwnerID: ownerID, - LastModelConfigID: modelConfigID, - Title: uniqueResponsesTitle(t, titlePrefix), - Status: database.ChatStatusWaiting, - MCPServerIDs: []uuid.UUID{}, - ClientType: database.ChatClientTypeApi, - }) -} - -func insertOpenAIResponsesMessages( - ctx context.Context, - t *testing.T, - db database.Store, - chatID uuid.UUID, - createdBy uuid.UUID, - modelConfigID uuid.UUID, - messages ...persistedResponsesMessage, -) { - t.Helper() - params := database.InsertChatMessagesParams{ChatID: chatID} - for _, message := range messages { - content, err := chatprompt.MarshalParts(message.parts) - require.NoError(t, err) - params.CreatedBy = append(params.CreatedBy, createdBy) - params.ModelConfigID = append(params.ModelConfigID, modelConfigID) - params.Role = append(params.Role, message.role) - params.Content = append(params.Content, string(content.RawMessage)) - params.ContentVersion = append(params.ContentVersion, chatprompt.CurrentContentVersion) - params.Visibility = append(params.Visibility, database.ChatMessageVisibilityBoth) - params.InputTokens = append(params.InputTokens, 0) - params.OutputTokens = append(params.OutputTokens, 0) - params.TotalTokens = append(params.TotalTokens, 0) - params.ReasoningTokens = append(params.ReasoningTokens, 0) - params.CacheCreationTokens = append(params.CacheCreationTokens, 0) - params.CacheReadTokens = append(params.CacheReadTokens, 0) - params.ContextLimit = append(params.ContextLimit, 0) - params.Compressed = append(params.Compressed, false) - params.TotalCostMicros = append(params.TotalCostMicros, 0) - params.RuntimeMs = append(params.RuntimeMs, 0) - params.ProviderResponseID = append(params.ProviderResponseID, message.providerResponseID) - } - // Keep this raw because dbgen.ChatMessage inserts one message at a time, - // while this helper needs to preserve variadic batch insert behavior. - _, err := db.InsertChatMessages(ctx, params) - require.NoError(t, err) -} - func requireResponsesChatWaiting( ctx context.Context, t *testing.T, @@ -537,47 +302,6 @@ func promptItemTypes(prompt []interface{}) []string { return types } -func promptItemRoles(prompt []interface{}) []string { - roles := make([]string, 0, len(prompt)) - for _, item := range prompt { - itemMap, ok := item.(map[string]interface{}) - if !ok { - continue - } - if role := chattest.StringResponseField(itemMap, "role"); role != "" { - roles = append(roles, role) - } - } - return roles -} - -func requirePromptItemWithTypeAndCallID( - t *testing.T, - prompt []interface{}, - itemType string, - callID string, -) map[string]interface{} { - t.Helper() - for _, item := range prompt { - itemMap, ok := item.(map[string]interface{}) - if !ok { - continue - } - if chattest.StringResponseField(itemMap, "type") == itemType && - chattest.StringResponseField(itemMap, "call_id") == callID { - return itemMap - } - } - promptJSON, err := json.Marshal(prompt) - require.NoError(t, err) - require.FailNowf(t, "prompt item missing", - "missing type=%q call_id=%q in prompt %s", itemType, callID, promptJSON) - return nil -} - -// requireNoResponsesProviderItemReplay rejects the explicit stale IDs and all -// provider-managed Responses item IDs. Chain mode should rely on -// previous_response_id, not replay rs_ or ws_ identifiers in prompt input. func requireNoResponsesProviderItemReplay( t *testing.T, prompt []interface{}, diff --git a/coderd/x/chatd/message_conversion.go b/coderd/x/chatd/message_conversion.go index 2771582c151dc..c9d4cc348e67a 100644 --- a/coderd/x/chatd/message_conversion.go +++ b/coderd/x/chatd/message_conversion.go @@ -200,9 +200,6 @@ func assistantMessage( if step.Runtime > 0 { msg.RuntimeMs = sql.NullInt64{Int64: step.Runtime.Milliseconds(), Valid: true} } - if step.ProviderResponseID != "" { - msg.ProviderResponseID = sql.NullString{String: step.ProviderResponseID, Valid: true} - } return msg } diff --git a/coderd/x/chatd/message_conversion_test.go b/coderd/x/chatd/message_conversion_test.go index c9566048bf1fb..4ec63a774c6b0 100644 --- a/coderd/x/chatd/message_conversion_test.go +++ b/coderd/x/chatd/message_conversion_test.go @@ -126,7 +126,7 @@ func TestBuildCommitStepMessages_ProviderExecutedResultsStayAssistantContent(t * require.True(t, parts[1].ProviderExecuted) } -func TestBuildCommitStepMessages_UsageCostRuntimeProviderResponseID(t *testing.T) { +func TestBuildCommitStepMessages_UsageCostRuntime(t *testing.T) { t.Parallel() inputPrice := decimal.NewFromFloat(2.5) @@ -142,11 +142,10 @@ func TestBuildCommitStepMessages_UsageCostRuntimeProviderResponseID(t *testing.T }, }, step: stepData{ - Content: []fantasy.Content{fantasy.TextContent{Text: "usage"}}, - Usage: fantasy.Usage{InputTokens: 100, OutputTokens: 20, TotalTokens: 120, ReasoningTokens: 3, CacheCreationTokens: 4, CacheReadTokens: 5}, - ContextLimit: sql.NullInt64{Int64: 4096, Valid: true}, - ProviderResponseID: "resp-123", - Runtime: 1500 * time.Millisecond, + Content: []fantasy.Content{fantasy.TextContent{Text: "usage"}}, + Usage: fantasy.Usage{InputTokens: 100, OutputTokens: 20, TotalTokens: 120, ReasoningTokens: 3, CacheCreationTokens: 4, CacheReadTokens: 5}, + ContextLimit: sql.NullInt64{Int64: 4096, Valid: true}, + Runtime: 1500 * time.Millisecond, }, }) require.NoError(t, err) @@ -160,7 +159,6 @@ func TestBuildCommitStepMessages_UsageCostRuntimeProviderResponseID(t *testing.T require.Equal(t, sql.NullInt64{Int64: 5, Valid: true}, msg.CacheReadTokens) require.Equal(t, sql.NullInt64{Int64: 4096, Valid: true}, msg.ContextLimit) require.Equal(t, sql.NullInt64{Int64: 1500, Valid: true}, msg.RuntimeMs) - require.Equal(t, sql.NullString{String: "resp-123", Valid: true}, msg.ProviderResponseID) require.True(t, msg.TotalCostMicros.Valid) require.Greater(t, msg.TotalCostMicros.Int64, int64(0)) } diff --git a/docs/admin/integrations/prometheus.md b/docs/admin/integrations/prometheus.md index d4cab1ac1c81d..03b625f8d3d43 100644 --- a/docs/admin/integrations/prometheus.md +++ b/docs/admin/integrations/prometheus.md @@ -229,7 +229,7 @@ deployment. They will always be available from the agent. | `coderd_chatd_prompt_size_bytes` | histogram | Estimated byte size of the prompt per LLM request. | `model` `provider` | | `coderd_chatd_steps_total` | counter | Total agentic loop steps across all chats. | `model` `provider` | | `coderd_chatd_stream_buffer_dropped_total` | counter | Number of chat stream buffer events dropped due to the per-chat buffer cap. | | -| `coderd_chatd_stream_retries_total` | counter | Total LLM stream retries. | `chain_broken` `kind` `model` `provider` | +| `coderd_chatd_stream_retries_total` | counter | Total LLM stream retries. | `kind` `model` `provider` | | `coderd_chatd_tool_errors_total` | counter | Total tool calls that returned an error result. | `model` `provider` `tool_name` | | `coderd_chatd_tool_result_size_bytes` | histogram | Size in bytes of each tool execution result. | `model` `provider` `tool_name` | | `coderd_chatd_tool_result_truncated_total` | counter | Total tool results truncated to fit the model context window. | `model` `provider` `tool_name` | diff --git a/scripts/metricsdocgen/generated_metrics b/scripts/metricsdocgen/generated_metrics index 9f3b9f629b0a0..d045636a8117a 100644 --- a/scripts/metricsdocgen/generated_metrics +++ b/scripts/metricsdocgen/generated_metrics @@ -291,7 +291,7 @@ coderd_chatd_steps_total{provider="",model=""} 0 coderd_chatd_stream_buffer_dropped_total 0 # HELP coderd_chatd_stream_retries_total Total LLM stream retries. # TYPE coderd_chatd_stream_retries_total counter -coderd_chatd_stream_retries_total{provider="",model="",kind="",chain_broken=""} 0 +coderd_chatd_stream_retries_total{provider="",model="",kind=""} 0 # HELP coderd_chatd_tool_errors_total Total tool calls that returned an error result. # TYPE coderd_chatd_tool_errors_total counter coderd_chatd_tool_errors_total{provider="",model="",tool_name=""} 0