diff --git a/coderd/apidoc/docs.go b/coderd/apidoc/docs.go index aae3e1f8a06..f0c0ade28c4 100644 --- a/coderd/apidoc/docs.go +++ b/coderd/apidoc/docs.go @@ -17113,7 +17113,8 @@ const docTemplate = `{ "config", "usage_limit", "missing_key", - "provider_disabled" + "provider_disabled", + "content_filter" ], "x-enum-varnames": [ "ChatErrorKindGeneric", @@ -17125,7 +17126,8 @@ const docTemplate = `{ "ChatErrorKindConfig", "ChatErrorKindUsageLimit", "ChatErrorKindMissingKey", - "ChatErrorKindProviderDisabled" + "ChatErrorKindProviderDisabled", + "ChatErrorKindContentFilter" ] }, "codersdk.ChatFileMetadata": { diff --git a/coderd/apidoc/swagger.json b/coderd/apidoc/swagger.json index 7f170c96ea7..e454158867b 100644 --- a/coderd/apidoc/swagger.json +++ b/coderd/apidoc/swagger.json @@ -15387,7 +15387,8 @@ "config", "usage_limit", "missing_key", - "provider_disabled" + "provider_disabled", + "content_filter" ], "x-enum-varnames": [ "ChatErrorKindGeneric", @@ -15399,7 +15400,8 @@ "ChatErrorKindConfig", "ChatErrorKindUsageLimit", "ChatErrorKindMissingKey", - "ChatErrorKindProviderDisabled" + "ChatErrorKindProviderDisabled", + "ChatErrorKindContentFilter" ] }, "codersdk.ChatFileMetadata": { diff --git a/coderd/x/chatd/chaterror/message.go b/coderd/x/chatd/chaterror/message.go index 3ebe6366e7f..a64488a8c89 100644 --- a/coderd/x/chatd/chaterror/message.go +++ b/coderd/x/chatd/chaterror/message.go @@ -64,6 +64,8 @@ func terminalMessage(classified ClassifiedError) string { " Contact your Coder administrator.", displayName, ) + case codersdk.ChatErrorKindContentFilter: + return ContentFilterMessage(classified.Provider, "") default: if !classified.Retryable && classified.StatusCode == 0 { return "The chat request failed unexpectedly." @@ -116,6 +118,20 @@ func retryMessage(classified ClassifiedError) string { } } +// ContentFilterMessage is the user-facing message for a response blocked by +// the provider's content filter. +func ContentFilterMessage(provider, category string) string { + subject := providerSubject(provider) + if category = strings.TrimSpace(category); category != "" { + return stringutil.Capitalize(fmt.Sprintf( + "%s blocked this response under its content policy (%s).", subject, category, + )) + } + return stringutil.Capitalize(fmt.Sprintf( + "%s blocked this response under its content policy.", subject, + )) +} + func providerSubject(provider string) string { if displayName := providerDisplayName(provider); displayName != "AI" && displayName != "" { return displayName diff --git a/coderd/x/chatd/chatloop/chatloop.go b/coderd/x/chatd/chatloop/chatloop.go index 33505aa593b..4e97bfc1862 100644 --- a/coderd/x/chatd/chatloop/chatloop.go +++ b/coderd/x/chatd/chatloop/chatloop.go @@ -47,6 +47,10 @@ var ( // StopAfterTools produces a successful result, indicating // the run should terminate cleanly after persistence. ErrStopAfterTool = xerrors.New("stop after tool") + // ErrContentFiltered is returned when the provider's safety + // classifiers blocked the response and the model produced no + // content, e.g. Anthropic's stop_reason "refusal". + ErrContentFiltered = xerrors.New("response blocked by provider content filter") errStreamSilenceTimeout = xerrors.New( "chat stream was silent for longer than the configured timeout", @@ -416,6 +420,12 @@ func GenerateAssistant(ctx context.Context, opts GenerateAssistantOptions) (Assi ctx, opts.Logger, provider, modelName, "assistant_helper", 0, result.finishReason, result.content, ) + // A content-filter finish with no content means the provider's + // safety classifiers blocked the whole response (e.g. Anthropic + // stop_reason "refusal"). + if len(result.content) == 0 && result.finishReason == fantasy.FinishReasonContentFilter { + return AssistantOutcome{}, contentFilterError(errorProvider, result.providerMetadata) + } step := PersistedStep{ Content: result.content, Usage: result.usage, @@ -450,6 +460,19 @@ func wrapProviderStreamError(provider string, err error) error { return xerrors.Errorf("stream response: %w", chaterror.WithClassification(err, classified)) } +func contentFilterError(provider string, metadata fantasy.ProviderMetadata) error { + classified := chaterror.ClassifiedError{ + Kind: codersdk.ChatErrorKindContentFilter, + Provider: provider, + Retryable: false, + } + if refusal := fantasyanthropic.GetRefusalMetadata(metadata); refusal != nil { + classified.Message = chaterror.ContentFilterMessage(provider, refusal.Category) + classified.Detail = strings.TrimSpace(refusal.Explanation) + } + return chaterror.WithClassification(ErrContentFiltered, classified) +} + // ExecuteLocalTools runs local tool calls and returns durable tool results. It // does not retry or persist. func ExecuteLocalTools(ctx context.Context, opts ExecuteLocalToolsOptions) (ToolExecutionOutcome, error) { diff --git a/coderd/x/chatd/chatloop/contentfilter_internal_test.go b/coderd/x/chatd/chatloop/contentfilter_internal_test.go new file mode 100644 index 00000000000..db7b571addc --- /dev/null +++ b/coderd/x/chatd/chatloop/contentfilter_internal_test.go @@ -0,0 +1,152 @@ +package chatloop + +import ( + "context" + "errors" + "testing" + + "charm.land/fantasy" + fantasyanthropic "charm.land/fantasy/providers/anthropic" + "github.com/stretchr/testify/require" + + "github.com/coder/coder/v2/coderd/x/chatd/chaterror" + "github.com/coder/coder/v2/coderd/x/chatd/chattest" + "github.com/coder/coder/v2/codersdk" +) + +func refusalProviderMetadataForTest(category, explanation string) fantasy.ProviderMetadata { + return fantasy.ProviderMetadata{ + fantasyanthropic.Name: &fantasyanthropic.RefusalMetadata{ + Category: category, + Explanation: explanation, + }, + } +} + +func TestContentFilterError(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + provider string + metadata fantasy.ProviderMetadata + wantMessage string + wantDetail string + }{ + { + name: "CategoryVerbatim", + provider: "anthropic", + metadata: refusalProviderMetadataForTest( + "harmful_content", "The response was blocked. See https://example.com for help.", + ), + wantMessage: "Anthropic blocked this response under its content policy (harmful_content).", + wantDetail: "The response was blocked. See https://example.com for help.", + }, + { + name: "NoMetadataFallsBackToDefault", + provider: "anthropic", + metadata: nil, + wantMessage: "Anthropic blocked this response under its content policy.", + }, + { + name: "WhitespaceCategory", + provider: "anthropic", + metadata: refusalProviderMetadataForTest(" ", ""), + wantMessage: "Anthropic blocked this response under its content policy.", + }, + { + name: "UnknownProvider", + provider: "", + metadata: refusalProviderMetadataForTest("cyber", ""), + wantMessage: "The AI provider blocked this response under its content policy (cyber).", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + err := contentFilterError(tt.provider, tt.metadata) + require.ErrorIs(t, err, ErrContentFiltered) + + classified := chaterror.Classify(err) + require.Equal(t, codersdk.ChatErrorKindContentFilter, classified.Kind) + require.Equal(t, tt.provider, classified.Provider) + require.False(t, classified.Retryable) + require.Equal(t, tt.wantMessage, classified.Message) + require.Equal(t, tt.wantDetail, classified.Detail) + + payload := chaterror.TerminalErrorPayload(classified) + require.NotNil(t, payload) + require.Equal(t, codersdk.ChatErrorKindContentFilter, payload.Kind) + }) + } +} + +func TestGenerateAssistant_ContentFilterRefusal(t *testing.T) { + t.Parallel() + + t.Run("EmptyContentSurfacesTerminalError", func(t *testing.T) { + t.Parallel() + + model := &chattest.FakeModel{ + ProviderName: "anthropic", + ModelName: "test-model", + StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { + return streamFromParts([]fantasy.StreamPart{{ + Type: fantasy.StreamPartTypeFinish, + FinishReason: fantasy.FinishReasonContentFilter, + ProviderMetadata: refusalProviderMetadataForTest( + "harmful_content", "The response was blocked.", + ), + }}), nil + }, + } + + outcome, err := GenerateAssistant(context.Background(), GenerateAssistantOptions{ + Model: model, + Messages: []fantasy.Message{ + textMessage(fantasy.MessageRoleUser, "hello"), + }, + }) + require.ErrorIs(t, err, ErrContentFiltered) + require.Empty(t, outcome.Step.Content) + + classified := chaterror.Classify(err) + require.Equal(t, codersdk.ChatErrorKindContentFilter, classified.Kind) + require.Equal(t, "anthropic", classified.Provider) + require.False(t, classified.Retryable) + require.Equal(t, "Anthropic blocked this response under its content policy (harmful_content).", classified.Message) + require.Equal(t, "The response was blocked.", classified.Detail) + }) + + t.Run("PartialContentIsPersistedNotErrored", func(t *testing.T) { + t.Parallel() + + model := &chattest.FakeModel{ + ProviderName: "anthropic", + ModelName: "test-model", + StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { + return streamFromParts([]fantasy.StreamPart{ + {Type: fantasy.StreamPartTypeTextStart, ID: "text-1"}, + {Type: fantasy.StreamPartTypeTextDelta, ID: "text-1", Delta: "partial"}, + {Type: fantasy.StreamPartTypeTextEnd, ID: "text-1"}, + { + Type: fantasy.StreamPartTypeFinish, + FinishReason: fantasy.FinishReasonContentFilter, + }, + }), nil + }, + } + + outcome, err := GenerateAssistant(context.Background(), GenerateAssistantOptions{ + Model: model, + Messages: []fantasy.Message{ + textMessage(fantasy.MessageRoleUser, "hello"), + }, + }) + require.NoError(t, err) + require.False(t, errors.Is(err, ErrContentFiltered)) + require.NotEmpty(t, outcome.Step.Content) + }) +} diff --git a/codersdk/chats.go b/codersdk/chats.go index 9e62d6f73a9..1196c54de63 100644 --- a/codersdk/chats.go +++ b/codersdk/chats.go @@ -1686,6 +1686,7 @@ const ( ChatErrorKindUsageLimit ChatErrorKind = "usage_limit" ChatErrorKindMissingKey ChatErrorKind = "missing_key" ChatErrorKindProviderDisabled ChatErrorKind = "provider_disabled" + ChatErrorKindContentFilter ChatErrorKind = "content_filter" ) // AllChatErrorKinds contains every ChatErrorKind value. @@ -1701,6 +1702,7 @@ var AllChatErrorKinds = []ChatErrorKind{ ChatErrorKindUsageLimit, ChatErrorKindMissingKey, ChatErrorKindProviderDisabled, + ChatErrorKindContentFilter, } // ChatError represents a terminal chat error in persisted chat state or the diff --git a/docs/reference/api/chats.md b/docs/reference/api/chats.md index a9bd9bd0426..f9a118f3d17 100644 --- a/docs/reference/api/chats.md +++ b/docs/reference/api/chats.md @@ -225,12 +225,12 @@ Status Code **200** #### Enumerated Values -| Property | Value(s) | -|---------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `client_type` | `api`, `ui` | -| `kind` | `auth`, `config`, `generic`, `instruction_file`, `mcp_config`, `mcp_server`, `missing_key`, `overloaded`, `provider_disabled`, `rate_limit`, `skill`, `stream_silence_timeout`, `timeout`, `usage_limit` | -| `status` | `error`, `excluded`, `interrupting`, `invalid`, `ok`, `oversize`, `requires_action`, `running`, `unreadable`, `waiting` | -| `plan_mode` | `plan` | +| Property | Value(s) | +|---------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `client_type` | `api`, `ui` | +| `kind` | `auth`, `config`, `content_filter`, `generic`, `instruction_file`, `mcp_config`, `mcp_server`, `missing_key`, `overloaded`, `provider_disabled`, `rate_limit`, `skill`, `stream_silence_timeout`, `timeout`, `usage_limit` | +| `status` | `error`, `excluded`, `interrupting`, `invalid`, `ok`, `oversize`, `requires_action`, `running`, `unreadable`, `waiting` | +| `plan_mode` | `plan` | To perform this operation, you must be authenticated. [Learn more](authentication.md). diff --git a/docs/reference/api/schemas.md b/docs/reference/api/schemas.md index 53547ca044e..f80d1418d0c 100644 --- a/docs/reference/api/schemas.md +++ b/docs/reference/api/schemas.md @@ -2578,9 +2578,9 @@ AuthorizationObject can represent a "set" of objects, such as: all workspaces in #### Enumerated Values -| Value(s) | -|-------------------------------------------------------------------------------------------------------------------------------------------------| -| `auth`, `config`, `generic`, `missing_key`, `overloaded`, `provider_disabled`, `rate_limit`, `stream_silence_timeout`, `timeout`, `usage_limit` | +| Value(s) | +|-------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `auth`, `config`, `content_filter`, `generic`, `missing_key`, `overloaded`, `provider_disabled`, `rate_limit`, `stream_silence_timeout`, `timeout`, `usage_limit` | ## codersdk.ChatFileMetadata diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts index 81eda3801b7..49e5831f453 100644 --- a/site/src/api/typesGenerated.ts +++ b/site/src/api/typesGenerated.ts @@ -2198,6 +2198,7 @@ export interface ChatError { export type ChatErrorKind = | "auth" | "config" + | "content_filter" | "generic" | "missing_key" | "overloaded" @@ -2210,6 +2211,7 @@ export type ChatErrorKind = export const ChatErrorKinds: ChatErrorKind[] = [ "auth", "config", + "content_filter", "generic", "missing_key", "overloaded", diff --git a/site/src/pages/AgentsPage/components/ChatConversation/LiveStreamTail.stories.tsx b/site/src/pages/AgentsPage/components/ChatConversation/LiveStreamTail.stories.tsx index e3fd42eae16..0cf96976d4a 100644 --- a/site/src/pages/AgentsPage/components/ChatConversation/LiveStreamTail.stories.tsx +++ b/site/src/pages/AgentsPage/components/ChatConversation/LiveStreamTail.stories.tsx @@ -124,6 +124,42 @@ export const TerminalOverloadedError: Story = { }, }; +/** Content-filter refusals render as terminal errors without a retry countdown or status link. */ +export const TerminalContentFilterError: Story = { + args: { + ...defaultArgs, + liveStatus: buildLiveStatus({ + persistedError: { + kind: "content_filter", + message: + "Anthropic blocked this response under its content policy (cyber).", + detail: + "This request triggered restrictions on violative cyber content and was blocked under Anthropic's Usage Policy. To learn more, see https://platform.claude.com/docs/en/build-with-claude/refusals-and-fallback.", + provider: "anthropic", + retryable: false, + }, + }), + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + expect( + canvas.getByRole("heading", { name: /response blocked/i }), + ).toBeVisible(); + expect( + canvas.getByText( + /anthropic blocked this response under its content policy \(cyber\)\./i, + ), + ).toBeVisible(); + expect( + canvas.getByText(/this request triggered restrictions/i), + ).toBeVisible(); + expect(canvas.queryByText(/retrying in/i)).not.toBeInTheDocument(); + expect( + canvas.queryByRole("link", { name: /status/i }), + ).not.toBeInTheDocument(); + }, +}; + /** * Transport timeouts render the per-provider "temporarily * unavailable" copy with a "Request timed out" heading rather than diff --git a/site/src/pages/AgentsPage/components/ChatConversation/chatStatusHelpers.ts b/site/src/pages/AgentsPage/components/ChatConversation/chatStatusHelpers.ts index 9389b6d11c6..51550c5042c 100644 --- a/site/src/pages/AgentsPage/components/ChatConversation/chatStatusHelpers.ts +++ b/site/src/pages/AgentsPage/components/ChatConversation/chatStatusHelpers.ts @@ -46,6 +46,8 @@ export const getErrorTitle = ( return "Chat interrupted"; case "provider_disabled": return "Provider disabled"; + case "content_filter": + return "Response blocked"; default: return mode === "retry" ? "Retrying request" : "Request failed"; }