From 70b426e08f472ed3ebe6c1908eaafa7717a4638a Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Mon, 10 Aug 2026 06:04:39 +0000 Subject: [PATCH 1/2] fix: report per-request Anthropic usage in chat token accounting --- coderd/x/chatd/chatd_test.go | 39 +++++++++++++++++++++++++ coderd/x/chatd/chattest/anthropic.go | 43 +++++++++++++++++++++------- go.mod | 8 ++++-- go.sum | 4 +-- 4 files changed, 78 insertions(+), 16 deletions(-) diff --git a/coderd/x/chatd/chatd_test.go b/coderd/x/chatd/chatd_test.go index 6cb1a5a6935..7e77dc6279d 100644 --- a/coderd/x/chatd/chatd_test.go +++ b/coderd/x/chatd/chatd_test.go @@ -6664,6 +6664,45 @@ func TestActiveServer_AnthropicUsageMatchesFinalDelta(t *testing.T) { require.Equal(t, sql.NullInt64{Int64: 150, Valid: true}, last.CacheReadTokens) } +func TestActiveServer_AnthropicPersistsPerRequestUsage(t *testing.T) { + t.Parallel() + + ctx := testutil.Context(t, testutil.WaitLong) + db, ps := dbtestutil.NewDB(t) + anthropicURL := chattest.NewAnthropic(t, func(_ *chattest.AnthropicRequest) chattest.AnthropicResponse { + return chattest.AnthropicStreamingResponse(chattest.AnthropicTextChunksWithMessageUsages( + chattest.AnthropicUsage{ + InputTokens: 2, + CacheReadInputTokens: 139956, + CacheCreationInputTokens: 7770, + }, + chattest.AnthropicUsage{ + InputTokens: 4, + CacheReadInputTokens: 287682, + CacheCreationInputTokens: 15556, + OutputTokens: 4996, + }, + "cached response", + )...) + }) + user, org, model := seedAnthropicChatDependencies(t, db, anthropicURL) + + server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) { + cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(chattest.NewMockAIBridgeTransport(t, anthropicURL, chattest.WithPreservePath())) + }) + chat := createChatThroughServer(ctx, t, db, server, org.ID, user.ID, model.ID, "hello") + waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusWaiting) + + messages := chatMessages(ctx, t, db, chat.ID) + last := messages[len(messages)-1] + require.Equal(t, database.ChatMessageRoleAssistant, last.Role) + require.Equal(t, sql.NullInt64{Int64: 139956, Valid: true}, last.CacheReadTokens) + require.Equal(t, sql.NullInt64{Int64: 7770, Valid: true}, last.CacheCreationTokens) + require.Equal(t, sql.NullInt64{Int64: 2, Valid: true}, last.InputTokens) + require.Equal(t, sql.NullInt64{Int64: 4996, Valid: true}, last.OutputTokens) + require.Equal(t, sql.NullInt64{Int64: 4998, Valid: true}, last.TotalTokens) +} + func TestActiveServer_ChatTurnDebugRunRecordsStreamStep(t *testing.T) { t.Parallel() diff --git a/coderd/x/chatd/chattest/anthropic.go b/coderd/x/chatd/chattest/anthropic.go index c8193a98c6c..6c1bfd5a491 100644 --- a/coderd/x/chatd/chattest/anthropic.go +++ b/coderd/x/chatd/chattest/anthropic.go @@ -375,6 +375,16 @@ func AnthropicTextChunks(deltas ...string) []AnthropicChunk { // the initial input and cache token counts, and the final message_delta // carries the output token count. func AnthropicTextChunksWithCacheUsage(usage AnthropicUsage, deltas ...string) []AnthropicChunk { + return AnthropicTextChunksWithMessageUsages( + usage, + AnthropicUsage{OutputTokens: usage.OutputTokens}, + deltas..., + ) +} + +// AnthropicTextChunksWithMessageUsages creates a streaming response with +// independent message_start and message_delta usage. +func AnthropicTextChunksWithMessageUsages(messageStartUsage, messageDeltaUsage AnthropicUsage, deltas ...string) []AnthropicChunk { if len(deltas) == 0 { return nil } @@ -383,13 +393,26 @@ func AnthropicTextChunksWithCacheUsage(usage AnthropicUsage, deltas ...string) [ model := "claude-3-opus-20240229" messageUsage := map[string]int{ - "input_tokens": usage.InputTokens, + "input_tokens": messageStartUsage.InputTokens, + } + if messageStartUsage.CacheCreationInputTokens != 0 { + messageUsage["cache_creation_input_tokens"] = messageStartUsage.CacheCreationInputTokens + } + if messageStartUsage.CacheReadInputTokens != 0 { + messageUsage["cache_read_input_tokens"] = messageStartUsage.CacheReadInputTokens + } + + deltaUsage := map[string]int{ + "output_tokens": messageDeltaUsage.OutputTokens, } - if usage.CacheCreationInputTokens != 0 { - messageUsage["cache_creation_input_tokens"] = usage.CacheCreationInputTokens + if messageDeltaUsage.InputTokens != 0 { + deltaUsage["input_tokens"] = messageDeltaUsage.InputTokens } - if usage.CacheReadInputTokens != 0 { - messageUsage["cache_read_input_tokens"] = usage.CacheReadInputTokens + if messageDeltaUsage.CacheCreationInputTokens != 0 { + deltaUsage["cache_creation_input_tokens"] = messageDeltaUsage.CacheCreationInputTokens + } + if messageDeltaUsage.CacheReadInputTokens != 0 { + deltaUsage["cache_read_input_tokens"] = messageDeltaUsage.CacheReadInputTokens } chunks := []AnthropicChunk{ @@ -432,9 +455,7 @@ func AnthropicTextChunksWithCacheUsage(usage AnthropicUsage, deltas ...string) [ AnthropicChunk{ Type: "message_delta", StopReason: "end_turn", - UsageMap: map[string]int{ - "output_tokens": usage.OutputTokens, - }, + UsageMap: deltaUsage, }, AnthropicChunk{ Type: "message_stop", @@ -556,6 +577,7 @@ func AnthropicToolCallChunks(toolName string, inputJSONDeltas ...string) []Anthr Type: "message", Role: "assistant", Model: model, + Usage: map[string]int{"input_tokens": 10}, }, }, { @@ -589,9 +611,8 @@ func AnthropicToolCallChunks(toolName string, inputJSONDeltas ...string) []Anthr AnthropicChunk{ Type: "message_delta", StopReason: "tool_use", - Usage: AnthropicUsage{ - InputTokens: 10, - OutputTokens: 5, + UsageMap: map[string]int{ + "output_tokens": 5, }, }, AnthropicChunk{ diff --git a/go.mod b/go.mod index 6e58d8ff074..c4e859d8e7f 100644 --- a/go.mod +++ b/go.mod @@ -94,10 +94,12 @@ replace github.com/spf13/afero => github.com/aslilac/afero v0.0.0-20250403163713 // 5) coder/fantasy bedrock: mirror the request region when prefixing // cross-region inference profiles so the model-ID prefix matches the // region the request is actually signed for. -// 6) go.mod replaces pointing anthropic-sdk-go and openai-go at the +// 6) coder/fantasy#50, report per-request Anthropic usage at Finish so +// multi-iteration requests do not inflate context estimates. +// 7) go.mod replaces pointing anthropic-sdk-go and openai-go at the // coder forks below. -// See: https://github.com/coder/fantasy/commits/bb10946892ef -replace charm.land/fantasy => github.com/coder/fantasy v0.0.0-20260810175832-bb10946892ef +// See: https://github.com/coder/fantasy/commits/02fe20885709 +replace charm.land/fantasy => github.com/coder/fantasy v0.0.0-20260811105534-02fe20885709 // coder/coder uses a fork of charmbracelet's fork of the Anthropic Go SDK // with performance improvements and Bedrock header cleanup. diff --git a/go.sum b/go.sum index 7a30aa47edc..4ebcf0430da 100644 --- a/go.sum +++ b/go.sum @@ -327,8 +327,8 @@ github.com/coder/bubbletea v1.2.2-0.20241212190825-007a1cdb2c41 h1:SBN/DA63+ZHwu github.com/coder/bubbletea v1.2.2-0.20241212190825-007a1cdb2c41/go.mod h1:I9ULxr64UaOSUv7hcb3nX4kowodJCVS7vt7VVJk/kW4= github.com/coder/clistat v1.2.1 h1:P9/10njXMyj5cWzIU5wkRsSy5LVQH49+tcGMsAgWX0w= github.com/coder/clistat v1.2.1/go.mod h1:m7SC0uj88eEERgvF8Kn6+w6XF21BeSr+15f7GoLAw0A= -github.com/coder/fantasy v0.0.0-20260810175832-bb10946892ef h1:p2FYbdIwvybIUctKHQ3/53+Wta8gO91NLTEuvyuWQj0= -github.com/coder/fantasy v0.0.0-20260810175832-bb10946892ef/go.mod h1:lcL8B/uroFq5YDHNFij2an0yarOtD7C3rmzD31i8oTA= +github.com/coder/fantasy v0.0.0-20260811105534-02fe20885709 h1:RJa0GtqP+xypYLNlHUtYw05faBcEcD/Wqq+C8JWJPas= +github.com/coder/fantasy v0.0.0-20260811105534-02fe20885709/go.mod h1:lcL8B/uroFq5YDHNFij2an0yarOtD7C3rmzD31i8oTA= github.com/coder/flog v1.1.0 h1:kbAes1ai8fIS5OeV+QAnKBQE22ty1jRF/mcAwHpLBa4= github.com/coder/flog v1.1.0/go.mod h1:UQlQvrkJBvnRGo69Le8E24Tcl5SJleAAR7gYEHzAmdQ= github.com/coder/go-httpstat v0.0.0-20230801153223-321c88088322 h1:m0lPZjlQ7vdVpRBPKfYIFlmgevoTkBxB10wv6l2gOaU= From 147e5a9199132260ced54c0f20d638d40242c8ea Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Wed, 12 Aug 2026 08:01:05 +0000 Subject: [PATCH 2/2] chore: pin fantasy to coder_2_33 merge commit for per-request usage fix --- go.mod | 4 ++-- go.sum | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index c4e859d8e7f..c02a25ea1dc 100644 --- a/go.mod +++ b/go.mod @@ -98,8 +98,8 @@ replace github.com/spf13/afero => github.com/aslilac/afero v0.0.0-20250403163713 // multi-iteration requests do not inflate context estimates. // 7) go.mod replaces pointing anthropic-sdk-go and openai-go at the // coder forks below. -// See: https://github.com/coder/fantasy/commits/02fe20885709 -replace charm.land/fantasy => github.com/coder/fantasy v0.0.0-20260811105534-02fe20885709 +// See: https://github.com/coder/fantasy/commits/6f8df3735907 +replace charm.land/fantasy => github.com/coder/fantasy v0.0.0-20260812075759-6f8df3735907 // coder/coder uses a fork of charmbracelet's fork of the Anthropic Go SDK // with performance improvements and Bedrock header cleanup. diff --git a/go.sum b/go.sum index 4ebcf0430da..ab74be38ddd 100644 --- a/go.sum +++ b/go.sum @@ -327,8 +327,8 @@ github.com/coder/bubbletea v1.2.2-0.20241212190825-007a1cdb2c41 h1:SBN/DA63+ZHwu github.com/coder/bubbletea v1.2.2-0.20241212190825-007a1cdb2c41/go.mod h1:I9ULxr64UaOSUv7hcb3nX4kowodJCVS7vt7VVJk/kW4= github.com/coder/clistat v1.2.1 h1:P9/10njXMyj5cWzIU5wkRsSy5LVQH49+tcGMsAgWX0w= github.com/coder/clistat v1.2.1/go.mod h1:m7SC0uj88eEERgvF8Kn6+w6XF21BeSr+15f7GoLAw0A= -github.com/coder/fantasy v0.0.0-20260811105534-02fe20885709 h1:RJa0GtqP+xypYLNlHUtYw05faBcEcD/Wqq+C8JWJPas= -github.com/coder/fantasy v0.0.0-20260811105534-02fe20885709/go.mod h1:lcL8B/uroFq5YDHNFij2an0yarOtD7C3rmzD31i8oTA= +github.com/coder/fantasy v0.0.0-20260812075759-6f8df3735907 h1:Mirl1lw+0MsndEL0feesyIqtxTdpiKrR8AH4zcA6SZU= +github.com/coder/fantasy v0.0.0-20260812075759-6f8df3735907/go.mod h1:lcL8B/uroFq5YDHNFij2an0yarOtD7C3rmzD31i8oTA= github.com/coder/flog v1.1.0 h1:kbAes1ai8fIS5OeV+QAnKBQE22ty1jRF/mcAwHpLBa4= github.com/coder/flog v1.1.0/go.mod h1:UQlQvrkJBvnRGo69Le8E24Tcl5SJleAAR7gYEHzAmdQ= github.com/coder/go-httpstat v0.0.0-20230801153223-321c88088322 h1:m0lPZjlQ7vdVpRBPKfYIFlmgevoTkBxB10wv6l2gOaU=