Thanks to visit codestin.com
Credit goes to github.com

Skip to content

fix: add cache write accounting to OpenAI interceptors and service tier to TokenUsage metadata for all interceptors #28714

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mtojek merged 2 commits into from
Aug 27, 2026
Merged

fix: add cache write accounting to OpenAI interceptors and service tier to TokenUsage metadata for all interceptors #28714

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0d157a0
fix: add cache write accounting to OpenAI interceptors (#28567)
pawbana Aug 27, 2026
56beb0a
feat: add service tier to TokenUsage recording (#28709)
pawbana Aug 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions aibridge/fixtures/anthropic/single_injected_tool.txtar
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ data: {"type":"message_stop" }

-- streaming/tool-call --
event: message_start
data: {"type":"message_start","message":{"id":"msg_01LZSVzMCLivzXrp6ZnTcmeG","type":"message","role":"assistant","model":"claude-sonnet-4-20250514","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":7763,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":1,"service_tier":"standard"}} }
data: {"type":"message_start","message":{"id":"msg_01LZSVzMCLivzXrp6ZnTcmeG","type":"message","role":"assistant","model":"claude-sonnet-4-20250514","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":7763,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":1,"service_tier":"priority"}} }

event: content_block_start
data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""} }
Expand Down Expand Up @@ -157,7 +157,7 @@ data: {"type":"message_stop" }
"cache_creation_input_tokens": 0,
"cache_read_input_tokens": 0,
"output_tokens": 129,
"service_tier": "standard"
"service_tier": "priority"
}
}

3 changes: 2 additions & 1 deletion aibridge/fixtures/openai/chatcompletions/simple.txtar
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -492,7 +492,7 @@ data: {"id":"chatcmpl-BwoiPTGRbKkY5rncfaM0s9KtWrq5N","object":"chat.completion.c

data: {"id":"chatcmpl-BwoiPTGRbKkY5rncfaM0s9KtWrq5N","object":"chat.completion.chunk","created":1753357673,"model":"gpt-4.1-2025-04-14","service_tier":"default","system_fingerprint":"fp_51e1070cf2","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null}

data: {"id":"chatcmpl-BwoiPTGRbKkY5rncfaM0s9KtWrq5N","object":"chat.completion.chunk","created":1753357673,"model":"gpt-4.1-2025-04-14","service_tier":"default","system_fingerprint":"fp_51e1070cf2","choices":[],"usage":{"prompt_tokens":19,"completion_tokens":238,"total_tokens":257,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}}}
data: {"id":"chatcmpl-BwoiPTGRbKkY5rncfaM0s9KtWrq5N","object":"chat.completion.chunk","created":1753357673,"model":"gpt-4.1-2025-04-14","service_tier":"default","system_fingerprint":"fp_51e1070cf2","choices":[],"usage":{"prompt_tokens":19,"completion_tokens":238,"total_tokens":257,"prompt_tokens_details":{"cached_tokens":0,"cache_write_tokens":5,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}}}

data: [DONE]

Expand Down Expand Up @@ -521,6 +521,7 @@ data: [DONE]
"total_tokens": 219,
"prompt_tokens_details": {
"cached_tokens": 0,
"cache_write_tokens": 5,
"audio_tokens": 0
},
"completion_tokens_details": {
Expand Down
156 changes: 79 additions & 77 deletions aibridge/fixtures/openai/chatcompletions/single_injected_tool.txtar
View file
Open in desktop

Large diffs are not rendered by default.

12 changes: 6 additions & 6 deletions aibridge/fixtures/openai/chatcompletions/streaming_cumulative_usage_injected_tool.txtar
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,19 +12,19 @@ Streaming response with cumulative usage on every JSON chunk and an injected too
}

-- streaming --
data: {"id":"chatcmpl-cumulative-tool","object":"chat.completion.chunk","created":1754479216,"model":"gpt-4.1","choices":[{"index":0,"delta":{"content":"Calling"}}],"usage":{"prompt_tokens":6000,"completion_tokens":10,"total_tokens":6010}}
data: {"id":"chatcmpl-cumulative-tool","object":"chat.completion.chunk","created":1754479216,"model":"gpt-4.1","choices":[{"index":0,"delta":{"content":"Calling"}}],"usage":{"prompt_tokens":6000,"completion_tokens":10,"total_tokens":6010,"prompt_tokens_details":{"cached_tokens":100,"cache_write_tokens":10,"audio_tokens":1},"completion_tokens_details":{"reasoning_tokens":1,"audio_tokens":1,"accepted_prediction_tokens":1,"rejected_prediction_tokens":1}}}

data: {"id":"chatcmpl-cumulative-tool","object":"chat.completion.chunk","created":1754479216,"model":"gpt-4.1","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call-cumulative","type":"function","function":{"name":"bmcp_coder_coder_list_workspaces","arguments":"{}"}}]}}],"usage":{"prompt_tokens":6000,"completion_tokens":20,"total_tokens":6020}}
data: {"id":"chatcmpl-cumulative-tool","object":"chat.completion.chunk","created":1754479216,"model":"gpt-4.1","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call-cumulative","type":"function","function":{"name":"bmcp_coder_coder_list_workspaces","arguments":"{}"}}]}}],"usage":{"prompt_tokens":6000,"completion_tokens":20,"total_tokens":6020,"prompt_tokens_details":{"cached_tokens":100,"cache_write_tokens":10,"audio_tokens":2},"completion_tokens_details":{"reasoning_tokens":2,"audio_tokens":2,"accepted_prediction_tokens":2,"rejected_prediction_tokens":2}}}

data: {"id":"chatcmpl-cumulative-tool","object":"chat.completion.chunk","created":1754479216,"model":"gpt-4.1","choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":6000,"completion_tokens":30,"total_tokens":6030}}
data: {"id":"chatcmpl-cumulative-tool","object":"chat.completion.chunk","created":1754479216,"model":"gpt-4.1","choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":6000,"completion_tokens":30,"total_tokens":6030,"prompt_tokens_details":{"cached_tokens":100,"cache_write_tokens":10,"audio_tokens":3},"completion_tokens_details":{"reasoning_tokens":4,"audio_tokens":5,"accepted_prediction_tokens":6,"rejected_prediction_tokens":7}}}

data: [DONE]

-- streaming/tool-call --
data: {"id":"chatcmpl-cumulative-final","object":"chat.completion.chunk","created":1754479217,"model":"gpt-4.1","choices":[{"index":0,"delta":{"content":"Done"}}],"usage":{"prompt_tokens":6000,"completion_tokens":10,"total_tokens":6010}}
data: {"id":"chatcmpl-cumulative-final","object":"chat.completion.chunk","created":1754479217,"model":"gpt-4.1","choices":[{"index":0,"delta":{"content":"Done"}}],"usage":{"prompt_tokens":6000,"completion_tokens":10,"total_tokens":6010,"prompt_tokens_details":{"cached_tokens":200,"cache_write_tokens":20,"audio_tokens":7},"completion_tokens_details":{"reasoning_tokens":8,"audio_tokens":9,"accepted_prediction_tokens":10,"rejected_prediction_tokens":11}}}

data: {"id":"chatcmpl-cumulative-final","object":"chat.completion.chunk","created":1754479217,"model":"gpt-4.1","choices":[{"index":0,"delta":{"content":"."}}],"usage":{"prompt_tokens":6000,"completion_tokens":20,"total_tokens":6020}}
data: {"id":"chatcmpl-cumulative-final","object":"chat.completion.chunk","created":1754479217,"model":"gpt-4.1","choices":[{"index":0,"delta":{"content":"."}}],"usage":{"prompt_tokens":6000,"completion_tokens":20,"total_tokens":6020,"prompt_tokens_details":{"cached_tokens":200,"cache_write_tokens":20,"audio_tokens":8},"completion_tokens_details":{"reasoning_tokens":9,"audio_tokens":10,"accepted_prediction_tokens":11,"rejected_prediction_tokens":12}}}

data: {"id":"chatcmpl-cumulative-final","object":"chat.completion.chunk","created":1754479217,"model":"gpt-4.1","choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":6000,"completion_tokens":30,"total_tokens":6030}}
data: {"id":"chatcmpl-cumulative-final","object":"chat.completion.chunk","created":1754479217,"model":"gpt-4.1","choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":6000,"completion_tokens":30,"total_tokens":6030,"prompt_tokens_details":{"cached_tokens":200,"cache_write_tokens":20,"audio_tokens":9},"completion_tokens_details":{"reasoning_tokens":10,"audio_tokens":11,"accepted_prediction_tokens":12,"rejected_prediction_tokens":13}}}

data: [DONE]
3 changes: 2 additions & 1 deletion aibridge/fixtures/openai/responses/blocking/cached_input_tokens.txtar
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,8 @@
"usage": {
"input_tokens": 12033,
"input_tokens_details": {
"cached_tokens": 11904
"cached_tokens": 11904,
"cache_write_tokens": 15
},
"output_tokens": 44,
"output_tokens_details": {
Expand Down
8 changes: 5 additions & 3 deletions aibridge/fixtures/openai/responses/blocking/single_injected_tool.txtar
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -749,7 +749,8 @@ Coder MCP tools automatically injected.
"usage": {
"input_tokens": 6371,
"input_tokens_details": {
"cached_tokens": 6144
"cached_tokens": 6144,
"cache_write_tokens": 7
},
"output_tokens": 75,
"output_tokens_details": {
Expand Down Expand Up @@ -810,7 +811,7 @@ Coder MCP tools automatically injected.
"summary": null
},
"safety_identifier": null,
"service_tier": "default",
"service_tier": "priority",
"status": "completed",
"store": true,
"temperature": 1,
Expand Down Expand Up @@ -1509,7 +1510,8 @@ Coder MCP tools automatically injected.
"usage": {
"input_tokens": 6756,
"input_tokens_details": {
"cached_tokens": 6144
"cached_tokens": 6144,
"cache_write_tokens": 11
},
"output_tokens": 231,
"output_tokens_details": {
Expand Down
6 changes: 4 additions & 2 deletions aibridge/fixtures/openai/responses/blocking/single_injected_tool_error.txtar
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -749,7 +749,8 @@ Coder MCP tools automatically injected, and errors invoking them are recorded.
"usage": {
"input_tokens": 6377,
"input_tokens_details": {
"cached_tokens": 6144
"cached_tokens": 6144,
"cache_write_tokens": 5
},
"output_tokens": 119,
"output_tokens_details": {
Expand Down Expand Up @@ -1509,7 +1510,8 @@ Coder MCP tools automatically injected, and errors invoking them are recorded.
"usage": {
"input_tokens": 6539,
"input_tokens_details": {
"cached_tokens": 6144
"cached_tokens": 6144,
"cache_write_tokens": 9
},
"output_tokens": 144,
"output_tokens_details": {
Expand Down
2 changes: 1 addition & 1 deletion aibridge/fixtures/openai/responses/streaming/cached_input_tokens.txtar
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,5 +43,5 @@ event: response.output_item.done
data: {"type":"response.output_item.done","item":{"id":"msg_05080461b406f3f501696a140a70d88195a2ce4c1a4eb39696","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"text":"Test response with cached tokens."}],"role":"assistant"},"output_index":0,"sequence_number":11}

event: response.completed
data: {"type":"response.completed","response":{"id":"resp_05080461b406f3f501696a1409d34c8195a40ff4b092145c35","object":"response","created_at":1768559625,"status":"completed","background":false,"completed_at":1768559627,"error":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.2-codex","output":[{"id":"msg_05080461b406f3f501696a140a70d88195a2ce4c1a4eb39696","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"text":"Test response with cached tokens."}],"role":"assistant"}],"parallel_tool_calls":true,"previous_response_id":null,"prompt_cache_key":"019bc657-f77b-7292-b5f4-2e8d6c2b0945","prompt_cache_retention":null,"reasoning":{"effort":"medium","summary":"detailed"},"service_tier":"default","store":false,"temperature":1.0,"tool_choice":"auto","tools":[],"truncation":"disabled","usage":{"input_tokens":16909,"input_tokens_details":{"cached_tokens":15744},"output_tokens":54,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":16963},"user":null,"metadata":{}},"sequence_number":12}
data: {"type":"response.completed","response":{"id":"resp_05080461b406f3f501696a1409d34c8195a40ff4b092145c35","object":"response","created_at":1768559625,"status":"completed","background":false,"completed_at":1768559627,"error":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.2-codex","output":[{"id":"msg_05080461b406f3f501696a140a70d88195a2ce4c1a4eb39696","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"text":"Test response with cached tokens."}],"role":"assistant"}],"parallel_tool_calls":true,"previous_response_id":null,"prompt_cache_key":"019bc657-f77b-7292-b5f4-2e8d6c2b0945","prompt_cache_retention":null,"reasoning":{"effort":"medium","summary":"detailed"},"service_tier":"default","store":false,"temperature":1.0,"tool_choice":"auto","tools":[],"truncation":"disabled","usage":{"input_tokens":16909,"input_tokens_details":{"cached_tokens":15744,"cache_write_tokens":30},"output_tokens":54,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":16963},"user":null,"metadata":{}},"sequence_number":12}

4 changes: 2 additions & 2 deletions aibridge/fixtures/openai/responses/streaming/single_injected_tool.txtar
View file
Open in desktop

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions aibridge/fixtures/openai/responses/streaming/single_injected_tool_error.txtar
View file
Open in desktop

Large diffs are not rendered by default.

40 changes: 24 additions & 16 deletions aibridge/intercept/chatcompletions/base.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -227,14 +227,21 @@ func (i *interceptionBase) hasInjectableTools() bool {
}

// recordTokenUsage records the token usage for a single completion, accounting
// for cached tokens included in the prompt token count.
func (i *interceptionBase) recordTokenUsage(ctx context.Context, msgID string, usage openai.CompletionUsage) {
// for cache read and write tokens included in the prompt token count.
func (i *interceptionBase) recordTokenUsage(ctx context.Context, msgID string, usage openai.CompletionUsage, serviceTier string) {
var metadata recorder.Metadata
if serviceTier != "" {
metadata = recorder.Metadata{recorder.MetadataKeyServiceTier: serviceTier}
}

_ = i.recorder.RecordTokenUsage(ctx, &recorder.TokenUsageRecord{
InterceptionID: i.ID().String(),
MsgID: msgID,
Input: calculateActualInputTokenUsage(usage),
Output: usage.CompletionTokens,
CacheReadInputTokens: usage.PromptTokensDetails.CachedTokens,
InterceptionID: i.ID().String(),
MsgID: msgID,
Input: calculateActualInputTokenUsage(usage),
Output: usage.CompletionTokens,
CacheReadInputTokens: usage.PromptTokensDetails.CachedTokens,
CacheWriteInputTokens: usage.PromptTokensDetails.CacheWriteTokens,
Metadata: metadata,
ExtraTokenTypes: map[string]int64{
"prompt_audio": usage.PromptTokensDetails.AudioTokens,
"completion_accepted_prediction": usage.CompletionTokensDetails.AcceptedPredictionTokens,
Expand All @@ -245,7 +252,7 @@ func (i *interceptionBase) recordTokenUsage(ctx context.Context, msgID string, u
})
}

func sumUsage(ref, in openai.CompletionUsage) openai.CompletionUsage {
func sumUsage(ref openai.CompletionUsage, in openai.CompletionUsage) openai.CompletionUsage {
return openai.CompletionUsage{
CompletionTokens: ref.CompletionTokens + in.CompletionTokens,
PromptTokens: ref.PromptTokens + in.PromptTokens,
Expand All @@ -257,17 +264,18 @@ func sumUsage(ref, in openai.CompletionUsage) openai.CompletionUsage {
RejectedPredictionTokens: ref.CompletionTokensDetails.RejectedPredictionTokens + in.CompletionTokensDetails.RejectedPredictionTokens,
},
PromptTokensDetails: openai.CompletionUsagePromptTokensDetails{
AudioTokens: ref.PromptTokensDetails.AudioTokens + in.PromptTokensDetails.AudioTokens,
CachedTokens: ref.PromptTokensDetails.CachedTokens + in.PromptTokensDetails.CachedTokens,
AudioTokens: ref.PromptTokensDetails.AudioTokens + in.PromptTokensDetails.AudioTokens,
CachedTokens: ref.PromptTokensDetails.CachedTokens + in.PromptTokensDetails.CachedTokens,
CacheWriteTokens: ref.PromptTokensDetails.CacheWriteTokens + in.PromptTokensDetails.CacheWriteTokens,
},
}
}

// calculateActualInputTokenUsage accounts for cached tokens which are included in [openai.CompletionUsage].PromptTokens.
// calculateActualInputTokenUsage calculates ordinary input tokens.
// in.PromptTokens contains sum of all prompt tokens including
// cache read and write tokens which are priced differently.
func calculateActualInputTokenUsage(in openai.CompletionUsage) int64 {
// Input *includes* the cached tokens, so we subtract them here to reflect actual input token usage.
// The original value can be reconstructed by adding CachedTokens back to Input.
// See https://platform.openai.com/docs/api-reference/usage/completions_object#usage/completions_object-input_tokens.
return max(0, in.PromptTokens /* The aggregated number of text input tokens used, including cached tokens. */ -
in.PromptTokensDetails.CachedTokens /* The aggregated number of text input tokens that has been cached from previous requests. */)
return max(0, in.PromptTokens-
in.PromptTokensDetails.CachedTokens-
in.PromptTokensDetails.CacheWriteTokens)
}
Loading
Loading