Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit ca8ef6c

Browse files
pawbanaibetitsmike
pawbana
and
ibetitsmike
authored
fix: add cache write accounting to OpenAI interceptors (#28716)
Cherry-picks: * ad10045 fixes token accumulation logic for chat completions PR #27967 * 3ca2ba4 adds cache write tokens accounting to OpenAI interceptors, PR #28567 Compatibility fix: * ae737db bumps OpenAI SDK + minimal changes, bump required by cherry picks, based on c97f4da3ac --------- Co-authored-by: Michael Suchacz <[email protected]>
1 parent 3076b0c commit ca8ef6c

22 files changed

Lines changed: 753 additions & 254 deletions

‎aibridge/fixtures/fixtures.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,9 @@ var (
6767

6868
//go:embed openai/chatcompletions/streaming_injected_tool_nonzero_index.txtar
6969
OaiChatStreamingInjectedToolNonzeroIndex []byte
70+
71+
//go:embed openai/chatcompletions/streaming_cumulative_usage_injected_tool.txtar
72+
OaiChatStreamingCumulativeUsageInjectedTool []byte
7073
)
7174

7275
var (

‎aibridge/fixtures/openai/chatcompletions/simple.txtar‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -492,7 +492,7 @@ data: {"id":"chatcmpl-BwoiPTGRbKkY5rncfaM0s9KtWrq5N","object":"chat.completion.c
492492

493493
data: {"id":"chatcmpl-BwoiPTGRbKkY5rncfaM0s9KtWrq5N","object":"chat.completion.chunk","created":1753357673,"model":"gpt-4.1-2025-04-14","service_tier":"default","system_fingerprint":"fp_51e1070cf2","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null}
494494

495-
data: {"id":"chatcmpl-BwoiPTGRbKkY5rncfaM0s9KtWrq5N","object":"chat.completion.chunk","created":1753357673,"model":"gpt-4.1-2025-04-14","service_tier":"default","system_fingerprint":"fp_51e1070cf2","choices":[],"usage":{"prompt_tokens":19,"completion_tokens":238,"total_tokens":257,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}}}
495+
data: {"id":"chatcmpl-BwoiPTGRbKkY5rncfaM0s9KtWrq5N","object":"chat.completion.chunk","created":1753357673,"model":"gpt-4.1-2025-04-14","service_tier":"default","system_fingerprint":"fp_51e1070cf2","choices":[],"usage":{"prompt_tokens":19,"completion_tokens":238,"total_tokens":257,"prompt_tokens_details":{"cached_tokens":0,"cache_write_tokens":5,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}}}
496496

497497
data: [DONE]
498498

@@ -521,6 +521,7 @@ data: [DONE]
521521
"total_tokens": 219,
522522
"prompt_tokens_details": {
523523
"cached_tokens": 0,
524+
"cache_write_tokens": 5,
524525
"audio_tokens": 0
525526
},
526527
"completion_tokens_details": {

‎aibridge/fixtures/openai/chatcompletions/single_injected_tool.txtar‎

Lines changed: 17 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ data: {"id":"chatcmpl-C1WTooFaxeQgtyLB1kg53t41aB0NV","object":"chat.completion.c
7272

7373
data: {"id":"chatcmpl-C1WTooFaxeQgtyLB1kg53t41aB0NV","object":"chat.completion.chunk","created":1754479216,"model":"gpt-4.1-2025-04-14","service_tier":"default","system_fingerprint":"fp_799e4ca3f1","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"tool_calls"}],"usage":null,"obfuscation":"sDj"}
7474

75-
data: {"id":"chatcmpl-C1WTooFaxeQgtyLB1kg53t41aB0NV","object":"chat.completion.chunk","created":1754479216,"model":"gpt-4.1-2025-04-14","service_tier":"default","system_fingerprint":"fp_799e4ca3f1","choices":[],"usage":{"prompt_tokens":4862,"completion_tokens":45,"total_tokens":4907,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}},"obfuscation":"8sIWE1chOW"}
75+
data: {"id":"chatcmpl-C1WTooFaxeQgtyLB1kg53t41aB0NV","object":"chat.completion.chunk","created":1754479216,"model":"gpt-4.1-2025-04-14","service_tier":"default","system_fingerprint":"fp_799e4ca3f1","choices":[],"usage":{"prompt_tokens":4862,"completion_tokens":45,"total_tokens":4907,"prompt_tokens_details":{"cached_tokens":100,"cache_write_tokens":20,"audio_tokens":3},"completion_tokens_details":{"reasoning_tokens":4,"audio_tokens":5,"accepted_prediction_tokens":6,"rejected_prediction_tokens":7}},"obfuscation":"8sIWE1chOW"}
7676

7777
data: [DONE]
7878

@@ -200,7 +200,7 @@ data: {"id":"chatcmpl-C1WTqhYgK7bV01bW98Lww3zqaf8ZF","object":"chat.completion.c
200200

201201
data: {"id":"chatcmpl-C1WTqhYgK7bV01bW98Lww3zqaf8ZF","object":"chat.completion.chunk","created":1754479218,"model":"gpt-4.1-2025-04-14","service_tier":"default","system_fingerprint":"fp_799e4ca3f1","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null,"obfuscation":"EFeFvdS8m"}
202202

203-
data: {"id":"chatcmpl-C1WTqhYgK7bV01bW98Lww3zqaf8ZF","object":"chat.completion.chunk","created":1754479218,"model":"gpt-4.1-2025-04-14","service_tier":"default","system_fingerprint":"fp_799e4ca3f1","choices":[],"usage":{"prompt_tokens":5049,"completion_tokens":60,"total_tokens":5109,"prompt_tokens_details":{"cached_tokens":4864,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}},"obfuscation":"0JQt7Fw"}
203+
data: {"id":"chatcmpl-C1WTqhYgK7bV01bW98Lww3zqaf8ZF","object":"chat.completion.chunk","created":1754479218,"model":"gpt-4.1-2025-04-14","service_tier":"default","system_fingerprint":"fp_799e4ca3f1","choices":[],"usage":{"prompt_tokens":5049,"completion_tokens":60,"total_tokens":5109,"prompt_tokens_details":{"cached_tokens":4864,"cache_write_tokens":10,"audio_tokens":9},"completion_tokens_details":{"reasoning_tokens":10,"audio_tokens":11,"accepted_prediction_tokens":12,"rejected_prediction_tokens":13}},"obfuscation":"0JQt7Fw"}
204204

205205
data: [DONE]
206206

@@ -237,16 +237,17 @@ data: [DONE]
237237
"usage": {
238238
"prompt_tokens": 4862,
239239
"completion_tokens": 45,
240-
"total_tokens": 4914,
240+
"total_tokens": 4907,
241241
"prompt_tokens_details": {
242-
"cached_tokens": 0,
243-
"audio_tokens": 0
242+
"cached_tokens": 100,
243+
"cache_write_tokens": 20,
244+
"audio_tokens": 3
244245
},
245246
"completion_tokens_details": {
246-
"reasoning_tokens": 0,
247-
"audio_tokens": 0,
248-
"accepted_prediction_tokens": 0,
249-
"rejected_prediction_tokens": 0
247+
"reasoning_tokens": 4,
248+
"audio_tokens": 5,
249+
"accepted_prediction_tokens": 6,
250+
"rejected_prediction_tokens": 7
250251
}
251252
},
252253
"service_tier": "default",
@@ -276,16 +277,17 @@ data: [DONE]
276277
"usage": {
277278
"prompt_tokens": 5049,
278279
"completion_tokens": 60,
279-
"total_tokens": 5119,
280+
"total_tokens": 5109,
280281
"prompt_tokens_details": {
281282
"cached_tokens": 4864,
282-
"audio_tokens": 0
283+
"cache_write_tokens": 10,
284+
"audio_tokens": 9
283285
},
284286
"completion_tokens_details": {
285-
"reasoning_tokens": 0,
286-
"audio_tokens": 0,
287-
"accepted_prediction_tokens": 0,
288-
"rejected_prediction_tokens": 0
287+
"reasoning_tokens": 10,
288+
"audio_tokens": 11,
289+
"accepted_prediction_tokens": 12,
290+
"rejected_prediction_tokens": 13
289291
}
290292
},
291293
"service_tier": "default",

‎aibridge/fixtures/openai/chatcompletions/streaming_cumulative_usage_injected_tool.txtar‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
Streaming response with cumulative usage on every JSON chunk and an injected tool call.
2+
3+
-- request --
4+
{
5+
"model": "gpt-4.1",
6+
"messages": [
7+
{
8+
"role": "user",
9+
"content": "list my coder workspaces"
10+
}
11+
]
12+
}
13+
14+
-- streaming --
15+
data: {"id":"chatcmpl-cumulative-tool","object":"chat.completion.chunk","created":1754479216,"model":"gpt-4.1","choices":[{"index":0,"delta":{"content":"Calling"}}],"usage":{"prompt_tokens":6000,"completion_tokens":10,"total_tokens":6010,"prompt_tokens_details":{"cached_tokens":100,"cache_write_tokens":10,"audio_tokens":1},"completion_tokens_details":{"reasoning_tokens":1,"audio_tokens":1,"accepted_prediction_tokens":1,"rejected_prediction_tokens":1}}}
16+
17+
data: {"id":"chatcmpl-cumulative-tool","object":"chat.completion.chunk","created":1754479216,"model":"gpt-4.1","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call-cumulative","type":"function","function":{"name":"bmcp_coder_coder_list_workspaces","arguments":"{}"}}]}}],"usage":{"prompt_tokens":6000,"completion_tokens":20,"total_tokens":6020,"prompt_tokens_details":{"cached_tokens":100,"cache_write_tokens":10,"audio_tokens":2},"completion_tokens_details":{"reasoning_tokens":2,"audio_tokens":2,"accepted_prediction_tokens":2,"rejected_prediction_tokens":2}}}
18+
19+
data: {"id":"chatcmpl-cumulative-tool","object":"chat.completion.chunk","created":1754479216,"model":"gpt-4.1","choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":6000,"completion_tokens":30,"total_tokens":6030,"prompt_tokens_details":{"cached_tokens":100,"cache_write_tokens":10,"audio_tokens":3},"completion_tokens_details":{"reasoning_tokens":4,"audio_tokens":5,"accepted_prediction_tokens":6,"rejected_prediction_tokens":7}}}
20+
21+
data: [DONE]
22+
23+
-- streaming/tool-call --
24+
data: {"id":"chatcmpl-cumulative-final","object":"chat.completion.chunk","created":1754479217,"model":"gpt-4.1","choices":[{"index":0,"delta":{"content":"Done"}}],"usage":{"prompt_tokens":6000,"completion_tokens":10,"total_tokens":6010,"prompt_tokens_details":{"cached_tokens":200,"cache_write_tokens":20,"audio_tokens":7},"completion_tokens_details":{"reasoning_tokens":8,"audio_tokens":9,"accepted_prediction_tokens":10,"rejected_prediction_tokens":11}}}
25+
26+
data: {"id":"chatcmpl-cumulative-final","object":"chat.completion.chunk","created":1754479217,"model":"gpt-4.1","choices":[{"index":0,"delta":{"content":"."}}],"usage":{"prompt_tokens":6000,"completion_tokens":20,"total_tokens":6020,"prompt_tokens_details":{"cached_tokens":200,"cache_write_tokens":20,"audio_tokens":8},"completion_tokens_details":{"reasoning_tokens":9,"audio_tokens":10,"accepted_prediction_tokens":11,"rejected_prediction_tokens":12}}}
27+
28+
data: {"id":"chatcmpl-cumulative-final","object":"chat.completion.chunk","created":1754479217,"model":"gpt-4.1","choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":6000,"completion_tokens":30,"total_tokens":6030,"prompt_tokens_details":{"cached_tokens":200,"cache_write_tokens":20,"audio_tokens":9},"completion_tokens_details":{"reasoning_tokens":10,"audio_tokens":11,"accepted_prediction_tokens":12,"rejected_prediction_tokens":13}}}
29+
30+
data: [DONE]

‎aibridge/fixtures/openai/responses/blocking/cached_input_tokens.txtar‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,8 @@
6868
"usage": {
6969
"input_tokens": 12033,
7070
"input_tokens_details": {
71-
"cached_tokens": 11904
71+
"cached_tokens": 11904,
72+
"cache_write_tokens": 15
7273
},
7374
"output_tokens": 44,
7475
"output_tokens_details": {

‎aibridge/fixtures/openai/responses/blocking/single_injected_tool.txtar‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -749,7 +749,8 @@ Coder MCP tools automatically injected.
749749
"usage": {
750750
"input_tokens": 6371,
751751
"input_tokens_details": {
752-
"cached_tokens": 6144
752+
"cached_tokens": 6144,
753+
"cache_write_tokens": 7
753754
},
754755
"output_tokens": 75,
755756
"output_tokens_details": {
@@ -1509,7 +1510,8 @@ Coder MCP tools automatically injected.
15091510
"usage": {
15101511
"input_tokens": 6756,
15111512
"input_tokens_details": {
1512-
"cached_tokens": 6144
1513+
"cached_tokens": 6144,
1514+
"cache_write_tokens": 11
15131515
},
15141516
"output_tokens": 231,
15151517
"output_tokens_details": {

‎aibridge/fixtures/openai/responses/blocking/single_injected_tool_error.txtar‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -749,7 +749,8 @@ Coder MCP tools automatically injected, and errors invoking them are recorded.
749749
"usage": {
750750
"input_tokens": 6377,
751751
"input_tokens_details": {
752-
"cached_tokens": 6144
752+
"cached_tokens": 6144,
753+
"cache_write_tokens": 5
753754
},
754755
"output_tokens": 119,
755756
"output_tokens_details": {
@@ -1509,7 +1510,8 @@ Coder MCP tools automatically injected, and errors invoking them are recorded.
15091510
"usage": {
15101511
"input_tokens": 6539,
15111512
"input_tokens_details": {
1512-
"cached_tokens": 6144
1513+
"cached_tokens": 6144,
1514+
"cache_write_tokens": 9
15131515
},
15141516
"output_tokens": 144,
15151517
"output_tokens_details": {

‎aibridge/fixtures/openai/responses/streaming/cached_input_tokens.txtar‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,5 +43,5 @@ event: response.output_item.done
4343
data: {"type":"response.output_item.done","item":{"id":"msg_05080461b406f3f501696a140a70d88195a2ce4c1a4eb39696","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"text":"Test response with cached tokens."}],"role":"assistant"},"output_index":0,"sequence_number":11}
4444

4545
event: response.completed
46-
data: {"type":"response.completed","response":{"id":"resp_05080461b406f3f501696a1409d34c8195a40ff4b092145c35","object":"response","created_at":1768559625,"status":"completed","background":false,"completed_at":1768559627,"error":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.2-codex","output":[{"id":"msg_05080461b406f3f501696a140a70d88195a2ce4c1a4eb39696","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"text":"Test response with cached tokens."}],"role":"assistant"}],"parallel_tool_calls":true,"previous_response_id":null,"prompt_cache_key":"019bc657-f77b-7292-b5f4-2e8d6c2b0945","prompt_cache_retention":null,"reasoning":{"effort":"medium","summary":"detailed"},"service_tier":"default","store":false,"temperature":1.0,"tool_choice":"auto","tools":[],"truncation":"disabled","usage":{"input_tokens":16909,"input_tokens_details":{"cached_tokens":15744},"output_tokens":54,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":16963},"user":null,"metadata":{}},"sequence_number":12}
46+
data: {"type":"response.completed","response":{"id":"resp_05080461b406f3f501696a1409d34c8195a40ff4b092145c35","object":"response","created_at":1768559625,"status":"completed","background":false,"completed_at":1768559627,"error":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.2-codex","output":[{"id":"msg_05080461b406f3f501696a140a70d88195a2ce4c1a4eb39696","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"text":"Test response with cached tokens."}],"role":"assistant"}],"parallel_tool_calls":true,"previous_response_id":null,"prompt_cache_key":"019bc657-f77b-7292-b5f4-2e8d6c2b0945","prompt_cache_retention":null,"reasoning":{"effort":"medium","summary":"detailed"},"service_tier":"default","store":false,"temperature":1.0,"tool_choice":"auto","tools":[],"truncation":"disabled","usage":{"input_tokens":16909,"input_tokens_details":{"cached_tokens":15744,"cache_write_tokens":30},"output_tokens":54,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":16963},"user":null,"metadata":{}},"sequence_number":12}
4747

‎aibridge/fixtures/openai/responses/streaming/single_injected_tool.txtar‎

Lines changed: 2 additions & 2 deletions
Large diffs are not rendered by default.

‎aibridge/fixtures/openai/responses/streaming/single_injected_tool_error.txtar‎

Lines changed: 2 additions & 2 deletions
Large diffs are not rendered by default.

0 commit comments

Comments
 (0)