diff --git a/aibridge/fixtures/fixtures.go b/aibridge/fixtures/fixtures.go index 835b7d2715a..7a30ccbd631 100644 --- a/aibridge/fixtures/fixtures.go +++ b/aibridge/fixtures/fixtures.go @@ -91,6 +91,9 @@ var ( //go:embed openai/responses/blocking/custom_tool.txtar OaiResponsesBlockingCustomTool []byte + //go:embed openai/responses/blocking/web_search.txtar + OaiResponsesBlockingWebSearch []byte + //go:embed openai/responses/blocking/conversation.txtar OaiResponsesBlockingConversation []byte @@ -123,6 +126,9 @@ var ( //go:embed openai/responses/streaming/builtin_tool.txtar OaiResponsesStreamingBuiltinTool []byte + //go:embed openai/responses/streaming/web_search.txtar + OaiResponsesStreamingWebSearch []byte + //go:embed openai/responses/streaming/multi_reasoning_builtin_tool.txtar OaiResponsesStreamingMultiReasoningBuiltinTool []byte diff --git a/aibridge/fixtures/openai/responses/blocking/web_search.txtar b/aibridge/fixtures/openai/responses/blocking/web_search.txtar new file mode 100644 index 00000000000..24982a31a1d --- /dev/null +++ b/aibridge/fixtures/openai/responses/blocking/web_search.txtar @@ -0,0 +1,114 @@ +-- request -- +{ + "input": [ + { + "role": "user", + "content": "Search the web for the Example domain." + } + ], + "model": "gpt-5.4", + "stream": false, + "tools": [ + { + "type": "web_search" + } + ] +} + +-- non-streaming -- +{ + "id": "resp_0b8f5f61bf0dee5f016a43ac7294d8819ca794d13e1744ac2b", + "object": "response", + "created_at": 1782819954, + "status": "completed", + "background": false, + "completed_at": 1782819963, + "error": null, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-5.4", + "output": [ + { + "id": "msg_0b8f5f61bf0dee5f016a43ac78e6c0819c9d1852c768844565", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "I'm invoking the web search tool now." + } + ], + "role": "assistant" + }, + { + "id": "ws_0b8f5f61bf0dee5f016a43ac7947bc819c945bff3bf2bcdbc9", + "type": "web_search_call", + "status": "completed", + "action": { + "type": "search", + "queries": [ + "example domain" + ], + "query": "example domain" + } + }, + { + "id": "msg_0b8f5f61bf0dee5f016a43ac7ae384819c8495d9f565ea2eed", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "The search ran successfully and returned normal web results." + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": null, + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [ + { + "type": "web_search" + } + ], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 50, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 30, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 80 + }, + "user": null, + "metadata": {} +} diff --git a/aibridge/fixtures/openai/responses/streaming/web_search.txtar b/aibridge/fixtures/openai/responses/streaming/web_search.txtar new file mode 100644 index 00000000000..806675beb1a --- /dev/null +++ b/aibridge/fixtures/openai/responses/streaming/web_search.txtar @@ -0,0 +1,60 @@ +-- request -- +{ + "input": [ + { + "role": "user", + "content": "Search the web for the Example domain." + } + ], + "model": "gpt-5.4", + "stream": true, + "tools": [ + { + "type": "web_search" + } + ] +} + +-- streaming -- +event: response.created +data: {"type":"response.created","response":{"id":"resp_0b8f5f61bf0dee5f016a43ac7294d8819ca794d13e1744ac2b","object":"response","created_at":1782819954,"status":"in_progress","background":false,"error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.4","output":[],"parallel_tool_calls":true,"previous_response_id":null,"reasoning":{"effort":null,"summary":null},"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"web_search"}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0} + +event: response.in_progress +data: {"type":"response.in_progress","response":{"id":"resp_0b8f5f61bf0dee5f016a43ac7294d8819ca794d13e1744ac2b","object":"response","created_at":1782819954,"status":"in_progress","background":false,"error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.4","output":[],"parallel_tool_calls":true,"previous_response_id":null,"reasoning":{"effort":null,"summary":null},"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"web_search"}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} + +event: response.output_item.added +data: {"type":"response.output_item.added","item":{"id":"ws_0b8f5f61bf0dee5f016a43ac7947bc819c945bff3bf2bcdbc9","type":"web_search_call","status":"in_progress"},"output_index":0,"sequence_number":2} + +event: response.web_search_call.in_progress +data: {"type":"response.web_search_call.in_progress","item_id":"ws_0b8f5f61bf0dee5f016a43ac7947bc819c945bff3bf2bcdbc9","output_index":0,"sequence_number":3} + +event: response.web_search_call.searching +data: {"type":"response.web_search_call.searching","item_id":"ws_0b8f5f61bf0dee5f016a43ac7947bc819c945bff3bf2bcdbc9","output_index":0,"sequence_number":4} + +event: response.web_search_call.completed +data: {"type":"response.web_search_call.completed","item_id":"ws_0b8f5f61bf0dee5f016a43ac7947bc819c945bff3bf2bcdbc9","output_index":0,"sequence_number":5} + +event: response.output_item.done +data: {"type":"response.output_item.done","item":{"id":"ws_0b8f5f61bf0dee5f016a43ac7947bc819c945bff3bf2bcdbc9","type":"web_search_call","status":"completed","action":{"type":"search","queries":["example domain"],"query":"example domain"}},"output_index":0,"sequence_number":6} + +event: response.output_item.added +data: {"type":"response.output_item.added","item":{"id":"msg_0b8f5f61bf0dee5f016a43ac7ae384819c8495d9f565ea2eed","type":"message","status":"in_progress","content":[],"role":"assistant"},"output_index":1,"sequence_number":7} + +event: response.content_part.added +data: {"type":"response.content_part.added","content_index":0,"item_id":"msg_0b8f5f61bf0dee5f016a43ac7ae384819c8495d9f565ea2eed","output_index":1,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":""},"sequence_number":8} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","content_index":0,"delta":"The search ran successfully.","item_id":"msg_0b8f5f61bf0dee5f016a43ac7ae384819c8495d9f565ea2eed","logprobs":[],"output_index":1,"sequence_number":9} + +event: response.output_text.done +data: {"type":"response.output_text.done","content_index":0,"item_id":"msg_0b8f5f61bf0dee5f016a43ac7ae384819c8495d9f565ea2eed","logprobs":[],"output_index":1,"sequence_number":10,"text":"The search ran successfully."} + +event: response.content_part.done +data: {"type":"response.content_part.done","content_index":0,"item_id":"msg_0b8f5f61bf0dee5f016a43ac7ae384819c8495d9f565ea2eed","output_index":1,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":"The search ran successfully."},"sequence_number":11} + +event: response.output_item.done +data: {"type":"response.output_item.done","item":{"id":"msg_0b8f5f61bf0dee5f016a43ac7ae384819c8495d9f565ea2eed","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"The search ran successfully."}],"role":"assistant"},"output_index":1,"sequence_number":12} + +event: response.completed +data: {"type":"response.completed","response":{"id":"resp_0b8f5f61bf0dee5f016a43ac7294d8819ca794d13e1744ac2b","object":"response","created_at":1782819954,"status":"completed","background":false,"completed_at":1782819963,"error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.4","output":[{"id":"ws_0b8f5f61bf0dee5f016a43ac7947bc819c945bff3bf2bcdbc9","type":"web_search_call","status":"completed","action":{"type":"search","queries":["example domain"],"query":"example domain"}},{"id":"msg_0b8f5f61bf0dee5f016a43ac7ae384819c8495d9f565ea2eed","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"The search ran successfully."}],"role":"assistant"}],"parallel_tool_calls":true,"previous_response_id":null,"reasoning":{"effort":null,"summary":null},"service_tier":"default","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"web_search"}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":50,"input_tokens_details":{"cached_tokens":0},"output_tokens":30,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":80},"user":null,"metadata":{}},"sequence_number":13} + diff --git a/aibridge/intercept/responses/base.go b/aibridge/intercept/responses/base.go index 1c03220df96..6513636fedb 100644 --- a/aibridge/intercept/responses/base.go +++ b/aibridge/intercept/responses/base.go @@ -257,25 +257,49 @@ func (i *responsesInterceptionBase) recordNonInjectedToolUsage(ctx context.Conte for _, item := range response.Output { var args recorder.ToolArgs - // recording other function types to be considered: https://github.com/coder/aibridge/issues/121 + // Whitelist the output item types that represent tool calls. Every + // other output type (message, reasoning, *_output, etc.) is skipped. + // Only function_call and custom_tool_call carry arguments we parse; + // the remaining built-in tool calls are recorded for visibility but + // have no uniform argument representation. switch item.Type { case string(constant.ValueOf[constant.FunctionCall]()): args = i.parseFunctionCallJSONArgs(ctx, item.Arguments) case string(constant.ValueOf[constant.CustomToolCall]()): args = item.Input + case string(constant.ValueOf[constant.WebSearchCall]()), + // computer_call has no SDK constant; only computer_call_output does. + "computer_call", + string(constant.ValueOf[constant.LocalShellCall]()), + string(constant.ValueOf[constant.ShellCall]()), + string(constant.ValueOf[constant.ApplyPatchCall]()), + string(constant.ValueOf[constant.CodeInterpreterCall]()), + string(constant.ValueOf[constant.McpCall]()), + string(constant.ValueOf[constant.FileSearchCall]()), + string(constant.ValueOf[constant.ImageGenerationCall]()): + // Built-in tool calls carry no uniform argument payload. default: continue } + // Built-in tools usually have no name, so fall back to the type. + toolName := item.Name + if toolName == "" { + toolName = item.Type + } + if err := i.recorder.RecordToolUsage(ctx, &recorder.ToolUsageRecord{ InterceptionID: i.ID().String(), MsgID: response.ID, - ToolCallID: item.CallID, - Tool: item.Name, - Args: args, - Injected: false, + // ItemID is always present; ToolCallID (call_id) is empty for + // hosted tools that the provider executes internally. + ItemID: item.ID, + ToolCallID: item.CallID, + Tool: toolName, + Args: args, + Injected: false, }); err != nil { - i.logger.Warn(ctx, "failed to record tool usage", slog.Error(err), slog.F("tool", item.Name)) + i.logger.Warn(ctx, "failed to record tool usage", slog.Error(err), slog.F("tool", toolName)) } } } diff --git a/aibridge/intercept/responses/base_internal_test.go b/aibridge/intercept/responses/base_internal_test.go index 265176d536e..69e245f619a 100644 --- a/aibridge/intercept/responses/base_internal_test.go +++ b/aibridge/intercept/responses/base_internal_test.go @@ -200,6 +200,89 @@ func TestRecordToolUsage(t *testing.T) { }, }, }, + { + // Function/agentic tools expose both id and call_id; both are captured. + name: "function_call_captures_both_ids", + response: &oairesponses.Response{ + ID: "resp_both", + Output: []oairesponses.ResponseOutputItemUnion{ + { + Type: "function_call", + ID: "fc_item_1", + CallID: "call_both", + Name: "get_weather", + Arguments: `{"location": "NYC"}`, + }, + }, + }, + expected: []*recorder.ToolUsageRecord{ + { + InterceptionID: id.String(), + MsgID: "resp_both", + ItemID: "fc_item_1", + ToolCallID: "call_both", + Tool: "get_weather", + Args: map[string]any{"location": "NYC"}, + Injected: false, + }, + }, + }, + { + // Hosted tools only have id (no call_id) and usually no name, so + // the type is recorded as the tool name and ToolCallID is empty. + name: "hosted_tool_uses_item_id_no_call_id", + response: &oairesponses.Response{ + ID: "resp_ws", + Output: []oairesponses.ResponseOutputItemUnion{ + { + Type: "web_search_call", + ID: "ws_abc", + }, + }, + }, + expected: []*recorder.ToolUsageRecord{ + { + InterceptionID: id.String(), + MsgID: "resp_ws", + ItemID: "ws_abc", + ToolCallID: "", + Tool: "web_search_call", + Injected: false, + }, + }, + }, + { + // Exercises every newly recorded tool type, the name-falls-back-to + // -type behavior, an explicit name override (mcp_call), and that + // non-tool output items (reasoning) are still skipped. + name: "all_additional_tool_types", + response: &oairesponses.Response{ + ID: "resp_all", + Output: []oairesponses.ResponseOutputItemUnion{ + {Type: "reasoning", ID: "rs_skip"}, + {Type: "web_search_call", ID: "ws_1"}, + {Type: "computer_call", ID: "cu_1", CallID: "call_cu"}, + {Type: "local_shell_call", ID: "ls_1", CallID: "call_ls"}, + {Type: "shell_call", ID: "sh_1", CallID: "call_sh"}, + {Type: "apply_patch_call", ID: "ap_1", CallID: "call_ap"}, + {Type: "code_interpreter_call", ID: "ci_1"}, + {Type: "mcp_call", ID: "mcp_1", Name: "fetch"}, + {Type: "file_search_call", ID: "fs_1"}, + {Type: "image_generation_call", ID: "ig_1"}, + }, + }, + expected: []*recorder.ToolUsageRecord{ + {InterceptionID: id.String(), MsgID: "resp_all", ItemID: "ws_1", Tool: "web_search_call"}, + {InterceptionID: id.String(), MsgID: "resp_all", ItemID: "cu_1", ToolCallID: "call_cu", Tool: "computer_call"}, + {InterceptionID: id.String(), MsgID: "resp_all", ItemID: "ls_1", ToolCallID: "call_ls", Tool: "local_shell_call"}, + {InterceptionID: id.String(), MsgID: "resp_all", ItemID: "sh_1", ToolCallID: "call_sh", Tool: "shell_call"}, + {InterceptionID: id.String(), MsgID: "resp_all", ItemID: "ap_1", ToolCallID: "call_ap", Tool: "apply_patch_call"}, + {InterceptionID: id.String(), MsgID: "resp_all", ItemID: "ci_1", Tool: "code_interpreter_call"}, + {InterceptionID: id.String(), MsgID: "resp_all", ItemID: "mcp_1", Tool: "fetch"}, + {InterceptionID: id.String(), MsgID: "resp_all", ItemID: "fs_1", Tool: "file_search_call"}, + {InterceptionID: id.String(), MsgID: "resp_all", ItemID: "ig_1", Tool: "image_generation_call"}, + }, + }, } for _, tc := range tests { diff --git a/aibridge/internal/integrationtest/responses_internal_test.go b/aibridge/internal/integrationtest/responses_internal_test.go index 7f94caa6b47..8a39985f679 100644 --- a/aibridge/internal/integrationtest/responses_internal_test.go +++ b/aibridge/internal/integrationtest/responses_internal_test.go @@ -73,6 +73,7 @@ func TestResponsesOutputMatchesUpstream(t *testing.T) { MsgID: "resp_0da6045a8b68fa5200695fa23dcc2c81a19c849f627abf8a31", Tool: "add", ToolCallID: "call_CJSaa2u51JG996575oVljuNq", + ItemID: "fc_0da6045a8b68fa5200695fa23e198081a19bf68887d47ae93d", Args: map[string]any{"a": float64(3), "b": float64(5)}, Injected: false, }, @@ -113,6 +114,7 @@ func TestResponsesOutputMatchesUpstream(t *testing.T) { MsgID: "resp_09c614364030cdf000696942589da081a0af07f5859acb7308", Tool: "code_exec", ToolCallID: "call_haf8njtwrVZ1754Gm6fjAtuA", + ItemID: "ctc_09c614364030cdf0006969425bf33481a09cc0f9522af2d980", Args: "print(\"hello world\")", Injected: false, }, @@ -127,6 +129,33 @@ func TestResponsesOutputMatchesUpstream(t *testing.T) { }, expectedClient: aibridge.ClientUnknown, }, + { + // web_search_call is a hosted tool executed server-side by the + // provider. It carries an item id but no call_id, so the recorded + // ToolCallID must be empty and the ItemID must be the output item's + // id. + name: "blocking_web_search", + fixture: fixtures.OaiResponsesBlockingWebSearch, + expectModel: "gpt-5.4", + expectPromptRecorded: "Search the web for the Example domain.", + expectToolRecorded: &recorder.ToolUsageRecord{ + MsgID: "resp_0b8f5f61bf0dee5f016a43ac7294d8819ca794d13e1744ac2b", + Tool: "web_search_call", + ToolCallID: "", + ItemID: "ws_0b8f5f61bf0dee5f016a43ac7947bc819c945bff3bf2bcdbc9", + Injected: false, + }, + expectTokenUsage: &recorder.TokenUsageRecord{ + MsgID: "resp_0b8f5f61bf0dee5f016a43ac7294d8819ca794d13e1744ac2b", + Input: 50, + Output: 30, + ExtraTokenTypes: map[string]int64{ + "output_reasoning": 0, + "total_tokens": 80, + }, + }, + expectedClient: aibridge.ClientUnknown, + }, { name: "blocking_conversation", fixture: fixtures.OaiResponsesBlockingConversation, @@ -205,6 +234,7 @@ func TestResponsesOutputMatchesUpstream(t *testing.T) { MsgID: "resp_0c3fb28cfcf463a500695fa2f0239481a095ec6ce3dfe4d458", Tool: "add", ToolCallID: "call_7VaiUXZYuuuwWwviCrckxq6t", + ItemID: "fc_0c3fb28cfcf463a500695fa2f0b0a881a0890103ba88b0628e", Args: map[string]any{"a": float64(3), "b": float64(5)}, Injected: false, }, @@ -247,6 +277,7 @@ func TestResponsesOutputMatchesUpstream(t *testing.T) { MsgID: "resp_0c26996bc41c2a0500696942e83634819fb71b2b8ff8a4a76c", Tool: "code_exec", ToolCallID: "call_2gSnF58IEhXLwlbnqbm5XKMd", + ItemID: "ctc_0c26996bc41c2a0500696942ee6db8819fa6e841317eecbfb2", Args: "print(\"hello world\")", Injected: false, }, @@ -261,6 +292,34 @@ func TestResponsesOutputMatchesUpstream(t *testing.T) { }, expectedClient: aibridge.ClientUnknown, }, + { + // web_search_call is a hosted tool executed server-side by the + // provider. It carries an item id but no call_id, so the recorded + // ToolCallID must be empty and the ItemID must be the output item's + // id. + name: "streaming_web_search", + fixture: fixtures.OaiResponsesStreamingWebSearch, + streaming: true, + expectModel: "gpt-5.4", + expectPromptRecorded: "Search the web for the Example domain.", + expectToolRecorded: &recorder.ToolUsageRecord{ + MsgID: "resp_0b8f5f61bf0dee5f016a43ac7294d8819ca794d13e1744ac2b", + Tool: "web_search_call", + ToolCallID: "", + ItemID: "ws_0b8f5f61bf0dee5f016a43ac7947bc819c945bff3bf2bcdbc9", + Injected: false, + }, + expectTokenUsage: &recorder.TokenUsageRecord{ + MsgID: "resp_0b8f5f61bf0dee5f016a43ac7294d8819ca794d13e1744ac2b", + Input: 50, + Output: 30, + ExtraTokenTypes: map[string]int64{ + "output_reasoning": 0, + "total_tokens": 80, + }, + }, + expectedClient: aibridge.ClientUnknown, + }, { name: "streaming_conversation", fixture: fixtures.OaiResponsesStreamingConversation, diff --git a/aibridge/recorder/types.go b/aibridge/recorder/types.go index 09dead839b1..49269a31c5e 100644 --- a/aibridge/recorder/types.go +++ b/aibridge/recorder/types.go @@ -87,10 +87,20 @@ type PromptUsageRecord struct { } type ToolUsageRecord struct { - InterceptionID string - MsgID string - Tool string - ToolCallID string + InterceptionID string + MsgID string + Tool string + // ToolCallID is the correlation ID used to match a tool call to its + // result (call_id in the Responses API, id in chat completions and + // Anthropic messages). It is empty for hosted Responses tools (e.g. + // web_search_call) which the provider executes internally. + ToolCallID string + // ItemID is the provider's unique ID for the output item that carried + // the tool call. It is specific to the OpenAI Responses API, where an + // output item has both an id and a call_id. It is empty for the chat + // completions and Anthropic messages APIs, which have no separate item + // ID concept. + ItemID string ServerURL *string Args ToolArgs Injected bool