Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 741da67

Browse files
rm-openairm-openai
authored
Realtime tracing (openai#1084)
Just sends tracing info to the server. --- [//]: # (BEGIN SAPLING FOOTER) * __->__ openai#1084 * openai#1082
1 parent a4499d4 commit 741da67

File tree

5 files changed

+299
-5
lines changed

5 files changed

+299
-5
lines changed

‎src/agents/realtime/config.py

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,8 @@ class RealtimeSessionModelSettings(TypedDict):
8383
tool_choice: NotRequired[ToolChoice]
8484
tools: NotRequired[list[Tool]]
8585

86+
tracing: NotRequired[RealtimeModelTracingConfig | None]
87+
8688

8789
class RealtimeGuardrailsSettings(TypedDict):
8890
"""Settings for output guardrails in realtime sessions."""
@@ -95,6 +97,19 @@ class RealtimeGuardrailsSettings(TypedDict):
9597
"""
9698

9799

100+
class RealtimeModelTracingConfig(TypedDict):
101+
"""Configuration for tracing in realtime model sessions."""
102+
103+
workflow_name: NotRequired[str]
104+
"""The workflow name to use for tracing."""
105+
106+
group_id: NotRequired[str]
107+
"""A group identifier to use for tracing, to link multiple traces together."""
108+
109+
metadata: NotRequired[dict[str, Any]]
110+
"""Additional metadata to include with the trace."""
111+
112+
98113
class RealtimeRunConfig(TypedDict):
99114
model_settings: NotRequired[RealtimeSessionModelSettings]
100115

@@ -104,6 +119,7 @@ class RealtimeRunConfig(TypedDict):
104119
guardrails_settings: NotRequired[RealtimeGuardrailsSettings]
105120
"""Settings for guardrail execution."""
106121

107-
# TODO (rm) Add tracing support
108-
# tracing: NotRequired[RealtimeTracingConfig | None]
122+
tracing_disabled: NotRequired[bool]
123+
"""Whether tracing is disabled for this run."""
124+
109125
# TODO (rm) Add history audio storage config

‎src/agents/realtime/model.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ class RealtimeModelConfig(TypedDict):
3838
"""
3939

4040
initial_model_settings: NotRequired[RealtimeSessionModelSettings]
41+
"""The initial model settings to use when connecting."""
4142

4243

4344
class RealtimeModel(abc.ABC):

‎src/agents/realtime/openai_realtime.py

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
import json
77
import os
88
from datetime import datetime
9-
from typing import Any, Callable
9+
from typing import Any, Callable, Literal
1010

1111
import websockets
1212
from openai.types.beta.realtime.conversation_item import ConversationItem
@@ -23,6 +23,7 @@
2323
from ..logger import logger
2424
from .config import (
2525
RealtimeClientMessage,
26+
RealtimeModelTracingConfig,
2627
RealtimeSessionModelSettings,
2728
RealtimeUserInput,
2829
)
@@ -73,6 +74,7 @@ def __init__(self) -> None:
7374
self._audio_length_ms: float = 0.0
7475
self._ongoing_response: bool = False
7576
self._current_audio_content_index: int | None = None
77+
self._tracing_config: RealtimeModelTracingConfig | Literal["auto"] | None = None
7678

7779
async def connect(self, options: RealtimeModelConfig) -> None:
7880
"""Establish a connection to the model and keep it alive."""
@@ -84,6 +86,11 @@ async def connect(self, options: RealtimeModelConfig) -> None:
8486
self.model = model_settings.get("model_name", self.model)
8587
api_key = await get_api_key(options.get("api_key"))
8688

89+
if "tracing" in model_settings:
90+
self._tracing_config = model_settings["tracing"]
91+
else:
92+
self._tracing_config = "auto"
93+
8794
if not api_key:
8895
raise UserError("API key is required but was not provided.")
8996

@@ -96,6 +103,15 @@ async def connect(self, options: RealtimeModelConfig) -> None:
96103
self._websocket = await websockets.connect(url, additional_headers=headers)
97104
self._websocket_task = asyncio.create_task(self._listen_for_messages())
98105

106+
async def _send_tracing_config(
107+
self, tracing_config: RealtimeModelTracingConfig | Literal["auto"] | None
108+
) -> None:
109+
"""Update tracing configuration via session.update event."""
110+
if tracing_config is not None:
111+
await self.send_event(
112+
{"type": "session.update", "other_data": {"session": {"tracing": tracing_config}}}
113+
)
114+
99115
def add_listener(self, listener: RealtimeModelListener) -> None:
100116
"""Add a listener to the model."""
101117
self._listeners.append(listener)
@@ -343,8 +359,7 @@ async def _handle_ws_event(self, event: dict[str, Any]):
343359
self._ongoing_response = False
344360
await self._emit_event(RealtimeModelTurnEndedEvent())
345361
elif parsed.type == "session.created":
346-
# TODO (rm) tracing stuff here
347-
pass
362+
await self._send_tracing_config(self._tracing_config)
348363
elif parsed.type == "error":
349364
await self._emit_event(RealtimeModelErrorEvent(error=parsed.error))
350365
elif parsed.type == "conversation.item.deleted":

‎src/agents/realtime/runner.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ async def run(
6969
"""
7070
model_settings = await self._get_model_settings(
7171
agent=self._starting_agent,
72+
disable_tracing=self._config.get("tracing_disabled", False) if self._config else False,
7273
initial_settings=model_config.get("initial_model_settings") if model_config else None,
7374
overrides=self._config.get("model_settings") if self._config else None,
7475
)
@@ -90,6 +91,7 @@ async def run(
9091
async def _get_model_settings(
9192
self,
9293
agent: RealtimeAgent,
94+
disable_tracing: bool,
9395
context: TContext | None = None,
9496
initial_settings: RealtimeSessionModelSettings | None = None,
9597
overrides: RealtimeSessionModelSettings | None = None,
@@ -110,4 +112,7 @@ async def _get_model_settings(
110112
if overrides:
111113
model_settings.update(overrides)
112114

115+
if disable_tracing:
116+
model_settings["tracing"] = None
117+
113118
return model_settings

‎tests/realtime/test_tracing.py

Lines changed: 257 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,257 @@
1+
from unittest.mock import AsyncMock, patch
2+
3+
import pytest
4+
5+
from agents.realtime.openai_realtime import OpenAIRealtimeWebSocketModel
6+
7+
8+
class TestRealtimeTracingIntegration:
9+
"""Test tracing configuration and session.update integration."""
10+
11+
@pytest.fixture
12+
def model(self):
13+
"""Create a fresh model instance for each test."""
14+
return OpenAIRealtimeWebSocketModel()
15+
16+
@pytest.fixture
17+
def mock_websocket(self):
18+
"""Create a mock websocket connection."""
19+
mock_ws = AsyncMock()
20+
mock_ws.send = AsyncMock()
21+
mock_ws.close = AsyncMock()
22+
return mock_ws
23+
24+
@pytest.mark.asyncio
25+
async def test_tracing_config_storage_and_defaults(self, model, mock_websocket):
26+
"""Test that tracing config is stored correctly and defaults to 'auto'."""
27+
# Test with explicit tracing config
28+
config_with_tracing = {
29+
"api_key": "test-key",
30+
"initial_model_settings": {
31+
"tracing": {
32+
"workflow_name": "test_workflow",
33+
"group_id": "group_123",
34+
"metadata": {"version": "1.0"},
35+
}
36+
},
37+
}
38+
39+
async def async_websocket(*args, **kwargs):
40+
return mock_websocket
41+
42+
with patch("websockets.connect", side_effect=async_websocket):
43+
with patch("asyncio.create_task") as mock_create_task:
44+
mock_task = AsyncMock()
45+
mock_create_task.return_value = mock_task
46+
mock_create_task.side_effect = lambda coro: (coro.close(), mock_task)[1]
47+
48+
await model.connect(config_with_tracing)
49+
50+
# Should store the tracing config
51+
assert model._tracing_config == {
52+
"workflow_name": "test_workflow",
53+
"group_id": "group_123",
54+
"metadata": {"version": "1.0"},
55+
}
56+
57+
# Test without tracing config - should default to "auto"
58+
model2 = OpenAIRealtimeWebSocketModel()
59+
config_no_tracing = {
60+
"api_key": "test-key",
61+
"initial_model_settings": {},
62+
}
63+
64+
with patch("websockets.connect", side_effect=async_websocket):
65+
with patch("asyncio.create_task") as mock_create_task:
66+
mock_create_task.side_effect = lambda coro: (coro.close(), mock_task)[1]
67+
68+
await model2.connect(config_no_tracing) # type: ignore[arg-type]
69+
assert model2._tracing_config == "auto"
70+
71+
@pytest.mark.asyncio
72+
async def test_send_tracing_config_on_session_created(self, model, mock_websocket):
73+
"""Test that tracing config is sent when session.created event is received."""
74+
config = {
75+
"api_key": "test-key",
76+
"initial_model_settings": {
77+
"tracing": {"workflow_name": "test_workflow", "group_id": "group_123"}
78+
},
79+
}
80+
81+
async def async_websocket(*args, **kwargs):
82+
return mock_websocket
83+
84+
with patch("websockets.connect", side_effect=async_websocket):
85+
with patch("asyncio.create_task") as mock_create_task:
86+
mock_task = AsyncMock()
87+
mock_create_task.side_effect = lambda coro: (coro.close(), mock_task)[1]
88+
89+
await model.connect(config)
90+
91+
# Simulate session.created event
92+
session_created_event = {
93+
"type": "session.created",
94+
"event_id": "event_123",
95+
"session": {"id": "session_456"},
96+
}
97+
98+
with patch.object(model, "send_event") as mock_send_event:
99+
await model._handle_ws_event(session_created_event)
100+
101+
# Should send session.update with tracing config
102+
mock_send_event.assert_called_once_with(
103+
{
104+
"type": "session.update",
105+
"other_data": {
106+
"session": {
107+
"tracing": {
108+
"workflow_name": "test_workflow",
109+
"group_id": "group_123",
110+
}
111+
}
112+
},
113+
}
114+
)
115+
116+
@pytest.mark.asyncio
117+
async def test_send_tracing_config_auto_mode(self, model, mock_websocket):
118+
"""Test that 'auto' tracing config is sent correctly."""
119+
config = {
120+
"api_key": "test-key",
121+
"initial_model_settings": {}, # No tracing config - defaults to "auto"
122+
}
123+
124+
async def async_websocket(*args, **kwargs):
125+
return mock_websocket
126+
127+
with patch("websockets.connect", side_effect=async_websocket):
128+
with patch("asyncio.create_task") as mock_create_task:
129+
mock_task = AsyncMock()
130+
mock_create_task.side_effect = lambda coro: (coro.close(), mock_task)[1]
131+
132+
await model.connect(config)
133+
134+
session_created_event = {
135+
"type": "session.created",
136+
"event_id": "event_123",
137+
"session": {"id": "session_456"},
138+
}
139+
140+
with patch.object(model, "send_event") as mock_send_event:
141+
await model._handle_ws_event(session_created_event)
142+
143+
# Should send session.update with "auto"
144+
mock_send_event.assert_called_once_with(
145+
{"type": "session.update", "other_data": {"session": {"tracing": "auto"}}}
146+
)
147+
148+
@pytest.mark.asyncio
149+
async def test_tracing_config_none_skips_session_update(self, model, mock_websocket):
150+
"""Test that None tracing config skips sending session.update."""
151+
# Manually set tracing config to None (this would happen if explicitly set)
152+
model._tracing_config = None
153+
154+
session_created_event = {
155+
"type": "session.created",
156+
"event_id": "event_123",
157+
"session": {"id": "session_456"},
158+
}
159+
160+
with patch.object(model, "send_event") as mock_send_event:
161+
await model._handle_ws_event(session_created_event)
162+
163+
# Should not send any session.update
164+
mock_send_event.assert_not_called()
165+
166+
@pytest.mark.asyncio
167+
async def test_tracing_config_with_metadata_serialization(self, model, mock_websocket):
168+
"""Test that complex metadata in tracing config is handled correctly."""
169+
complex_metadata = {
170+
"user_id": "user_123",
171+
"session_type": "demo",
172+
"features": ["audio", "tools"],
173+
"config": {"timeout": 30, "retries": 3},
174+
}
175+
176+
config = {
177+
"api_key": "test-key",
178+
"initial_model_settings": {
179+
"tracing": {"workflow_name": "complex_workflow", "metadata": complex_metadata}
180+
},
181+
}
182+
183+
async def async_websocket(*args, **kwargs):
184+
return mock_websocket
185+
186+
with patch("websockets.connect", side_effect=async_websocket):
187+
with patch("asyncio.create_task") as mock_create_task:
188+
mock_task = AsyncMock()
189+
mock_create_task.side_effect = lambda coro: (coro.close(), mock_task)[1]
190+
191+
await model.connect(config)
192+
193+
session_created_event = {
194+
"type": "session.created",
195+
"event_id": "event_123",
196+
"session": {"id": "session_456"},
197+
}
198+
199+
with patch.object(model, "send_event") as mock_send_event:
200+
await model._handle_ws_event(session_created_event)
201+
202+
# Should send session.update with complete tracing config including metadata
203+
expected_call = {
204+
"type": "session.update",
205+
"other_data": {
206+
"session": {
207+
"tracing": {
208+
"workflow_name": "complex_workflow",
209+
"metadata": complex_metadata,
210+
}
211+
}
212+
},
213+
}
214+
mock_send_event.assert_called_once_with(expected_call)
215+
216+
@pytest.mark.asyncio
217+
async def test_tracing_disabled_prevents_tracing(self, mock_websocket):
218+
"""Test that tracing_disabled=True prevents tracing configuration."""
219+
from agents.realtime.agent import RealtimeAgent
220+
from agents.realtime.runner import RealtimeRunner
221+
222+
# Create a test agent and runner with tracing disabled
223+
agent = RealtimeAgent(name="test_agent", instructions="test")
224+
225+
runner = RealtimeRunner(
226+
starting_agent=agent,
227+
config={"tracing_disabled": True}
228+
)
229+
230+
# Test the _get_model_settings method directly since that's where the logic is
231+
model_settings = await runner._get_model_settings(
232+
agent=agent,
233+
disable_tracing=True, # This should come from config["tracing_disabled"]
234+
initial_settings=None,
235+
overrides=None
236+
)
237+
238+
# When tracing is disabled, model settings should have tracing=None
239+
assert model_settings["tracing"] is None
240+
241+
# Also test that the runner passes disable_tracing=True correctly
242+
with patch.object(runner, '_get_model_settings') as mock_get_settings:
243+
mock_get_settings.return_value = {"tracing": None}
244+
245+
with patch('agents.realtime.session.RealtimeSession') as mock_session_class:
246+
mock_session = AsyncMock()
247+
mock_session_class.return_value = mock_session
248+
249+
await runner.run()
250+
251+
# Verify that _get_model_settings was called with disable_tracing=True
252+
mock_get_settings.assert_called_once_with(
253+
agent=agent,
254+
disable_tracing=True,
255+
initial_settings=None,
256+
overrides=None
257+
)

0 commit comments

Comments
 (0)