Thanks to visit codestin.com
Credit goes to github.com

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 8 additions & 3 deletions aibridge/intercept/responses/streaming.go
Original file line number Diff line number Diff line change
Expand Up @@ -241,16 +241,21 @@ func (i *StreamingResponsesInterceptor) ProcessRequest(w http.ResponseWriter, r
return err
}

// Record token usage for every iteration, whether or not tools are
// injected. Usage is reported by upstream independently of the MCP
// proxy, so gating this on the proxy drops usage entirely for
// deployments that run without one.
if completedResponse != nil {
i.recordTokenUsage(ctx, completedResponse)
}

if i.mcpProxy != nil && completedResponse != nil {
pending := i.getPendingInjectedToolCalls(completedResponse)
shouldLoop, innerLoopErr = i.handleInnerAgenticLoop(ctx, pending, completedResponse)
if innerLoopErr != nil {
i.sendCustomErr(ctx, w, http.StatusInternalServerError, innerLoopErr)
shouldLoop = false
}

// Record token usage for each inner loop iteration
i.recordTokenUsage(ctx, completedResponse)
}

i.recordModelThoughts(ctx, completedResponse)
Expand Down
62 changes: 62 additions & 0 deletions aibridge/internal/integrationtest/bridge_internal_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2486,3 +2486,65 @@ func extractSigV4Field(authHeader, prefix string) string {
}
return strings.TrimSpace(val)
}

// TestTokenUsageRecordedWithoutMCPProxier asserts that an interception records
// token usage when no MCP server proxier is configured. Upstream reports usage
// independently of tool injection, and coderd/aibridged tolerates a nil
// proxier when proxier construction fails, so usage must not depend on one.
func TestTokenUsageRecordedWithoutMCPProxier(t *testing.T) {
t.Parallel()

cases := []struct {
name string
fixture []byte
path string
expectedInputTokens, expectedOutputTokens int64
}{
{
name: "openai responses",
fixture: fixtures.OaiResponsesStreamingSimple,
path: pathOpenAIResponses,
expectedInputTokens: 11,
expectedOutputTokens: 18,
},
{
name: "anthropic messages",
fixture: fixtures.AntSimple,
path: pathAnthropicMessages,
expectedInputTokens: 18,
expectedOutputTokens: 241,
},
}

for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()

ctx, cancel := context.WithTimeout(t.Context(), testutil.WaitLong)
t.Cleanup(cancel)

fix := fixtures.Parse(t, tc.fixture)
upstream := testutil.NewMockUpstream(ctx, t, testutil.NewFixtureResponse(fix))

bridgeServer := newBridgeTestServer(ctx, t, upstream.URL, func(c *bridgeConfig) {
c.noMCPProxy = true
})

inputBefore := bridgeServer.Recorder.TotalInputTokens()
outputBefore := bridgeServer.Recorder.TotalOutputTokens()

reqBody, err := sjson.SetBytes(fix.Request(), "stream", true)
require.NoError(t, err)
resp, err := bridgeServer.makeRequest(t, http.MethodPost, tc.path, reqBody)
require.NoError(t, err)
defer resp.Body.Close()
require.Equal(t, http.StatusOK, resp.StatusCode)
_, err = io.ReadAll(resp.Body)
require.NoError(t, err)

require.NotEmpty(t, bridgeServer.Recorder.RecordedTokenUsages(), "token usage must be recorded without an MCP proxier")
assert.EqualValues(t, tc.expectedInputTokens, bridgeServer.Recorder.TotalInputTokens()-inputBefore, "input tokens miscalculated")
assert.EqualValues(t, tc.expectedOutputTokens, bridgeServer.Recorder.TotalOutputTokens()-outputBefore, "output tokens miscalculated")
})
}
}
11 changes: 7 additions & 4 deletions aibridge/internal/integrationtest/setupbridge.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,9 +50,12 @@ type bridgeConfig struct {
metrics *metrics.Metrics
tracer trace.Tracer
mcpProxy mcp.ServerProxier
userID string
metadata recorder.Metadata
logger slog.Logger
// noMCPProxy leaves the proxier nil instead of falling back to
// NoopMCPManager, which is non-nil and reports zero tools.
noMCPProxy bool
userID string
metadata recorder.Metadata
logger slog.Logger
}

// bridgeTestServer wraps an httptest.Server running a RequestBridge.
Expand Down Expand Up @@ -150,7 +153,7 @@ func newBridgeTestServer(
cfg.tracer = defaultTracer
}
cfg.logger = newLogger(t)
if cfg.mcpProxy == nil {
if cfg.mcpProxy == nil && !cfg.noMCPProxy {
cfg.mcpProxy = newNoopMCPManager()
}

Expand Down
Loading