From 9eb2b42874a812aa96fa6575352619f4c19ef2b6 Mon Sep 17 00:00:00 2001 From: Susana Ferreira Date: Mon, 17 Aug 2026 14:08:28 +0100 Subject: [PATCH] fix: label unpriced token usage metric by provider name and type (#28210) The `provider` label was inconsistent between AI Gateway metrics. Every metric emitted by the gateway labels `provider` with the provider instance name, for example `anthropic-eu`, while `coder_ai_gateway_cost_control_unpriced_token_usage_records_total` used the provider type, for example `anthropic`. The two could not be correlated on `provider`. The metric was also inconsistent with itself: the path where a provider fails to resolve labelled by instance name, and the path where a model has no price labelled by type. The type is still worth exposing, since prices are keyed on `(provider_type, model)` and that is what an operator needs to add a price. - Label the metric with `provider` (the instance name, consistent with the other gateway metrics) and add `provider_type` (the configured type the price is keyed on). - Use `unknown` for `provider_type` when the provider does not resolve to a configured type. - Log the unresolved-provider case at `warn` instead of `info`. A missing price is an expected steady state, but a provider that cannot be resolved is not. - Update the metrics docs and the `metricsdocgen` fixture. Closes [AIGOV-574](https://linear.app/codercom/issue/AIGOV-574) > [!NOTE] > Initially generated by Claude Opus 5, modified and reviewed by @ssncferreira (cherry picked from commit 95328f1ead6bdf275664678a92033a771c8a8db1) --- coderd/aibridgedserver/aibridgedserver_test.go | 8 ++++---- coderd/aibridgedserver/cost.go | 10 +++++++--- coderd/aibridgedserver/metrics.go | 5 +++-- docs/admin/integrations/prometheus.md | 2 +- docs/ai-coder/ai-gateway/cost-controls.md | 7 ++++--- docs/ai-coder/ai-gateway/monitoring.md | 12 ++++++------ scripts/metricsdocgen/metrics | 4 ++-- 7 files changed, 27 insertions(+), 21 deletions(-) diff --git a/coderd/aibridgedserver/aibridgedserver_test.go b/coderd/aibridgedserver/aibridgedserver_test.go index 8fa4776fb92..9b63d66b64d 100644 --- a/coderd/aibridgedserver/aibridgedserver_test.go +++ b/coderd/aibridgedserver/aibridgedserver_test.go @@ -1762,7 +1762,7 @@ func TestRecordTokenUsage(t *testing.T) { // A priced model does not increment unpriced_token_usage_records_total. assertMetrics: func(t *testing.T, reg *prometheus.Registry) { require.Nil(t, promhelp.MetricValue(t, reg, "cost_control_unpriced_token_usage_records_total", - prometheus.Labels{"provider": "anthropic", "model": "claude-sonnet-4-6"})) + prometheus.Labels{"provider": "anthropic-eu", "provider_type": "anthropic", "model": "claude-sonnet-4-6"})) }, }, { @@ -1909,7 +1909,7 @@ func TestRecordTokenUsage(t *testing.T) { // A missing price row increments unpriced_token_usage_records_total. assertMetrics: func(t *testing.T, reg *prometheus.Registry) { require.Equal(t, 1, promhelp.CounterValue(t, reg, "cost_control_unpriced_token_usage_records_total", - prometheus.Labels{"provider": "anthropic", "model": "claude-sonnet-4-6"})) + prometheus.Labels{"provider": "anthropic-eu", "provider_type": "anthropic", "model": "claude-sonnet-4-6"})) }, }, { @@ -2089,7 +2089,7 @@ func TestRecordTokenUsage(t *testing.T) { // A missing price row increments unpriced_token_usage_records_total. assertMetrics: func(t *testing.T, reg *prometheus.Registry) { require.Equal(t, 1, promhelp.CounterValue(t, reg, "cost_control_unpriced_token_usage_records_total", - prometheus.Labels{"provider": "anthropic", "model": "claude-sonnet-4-6"})) + prometheus.Labels{"provider": "anthropic-eu", "provider_type": "anthropic", "model": "claude-sonnet-4-6"})) }, }, { @@ -2256,7 +2256,7 @@ func TestRecordTokenUsage(t *testing.T) { // The metric names the provider that failed to resolve. assertMetrics: func(t *testing.T, reg *prometheus.Registry) { require.Equal(t, 1, promhelp.CounterValue(t, reg, "cost_control_unpriced_token_usage_records_total", - prometheus.Labels{"provider": "anthropic-eu", "model": "claude-sonnet-4-6"})) + prometheus.Labels{"provider": "anthropic-eu", "provider_type": "unknown", "model": "claude-sonnet-4-6"})) }, }, { diff --git a/coderd/aibridgedserver/cost.go b/coderd/aibridgedserver/cost.go index ba56412a455..0470e66b09a 100644 --- a/coderd/aibridgedserver/cost.go +++ b/coderd/aibridgedserver/cost.go @@ -19,6 +19,10 @@ import ( // tokens. const tokensPerMillion = 1_000_000 +// unknownProviderType labels a metric whose provider did not resolve to a +// configured type. +const unknownProviderType = "unknown" + // tokenUsageCost holds the cost-attribution columns snapshotted onto a token // usage record. A field left unset (Valid == false) is recorded as SQL NULL; a // price or cost of 0 is recorded as 0, which is distinct from NULL. @@ -71,10 +75,10 @@ func (s *Server) resolveTokenUsageCost(ctx context.Context, intc database.AIBrid switch { case errors.Is(err, sql.ErrNoRows): // Only reachable if the provider was deleted mid-request. - s.logger.Info(ctx, "no configured provider found for interception, recording token usage with NULL cost", + s.logger.Warn(ctx, "no configured provider found for interception, recording token usage with NULL cost", slog.F("provider_name", intc.ProviderName), slog.F("model", intc.Model)) if s.metrics != nil { - s.metrics.UnpricedTokenUsageRecords.WithLabelValues(intc.ProviderName, intc.Model).Inc() + s.metrics.UnpricedTokenUsageRecords.WithLabelValues(intc.ProviderName, unknownProviderType, intc.Model).Inc() } return result, nil case err != nil: @@ -93,7 +97,7 @@ func (s *Server) resolveTokenUsageCost(ctx context.Context, intc database.AIBrid s.logger.Info(ctx, "no price found for model, recording token usage with NULL cost", slog.F("provider", configuredType), slog.F("model", intc.Model)) if s.metrics != nil { - s.metrics.UnpricedTokenUsageRecords.WithLabelValues(configuredType, intc.Model).Inc() + s.metrics.UnpricedTokenUsageRecords.WithLabelValues(intc.ProviderName, configuredType, intc.Model).Inc() } return result, nil case err != nil: diff --git a/coderd/aibridgedserver/metrics.go b/coderd/aibridgedserver/metrics.go index b88b8789a6f..c613b2f2f3e 100644 --- a/coderd/aibridgedserver/metrics.go +++ b/coderd/aibridgedserver/metrics.go @@ -54,8 +54,9 @@ func NewMetrics(reg prometheus.Registerer) *Metrics { UnpricedTokenUsageRecords: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ Subsystem: "cost_control", Name: "unpriced_token_usage_records_total", - Help: "The number of recorded AI token-usage records for which no (provider, model) price was found.", - }, []string{"provider", "model"}), + Help: "The number of recorded AI token-usage records for which no (provider_type, model) price was found. " + + "provider is the provider instance name, and provider_type is its configured type.", + }, []string{"provider", "provider_type", "model"}), // Pessimistic cardinality: 3 outcomes, 8 buckets + 3 extra series // (count, sum, +Inf) = up to 33. EnforcementDuration: promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{ diff --git a/docs/admin/integrations/prometheus.md b/docs/admin/integrations/prometheus.md index 408e87882b2..b67cfb7e5ad 100644 --- a/docs/admin/integrations/prometheus.md +++ b/docs/admin/integrations/prometheus.md @@ -125,7 +125,7 @@ The `coder_ai_gateway_cost_control_*` metrics are exported only by `coderd`. | `coder_ai_gateway_cost_control_blocked_requests_total` | counter | The number of AI requests blocked because the initiator's budget was exceeded. | `group_id` | | `coder_ai_gateway_cost_control_blocked_users` | gauge | The number of users currently over their AI budget. | `group_id` | | `coder_ai_gateway_cost_control_enforcement_duration_seconds` | histogram | The duration of AI budget enforcement checks, in seconds (outcome: allowed, blocked, error). | `outcome` | -| `coder_ai_gateway_cost_control_unpriced_token_usage_records_total` | counter | The number of recorded AI token-usage records for which no (provider, model) price was found. | `model` `provider` | +| `coder_ai_gateway_cost_control_unpriced_token_usage_records_total` | counter | The number of recorded AI token-usage records for which no (provider_type, model) price was found. provider is the provider instance name, and provider_type is its configured type. | `model` `provider_type` `provider` | | `coder_ai_gateway_injected_tool_invocations_total` | counter | The number of times an injected MCP tool was invoked by AI Gateway. | `model` `name` `provider` `server` | | `coder_ai_gateway_interceptions_duration_seconds` | histogram | The total duration of intercepted requests, in seconds. The majority of this time will be the upstream processing of the request. AI Gateway has no control over upstream processing time, so it's just an illustrative metric. | `model` `provider` | | `coder_ai_gateway_interceptions_inflight` | gauge | The number of intercepted requests which are being processed. | `model` `provider` `route` | diff --git a/docs/ai-coder/ai-gateway/cost-controls.md b/docs/ai-coder/ai-gateway/cost-controls.md index 6b8b81be116..0b563cc3b26 100644 --- a/docs/ai-coder/ai-gateway/cost-controls.md +++ b/docs/ai-coder/ai-gateway/cost-controls.md @@ -210,9 +210,10 @@ Replace `` with your Coder minor version, for example `2.36`. > effectively unlimited. Monitor `coder_ai_gateway_cost_control_unpriced_token_usage_records_total`, -labeled by `provider` and `model`, to detect unpriced usage. Any non-zero value -means spend is under-counted. Because the price book ships with the release, a -newly launched model can remain unpriced until you upgrade Coder. +labeled by `provider`, `provider_type`, and `model`, to detect unpriced usage. +The `(provider_type, model)` tuple identifies the missing price. Any non-zero +value means spend is under-counted. Because the price book ships with the +release, a newly launched model can remain unpriced until you upgrade Coder. ## Monitor spend diff --git a/docs/ai-coder/ai-gateway/monitoring.md b/docs/ai-coder/ai-gateway/monitoring.md index df648ef8ec9..b56a6d8fbec 100644 --- a/docs/ai-coder/ai-gateway/monitoring.md +++ b/docs/ai-coder/ai-gateway/monitoring.md @@ -55,12 +55,12 @@ Budget enforcement runs in `coderd`. Cost control metrics are exported only from the `coderd` Prometheus listener. Standalone replicas do not export them. -| Metric | Type | Labels | Purpose | -|--------------------------------------------------------------------|-----------|---------------------|------------------------------------------------------------------------------------------| -| `coder_ai_gateway_cost_control_blocked_requests_total` | counter | `group_id` | AI requests blocked because the initiator's budget was exceeded. | -| `coder_ai_gateway_cost_control_blocked_users` | gauge | `group_id` | Users currently over their AI budget. | -| `coder_ai_gateway_cost_control_enforcement_duration_seconds` | histogram | `outcome` | Duration of AI budget enforcement checks. `outcome` is `allowed`, `blocked`, or `error`. | -| `coder_ai_gateway_cost_control_unpriced_token_usage_records_total` | counter | `model`, `provider` | Recorded token-usage records for which no model price was found. | +| Metric | Type | Labels | Purpose | +|--------------------------------------------------------------------|-----------|--------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `coder_ai_gateway_cost_control_blocked_requests_total` | counter | `group_id` | AI requests blocked because the initiator's budget was exceeded. | +| `coder_ai_gateway_cost_control_blocked_users` | gauge | `group_id` | Users currently over their AI budget. | +| `coder_ai_gateway_cost_control_enforcement_duration_seconds` | histogram | `outcome` | Duration of AI budget enforcement checks. `outcome` is `allowed`, `blocked`, or `error`. | +| `coder_ai_gateway_cost_control_unpriced_token_usage_records_total` | counter | `model`, `provider`, `provider_type` | Recorded token-usage records for which no model price was found. `provider` is the provider instance name, and `provider_type` is the configured type the price is keyed on, or `unknown` when the provider could not be resolved. | ### AI Gateway Proxy metrics diff --git a/scripts/metricsdocgen/metrics b/scripts/metricsdocgen/metrics index 755068744c8..ccce769144d 100644 --- a/scripts/metricsdocgen/metrics +++ b/scripts/metricsdocgen/metrics @@ -166,9 +166,9 @@ coder_ai_gateway_cost_control_enforcement_duration_seconds_bucket{outcome="allow coder_ai_gateway_cost_control_enforcement_duration_seconds_bucket{outcome="allowed",le="+Inf"} 0 coder_ai_gateway_cost_control_enforcement_duration_seconds_sum{outcome="allowed"} 0 coder_ai_gateway_cost_control_enforcement_duration_seconds_count{outcome="allowed"} 0 -# HELP coder_ai_gateway_cost_control_unpriced_token_usage_records_total The number of recorded AI token-usage records for which no (provider, model) price was found. +# HELP coder_ai_gateway_cost_control_unpriced_token_usage_records_total The number of recorded AI token-usage records for which no (provider_type, model) price was found. provider is the provider instance name, and provider_type is its configured type. # TYPE coder_ai_gateway_cost_control_unpriced_token_usage_records_total counter -coder_ai_gateway_cost_control_unpriced_token_usage_records_total{model="gpt-5-nano",provider="openai"} 0 +coder_ai_gateway_cost_control_unpriced_token_usage_records_total{model="gpt-5-nano",provider="openai",provider_type="openai"} 0 # HELP coder_ai_gateway_injected_tool_invocations_total The number of times an injected MCP tool was invoked by AI Gateway. # TYPE coder_ai_gateway_injected_tool_invocations_total counter coder_ai_gateway_injected_tool_invocations_total{model="gpt-5-nano",name="coder_list_templates",provider="openai",server="https://xxx.pit-1.try.coder.app/api/experimental/mcp/http"} 1