diff --git a/coderd/aibridgedserver/aibridgedserver_test.go b/coderd/aibridgedserver/aibridgedserver_test.go index 42915b95892fe..31fcbddd703b9 100644 --- a/coderd/aibridgedserver/aibridgedserver_test.go +++ b/coderd/aibridgedserver/aibridgedserver_test.go @@ -1763,7 +1763,7 @@ func TestRecordTokenUsage(t *testing.T) { // A priced model does not increment unpriced_token_usage_records_total. assertMetrics: func(t *testing.T, reg *prometheus.Registry) { require.Nil(t, promhelp.MetricValue(t, reg, "cost_control_unpriced_token_usage_records_total", - prometheus.Labels{"provider": "anthropic", "model": "claude-sonnet-4-6"})) + prometheus.Labels{"provider": "anthropic-eu", "provider_type": "anthropic", "model": "claude-sonnet-4-6"})) }, }, { @@ -1910,7 +1910,7 @@ func TestRecordTokenUsage(t *testing.T) { // A missing price row increments unpriced_token_usage_records_total. assertMetrics: func(t *testing.T, reg *prometheus.Registry) { require.Equal(t, 1, promhelp.CounterValue(t, reg, "cost_control_unpriced_token_usage_records_total", - prometheus.Labels{"provider": "anthropic", "model": "claude-sonnet-4-6"})) + prometheus.Labels{"provider": "anthropic-eu", "provider_type": "anthropic", "model": "claude-sonnet-4-6"})) }, }, { @@ -2159,7 +2159,7 @@ func TestRecordTokenUsage(t *testing.T) { // A missing price row increments unpriced_token_usage_records_total. assertMetrics: func(t *testing.T, reg *prometheus.Registry) { require.Equal(t, 1, promhelp.CounterValue(t, reg, "cost_control_unpriced_token_usage_records_total", - prometheus.Labels{"provider": "anthropic", "model": "claude-sonnet-4-6"})) + prometheus.Labels{"provider": "anthropic-eu", "provider_type": "anthropic", "model": "claude-sonnet-4-6"})) }, }, { @@ -2326,7 +2326,7 @@ func TestRecordTokenUsage(t *testing.T) { // The metric names the provider that failed to resolve. assertMetrics: func(t *testing.T, reg *prometheus.Registry) { require.Equal(t, 1, promhelp.CounterValue(t, reg, "cost_control_unpriced_token_usage_records_total", - prometheus.Labels{"provider": "anthropic-eu", "model": "claude-sonnet-4-6"})) + prometheus.Labels{"provider": "anthropic-eu", "provider_type": "unknown", "model": "claude-sonnet-4-6"})) }, }, { diff --git a/coderd/aibridgedserver/cost.go b/coderd/aibridgedserver/cost.go index 08b84aef6c8a3..1a4b072971498 100644 --- a/coderd/aibridgedserver/cost.go +++ b/coderd/aibridgedserver/cost.go @@ -37,6 +37,10 @@ var errTokenUsageOutOfRange = xerrors.New("reported token usage is out of range" // provider-reported token counts. var errCostOutOfRange = xerrors.New("computed cost is out of range") +// unknownProviderType labels a metric whose provider did not resolve to a +// configured type. +const unknownProviderType = "unknown" + // validateTokenUsage rejects an interception whose reported token counts fall // outside [0, maxAllowedTokenUsage]. func validateTokenUsage(in *proto.RecordTokenUsageRequest) error { @@ -109,10 +113,10 @@ func (s *Server) resolveTokenUsageCost(ctx context.Context, intc database.AIBrid switch { case errors.Is(err, sql.ErrNoRows): // Only reachable if the provider was deleted mid-request. - s.logger.Info(ctx, "no configured provider found for interception, recording token usage with NULL cost", + s.logger.Warn(ctx, "no configured provider found for interception, recording token usage with NULL cost", slog.F("provider_name", intc.ProviderName), slog.F("model", intc.Model)) if s.metrics != nil { - s.metrics.UnpricedTokenUsageRecords.WithLabelValues(intc.ProviderName, intc.Model).Inc() + s.metrics.UnpricedTokenUsageRecords.WithLabelValues(intc.ProviderName, unknownProviderType, intc.Model).Inc() } return result, nil case err != nil: @@ -131,7 +135,7 @@ func (s *Server) resolveTokenUsageCost(ctx context.Context, intc database.AIBrid s.logger.Info(ctx, "no price found for model, recording token usage with NULL cost", slog.F("provider", configuredType), slog.F("model", intc.Model)) if s.metrics != nil { - s.metrics.UnpricedTokenUsageRecords.WithLabelValues(configuredType, intc.Model).Inc() + s.metrics.UnpricedTokenUsageRecords.WithLabelValues(intc.ProviderName, configuredType, intc.Model).Inc() } return result, nil case err != nil: diff --git a/coderd/aibridgedserver/metrics.go b/coderd/aibridgedserver/metrics.go index b88b8789a6f6b..c613b2f2f3e91 100644 --- a/coderd/aibridgedserver/metrics.go +++ b/coderd/aibridgedserver/metrics.go @@ -54,8 +54,9 @@ func NewMetrics(reg prometheus.Registerer) *Metrics { UnpricedTokenUsageRecords: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ Subsystem: "cost_control", Name: "unpriced_token_usage_records_total", - Help: "The number of recorded AI token-usage records for which no (provider, model) price was found.", - }, []string{"provider", "model"}), + Help: "The number of recorded AI token-usage records for which no (provider_type, model) price was found. " + + "provider is the provider instance name, and provider_type is its configured type.", + }, []string{"provider", "provider_type", "model"}), // Pessimistic cardinality: 3 outcomes, 8 buckets + 3 extra series // (count, sum, +Inf) = up to 33. EnforcementDuration: promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{ diff --git a/docs/admin/integrations/prometheus.md b/docs/admin/integrations/prometheus.md index 0d580d68eb664..5d8551d99183b 100644 --- a/docs/admin/integrations/prometheus.md +++ b/docs/admin/integrations/prometheus.md @@ -125,7 +125,7 @@ The `coder_ai_gateway_cost_control_*` metrics are exported only by `coderd`. | `coder_ai_gateway_cost_control_blocked_requests_total` | counter | The number of AI requests blocked because the initiator's budget was exceeded. | `group_id` | | `coder_ai_gateway_cost_control_blocked_users` | gauge | The number of users currently over their AI budget. | `group_id` | | `coder_ai_gateway_cost_control_enforcement_duration_seconds` | histogram | The duration of AI budget enforcement checks, in seconds (outcome: allowed, blocked, error). | `outcome` | -| `coder_ai_gateway_cost_control_unpriced_token_usage_records_total` | counter | The number of recorded AI token-usage records for which no (provider, model) price was found. | `model` `provider` | +| `coder_ai_gateway_cost_control_unpriced_token_usage_records_total` | counter | The number of recorded AI token-usage records for which no (provider_type, model) price was found. provider is the provider instance name, and provider_type is its configured type. | `model` `provider_type` `provider` | | `coder_ai_gateway_injected_tool_invocations_total` | counter | The number of times an injected MCP tool was invoked by AI Gateway. | `model` `name` `provider` `server` | | `coder_ai_gateway_interceptions_duration_seconds` | histogram | The total duration of intercepted requests, in seconds. The majority of this time will be the upstream processing of the request. AI Gateway has no control over upstream processing time, so it's just an illustrative metric. | `model` `provider` | | `coder_ai_gateway_interceptions_inflight` | gauge | The number of intercepted requests which are being processed. | `model` `provider` `route` | diff --git a/docs/ai-coder/ai-gateway/cost-controls.md b/docs/ai-coder/ai-gateway/cost-controls.md index 5e95b82644096..741d6e5134af6 100644 --- a/docs/ai-coder/ai-gateway/cost-controls.md +++ b/docs/ai-coder/ai-gateway/cost-controls.md @@ -208,10 +208,11 @@ Replace `` with your Coder minor version, for example `2.36`. > effectively unlimited. Monitor `coder_ai_gateway_cost_control_unpriced_token_usage_records_total`, -labeled by `provider` and `model`, to detect unpriced usage. Any non-zero value -means spend is under-counted. Because the price book ships with the release, a -newly launched model is unpriced until you upgrade Coder or set a price for it -yourself. +labeled by `provider`, `provider_type`, and `model`, to detect unpriced usage. +Use the `(provider_type, model)` tuple to find the price to set. Any non-zero +value means spend is under-counted. Because the price book ships with the +release, a newly launched model is unpriced until you upgrade Coder or set a +price for it yourself. ### Set model prices diff --git a/docs/ai-coder/ai-gateway/monitoring.md b/docs/ai-coder/ai-gateway/monitoring.md index da45b643ee0af..937cf7360d635 100644 --- a/docs/ai-coder/ai-gateway/monitoring.md +++ b/docs/ai-coder/ai-gateway/monitoring.md @@ -54,12 +54,12 @@ Budget enforcement runs in `coderd`. Cost control metrics are exported only from the `coderd` Prometheus listener. Standalone replicas do not export them. -| Metric | Type | Labels | Purpose | -|--------------------------------------------------------------------|-----------|---------------------|------------------------------------------------------------------------------------------| -| `coder_ai_gateway_cost_control_blocked_requests_total` | counter | `group_id` | AI requests blocked because the initiator's budget was exceeded. | -| `coder_ai_gateway_cost_control_blocked_users` | gauge | `group_id` | Users currently over their AI budget. | -| `coder_ai_gateway_cost_control_enforcement_duration_seconds` | histogram | `outcome` | Duration of AI budget enforcement checks. `outcome` is `allowed`, `blocked`, or `error`. | -| `coder_ai_gateway_cost_control_unpriced_token_usage_records_total` | counter | `model`, `provider` | Recorded token-usage records for which no model price was found. | +| Metric | Type | Labels | Purpose | +|--------------------------------------------------------------------|-----------|--------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `coder_ai_gateway_cost_control_blocked_requests_total` | counter | `group_id` | AI requests blocked because the initiator's budget was exceeded. | +| `coder_ai_gateway_cost_control_blocked_users` | gauge | `group_id` | Users currently over their AI budget. | +| `coder_ai_gateway_cost_control_enforcement_duration_seconds` | histogram | `outcome` | Duration of AI budget enforcement checks. `outcome` is `allowed`, `blocked`, or `error`. | +| `coder_ai_gateway_cost_control_unpriced_token_usage_records_total` | counter | `model`, `provider`, `provider_type` | Recorded token-usage records for which no model price was found. `provider` is the provider instance name, and `provider_type` is the configured type the price is keyed on, or `unknown` when the provider could not be resolved. | ### AI Gateway Proxy metrics diff --git a/scripts/metricsdocgen/metrics b/scripts/metricsdocgen/metrics index 755068744c8ff..ccce769144d45 100644 --- a/scripts/metricsdocgen/metrics +++ b/scripts/metricsdocgen/metrics @@ -166,9 +166,9 @@ coder_ai_gateway_cost_control_enforcement_duration_seconds_bucket{outcome="allow coder_ai_gateway_cost_control_enforcement_duration_seconds_bucket{outcome="allowed",le="+Inf"} 0 coder_ai_gateway_cost_control_enforcement_duration_seconds_sum{outcome="allowed"} 0 coder_ai_gateway_cost_control_enforcement_duration_seconds_count{outcome="allowed"} 0 -# HELP coder_ai_gateway_cost_control_unpriced_token_usage_records_total The number of recorded AI token-usage records for which no (provider, model) price was found. +# HELP coder_ai_gateway_cost_control_unpriced_token_usage_records_total The number of recorded AI token-usage records for which no (provider_type, model) price was found. provider is the provider instance name, and provider_type is its configured type. # TYPE coder_ai_gateway_cost_control_unpriced_token_usage_records_total counter -coder_ai_gateway_cost_control_unpriced_token_usage_records_total{model="gpt-5-nano",provider="openai"} 0 +coder_ai_gateway_cost_control_unpriced_token_usage_records_total{model="gpt-5-nano",provider="openai",provider_type="openai"} 0 # HELP coder_ai_gateway_injected_tool_invocations_total The number of times an injected MCP tool was invoked by AI Gateway. # TYPE coder_ai_gateway_injected_tool_invocations_total counter coder_ai_gateway_injected_tool_invocations_total{model="gpt-5-nano",name="coder_list_templates",provider="openai",server="https://xxx.pit-1.try.coder.app/api/experimental/mcp/http"} 1