Thanks to visit codestin.com
Credit goes to github.com

Skip to content

fix: label unpriced token usage metric by provider name and type #28210

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ssncferreira merged 1 commit into from
Aug 17, 2026
+28 −22
Merged

fix: label unpriced token usage metric by provider name and type #28210

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions coderd/aibridgedserver/aibridgedserver_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1763,7 +1763,7 @@ func TestRecordTokenUsage(t *testing.T) {
// A priced model does not increment unpriced_token_usage_records_total.
assertMetrics: func(t *testing.T, reg *prometheus.Registry) {
require.Nil(t, promhelp.MetricValue(t, reg, "cost_control_unpriced_token_usage_records_total",
prometheus.Labels{"provider": "anthropic", "model": "claude-sonnet-4-6"}))
prometheus.Labels{"provider": "anthropic-eu", "provider_type": "anthropic", "model": "claude-sonnet-4-6"}))
},
},
{
Expand Down Expand Up @@ -1910,7 +1910,7 @@ func TestRecordTokenUsage(t *testing.T) {
// A missing price row increments unpriced_token_usage_records_total.
assertMetrics: func(t *testing.T, reg *prometheus.Registry) {
require.Equal(t, 1, promhelp.CounterValue(t, reg, "cost_control_unpriced_token_usage_records_total",
prometheus.Labels{"provider": "anthropic", "model": "claude-sonnet-4-6"}))
prometheus.Labels{"provider": "anthropic-eu", "provider_type": "anthropic", "model": "claude-sonnet-4-6"}))
},
},
{
Expand Down Expand Up @@ -2159,7 +2159,7 @@ func TestRecordTokenUsage(t *testing.T) {
// A missing price row increments unpriced_token_usage_records_total.
assertMetrics: func(t *testing.T, reg *prometheus.Registry) {
require.Equal(t, 1, promhelp.CounterValue(t, reg, "cost_control_unpriced_token_usage_records_total",
prometheus.Labels{"provider": "anthropic", "model": "claude-sonnet-4-6"}))
prometheus.Labels{"provider": "anthropic-eu", "provider_type": "anthropic", "model": "claude-sonnet-4-6"}))
},
},
{
Expand Down Expand Up @@ -2326,7 +2326,7 @@ func TestRecordTokenUsage(t *testing.T) {
// The metric names the provider that failed to resolve.
assertMetrics: func(t *testing.T, reg *prometheus.Registry) {
require.Equal(t, 1, promhelp.CounterValue(t, reg, "cost_control_unpriced_token_usage_records_total",
prometheus.Labels{"provider": "anthropic-eu", "model": "claude-sonnet-4-6"}))
prometheus.Labels{"provider": "anthropic-eu", "provider_type": "unknown", "model": "claude-sonnet-4-6"}))
},
},
{
Expand Down
10 changes: 7 additions & 3 deletions coderd/aibridgedserver/cost.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,10 @@ var errTokenUsageOutOfRange = xerrors.New("reported token usage is out of range"
// provider-reported token counts.
var errCostOutOfRange = xerrors.New("computed cost is out of range")

// unknownProviderType labels a metric whose provider did not resolve to a
// configured type.
const unknownProviderType = "unknown"

// validateTokenUsage rejects an interception whose reported token counts fall
// outside [0, maxAllowedTokenUsage].
func validateTokenUsage(in *proto.RecordTokenUsageRequest) error {
Expand Down Expand Up @@ -109,10 +113,10 @@ func (s *Server) resolveTokenUsageCost(ctx context.Context, intc database.AIBrid
switch {
case errors.Is(err, sql.ErrNoRows):
// Only reachable if the provider was deleted mid-request.
s.logger.Info(ctx, "no configured provider found for interception, recording token usage with NULL cost",
s.logger.Warn(ctx, "no configured provider found for interception, recording token usage with NULL cost",
slog.F("provider_name", intc.ProviderName), slog.F("model", intc.Model))
if s.metrics != nil {
s.metrics.UnpricedTokenUsageRecords.WithLabelValues(intc.ProviderName, intc.Model).Inc()
s.metrics.UnpricedTokenUsageRecords.WithLabelValues(intc.ProviderName, unknownProviderType, intc.Model).Inc()
}
return result, nil
case err != nil:
Expand All @@ -131,7 +135,7 @@ func (s *Server) resolveTokenUsageCost(ctx context.Context, intc database.AIBrid
s.logger.Info(ctx, "no price found for model, recording token usage with NULL cost",
slog.F("provider", configuredType), slog.F("model", intc.Model))
if s.metrics != nil {
s.metrics.UnpricedTokenUsageRecords.WithLabelValues(configuredType, intc.Model).Inc()
s.metrics.UnpricedTokenUsageRecords.WithLabelValues(intc.ProviderName, configuredType, intc.Model).Inc()
}
return result, nil
case err != nil:
Expand Down
5 changes: 3 additions & 2 deletions coderd/aibridgedserver/metrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,9 @@ func NewMetrics(reg prometheus.Registerer) *Metrics {
UnpricedTokenUsageRecords: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{
Subsystem: "cost_control",
Name: "unpriced_token_usage_records_total",
Help: "The number of recorded AI token-usage records for which no (provider, model) price was found.",
}, []string{"provider", "model"}),
Help: "The number of recorded AI token-usage records for which no (provider_type, model) price was found. " +
"provider is the provider instance name, and provider_type is its configured type.",
}, []string{"provider", "provider_type", "model"}),
// Pessimistic cardinality: 3 outcomes, 8 buckets + 3 extra series
// (count, sum, +Inf) = up to 33.
EnforcementDuration: promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{
Expand Down
2 changes: 1 addition & 1 deletion docs/admin/integrations/prometheus.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ The `coder_ai_gateway_cost_control_*` metrics are exported only by `coderd`.
| `coder_ai_gateway_cost_control_blocked_requests_total` | counter | The number of AI requests blocked because the initiator's budget was exceeded. | `group_id` |
| `coder_ai_gateway_cost_control_blocked_users` | gauge | The number of users currently over their AI budget. | `group_id` |
| `coder_ai_gateway_cost_control_enforcement_duration_seconds` | histogram | The duration of AI budget enforcement checks, in seconds (outcome: allowed, blocked, error). | `outcome` |
| `coder_ai_gateway_cost_control_unpriced_token_usage_records_total` | counter | The number of recorded AI token-usage records for which no (provider, model) price was found. | `model` `provider` |
| `coder_ai_gateway_cost_control_unpriced_token_usage_records_total` | counter | The number of recorded AI token-usage records for which no (provider_type, model) price was found. provider is the provider instance name, and provider_type is its configured type. | `model` `provider_type` `provider` |
| `coder_ai_gateway_injected_tool_invocations_total` | counter | The number of times an injected MCP tool was invoked by AI Gateway. | `model` `name` `provider` `server` |
| `coder_ai_gateway_interceptions_duration_seconds` | histogram | The total duration of intercepted requests, in seconds. The majority of this time will be the upstream processing of the request. AI Gateway has no control over upstream processing time, so it's just an illustrative metric. | `model` `provider` |
| `coder_ai_gateway_interceptions_inflight` | gauge | The number of intercepted requests which are being processed. | `model` `provider` `route` |
Expand Down
9 changes: 5 additions & 4 deletions docs/ai-coder/ai-gateway/cost-controls.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -208,10 +208,11 @@ Replace `<VERSION>` with your Coder minor version, for example `2.36`.
> effectively unlimited.

Monitor `coder_ai_gateway_cost_control_unpriced_token_usage_records_total`,
labeled by `provider` and `model`, to detect unpriced usage. Any non-zero value
means spend is under-counted. Because the price book ships with the release, a
newly launched model is unpriced until you upgrade Coder or set a price for it
yourself.
labeled by `provider`, `provider_type`, and `model`, to detect unpriced usage.
Use the `(provider_type, model)` tuple to find the price to set. Any non-zero
value means spend is under-counted. Because the price book ships with the
release, a newly launched model is unpriced until you upgrade Coder or set a
price for it yourself.

### Set model prices

Expand Down
12 changes: 6 additions & 6 deletions docs/ai-coder/ai-gateway/monitoring.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,12 +54,12 @@ Budget enforcement runs in `coderd`.
Cost control metrics are exported only from the `coderd` Prometheus listener.
Standalone replicas do not export them.

| Metric | Type | Labels | Purpose |
|--------------------------------------------------------------------|-----------|---------------------|------------------------------------------------------------------------------------------|
| `coder_ai_gateway_cost_control_blocked_requests_total` | counter | `group_id` | AI requests blocked because the initiator's budget was exceeded. |
| `coder_ai_gateway_cost_control_blocked_users` | gauge | `group_id` | Users currently over their AI budget. |
| `coder_ai_gateway_cost_control_enforcement_duration_seconds` | histogram | `outcome` | Duration of AI budget enforcement checks. `outcome` is `allowed`, `blocked`, or `error`. |
| `coder_ai_gateway_cost_control_unpriced_token_usage_records_total` | counter | `model`, `provider` | Recorded token-usage records for which no model price was found. |
| Metric | Type | Labels | Purpose |
|--------------------------------------------------------------------|-----------|--------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `coder_ai_gateway_cost_control_blocked_requests_total` | counter | `group_id` | AI requests blocked because the initiator's budget was exceeded. |
| `coder_ai_gateway_cost_control_blocked_users` | gauge | `group_id` | Users currently over their AI budget. |
| `coder_ai_gateway_cost_control_enforcement_duration_seconds` | histogram | `outcome` | Duration of AI budget enforcement checks. `outcome` is `allowed`, `blocked`, or `error`. |
| `coder_ai_gateway_cost_control_unpriced_token_usage_records_total` | counter | `model`, `provider`, `provider_type` | Recorded token-usage records for which no model price was found. `provider` is the provider instance name, and `provider_type` is the configured type the price is keyed on, or `unknown` when the provider could not be resolved. |

### AI Gateway Proxy metrics

Expand Down
4 changes: 2 additions & 2 deletions scripts/metricsdocgen/metrics
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,9 +166,9 @@ coder_ai_gateway_cost_control_enforcement_duration_seconds_bucket{outcome="allow
coder_ai_gateway_cost_control_enforcement_duration_seconds_bucket{outcome="allowed",le="+Inf"} 0
coder_ai_gateway_cost_control_enforcement_duration_seconds_sum{outcome="allowed"} 0
coder_ai_gateway_cost_control_enforcement_duration_seconds_count{outcome="allowed"} 0
# HELP coder_ai_gateway_cost_control_unpriced_token_usage_records_total The number of recorded AI token-usage records for which no (provider, model) price was found.
# HELP coder_ai_gateway_cost_control_unpriced_token_usage_records_total The number of recorded AI token-usage records for which no (provider_type, model) price was found. provider is the provider instance name, and provider_type is its configured type.
# TYPE coder_ai_gateway_cost_control_unpriced_token_usage_records_total counter
coder_ai_gateway_cost_control_unpriced_token_usage_records_total{model="gpt-5-nano",provider="openai"} 0
coder_ai_gateway_cost_control_unpriced_token_usage_records_total{model="gpt-5-nano",provider="openai",provider_type="openai"} 0
# HELP coder_ai_gateway_injected_tool_invocations_total The number of times an injected MCP tool was invoked by AI Gateway.
# TYPE coder_ai_gateway_injected_tool_invocations_total counter
coder_ai_gateway_injected_tool_invocations_total{model="gpt-5-nano",name="coder_list_templates",provider="openai",server="https://xxx.pit-1.try.coder.app/api/experimental/mcp/http"} 1
Expand Down
Loading