diff --git a/Makefile b/Makefile index 4070f8924107e..4a94a7237c5a1 100644 --- a/Makefile +++ b/Makefile @@ -72,6 +72,7 @@ endif examples/examples.gen.json \ docs/manifest.json \ docs/admin/integrations/prometheus.md \ + docs/ai-coder/ai-gateway/monitoring.md \ docs/admin/security/audit-logs.md \ docs/admin/setup/configuration-reference.md \ docs/reference/cli/index.md \ @@ -1015,6 +1016,7 @@ GEN_FILES := \ coderd/rbac/scopes_constants_gen.go \ codersdk/apikey_scopes_gen.go \ docs/admin/integrations/prometheus.md \ + docs/ai-coder/ai-gateway/monitoring.md \ docs/reference/cli/index.md \ docs/admin/security/audit-logs.md \ docs/install/releases/feature-stages.md \ @@ -1114,6 +1116,7 @@ gen/mark-fresh: site/src/api/countriesGenerated.ts \ site/src/api/chatModelOptionsGenerated.json \ docs/admin/integrations/prometheus.md \ + docs/ai-coder/ai-gateway/monitoring.md \ docs/reference/cli/index.md \ docs/admin/security/audit-logs.md \ docs/install/releases/feature-stages.md \ @@ -1330,7 +1333,17 @@ scripts/metricsdocgen/generated_metrics: $(GO_SRC_FILES) | _gen _gen/bin/metrics docs/admin/integrations/prometheus.md: node_modules/.installed scripts/metricsdocgen/main.go scripts/metricsdocgen/metrics scripts/metricsdocgen/generated_metrics | _gen _gen/bin/metricsdocgen tmpdir=$$(mktemp -d -p _gen) && tmpfile=$$(realpath "$$tmpdir")/$(notdir $@) && cp "$@" "$$tmpfile" && \ - _gen/bin/metricsdocgen --prometheus-doc-file="$$tmpfile" && \ + _gen/bin/metricsdocgen --doc-file="$$tmpfile" --doc-target="$@" && \ + pnpm exec markdownlint-cli2 --fix "$$tmpfile" && \ + pnpm exec markdown-table-formatter "$$tmpfile" && \ + mv "$$tmpfile" "$@" && rm -rf "$$tmpdir" + +docs/ai-coder/ai-gateway/monitoring.md: node_modules/.installed scripts/metricsdocgen/main.go scripts/metricsdocgen/metrics scripts/metricsdocgen/generated_metrics | _gen _gen/bin/metricsdocgen + tmpdir=$$(mktemp -d -p _gen) && tmpfile=$$(realpath "$$tmpdir")/$(notdir $@) && cp "$@" "$$tmpfile" && \ + _gen/bin/metricsdocgen --doc-file="$$tmpfile" --doc-target="$@" \ + --section='common=prefix:coder_ai_gateway_,!coder_ai_gateway_cost_control_,!coder_ai_gateway_proxy_' \ + --section='cost-control=prefix:coder_ai_gateway_cost_control_' \ + --section='proxy=prefix:coder_ai_gateway_proxy_' && \ pnpm exec markdownlint-cli2 --fix "$$tmpfile" && \ pnpm exec markdown-table-formatter "$$tmpfile" && \ mv "$$tmpfile" "$@" && rm -rf "$$tmpdir" diff --git a/aibridge/metrics/metrics.go b/aibridge/metrics/metrics.go index 8584261da25ed..0973ad84051c1 100644 --- a/aibridge/metrics/metrics.go +++ b/aibridge/metrics/metrics.go @@ -9,6 +9,9 @@ import ( var baseLabels = []string{"provider", "model"} const ( + // PrometheusMetricPrefix is the canonical prefix applied to every metric + // defined in this package. The registration wiring wraps the registerer + // with it, so the metric options below omit it. PrometheusMetricPrefix = "coder_ai_gateway_" InterceptionCountStatusFailed = "failed" @@ -69,9 +72,8 @@ func NewMetrics(reg prometheus.Registerer) *Metrics { InterceptionDuration: promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{ Subsystem: "interceptions", Name: "duration_seconds", - Help: "The total duration of intercepted requests, in seconds. " + - "The majority of this time will be the upstream processing of the request. " + - "AI Gateway has no control over upstream processing time, so it's just an illustrative metric.", + Help: "The total time each intercepted request spent inside AI Gateway, in seconds. " + + "Dominated by upstream provider latency; use distributed tracing spans to isolate AI Gateway's own overhead.", // TODO: add docs around determining aibridge's *own* latency with distributed traces // once https://github.com/coder/aibridge/issues/26 lands. Buckets: []float64{0.5, 2, 5, 15, 30, 60, 120}, diff --git a/cli/server.go b/cli/server.go index b0f344694a113..4795ad56179db 100644 --- a/cli/server.go +++ b/cli/server.go @@ -1187,7 +1187,7 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd. // https://linear.app/codercom/issue/AIGOV-447/remove-legacy-ai-gateway-metric-aliases aibridgeReg := prometheusmetrics.NewMetricAliasRegisterer(coderAPI.PrometheusRegistry, aibridgemetrics.PrometheusMetricPrefix, "coder_aibridged_") aibridgeMetrics := aibridge.NewMetrics(aibridgeReg) - costControlReg := prometheus.WrapRegistererWithPrefix("coder_ai_gateway_", coderAPI.PrometheusRegistry) + costControlReg := prometheus.WrapRegistererWithPrefix(aibridgemetrics.PrometheusMetricPrefix, coderAPI.PrometheusRegistry) coderAPI.AIGatewayServerMetrics = aibridgedserver.NewMetrics(costControlReg) if vals.Prometheus.Enable { budgetPeriod := codersdk.NewAIBudgetPeriodFromString(vals.AI.BridgeConfig.BudgetPeriod) diff --git a/coderd/aibridgedserver/metrics.go b/coderd/aibridgedserver/metrics.go index c613b2f2f3e91..a7df42cf087f1 100644 --- a/coderd/aibridgedserver/metrics.go +++ b/coderd/aibridgedserver/metrics.go @@ -55,7 +55,8 @@ func NewMetrics(reg prometheus.Registerer) *Metrics { Subsystem: "cost_control", Name: "unpriced_token_usage_records_total", Help: "The number of recorded AI token-usage records for which no (provider_type, model) price was found. " + - "provider is the provider instance name, and provider_type is its configured type.", + "provider is the provider instance name, and provider_type is its configured type, " + + "or unknown when the provider could not be resolved.", }, []string{"provider", "provider_type", "model"}), // Pessimistic cardinality: 3 outcomes, 8 buckets + 3 extra series // (count, sum, +Inf) = up to 33. diff --git a/docs/admin/integrations/prometheus.md b/docs/admin/integrations/prometheus.md index bb03362fc4689..3376cc72f953b 100644 --- a/docs/admin/integrations/prometheus.md +++ b/docs/admin/integrations/prometheus.md @@ -125,9 +125,9 @@ The `coder_ai_gateway_cost_control_*` metrics are exported only by `coderd`. | `coder_ai_gateway_cost_control_blocked_requests_total` | counter | The number of AI requests blocked because the initiator's budget was exceeded. | `group_id` | | `coder_ai_gateway_cost_control_blocked_users` | gauge | The number of users currently over their AI budget. | `group_id` | | `coder_ai_gateway_cost_control_enforcement_duration_seconds` | histogram | The duration of AI budget enforcement checks, in seconds (outcome: allowed, blocked, error). | `outcome` | -| `coder_ai_gateway_cost_control_unpriced_token_usage_records_total` | counter | The number of recorded AI token-usage records for which no (provider_type, model) price was found. provider is the provider instance name, and provider_type is its configured type. | `model` `provider_type` `provider` | +| `coder_ai_gateway_cost_control_unpriced_token_usage_records_total` | counter | The number of recorded AI token-usage records for which no (provider_type, model) price was found. provider is the provider instance name, and provider_type is its configured type, or unknown when the provider could not be resolved. | `model` `provider_type` `provider` | | `coder_ai_gateway_injected_tool_invocations_total` | counter | The number of times an injected MCP tool was invoked by AI Gateway. | `model` `name` `provider` `server` | -| `coder_ai_gateway_interceptions_duration_seconds` | histogram | The total duration of intercepted requests, in seconds. The majority of this time will be the upstream processing of the request. AI Gateway has no control over upstream processing time, so it's just an illustrative metric. | `model` `provider` | +| `coder_ai_gateway_interceptions_duration_seconds` | histogram | The total time each intercepted request spent inside AI Gateway, in seconds. Dominated by upstream provider latency; use distributed tracing spans to isolate AI Gateway's own overhead. | `model` `provider` | | `coder_ai_gateway_interceptions_inflight` | gauge | The number of intercepted requests which are being processed. | `model` `provider` `route` | | `coder_ai_gateway_interceptions_total` | counter | The count of intercepted requests. | `client` `initiator_id` `method` `model` `provider` `route` `status` | | `coder_ai_gateway_key_pool_exhaustions_total` | counter | The number of times the key pool was exhausted with no usable key (outcome: rate_limited, auth_failed). | `outcome` `provider` | @@ -140,10 +140,10 @@ The `coder_ai_gateway_cost_control_*` metrics are exported only by `coderd`. | `coder_ai_gateway_provider_info` | gauge | One series per configured AI provider. Value is always 1; the status label (enabled, disabled, error) carries the alertable signal. | `provider_name` `provider_type` `status` | | `coder_ai_gateway_providers_last_reload_success_timestamp_seconds` | gauge | Unix timestamp of the last provider reload that successfully refreshed the pool. A gap against the providers_last_reload_timestamp_seconds gauge means the loop is firing but the refresh function is failing. | | | `coder_ai_gateway_providers_last_reload_timestamp_seconds` | gauge | Unix timestamp of the last provider reload attempt, success or failure. | | -| `coder_ai_gateway_proxy_connect_sessions_total` | counter | Total number of CONNECT sessions established. | `type` | +| `coder_ai_gateway_proxy_connect_sessions_total` | counter | Total number of CONNECT sessions established (type: mitm, tunneled). | `type` | | `coder_ai_gateway_proxy_inflight_mitm_requests` | gauge | Number of MITM requests currently being processed. | `provider` | | `coder_ai_gateway_proxy_mitm_requests_total` | counter | Total number of MITM requests handled by the proxy. | `provider` | -| `coder_ai_gateway_proxy_mitm_responses_total` | counter | Total number of MITM responses by HTTP status code class. | `code` `provider` | +| `coder_ai_gateway_proxy_mitm_responses_total` | counter | Total number of MITM responses by complete HTTP status code. | `code` `provider` | | `coder_ai_gateway_proxy_provider_info` | gauge | One series per configured AI provider. Value is always 1; the status label (enabled, disabled, error) carries the alertable signal. | `provider_name` `provider_type` `status` | | `coder_ai_gateway_proxy_providers_last_reload_success_timestamp_seconds` | gauge | Unix timestamp of the last provider reload that successfully refreshed the router. A gap against the providers_last_reload_timestamp_seconds gauge means the loop is firing but the refresh function is failing. | | | `coder_ai_gateway_proxy_providers_last_reload_timestamp_seconds` | gauge | Unix timestamp of the last provider reload attempt, success or failure. | | diff --git a/docs/ai-coder/ai-gateway/monitoring.md b/docs/ai-coder/ai-gateway/monitoring.md index 5f888c5961c47..57ab70836f1d1 100644 --- a/docs/ai-coder/ai-gateway/monitoring.md +++ b/docs/ai-coder/ai-gateway/monitoring.md @@ -28,26 +28,30 @@ Refer to [provider configuration](./providers.md) for the provider reload lifecy The `provider` label is the provider instance name. Some metrics use the explicit `provider_name` or `provider_type` labels for clarity. -| Metric | Type | Labels | Purpose | -|--------------------------------------------------------------------|-----------|----------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------| -| `coder_ai_gateway_interceptions_total` | counter | `client`, `initiator_id`, `method`, `model`, `provider`, `route`, `status` | Intercepted requests. | -| `coder_ai_gateway_interceptions_inflight` | gauge | `model`, `provider`, `route` | Intercepted requests currently being processed. | -| `coder_ai_gateway_interceptions_duration_seconds` | histogram | `model`, `provider` | Total intercepted request duration, including upstream processing. | -| `coder_ai_gateway_passthrough_total` | counter | `method`, `provider`, `route` | Requests passed through to an upstream provider without interception. | -| `coder_ai_gateway_prompts_total` | counter | `client`, `initiator_id`, `model`, `provider` | Prompts issued by users. | -| `coder_ai_gateway_tokens_total` | counter | `client`, `initiator_id`, `model`, `provider`, `type` | Tokens used by intercepted requests. | -| `coder_ai_gateway_injected_tool_invocations_total` | counter | `model`, `name`, `provider`, `server` | Invocations of MCP tools injected by AI Gateway. | -| `coder_ai_gateway_non_injected_tool_selections_total` | counter | `model`, `name`, `provider` | Tools selected by a model for the client to invoke. | -| `coder_ai_gateway_circuit_breaker_state` | gauge | `endpoint`, `model`, `provider` | Current circuit-breaker state: `0` for closed, `0.5` for half-open, and `1` for open. | -| `coder_ai_gateway_circuit_breaker_trips_total` | counter | `endpoint`, `model`, `provider` | Times a circuit breaker transitioned to the open state. | -| `coder_ai_gateway_circuit_breaker_rejects_total` | counter | `endpoint`, `model`, `provider` | Requests rejected because a circuit breaker was open. | -| `coder_ai_gateway_key_pool_state` | gauge | `provider`, `state` | Provider keys in each state: `valid`, `temporary`, or `permanent`. | -| `coder_ai_gateway_key_pool_state_transitions_total` | counter | `provider`, `reason` | Provider key state transitions during failover. | -| `coder_ai_gateway_key_pool_exhaustions_total` | counter | `outcome`, `provider` | Times a provider key pool had no usable key. | -| `coder_ai_gateway_key_pool_failover_attempts` | histogram | `provider` | Keys attempted before a request succeeded or exhausted the provider key pool. | -| `coder_ai_gateway_provider_info` | gauge | `provider_name`, `provider_type`, `status` | Build status of each configured provider, including disabled and errored ones. Value is always `1`; `status` is `enabled`, `disabled`, or `error`. | -| `coder_ai_gateway_providers_last_reload_timestamp_seconds` | gauge | | Unix timestamp of the last attempt to rebuild the Gateway provider pool. | -| `coder_ai_gateway_providers_last_reload_success_timestamp_seconds` | gauge | | Unix timestamp of the last successful rebuild of the Gateway provider pool. | + + +| Name | Type | Description | Labels | +|--------------------------------------------------------------------|-----------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------| +| `coder_ai_gateway_circuit_breaker_rejects_total` | counter | Total number of requests rejected due to open circuit breaker. | `endpoint` `model` `provider` | +| `coder_ai_gateway_circuit_breaker_state` | gauge | Current state of the circuit breaker (0=closed, 0.5=half-open, 1=open). | `endpoint` `model` `provider` | +| `coder_ai_gateway_circuit_breaker_trips_total` | counter | Total number of times the circuit breaker transitioned to open state. | `endpoint` `model` `provider` | +| `coder_ai_gateway_injected_tool_invocations_total` | counter | The number of times an injected MCP tool was invoked by AI Gateway. | `model` `name` `provider` `server` | +| `coder_ai_gateway_interceptions_duration_seconds` | histogram | The total time each intercepted request spent inside AI Gateway, in seconds. Dominated by upstream provider latency; use distributed tracing spans to isolate AI Gateway's own overhead. | `model` `provider` | +| `coder_ai_gateway_interceptions_inflight` | gauge | The number of intercepted requests which are being processed. | `model` `provider` `route` | +| `coder_ai_gateway_interceptions_total` | counter | The count of intercepted requests. | `client` `initiator_id` `method` `model` `provider` `route` `status` | +| `coder_ai_gateway_key_pool_exhaustions_total` | counter | The number of times the key pool was exhausted with no usable key (outcome: rate_limited, auth_failed). | `outcome` `provider` | +| `coder_ai_gateway_key_pool_failover_attempts` | histogram | The number of keys attempted before success or exhaustion, per interception for bridged requests and per request for passthrough requests. | `provider` | +| `coder_ai_gateway_key_pool_state` | gauge | The number of keys currently in each state (state: valid, temporary, permanent). | `provider` `state` | +| `coder_ai_gateway_key_pool_state_transitions_total` | counter | The number of API key state transitions during failover (reason: rate_limited, unauthorized). | `provider` `reason` | +| `coder_ai_gateway_non_injected_tool_selections_total` | counter | The number of times an AI model selected a tool to be invoked by the client. | `model` `name` `provider` | +| `coder_ai_gateway_passthrough_total` | counter | The count of requests which were not intercepted but passed through to the upstream. | `method` `provider` `route` | +| `coder_ai_gateway_prompts_total` | counter | The number of prompts issued by users (initiators). | `client` `initiator_id` `model` `provider` | +| `coder_ai_gateway_provider_info` | gauge | One series per configured AI provider. Value is always 1; the status label (enabled, disabled, error) carries the alertable signal. | `provider_name` `provider_type` `status` | +| `coder_ai_gateway_providers_last_reload_success_timestamp_seconds` | gauge | Unix timestamp of the last provider reload that successfully refreshed the pool. A gap against the providers_last_reload_timestamp_seconds gauge means the loop is firing but the refresh function is failing. | | +| `coder_ai_gateway_providers_last_reload_timestamp_seconds` | gauge | Unix timestamp of the last provider reload attempt, success or failure. | | +| `coder_ai_gateway_tokens_total` | counter | The number of tokens used by intercepted requests. | `client` `initiator_id` `model` `provider` `type` | + + Histograms also emit the standard `_bucket`, `_sum`, and `_count` series. @@ -57,26 +61,34 @@ Budget enforcement runs in `coderd`. Cost control metrics are exported only from the `coderd` Prometheus listener. Standalone replicas do not export them. -| Metric | Type | Labels | Purpose | -|--------------------------------------------------------------------|-----------|--------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `coder_ai_gateway_cost_control_blocked_requests_total` | counter | `group_id` | AI requests blocked because the initiator's budget was exceeded. | -| `coder_ai_gateway_cost_control_blocked_users` | gauge | `group_id` | Users currently over their AI budget. | -| `coder_ai_gateway_cost_control_enforcement_duration_seconds` | histogram | `outcome` | Duration of AI budget enforcement checks. `outcome` is `allowed`, `blocked`, or `error`. | -| `coder_ai_gateway_cost_control_unpriced_token_usage_records_total` | counter | `model`, `provider`, `provider_type` | Recorded token-usage records for which no model price was found. `provider` is the provider instance name, and `provider_type` is the configured type the price is keyed on, or `unknown` when the provider could not be resolved. | + + +| Name | Type | Description | Labels | +|--------------------------------------------------------------------|-----------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------| +| `coder_ai_gateway_cost_control_blocked_requests_total` | counter | The number of AI requests blocked because the initiator's budget was exceeded. | `group_id` | +| `coder_ai_gateway_cost_control_blocked_users` | gauge | The number of users currently over their AI budget. | `group_id` | +| `coder_ai_gateway_cost_control_enforcement_duration_seconds` | histogram | The duration of AI budget enforcement checks, in seconds (outcome: allowed, blocked, error). | `outcome` | +| `coder_ai_gateway_cost_control_unpriced_token_usage_records_total` | counter | The number of recorded AI token-usage records for which no (provider_type, model) price was found. provider is the provider instance name, and provider_type is its configured type, or unknown when the provider could not be resolved. | `model` `provider_type` `provider` | + + ### AI Gateway Proxy metrics AI Gateway Proxy exports metrics from the `coderd` Prometheus listener. -| Metric | Type | Labels | Purpose | -|--------------------------------------------------------------------------|---------|--------------------------------------------|-----------------------------------------------------------------------------------------------------------------| -| `coder_ai_gateway_proxy_connect_sessions_total` | counter | `type` | CONNECT sessions established, classified as `mitm` or `tunneled`. | -| `coder_ai_gateway_proxy_mitm_requests_total` | counter | `provider` | MITM requests handled by AI Gateway Proxy. | -| `coder_ai_gateway_proxy_inflight_mitm_requests` | gauge | `provider` | MITM requests currently being processed. | -| `coder_ai_gateway_proxy_mitm_responses_total` | counter | `code`, `provider` | MITM responses by HTTP status code. | -| `coder_ai_gateway_proxy_provider_info` | gauge | `provider_name`, `provider_type`, `status` | Routing status of each configured provider. Value is always `1`; `status` is `enabled`, `disabled`, or `error`. | -| `coder_ai_gateway_proxy_providers_last_reload_timestamp_seconds` | gauge | | Unix timestamp of the last attempt to rebuild the proxy routing snapshot. | -| `coder_ai_gateway_proxy_providers_last_reload_success_timestamp_seconds` | gauge | | Unix timestamp of the last successful rebuild of the proxy routing snapshot. | + + +| Name | Type | Description | Labels | +|--------------------------------------------------------------------------|---------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------| +| `coder_ai_gateway_proxy_connect_sessions_total` | counter | Total number of CONNECT sessions established (type: mitm, tunneled). | `type` | +| `coder_ai_gateway_proxy_inflight_mitm_requests` | gauge | Number of MITM requests currently being processed. | `provider` | +| `coder_ai_gateway_proxy_mitm_requests_total` | counter | Total number of MITM requests handled by the proxy. | `provider` | +| `coder_ai_gateway_proxy_mitm_responses_total` | counter | Total number of MITM responses by complete HTTP status code. | `code` `provider` | +| `coder_ai_gateway_proxy_provider_info` | gauge | One series per configured AI provider. Value is always 1; the status label (enabled, disabled, error) carries the alertable signal. | `provider_name` `provider_type` `status` | +| `coder_ai_gateway_proxy_providers_last_reload_success_timestamp_seconds` | gauge | Unix timestamp of the last provider reload that successfully refreshed the router. A gap against the providers_last_reload_timestamp_seconds gauge means the loop is firing but the refresh function is failing. | | +| `coder_ai_gateway_proxy_providers_last_reload_timestamp_seconds` | gauge | Unix timestamp of the last provider reload attempt, success or failure. | | + + Refer to the [Prometheus reference](../../admin/integrations/prometheus.md) for these metrics alongside the other metrics that Coder components export. diff --git a/enterprise/aibridgeproxyd/metrics.go b/enterprise/aibridgeproxyd/metrics.go index 4f169434d3412..88623af93d90d 100644 --- a/enterprise/aibridgeproxyd/metrics.go +++ b/enterprise/aibridgeproxyd/metrics.go @@ -6,6 +6,11 @@ import ( ) const ( + // PrometheusMetricPrefix is the canonical prefix applied to every metric + // defined in this package. The registration wiring wraps the registerer + // with it, so the metric options below omit it. + PrometheusMetricPrefix = "coder_ai_gateway_proxy_" + RequestTypeMITM = "mitm" RequestTypeTunneled = "tunneled" ) @@ -56,7 +61,7 @@ func NewMetrics(reg prometheus.Registerer) *Metrics { ConnectSessionsTotal: factory.NewCounterVec(prometheus.CounterOpts{ Name: "connect_sessions_total", - Help: "Total number of CONNECT sessions established.", + Help: "Total number of CONNECT sessions established (type: mitm, tunneled).", }, []string{"type"}), MITMRequestsTotal: factory.NewCounterVec(prometheus.CounterOpts{ @@ -71,7 +76,7 @@ func NewMetrics(reg prometheus.Registerer) *Metrics { MITMResponsesTotal: factory.NewCounterVec(prometheus.CounterOpts{ Name: "mitm_responses_total", - Help: "Total number of MITM responses by HTTP status code class.", + Help: "Total number of MITM responses by complete HTTP status code.", }, []string{"code", "provider"}), ProviderInfo: factory.NewGaugeVec(prometheus.GaugeOpts{ diff --git a/enterprise/cli/aibridgeproxyd.go b/enterprise/cli/aibridgeproxyd.go index 4b1fdefe7c8d7..be2b082c4928c 100644 --- a/enterprise/cli/aibridgeproxyd.go +++ b/enterprise/cli/aibridgeproxyd.go @@ -48,7 +48,7 @@ func newAIBridgeProxyDaemon(coderAPI *coderd.API) (io.Closer, error) { // TODO(deprecation): Remove "coder_aibridgeproxyd_" in v2.37. // See AIGOV-447: // https://linear.app/codercom/issue/AIGOV-447/remove-legacy-ai-gateway-metric-aliases - reg := prometheusmetrics.NewMetricAliasRegisterer(coderAPI.PrometheusRegistry, "coder_ai_gateway_proxy_", "coder_aibridgeproxyd_") + reg := prometheusmetrics.NewMetricAliasRegisterer(coderAPI.PrometheusRegistry, aibridgeproxyd.PrometheusMetricPrefix, "coder_aibridgeproxyd_") metrics := aibridgeproxyd.NewMetrics(reg) var newDumper func(provider, requestID string) aibridgeproxyd.RoundTripDumper diff --git a/scripts/metricsdocgen/README.md b/scripts/metricsdocgen/README.md index e418d235fdb91..07ec072201bf0 100644 --- a/scripts/metricsdocgen/README.md +++ b/scripts/metricsdocgen/README.md @@ -1,6 +1,6 @@ # Metrics Documentation Generator -This tool generates the Prometheus metrics documentation at [`docs/admin/integrations/prometheus.md`](https://coder.com/docs/admin/integrations/prometheus#available-metrics). +This tool generates Prometheus metric tables in [`docs/admin/integrations/prometheus.md`](https://coder.com/docs/admin/integrations/prometheus#available-metrics) and [`docs/ai-coder/ai-gateway/monitoring.md`](https://coder.com/docs/ai-coder/ai-gateway/monitoring#prometheus-metrics). ## How It Works @@ -18,7 +18,6 @@ Contains metrics that are **not** directly defined in the coder source code: - `go_*`: Go runtime metrics - `process_*`: Process metrics from prometheus/client_golang - `promhttp_*`: Prometheus HTTP handler metrics -- `coder_ai_gateway_*`: AI Gateway metrics are registered through prefixed registerer that the scanner does not resolve. > [!Note] > This file also contains edge cases where metric metadata cannot be accurately extracted by the scanner (e.g., labels determined by runtime logic). @@ -37,16 +36,55 @@ Contains metrics extracted from the coder source code by the AST scanner (`scann make scripts/metricsdocgen/generated_metrics ``` +Metrics registered through a prefixed registerer, such as the `coder_ai_gateway_*` families, are +defined in the source without their prefix. `scanner/scanner.go` maps each defining file to the +prefix constant its registration wiring uses, so the scanned name matches the scraped name. A +scanned metric whose name carries none of the known namespaces fails the scan, because that means +its prefix is applied at registration and the defining file has no mapping. + +## Sections + +A document holds either a single unnamed table or several named ones. + +Without `--section`, the tool replaces the content between the document's unnamed marker pair with +one table containing every metric. `docs/admin/integrations/prometheus.md` uses this form. + +With one or more `--section NAME=prefix:PREFIX[,PREFIX...][,!EXCLUDED_PREFIX...]` flags, each +section renders its own table from the metrics its filter selects. Prefixed with `!`, a value +excludes the metrics it matches. A section that matches no metric is an error rather than an empty +table. `docs/ai-coder/ai-gateway/monitoring.md` uses this form with the `common`, `cost-control`, +and `proxy` sections. + +Each named section replaces the content between its own marker pair, which the document must +already contain: + +```markdown + + + +``` + +A named marker pair in the document that no `--section` flag claims is an error too, so removing a +section from the Makefile without deleting its markers cannot leave a stale table behind. The same +holds for a marker pair that names another make target, for a repeated marker pair, and, in either +direction, for a pair belonging to the mode that is not in use: a named pair in a document generated +without `--section`, or the unnamed pair in a document generated with it. Every one of those pairs +would keep its table under a "DO NOT EDIT" header that regeneration never reaches. + +A closing marker with no opening one is rejected too, for a different reason: it carries no table +and no "DO NOT EDIT" header, but it reads as a section boundary that nothing owns. + +The make targets write to a temporary copy of the document, so they pass `--doc-target` to keep the +markers naming the make target rather than the temporary path. + ## Updating Metrics Documentation To regenerate the documentation after code changes: ```bash make docs/admin/integrations/prometheus.md +make docs/ai-coder/ai-gateway/monitoring.md ``` -This will: - -- Run the scanner to update `generated_metrics` -- Merge `metrics` and `generated_metrics` metric files -- Update the documentation file +These targets rerun the scanner, merge the `metrics` and `generated_metrics` files, and update the +generated metric tables. diff --git a/scripts/metricsdocgen/generated_metrics b/scripts/metricsdocgen/generated_metrics index 739a4f84f1f63..4d55b944908d4 100644 --- a/scripts/metricsdocgen/generated_metrics +++ b/scripts/metricsdocgen/generated_metrics @@ -7,6 +7,93 @@ agent_boundary_log_proxy_batches_forwarded_total 0 # HELP agent_boundary_log_proxy_logs_dropped_total Total number of individual boundary log entries dropped before reaching coderd. Reason: buffer_full = the agent's internal buffer is full; forward_failed = the agent failed to send the batch to coderd; boundary_channel_full = boundary's internal send channel overflowed, meaning boundary is generating logs faster than it can batch and send them; boundary_batch_full = boundary's outgoing batch buffer overflowed after a failed flush, meaning boundary could not write to the agent's socket. # TYPE agent_boundary_log_proxy_logs_dropped_total counter agent_boundary_log_proxy_logs_dropped_total{reason=""} 0 +# HELP coder_ai_gateway_circuit_breaker_rejects_total Total number of requests rejected due to open circuit breaker. +# TYPE coder_ai_gateway_circuit_breaker_rejects_total counter +coder_ai_gateway_circuit_breaker_rejects_total{provider="",endpoint="",model=""} 0 +# HELP coder_ai_gateway_circuit_breaker_state Current state of the circuit breaker (0=closed, 0.5=half-open, 1=open). +# TYPE coder_ai_gateway_circuit_breaker_state gauge +coder_ai_gateway_circuit_breaker_state{provider="",endpoint="",model=""} 0 +# HELP coder_ai_gateway_circuit_breaker_trips_total Total number of times the circuit breaker transitioned to open state. +# TYPE coder_ai_gateway_circuit_breaker_trips_total counter +coder_ai_gateway_circuit_breaker_trips_total{provider="",endpoint="",model=""} 0 +# HELP coder_ai_gateway_cost_control_blocked_requests_total The number of AI requests blocked because the initiator's budget was exceeded. +# TYPE coder_ai_gateway_cost_control_blocked_requests_total counter +coder_ai_gateway_cost_control_blocked_requests_total{group_id=""} 0 +# HELP coder_ai_gateway_cost_control_blocked_users The number of users currently over their AI budget. +# TYPE coder_ai_gateway_cost_control_blocked_users gauge +coder_ai_gateway_cost_control_blocked_users{group_id=""} 0 +# HELP coder_ai_gateway_cost_control_enforcement_duration_seconds The duration of AI budget enforcement checks, in seconds (outcome: allowed, blocked, error). +# TYPE coder_ai_gateway_cost_control_enforcement_duration_seconds histogram +coder_ai_gateway_cost_control_enforcement_duration_seconds{outcome=""} 0 +# HELP coder_ai_gateway_cost_control_unpriced_token_usage_records_total The number of recorded AI token-usage records for which no (provider_type, model) price was found. provider is the provider instance name, and provider_type is its configured type, or unknown when the provider could not be resolved. +# TYPE coder_ai_gateway_cost_control_unpriced_token_usage_records_total counter +coder_ai_gateway_cost_control_unpriced_token_usage_records_total{provider="",provider_type="",model=""} 0 +# HELP coder_ai_gateway_injected_tool_invocations_total The number of times an injected MCP tool was invoked by AI Gateway. +# TYPE coder_ai_gateway_injected_tool_invocations_total counter +coder_ai_gateway_injected_tool_invocations_total{provider="",model="",server="",name=""} 0 +# HELP coder_ai_gateway_interceptions_duration_seconds The total time each intercepted request spent inside AI Gateway, in seconds. Dominated by upstream provider latency; use distributed tracing spans to isolate AI Gateway's own overhead. +# TYPE coder_ai_gateway_interceptions_duration_seconds histogram +coder_ai_gateway_interceptions_duration_seconds{provider="",model=""} 0 +# HELP coder_ai_gateway_interceptions_inflight The number of intercepted requests which are being processed. +# TYPE coder_ai_gateway_interceptions_inflight gauge +coder_ai_gateway_interceptions_inflight{provider="",model="",route=""} 0 +# HELP coder_ai_gateway_interceptions_total The count of intercepted requests. +# TYPE coder_ai_gateway_interceptions_total counter +coder_ai_gateway_interceptions_total{provider="",model="",status="",route="",method="",initiator_id="",client=""} 0 +# HELP coder_ai_gateway_key_pool_exhaustions_total The number of times the key pool was exhausted with no usable key (outcome: rate_limited, auth_failed). +# TYPE coder_ai_gateway_key_pool_exhaustions_total counter +coder_ai_gateway_key_pool_exhaustions_total{provider="",outcome=""} 0 +# HELP coder_ai_gateway_key_pool_failover_attempts The number of keys attempted before success or exhaustion, per interception for bridged requests and per request for passthrough requests. +# TYPE coder_ai_gateway_key_pool_failover_attempts histogram +coder_ai_gateway_key_pool_failover_attempts{provider=""} 0 +# HELP coder_ai_gateway_key_pool_state The number of keys currently in each state (state: valid, temporary, permanent). +# TYPE coder_ai_gateway_key_pool_state gauge +coder_ai_gateway_key_pool_state{provider="",state=""} 0 +# HELP coder_ai_gateway_key_pool_state_transitions_total The number of API key state transitions during failover (reason: rate_limited, unauthorized). +# TYPE coder_ai_gateway_key_pool_state_transitions_total counter +coder_ai_gateway_key_pool_state_transitions_total{provider="",reason=""} 0 +# HELP coder_ai_gateway_non_injected_tool_selections_total The number of times an AI model selected a tool to be invoked by the client. +# TYPE coder_ai_gateway_non_injected_tool_selections_total counter +coder_ai_gateway_non_injected_tool_selections_total{provider="",model="",name=""} 0 +# HELP coder_ai_gateway_passthrough_total The count of requests which were not intercepted but passed through to the upstream. +# TYPE coder_ai_gateway_passthrough_total counter +coder_ai_gateway_passthrough_total{provider="",route="",method=""} 0 +# HELP coder_ai_gateway_prompts_total The number of prompts issued by users (initiators). +# TYPE coder_ai_gateway_prompts_total counter +coder_ai_gateway_prompts_total{provider="",model="",initiator_id="",client=""} 0 +# HELP coder_ai_gateway_provider_info One series per configured AI provider. Value is always 1; the status label (enabled, disabled, error) carries the alertable signal. +# TYPE coder_ai_gateway_provider_info gauge +coder_ai_gateway_provider_info{provider_name="",provider_type="",status=""} 0 +# HELP coder_ai_gateway_providers_last_reload_success_timestamp_seconds Unix timestamp of the last provider reload that successfully refreshed the pool. A gap against the providers_last_reload_timestamp_seconds gauge means the loop is firing but the refresh function is failing. +# TYPE coder_ai_gateway_providers_last_reload_success_timestamp_seconds gauge +coder_ai_gateway_providers_last_reload_success_timestamp_seconds 0 +# HELP coder_ai_gateway_providers_last_reload_timestamp_seconds Unix timestamp of the last provider reload attempt, success or failure. +# TYPE coder_ai_gateway_providers_last_reload_timestamp_seconds gauge +coder_ai_gateway_providers_last_reload_timestamp_seconds 0 +# HELP coder_ai_gateway_proxy_connect_sessions_total Total number of CONNECT sessions established (type: mitm, tunneled). +# TYPE coder_ai_gateway_proxy_connect_sessions_total counter +coder_ai_gateway_proxy_connect_sessions_total{type=""} 0 +# HELP coder_ai_gateway_proxy_inflight_mitm_requests Number of MITM requests currently being processed. +# TYPE coder_ai_gateway_proxy_inflight_mitm_requests gauge +coder_ai_gateway_proxy_inflight_mitm_requests{provider=""} 0 +# HELP coder_ai_gateway_proxy_mitm_requests_total Total number of MITM requests handled by the proxy. +# TYPE coder_ai_gateway_proxy_mitm_requests_total counter +coder_ai_gateway_proxy_mitm_requests_total{provider=""} 0 +# HELP coder_ai_gateway_proxy_mitm_responses_total Total number of MITM responses by complete HTTP status code. +# TYPE coder_ai_gateway_proxy_mitm_responses_total counter +coder_ai_gateway_proxy_mitm_responses_total{code="",provider=""} 0 +# HELP coder_ai_gateway_proxy_provider_info One series per configured AI provider. Value is always 1; the status label (enabled, disabled, error) carries the alertable signal. +# TYPE coder_ai_gateway_proxy_provider_info gauge +coder_ai_gateway_proxy_provider_info{provider_name="",provider_type="",status=""} 0 +# HELP coder_ai_gateway_proxy_providers_last_reload_success_timestamp_seconds Unix timestamp of the last provider reload that successfully refreshed the router. A gap against the providers_last_reload_timestamp_seconds gauge means the loop is firing but the refresh function is failing. +# TYPE coder_ai_gateway_proxy_providers_last_reload_success_timestamp_seconds gauge +coder_ai_gateway_proxy_providers_last_reload_success_timestamp_seconds 0 +# HELP coder_ai_gateway_proxy_providers_last_reload_timestamp_seconds Unix timestamp of the last provider reload attempt, success or failure. +# TYPE coder_ai_gateway_proxy_providers_last_reload_timestamp_seconds gauge +coder_ai_gateway_proxy_providers_last_reload_timestamp_seconds 0 +# HELP coder_ai_gateway_tokens_total The number of tokens used by intercepted requests. +# TYPE coder_ai_gateway_tokens_total counter +coder_ai_gateway_tokens_total{provider="",model="",type="",initiator_id="",client=""} 0 # HELP coder_derp_server_accepts_total Total DERP connections accepted. # TYPE coder_derp_server_accepts_total counter coder_derp_server_accepts_total 0 diff --git a/scripts/metricsdocgen/main.go b/scripts/metricsdocgen/main.go index 302320e25e236..5566fc03b89a9 100644 --- a/scripts/metricsdocgen/main.go +++ b/scripts/metricsdocgen/main.go @@ -4,50 +4,105 @@ import ( "bytes" "errors" "flag" + "fmt" "io" "log" + "maps" "os" - "sort" + "regexp" + "slices" "strings" dto "github.com/prometheus/client_model/go" "github.com/prometheus/common/expfmt" "golang.org/x/xerrors" - "github.com/coder/coder/v2/coderd/util/maps" "github.com/coder/coder/v2/scripts/atomicwrite" ) var ( staticMetricsFile string - prometheusDocFile string + docFile string + docTarget string generatedMetricsFile string dryRun bool - - generatorPrefix = []byte("") - generatorSuffix = []byte("") + sections sectionFlags + + // generatedMarker and generatedEndMarker match the opening and closing + // generated markers, named or unnamed, and capture the make target and the + // section name. Matching markers that name another target is what turns a + // stale pair into an error instead of a table that regeneration never + // reaches. + generatedMarker = regexp.MustCompile(``) + generatedEndMarker = regexp.MustCompile(``) ) +type metricFilter struct { + prefixes []string + excludePrefixes []string +} + +type documentSection struct { + name string + filter metricFilter +} + +type sectionFlags []documentSection + +func (s *sectionFlags) String() string { + values := make([]string, 0, len(*s)) + for _, section := range *s { + values = append(values, section.name) + } + return strings.Join(values, ",") +} + +func (s *sectionFlags) Set(value string) error { + section, err := parseSection(value) + if err != nil { + return err + } + if slices.ContainsFunc(*s, func(existing documentSection) bool { + return existing.name == section.name + }) { + return xerrors.Errorf("duplicate section %q", section.name) + } + *s = append(*s, section) + return nil +} + func main() { flag.StringVar(&staticMetricsFile, "static-metrics", "scripts/metricsdocgen/metrics", "Path to static metrics file (manually maintained)") flag.StringVar(&generatedMetricsFile, "generated-metrics", "scripts/metricsdocgen/generated_metrics", "Path to generated metrics file (from scanner)") - flag.StringVar(&prometheusDocFile, "prometheus-doc-file", "docs/admin/integrations/prometheus.md", "Path to Prometheus doc file") + flag.StringVar(&docFile, "doc-file", "docs/admin/integrations/prometheus.md", "Path to the Markdown document to update") + flag.StringVar(&docTarget, "doc-target", "", "Make target named in the generated markers (defaults to --doc-file). Set this when --doc-file points at a temporary copy") + flag.Var(§ions, "section", "Named generated section, repeatable: NAME=prefix:PREFIX[,PREFIX...][,!EXCLUDED_PREFIX...]. "+ + "Each section replaces the content between its own `` and "+ + "`` markers. Without this flag the document's single unnamed marker pair is updated instead.") flag.BoolVar(&dryRun, "dry-run", false, "Dry run") flag.Parse() + if docTarget == "" { + docTarget = docFile + } + metrics, err := readAndMergeMetrics() if err != nil { log.Fatal("can't read metrics: ", err) } - doc, err := readPrometheusDoc() + doc, err := readDoc() if err != nil { - log.Fatal("can't read Prometheus doc: ", err) + log.Fatalf("can't read %s: %v", docFile, err) } - doc, err = updatePrometheusDoc(doc, metrics) + if len(sections) == 0 { + doc, err = updateDefaultSection(doc, metrics, docTarget) + } else { + doc, err = updateDocumentSections(doc, metrics, docTarget, sections) + } if err != nil { - log.Fatal("can't update Prometheus doc: ", err) + log.Fatalf("can't update %s: %v", docFile, err) } if dryRun { @@ -55,12 +110,54 @@ func main() { return } - err = writePrometheusDoc(doc) + err = writeDoc(doc) if err != nil { - log.Fatal("can't write updated Prometheus doc: ", err) + log.Fatalf("can't write updated %s: %v", docFile, err) } } +func parseSection(value string) (documentSection, error) { + name, filterSpec, ok := strings.Cut(value, "=") + if !ok || name == "" || filterSpec == "" { + return documentSection{}, xerrors.New("section must have the form NAME=prefix:PREFIX") + } + for _, r := range name { + if (r < 'a' || r > 'z') && (r < 'A' || r > 'Z') && (r < '0' || r > '9') && r != '-' && r != '_' { + return documentSection{}, xerrors.Errorf("section name %q may contain only letters, numbers, hyphens, and underscores", name) + } + } + + kind, valuesSpec, ok := strings.Cut(filterSpec, ":") + if !ok || valuesSpec == "" { + return documentSection{}, xerrors.Errorf("section %q filter must have the form prefix:VALUE", name) + } + values := strings.Split(valuesSpec, ",") + for _, value := range values { + if value == "" { + return documentSection{}, xerrors.Errorf("section %q contains an empty filter value", name) + } + } + + section := documentSection{name: name} + if kind != "prefix" { + return documentSection{}, xerrors.Errorf("section %q has unknown filter kind %q", name, kind) + } + for _, value := range values { + if excluded, ok := strings.CutPrefix(value, "!"); ok { + if excluded == "" { + return documentSection{}, xerrors.Errorf("section %q contains an empty excluded prefix", name) + } + section.filter.excludePrefixes = append(section.filter.excludePrefixes, excluded) + continue + } + section.filter.prefixes = append(section.filter.prefixes, value) + } + if len(section.filter.prefixes) == 0 { + return documentSection{}, xerrors.Errorf("section %q requires at least one included prefix", name) + } + return section, nil +} + // readMetricsFromFile reads metrics from a single Prometheus text format file. func readMetricsFromFile(path string) ([]*dto.MetricFamily, error) { f, err := os.Open(path) @@ -104,90 +201,197 @@ func readAndMergeMetrics() ([]*dto.MetricFamily, error) { return nil, xerrors.Errorf("reading static metrics: %w", err) } - // Merge metrics, using a map to deduplicate by name. metricsByName := make(map[string]*dto.MetricFamily) - - // Add generated metrics first. for _, m := range generatedMetrics { - metricsByName[*m.Name] = m + metricsByName[m.GetName()] = m } - - // Static metrics overwrite generated metrics if they exist. for _, m := range staticMetrics { - metricsByName[*m.Name] = m + metricsByName[m.GetName()] = m } - // Convert back to slice and sort. - var metrics []*dto.MetricFamily - for _, m := range metricsByName { - metrics = append(metrics, m) - } + return slices.SortedFunc(maps.Values(metricsByName), func(a, b *dto.MetricFamily) int { + return strings.Compare(a.GetName(), b.GetName()) + }), nil +} - sort.Slice(metrics, func(i, j int) bool { - return *metrics[i].Name < *metrics[j].Name - }) +func readDoc() ([]byte, error) { + doc, err := os.ReadFile(docFile) + if err != nil { + return nil, err + } + return doc, nil +} - return metrics, nil +// updateDefaultSection rewrites the document's single unnamed marker pair. +func updateDefaultSection(doc []byte, metricFamilies []*dto.MetricFamily, docTarget string) ([]byte, error) { + if err := validateGeneratedMarkers(doc, docTarget, nil); err != nil { + return nil, err + } + return replaceGeneratedSection(doc, defaultGeneratorPrefix(docTarget), defaultGeneratorSuffix(docTarget), metricFamilies) } -func readPrometheusDoc() ([]byte, error) { - doc, err := os.ReadFile(prometheusDocFile) - if err != nil { +// updateDocumentSections rewrites one marker pair per requested section. +func updateDocumentSections(doc []byte, metricFamilies []*dto.MetricFamily, docTarget string, sections []documentSection) ([]byte, error) { + if err := validateGeneratedMarkers(doc, docTarget, sections); err != nil { return nil, err } + var err error + for _, section := range sections { + filtered := filterMetricFamilies(metricFamilies, section.filter) + if len(filtered) == 0 { + return nil, xerrors.Errorf("section %q matches no metrics", section.name) + } + doc, err = replaceGeneratedSection(doc, namedGeneratorPrefix(docTarget, section.name), namedGeneratorSuffix(docTarget, section.name), filtered) + if err != nil { + return nil, xerrors.Errorf("updating section %q: %w", section.name, err) + } + } return doc, nil } -func updatePrometheusDoc(doc []byte, metricFamilies []*dto.MetricFamily) ([]byte, error) { - i := bytes.Index(doc, generatorPrefix) - if i < 0 { - return nil, xerrors.New("generator prefix tag not found") +// validateGeneratedMarkers rejects every generated marker in the document that +// the current invocation does not rewrite. A marker naming another make target, +// a named section that no --section flag claims, a marker of the mode that is +// not in use, and a repeated marker all keep a stale table under a header that +// reads "DO NOT EDIT", because only the requested markers get replaced and only +// the first occurrence of each is found. A closing marker with no opening one +// is rejected as well, since it reads as a section boundary that nothing owns. +func validateGeneratedMarkers(doc []byte, docTarget string, sections []documentSection) error { + opened := map[string]int{} + for _, match := range generatedMarker.FindAllSubmatch(doc, -1) { + target, name := string(match[1]), string(match[2]) + if target != docTarget { + return xerrors.Errorf("generated marker %q names make target %q, want %q", match[0], target, docTarget) + } + switch { + case name == "" && len(sections) > 0: + return xerrors.Errorf("document has an unnamed generated section, which --section mode never rewrites") + case name != "" && len(sections) == 0: + return xerrors.Errorf("document has generated section %q, which default mode never rewrites", name) + case name != "" && !slices.ContainsFunc(sections, func(section documentSection) bool { + return section.name == name + }): + return xerrors.Errorf("document has a generated section that no --section flag claims: %q", name) + } + opened[name]++ + if opened[name] > 1 { + return xerrors.Errorf("document has %d generated markers for the same section: %q", opened[name], match[0]) + } + } + + closed := map[string]int{} + for _, match := range generatedEndMarker.FindAllSubmatch(doc, -1) { + target, name := string(match[1]), string(match[2]) + if target != docTarget { + return xerrors.Errorf("generated marker %q names make target %q, want %q", match[0], target, docTarget) + } + if opened[name] == 0 { + return xerrors.Errorf("document has a closing marker with no opening marker: %q", match[0]) + } + closed[name]++ + if closed[name] > 1 { + return xerrors.Errorf("document has %d closing markers for the same section: %q", closed[name], match[0]) + } + } + return nil +} + +func defaultGeneratorPrefix(docTarget string) []byte { + return fmt.Appendf(nil, "", docTarget) +} + +func defaultGeneratorSuffix(docTarget string) []byte { + return fmt.Appendf(nil, "", docTarget) +} + +func namedGeneratorPrefix(docTarget, name string) []byte { + return fmt.Appendf(nil, "", docTarget, name) +} + +func namedGeneratorSuffix(docTarget, name string) []byte { + return fmt.Appendf(nil, "", docTarget, name) +} + +func filterMetricFamilies(metricFamilies []*dto.MetricFamily, filter metricFilter) []*dto.MetricFamily { + filtered := make([]*dto.MetricFamily, 0, len(metricFamilies)) + for _, metricFamily := range metricFamilies { + name := metricFamily.GetName() + matches := func(prefix string) bool { return strings.HasPrefix(name, prefix) } + if !slices.ContainsFunc(filter.prefixes, matches) { + continue + } + if slices.ContainsFunc(filter.excludePrefixes, matches) { + continue + } + filtered = append(filtered, metricFamily) } - tableStartIndex := i + len(generatorPrefix) + 1 + return filtered +} - j := bytes.Index(doc[tableStartIndex:], generatorSuffix) +func replaceGeneratedSection(doc, prefix, suffix []byte, metricFamilies []*dto.MetricFamily) ([]byte, error) { + // Both entry points run validateGeneratedMarkers first, so a repeated prefix + // never reaches here through the CLI. The check keeps the invariant local: + // rewriting the first pair of several silently leaves the rest stale. + switch count := bytes.Count(doc, prefix); { + case count == 0: + return nil, xerrors.Errorf("generator prefix tag %q not found", prefix) + case count > 1: + return nil, xerrors.Errorf("generator prefix tag %q found %d times, want 1", prefix, count) + } + i := bytes.Index(doc, prefix) + tableStartIndex := i + len(prefix) + if tableStartIndex < len(doc) && doc[tableStartIndex] == '\n' { + tableStartIndex++ + } + + j := bytes.Index(doc[tableStartIndex:], suffix) if j < 0 { - return nil, xerrors.New("generator suffix tag not found") + return nil, xerrors.Errorf("generator suffix tag %q not found", suffix) } tableEndIndex := tableStartIndex + j var buffer bytes.Buffer _, _ = buffer.Write(doc[:tableStartIndex]) - _ = buffer.WriteByte('\n') + _, _ = buffer.Write(renderMetricTable(metricFamilies)) + _, _ = buffer.Write(doc[tableEndIndex:]) + return buffer.Bytes(), nil +} - _, _ = buffer.WriteString("| Name | Type | Description | Labels |\n") +func renderMetricTable(metricFamilies []*dto.MetricFamily) []byte { + var buffer bytes.Buffer + _, _ = buffer.WriteString("\n| Name | Type | Description | Labels |\n") _, _ = buffer.WriteString("| - | - | - | - |\n") for _, mf := range metricFamilies { - _, _ = buffer.WriteString("| ") - _, _ = buffer.Write([]byte("`" + *mf.Name + "`")) - _, _ = buffer.WriteString(" | ") - _, _ = buffer.Write([]byte(strings.ToLower(mf.Type.String()))) + _, _ = buffer.WriteString("| `") + _, _ = buffer.WriteString(mf.GetName()) + _, _ = buffer.WriteString("` | ") + _, _ = buffer.WriteString(strings.ToLower(mf.GetType().String())) _, _ = buffer.WriteString(" | ") - if mf.Help != nil { - _, _ = buffer.Write([]byte(*mf.Help)) - } + _, _ = buffer.WriteString(markdownCell(mf.GetHelp())) _, _ = buffer.WriteString(" | ") labels := map[string]struct{}{} - metrics := mf.GetMetric() - for _, m := range metrics { - for _, label := range m.Label { - labels["`"+*label.Name+"`"] = struct{}{} + for _, metric := range mf.GetMetric() { + for _, label := range metric.GetLabel() { + labels["`"+label.GetName()+"`"] = struct{}{} } } - - if len(labels) > 0 { - _, _ = buffer.WriteString(strings.Join(maps.SortedKeys(labels), " ")) - } - + _, _ = buffer.WriteString(strings.Join(slices.Sorted(maps.Keys(labels)), " ")) _, _ = buffer.WriteString(" |\n") } - _ = buffer.WriteByte('\n') - _, _ = buffer.Write(doc[tableEndIndex:]) - return buffer.Bytes(), nil + return buffer.Bytes() +} + +// markdownCell makes a value safe to embed in a Markdown table cell. A pipe +// would shift the columns and a line break would split the row. +func markdownCell(value string) string { + value = strings.ReplaceAll(value, "\r\n", " ") + value = strings.ReplaceAll(value, "\n", " ") + value = strings.ReplaceAll(value, "|", "\\|") + return value } -func writePrometheusDoc(doc []byte) error { - return atomicwrite.File(prometheusDocFile, doc) +func writeDoc(doc []byte) error { + return atomicwrite.File(docFile, doc) } diff --git a/scripts/metricsdocgen/main_test.go b/scripts/metricsdocgen/main_test.go new file mode 100644 index 0000000000000..a66091cd7bcf0 --- /dev/null +++ b/scripts/metricsdocgen/main_test.go @@ -0,0 +1,431 @@ +package main + +import ( + "bytes" + "strconv" + "testing" + + dto "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/require" +) + +const testDocPath = "docs/ai-coder/ai-gateway/monitoring.md" + +func TestParseSection(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + value string + want documentSection + wantErr string + }{ + { + name: "prefix", + value: "common=prefix:coder_ai_gateway_,!coder_ai_gateway_proxy_", + want: documentSection{ + name: "common", + filter: metricFilter{ + prefixes: []string{"coder_ai_gateway_"}, + excludePrefixes: []string{"coder_ai_gateway_proxy_"}, + }, + }, + }, + { + name: "multiple prefixes and exclusions", + value: "common=prefix:coder_ai_gateway_,coder_other_,!coder_ai_gateway_proxy_,!coder_ai_gateway_cost_control_", + want: documentSection{ + name: "common", + filter: metricFilter{ + prefixes: []string{"coder_ai_gateway_", "coder_other_"}, + excludePrefixes: []string{ + "coder_ai_gateway_proxy_", + "coder_ai_gateway_cost_control_", + }, + }, + }, + }, + { + name: "missing filter", + value: "common", + wantErr: "section must have the form", + }, + { + name: "invalid name", + value: "bad name=prefix:metric_", + wantErr: "may contain only", + }, + { + name: "invalid kind", + value: "common=contains:metric", + wantErr: "unknown filter kind", + }, + { + name: "missing filter kind", + value: "common=coder_ai_gateway_", + wantErr: "must have the form prefix:VALUE", + }, + { + name: "empty filter value", + value: "common=prefix:coder_ai_gateway_,,coder_other_", + wantErr: "empty filter value", + }, + { + name: "empty excluded prefix", + value: "common=prefix:coder_ai_gateway_,!", + wantErr: "empty excluded prefix", + }, + { + name: "only exclusions", + value: "common=prefix:!coder_ai_gateway_proxy_", + wantErr: "requires at least one included prefix", + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + + got, err := parseSection(testCase.value) + if testCase.wantErr != "" { + require.ErrorContains(t, err, testCase.wantErr) + return + } + require.NoError(t, err) + require.Equal(t, testCase.want, got) + }) + } +} + +func TestSectionFlagsSetRejectsDuplicates(t *testing.T) { + t.Parallel() + + var flags sectionFlags + require.NoError(t, flags.Set("common=prefix:coder_ai_gateway_")) + require.ErrorContains(t, flags.Set("common=prefix:coder_other_"), "duplicate section") + require.NoError(t, flags.Set("proxy=prefix:coder_ai_gateway_proxy_")) + require.Equal(t, "common,proxy", flags.String()) +} + +func TestFilterMetricFamilies(t *testing.T) { + t.Parallel() + + metrics := []*dto.MetricFamily{ + metricFamily("coder_ai_gateway_cost_control_blocked_users", dto.MetricType_GAUGE, "Blocked users.", "group_id"), + metricFamily("coder_ai_gateway_interceptions_total", dto.MetricType_COUNTER, "Intercepted requests.", "provider"), + metricFamily("coder_ai_gateway_proxy_mitm_requests_total", dto.MetricType_COUNTER, "Proxy requests.", "provider"), + } + + byPrefix := filterMetricFamilies(metrics, metricFilter{prefixes: []string{"coder_ai_gateway_proxy_"}}) + require.Equal(t, []string{"coder_ai_gateway_proxy_mitm_requests_total"}, metricNames(byPrefix)) + + byExcludedPrefix := filterMetricFamilies(metrics, metricFilter{ + prefixes: []string{"coder_ai_gateway_"}, + excludePrefixes: []string{ + "coder_ai_gateway_cost_control_", + "coder_ai_gateway_proxy_", + }, + }) + require.Equal(t, []string{"coder_ai_gateway_interceptions_total"}, metricNames(byExcludedPrefix)) + + require.Empty(t, filterMetricFamilies(metrics, metricFilter{prefixes: []string{"coder_absent_"}})) +} + +func TestUpdateDocumentSections(t *testing.T) { + t.Parallel() + + commonPrefix := string(namedGeneratorPrefix(testDocPath, "common")) + commonSuffix := string(namedGeneratorSuffix(testDocPath, "common")) + proxyPrefix := string(namedGeneratorPrefix(testDocPath, "proxy")) + proxySuffix := string(namedGeneratorSuffix(testDocPath, "proxy")) + doc := []byte("before\n" + commonPrefix + "\nstale common\n" + commonSuffix + "\nmiddle\n" + proxyPrefix + "\nstale proxy\n" + proxySuffix + "\nafter\n") + metrics := []*dto.MetricFamily{ + metricFamily("coder_ai_gateway_interceptions_total", dto.MetricType_COUNTER, "Intercepted requests.", "provider", "model"), + metricFamily("coder_ai_gateway_proxy_mitm_requests_total", dto.MetricType_COUNTER, "Proxy requests.", "provider"), + } + sections := []documentSection{ + {name: "common", filter: metricFilter{ + prefixes: []string{"coder_ai_gateway_"}, + excludePrefixes: []string{"coder_ai_gateway_proxy_"}, + }}, + {name: "proxy", filter: metricFilter{prefixes: []string{"coder_ai_gateway_proxy_"}}}, + } + + updated, err := updateDocumentSections(doc, metrics, testDocPath, sections) + require.NoError(t, err) + require.NotContains(t, string(updated), "stale common") + require.NotContains(t, string(updated), "stale proxy") + + commonBody := sectionBody(t, updated, commonPrefix, commonSuffix) + require.Contains(t, commonBody, "| `coder_ai_gateway_interceptions_total` | counter | Intercepted requests. | `model` `provider` |") + require.NotContains(t, commonBody, "coder_ai_gateway_proxy_mitm_requests_total") + + proxyBody := sectionBody(t, updated, proxyPrefix, proxySuffix) + require.Contains(t, proxyBody, "| `coder_ai_gateway_proxy_mitm_requests_total` | counter | Proxy requests. | `provider` |") + require.NotContains(t, proxyBody, "coder_ai_gateway_interceptions_total`") + + updatedAgain, err := updateDocumentSections(updated, metrics, testDocPath, sections) + require.NoError(t, err) + require.Equal(t, updated, updatedAgain) +} + +func TestUpdateDocumentSectionsRejectsEmptySection(t *testing.T) { + t.Parallel() + + prefix := string(namedGeneratorPrefix(testDocPath, "bogus")) + suffix := string(namedGeneratorSuffix(testDocPath, "bogus")) + doc := []byte(prefix + "\nstale\n" + suffix + "\n") + metrics := []*dto.MetricFamily{ + metricFamily("coder_ai_gateway_interceptions_total", dto.MetricType_COUNTER, "Intercepted requests.", "provider"), + } + sections := []documentSection{ + {name: "bogus", filter: metricFilter{prefixes: []string{"coder_absent_"}}}, + } + + _, err := updateDocumentSections(doc, metrics, testDocPath, sections) + require.ErrorContains(t, err, `section "bogus" matches no metrics`) +} + +func TestUpdateDocumentSectionsRejectsOrphanSection(t *testing.T) { + t.Parallel() + + commonPrefix := string(namedGeneratorPrefix(testDocPath, "common")) + commonSuffix := string(namedGeneratorSuffix(testDocPath, "common")) + orphanPrefix := string(namedGeneratorPrefix(testDocPath, "circuit-breakers")) + orphanSuffix := string(namedGeneratorSuffix(testDocPath, "circuit-breakers")) + doc := []byte(commonPrefix + "\nstale common\n" + commonSuffix + "\n" + orphanPrefix + "\nstale orphan\n" + orphanSuffix + "\n") + metrics := []*dto.MetricFamily{ + metricFamily("coder_ai_gateway_interceptions_total", dto.MetricType_COUNTER, "Intercepted requests.", "provider"), + } + sections := []documentSection{ + {name: "common", filter: metricFilter{prefixes: []string{"coder_ai_gateway_"}}}, + } + + _, err := updateDocumentSections(doc, metrics, testDocPath, sections) + require.ErrorContains(t, err, `no --section flag claims: "circuit-breakers"`) +} + +// A closing marker with no opening one reads as a section boundary that no +// invocation owns. +func TestUpdateDocumentSectionsRejectsStrayClosingMarker(t *testing.T) { + t.Parallel() + + prefix := string(namedGeneratorPrefix(testDocPath, "common")) + suffix := string(namedGeneratorSuffix(testDocPath, "common")) + stray := string(namedGeneratorSuffix(testDocPath, "circuit-breakers")) + doc := []byte(prefix + "\nstale\n" + suffix + "\n" + stray + "\n") + metrics := []*dto.MetricFamily{ + metricFamily("coder_ai_gateway_interceptions_total", dto.MetricType_COUNTER, "Intercepted requests.", "provider"), + } + sections := []documentSection{ + {name: "common", filter: metricFilter{prefixes: []string{"coder_ai_gateway_"}}}, + } + + _, err := updateDocumentSections(doc, metrics, testDocPath, sections) + require.ErrorContains(t, err, "closing marker with no opening marker") +} + +// A second closing marker leaves the text between it and the pair it follows +// inside no section at all. +func TestUpdateDocumentSectionsRejectsDuplicateClosingMarker(t *testing.T) { + t.Parallel() + + prefix := string(namedGeneratorPrefix(testDocPath, "common")) + suffix := string(namedGeneratorSuffix(testDocPath, "common")) + doc := []byte(prefix + "\nstale\n" + suffix + "\nbetween\n" + suffix + "\n") + metrics := []*dto.MetricFamily{ + metricFamily("coder_ai_gateway_interceptions_total", dto.MetricType_COUNTER, "Intercepted requests.", "provider"), + } + sections := []documentSection{ + {name: "common", filter: metricFilter{prefixes: []string{"coder_ai_gateway_"}}}, + } + + _, err := updateDocumentSections(doc, metrics, testDocPath, sections) + require.ErrorContains(t, err, "2 closing markers for the same section") +} + +// A document that carries the same section twice would keep every pair after +// the first stale, because only the first pair is rewritten. +func TestUpdateDocumentSectionsRejectsDuplicateSection(t *testing.T) { + t.Parallel() + + prefix := string(namedGeneratorPrefix(testDocPath, "common")) + suffix := string(namedGeneratorSuffix(testDocPath, "common")) + pair := prefix + "\nstale\n" + suffix + "\n" + doc := []byte(pair + pair) + metrics := []*dto.MetricFamily{ + metricFamily("coder_ai_gateway_interceptions_total", dto.MetricType_COUNTER, "Intercepted requests.", "provider"), + } + sections := []documentSection{ + {name: "common", filter: metricFilter{prefixes: []string{"coder_ai_gateway_"}}}, + } + + _, err := updateDocumentSections(doc, metrics, testDocPath, sections) + require.ErrorContains(t, err, "2 generated markers for the same section") +} + +// A marker naming another make target is never rewritten, and its section name +// alone would make it look claimed. +func TestUpdateDocumentSectionsRejectsForeignTargetMarker(t *testing.T) { + t.Parallel() + + prefix := string(namedGeneratorPrefix(testDocPath, "common")) + suffix := string(namedGeneratorSuffix(testDocPath, "common")) + foreignPrefix := string(namedGeneratorPrefix("docs/other/target.md", "common")) + foreignSuffix := string(namedGeneratorSuffix("docs/other/target.md", "common")) + doc := []byte(prefix + "\nstale\n" + suffix + "\n" + foreignPrefix + "\nstale foreign\n" + foreignSuffix + "\n") + metrics := []*dto.MetricFamily{ + metricFamily("coder_ai_gateway_interceptions_total", dto.MetricType_COUNTER, "Intercepted requests.", "provider"), + } + sections := []documentSection{ + {name: "common", filter: metricFilter{prefixes: []string{"coder_ai_gateway_"}}}, + } + + _, err := updateDocumentSections(doc, metrics, testDocPath, sections) + require.ErrorContains(t, err, `names make target "docs/other/target.md"`) + // The opener literal pins the check to the opener branch, which shares its + // error template with the closer branch. + require.ErrorContains(t, err, strconv.Quote(foreignPrefix)) +} + +// A closing marker naming another make target is never rewritten either, and +// the opening marker of its pair can be the current target's own. +func TestUpdateDocumentSectionsRejectsForeignTargetClosingMarker(t *testing.T) { + t.Parallel() + + prefix := string(namedGeneratorPrefix(testDocPath, "common")) + suffix := string(namedGeneratorSuffix(testDocPath, "common")) + foreignSuffix := string(namedGeneratorSuffix("docs/other/target.md", "common")) + doc := []byte(prefix + "\nstale\n" + suffix + "\n" + foreignSuffix + "\n") + metrics := []*dto.MetricFamily{ + metricFamily("coder_ai_gateway_interceptions_total", dto.MetricType_COUNTER, "Intercepted requests.", "provider"), + } + sections := []documentSection{ + {name: "common", filter: metricFilter{prefixes: []string{"coder_ai_gateway_"}}}, + } + + _, err := updateDocumentSections(doc, metrics, testDocPath, sections) + require.ErrorContains(t, err, `names make target "docs/other/target.md"`) + // The full closer literal is what separates this branch from the opener-side + // check, which cannot name a closing marker. + require.ErrorContains(t, err, strconv.Quote(foreignSuffix)) +} + +// The unnamed pair belongs to default mode, so --section mode never rewrites it. +func TestUpdateDocumentSectionsRejectsDefaultMarker(t *testing.T) { + t.Parallel() + + prefix := string(namedGeneratorPrefix(testDocPath, "common")) + suffix := string(namedGeneratorSuffix(testDocPath, "common")) + defaultPrefix := string(defaultGeneratorPrefix(testDocPath)) + defaultSuffix := string(defaultGeneratorSuffix(testDocPath)) + doc := []byte(prefix + "\nstale\n" + suffix + "\n" + defaultPrefix + "\nstale default\n" + defaultSuffix + "\n") + metrics := []*dto.MetricFamily{ + metricFamily("coder_ai_gateway_interceptions_total", dto.MetricType_COUNTER, "Intercepted requests.", "provider"), + } + sections := []documentSection{ + {name: "common", filter: metricFilter{prefixes: []string{"coder_ai_gateway_"}}}, + } + + _, err := updateDocumentSections(doc, metrics, testDocPath, sections) + require.ErrorContains(t, err, "unnamed generated section") +} + +// Default mode rewrites the unnamed pair only, so a named pair left in the +// document is as stale as an unclaimed section is in --section mode. +func TestUpdateDefaultSectionRejectsNamedMarker(t *testing.T) { + t.Parallel() + + prefix := string(defaultGeneratorPrefix(testDocPath)) + suffix := string(defaultGeneratorSuffix(testDocPath)) + stalePrefix := string(namedGeneratorPrefix(testDocPath, "stale")) + staleSuffix := string(namedGeneratorSuffix(testDocPath, "stale")) + doc := []byte(stalePrefix + "\nstale named\n" + staleSuffix + "\n" + prefix + "\nstale\n" + suffix + "\n") + metrics := []*dto.MetricFamily{ + metricFamily("coder_metric_total", dto.MetricType_COUNTER, "Requests.", "provider"), + } + + _, err := updateDefaultSection(doc, metrics, testDocPath) + require.ErrorContains(t, err, `document has generated section "stale", which default mode never rewrites`) +} + +// Two unnamed pairs leave everything after the first stale. +func TestUpdateDefaultSectionRejectsDuplicateMarkers(t *testing.T) { + t.Parallel() + + prefix := string(defaultGeneratorPrefix(testDocPath)) + suffix := string(defaultGeneratorSuffix(testDocPath)) + pair := prefix + "\nstale\n" + suffix + "\n" + doc := []byte(pair + pair) + metrics := []*dto.MetricFamily{ + metricFamily("coder_metric_total", dto.MetricType_COUNTER, "Requests.", "provider"), + } + + _, err := updateDefaultSection(doc, metrics, testDocPath) + require.ErrorContains(t, err, "2 generated markers for the same section") +} + +func TestUpdateDefaultSectionPreservesDefaultMarkers(t *testing.T) { + t.Parallel() + + const defaultDocPath = "docs/admin/integrations/prometheus.md" + prefix := string(defaultGeneratorPrefix(defaultDocPath)) + suffix := string(defaultGeneratorSuffix(defaultDocPath)) + doc := []byte("before\n" + prefix + "\nstale\n" + suffix + "\nafter\n") + metrics := []*dto.MetricFamily{ + metricFamily("z_metric", dto.MetricType_GAUGE, "Last.", "z", "a"), + metricFamily("a_metric", dto.MetricType_COUNTER, "First."), + } + + updated, err := updateDefaultSection(doc, metrics, defaultDocPath) + require.NoError(t, err) + require.Contains(t, string(updated), prefix) + require.Contains(t, string(updated), suffix) + require.NotContains(t, string(updated), "stale") + require.Less(t, bytes.Index(updated, []byte("z_metric")), bytes.Index(updated, []byte(suffix))) + require.Contains(t, string(updated), "| `z_metric` | gauge | Last. | `a` `z` |") + // A metric without labels renders an empty Labels cell. + require.Contains(t, string(updated), "| `a_metric` | counter | First. | |") +} + +func TestRenderMetricTableEscapesCellSeparators(t *testing.T) { + t.Parallel() + + table := string(renderMetricTable([]*dto.MetricFamily{ + metricFamily("coder_metric_total", dto.MetricType_COUNTER, "Requests by a|b.\nSecond line.", "provider"), + })) + require.Contains(t, table, "| `coder_metric_total` | counter | Requests by a\\|b. Second line. | `provider` |") +} + +func sectionBody(t *testing.T, doc []byte, prefix, suffix string) string { + t.Helper() + + start := bytes.Index(doc, []byte(prefix)) + require.GreaterOrEqual(t, start, 0) + start += len(prefix) + end := bytes.Index(doc[start:], []byte(suffix)) + require.GreaterOrEqual(t, end, 0) + return string(doc[start : start+end]) +} + +func metricFamily(name string, metricType dto.MetricType, help string, labels ...string) *dto.MetricFamily { + metric := &dto.Metric{} + for _, label := range labels { + labelValue := "" + metric.Label = append(metric.Label, &dto.LabelPair{Name: &label, Value: &labelValue}) + } + return &dto.MetricFamily{ + Name: &name, + Type: &metricType, + Help: &help, + Metric: []*dto.Metric{metric}, + } +} + +func metricNames(metrics []*dto.MetricFamily) []string { + names := make([]string, 0, len(metrics)) + for _, metric := range metrics { + names = append(names, metric.GetName()) + } + return names +} diff --git a/scripts/metricsdocgen/metrics b/scripts/metricsdocgen/metrics index ccce769144d45..f69ca54736a06 100644 --- a/scripts/metricsdocgen/metrics +++ b/scripts/metricsdocgen/metrics @@ -147,127 +147,3 @@ promhttp_metric_handler_requests_in_flight 1 promhttp_metric_handler_requests_total{code="200"} 2 promhttp_metric_handler_requests_total{code="500"} 0 promhttp_metric_handler_requests_total{code="503"} 0 -# HELP coder_ai_gateway_cost_control_blocked_requests_total The number of AI requests blocked because the initiator's budget was exceeded. -# TYPE coder_ai_gateway_cost_control_blocked_requests_total counter -coder_ai_gateway_cost_control_blocked_requests_total{group_id="a1b2c3d4-e5f6-7890-abcd-ef1234567890"} 0 -# HELP coder_ai_gateway_cost_control_blocked_users The number of users currently over their AI budget. -# TYPE coder_ai_gateway_cost_control_blocked_users gauge -coder_ai_gateway_cost_control_blocked_users{group_id="a1b2c3d4-e5f6-7890-abcd-ef1234567890"} 0 -# HELP coder_ai_gateway_cost_control_enforcement_duration_seconds The duration of AI budget enforcement checks, in seconds (outcome: allowed, blocked, error). -# TYPE coder_ai_gateway_cost_control_enforcement_duration_seconds histogram -coder_ai_gateway_cost_control_enforcement_duration_seconds_bucket{outcome="allowed",le="0.005"} 0 -coder_ai_gateway_cost_control_enforcement_duration_seconds_bucket{outcome="allowed",le="0.01"} 0 -coder_ai_gateway_cost_control_enforcement_duration_seconds_bucket{outcome="allowed",le="0.025"} 0 -coder_ai_gateway_cost_control_enforcement_duration_seconds_bucket{outcome="allowed",le="0.05"} 0 -coder_ai_gateway_cost_control_enforcement_duration_seconds_bucket{outcome="allowed",le="0.1"} 0 -coder_ai_gateway_cost_control_enforcement_duration_seconds_bucket{outcome="allowed",le="0.25"} 0 -coder_ai_gateway_cost_control_enforcement_duration_seconds_bucket{outcome="allowed",le="0.5"} 0 -coder_ai_gateway_cost_control_enforcement_duration_seconds_bucket{outcome="allowed",le="1"} 0 -coder_ai_gateway_cost_control_enforcement_duration_seconds_bucket{outcome="allowed",le="+Inf"} 0 -coder_ai_gateway_cost_control_enforcement_duration_seconds_sum{outcome="allowed"} 0 -coder_ai_gateway_cost_control_enforcement_duration_seconds_count{outcome="allowed"} 0 -# HELP coder_ai_gateway_cost_control_unpriced_token_usage_records_total The number of recorded AI token-usage records for which no (provider_type, model) price was found. provider is the provider instance name, and provider_type is its configured type. -# TYPE coder_ai_gateway_cost_control_unpriced_token_usage_records_total counter -coder_ai_gateway_cost_control_unpriced_token_usage_records_total{model="gpt-5-nano",provider="openai",provider_type="openai"} 0 -# HELP coder_ai_gateway_injected_tool_invocations_total The number of times an injected MCP tool was invoked by AI Gateway. -# TYPE coder_ai_gateway_injected_tool_invocations_total counter -coder_ai_gateway_injected_tool_invocations_total{model="gpt-5-nano",name="coder_list_templates",provider="openai",server="https://xxx.pit-1.try.coder.app/api/experimental/mcp/http"} 1 -# HELP coder_ai_gateway_interceptions_duration_seconds The total duration of intercepted requests, in seconds. The majority of this time will be the upstream processing of the request. AI Gateway has no control over upstream processing time, so it's just an illustrative metric. -# TYPE coder_ai_gateway_interceptions_duration_seconds histogram -coder_ai_gateway_interceptions_duration_seconds_bucket{model="gpt-5-nano",provider="openai",le="0.5"} 0 -coder_ai_gateway_interceptions_duration_seconds_bucket{model="gpt-5-nano",provider="openai",le="2"} 0 -coder_ai_gateway_interceptions_duration_seconds_bucket{model="gpt-5-nano",provider="openai",le="5"} 3 -coder_ai_gateway_interceptions_duration_seconds_bucket{model="gpt-5-nano",provider="openai",le="15"} 6 -coder_ai_gateway_interceptions_duration_seconds_bucket{model="gpt-5-nano",provider="openai",le="30"} 6 -coder_ai_gateway_interceptions_duration_seconds_bucket{model="gpt-5-nano",provider="openai",le="60"} 6 -coder_ai_gateway_interceptions_duration_seconds_bucket{model="gpt-5-nano",provider="openai",le="120"} 6 -coder_ai_gateway_interceptions_duration_seconds_bucket{model="gpt-5-nano",provider="openai",le="+Inf"} 6 -coder_ai_gateway_interceptions_duration_seconds_sum{model="gpt-5-nano",provider="openai"} 34.120188692 -coder_ai_gateway_interceptions_duration_seconds_count{model="gpt-5-nano",provider="openai"} 6 -# HELP coder_ai_gateway_interceptions_inflight The number of intercepted requests which are being processed. -# TYPE coder_ai_gateway_interceptions_inflight gauge -coder_ai_gateway_interceptions_inflight{model="gpt-5-nano",provider="openai",route="/v1/chat/completions"} 0 -# HELP coder_ai_gateway_interceptions_total The count of intercepted requests. -# TYPE coder_ai_gateway_interceptions_total counter -coder_ai_gateway_interceptions_total{client="Codex",initiator_id="95f6752b-08cc-4cf1-97f7-c2165e3519c5",method="POST",model="gpt-5-nano",provider="openai",route="/v1/chat/completions",status="completed"} 6 -# HELP coder_ai_gateway_key_pool_state The number of keys currently in each state (state: valid, temporary, permanent). -# TYPE coder_ai_gateway_key_pool_state gauge -coder_ai_gateway_key_pool_state{provider="openai",state="valid"} 2 -coder_ai_gateway_key_pool_state{provider="openai",state="temporary"} 0 -coder_ai_gateway_key_pool_state{provider="openai",state="permanent"} 0 -# HELP coder_ai_gateway_key_pool_state_transitions_total The number of API key state transitions during failover (reason: rate_limited, unauthorized). -# TYPE coder_ai_gateway_key_pool_state_transitions_total counter -coder_ai_gateway_key_pool_state_transitions_total{provider="openai",reason="rate_limited"} 1 -# HELP coder_ai_gateway_key_pool_exhaustions_total The number of times the key pool was exhausted with no usable key (outcome: rate_limited, auth_failed). -# TYPE coder_ai_gateway_key_pool_exhaustions_total counter -coder_ai_gateway_key_pool_exhaustions_total{provider="openai",outcome="rate_limited"} 1 -# HELP coder_ai_gateway_key_pool_failover_attempts The number of keys attempted before success or exhaustion, per interception for bridged requests and per request for passthrough requests. -# TYPE coder_ai_gateway_key_pool_failover_attempts histogram -coder_ai_gateway_key_pool_failover_attempts_bucket{provider="openai",le="1"} 0 -coder_ai_gateway_key_pool_failover_attempts_bucket{provider="openai",le="2"} 1 -coder_ai_gateway_key_pool_failover_attempts_bucket{provider="openai",le="3"} 1 -coder_ai_gateway_key_pool_failover_attempts_bucket{provider="openai",le="4"} 1 -coder_ai_gateway_key_pool_failover_attempts_bucket{provider="openai",le="5"} 1 -coder_ai_gateway_key_pool_failover_attempts_bucket{provider="openai",le="10"} 1 -coder_ai_gateway_key_pool_failover_attempts_bucket{provider="openai",le="25"} 1 -coder_ai_gateway_key_pool_failover_attempts_bucket{provider="openai",le="+Inf"} 1 -coder_ai_gateway_key_pool_failover_attempts_sum{provider="openai"} 2 -coder_ai_gateway_key_pool_failover_attempts_count{provider="openai"} 1 -# HELP coder_ai_gateway_non_injected_tool_selections_total The number of times an AI model selected a tool to be invoked by the client. -# TYPE coder_ai_gateway_non_injected_tool_selections_total counter -coder_ai_gateway_non_injected_tool_selections_total{model="gpt-5-nano",name="read_file",provider="openai"} 2 -# HELP coder_ai_gateway_prompts_total The number of prompts issued by users (initiators). -# TYPE coder_ai_gateway_prompts_total counter -coder_ai_gateway_prompts_total{client="Codex",initiator_id="95f6752b-08cc-4cf1-97f7-c2165e3519c5",model="gpt-5-nano",provider="openai"} 4 -# HELP coder_ai_gateway_tokens_total The number of tokens used by intercepted requests. -# TYPE coder_ai_gateway_tokens_total counter -coder_ai_gateway_tokens_total{client="Codex",initiator_id="95f6752b-08cc-4cf1-97f7-c2165e3519c5",model="gpt-5-nano",provider="openai",type="completion_accepted_prediction"} 0 -coder_ai_gateway_tokens_total{client="Codex",initiator_id="95f6752b-08cc-4cf1-97f7-c2165e3519c5",model="gpt-5-nano",provider="openai",type="completion_audio"} 0 -coder_ai_gateway_tokens_total{client="Codex",initiator_id="95f6752b-08cc-4cf1-97f7-c2165e3519c5",model="gpt-5-nano",provider="openai",type="completion_reasoning"} 1664 -coder_ai_gateway_tokens_total{client="Codex",initiator_id="95f6752b-08cc-4cf1-97f7-c2165e3519c5",model="gpt-5-nano",provider="openai",type="completion_rejected_prediction"} 0 -coder_ai_gateway_tokens_total{client="Codex",initiator_id="95f6752b-08cc-4cf1-97f7-c2165e3519c5",model="gpt-5-nano",provider="openai",type="input"} 13823 -coder_ai_gateway_tokens_total{client="Codex",initiator_id="95f6752b-08cc-4cf1-97f7-c2165e3519c5",model="gpt-5-nano",provider="openai",type="output"} 2014 -coder_ai_gateway_tokens_total{client="Codex",initiator_id="95f6752b-08cc-4cf1-97f7-c2165e3519c5",model="gpt-5-nano",provider="openai",type="prompt_audio"} 0 -coder_ai_gateway_tokens_total{client="Codex",initiator_id="95f6752b-08cc-4cf1-97f7-c2165e3519c5",model="gpt-5-nano",provider="openai",type="prompt_cached"} 31872 -# HELP coder_ai_gateway_circuit_breaker_rejects_total Total number of requests rejected due to open circuit breaker. -# TYPE coder_ai_gateway_circuit_breaker_rejects_total counter -coder_ai_gateway_circuit_breaker_rejects_total{provider="",endpoint="",model=""} 0 -# HELP coder_ai_gateway_circuit_breaker_state Current state of the circuit breaker (0=closed, 0.5=half-open, 1=open). -# TYPE coder_ai_gateway_circuit_breaker_state gauge -coder_ai_gateway_circuit_breaker_state{provider="",endpoint="",model=""} 0 -# HELP coder_ai_gateway_circuit_breaker_trips_total Total number of times the circuit breaker transitioned to open state. -# TYPE coder_ai_gateway_circuit_breaker_trips_total counter -coder_ai_gateway_circuit_breaker_trips_total{provider="",endpoint="",model=""} 0 -# HELP coder_ai_gateway_passthrough_total The count of requests which were not intercepted but passed through to the upstream. -# TYPE coder_ai_gateway_passthrough_total counter -coder_ai_gateway_passthrough_total{provider="",route="",method=""} 0 -# HELP coder_ai_gateway_proxy_connect_sessions_total Total number of CONNECT sessions established. -# TYPE coder_ai_gateway_proxy_connect_sessions_total counter -coder_ai_gateway_proxy_connect_sessions_total{type=""} 0 -# HELP coder_ai_gateway_proxy_inflight_mitm_requests Number of MITM requests currently being processed. -# TYPE coder_ai_gateway_proxy_inflight_mitm_requests gauge -coder_ai_gateway_proxy_inflight_mitm_requests{provider=""} 0 -# HELP coder_ai_gateway_proxy_mitm_requests_total Total number of MITM requests handled by the proxy. -# TYPE coder_ai_gateway_proxy_mitm_requests_total counter -coder_ai_gateway_proxy_mitm_requests_total{provider=""} 0 -# HELP coder_ai_gateway_proxy_mitm_responses_total Total number of MITM responses by HTTP status code class. -# TYPE coder_ai_gateway_proxy_mitm_responses_total counter -coder_ai_gateway_proxy_mitm_responses_total{code="",provider=""} 0 -# HELP coder_ai_gateway_provider_info One series per configured AI provider. Value is always 1; the status label (enabled, disabled, error) carries the alertable signal. -# TYPE coder_ai_gateway_provider_info gauge -coder_ai_gateway_provider_info{provider_name="",provider_type="",status=""} 0 -# HELP coder_ai_gateway_providers_last_reload_timestamp_seconds Unix timestamp of the last provider reload attempt, success or failure. -# TYPE coder_ai_gateway_providers_last_reload_timestamp_seconds gauge -coder_ai_gateway_providers_last_reload_timestamp_seconds 0 -# HELP coder_ai_gateway_providers_last_reload_success_timestamp_seconds Unix timestamp of the last provider reload that successfully refreshed the pool. A gap against the providers_last_reload_timestamp_seconds gauge means the loop is firing but the refresh function is failing. -# TYPE coder_ai_gateway_providers_last_reload_success_timestamp_seconds gauge -coder_ai_gateway_providers_last_reload_success_timestamp_seconds 0 -# HELP coder_ai_gateway_proxy_provider_info One series per configured AI provider. Value is always 1; the status label (enabled, disabled, error) carries the alertable signal. -# TYPE coder_ai_gateway_proxy_provider_info gauge -coder_ai_gateway_proxy_provider_info{provider_name="",provider_type="",status=""} 0 -# HELP coder_ai_gateway_proxy_providers_last_reload_timestamp_seconds Unix timestamp of the last provider reload attempt, success or failure. -# TYPE coder_ai_gateway_proxy_providers_last_reload_timestamp_seconds gauge -coder_ai_gateway_proxy_providers_last_reload_timestamp_seconds 0 -# HELP coder_ai_gateway_proxy_providers_last_reload_success_timestamp_seconds Unix timestamp of the last provider reload that successfully refreshed the router. A gap against the providers_last_reload_timestamp_seconds gauge means the loop is firing but the refresh function is failing. -# TYPE coder_ai_gateway_proxy_providers_last_reload_success_timestamp_seconds gauge -coder_ai_gateway_proxy_providers_last_reload_success_timestamp_seconds 0 diff --git a/scripts/metricsdocgen/scanner/scanner.go b/scripts/metricsdocgen/scanner/scanner.go index 619469285d73d..d0ae5b7bc433c 100644 --- a/scripts/metricsdocgen/scanner/scanner.go +++ b/scripts/metricsdocgen/scanner/scanner.go @@ -15,9 +15,10 @@ import ( "io" "io/fs" "log" + "maps" "os" "path/filepath" - "sort" + "slices" "strings" "golang.org/x/term" @@ -28,6 +29,7 @@ import ( // Add or remove directories here to control the scanner's scope. var scanDirs = []string{ "agent", + "aibridge", "coderd", "enterprise", "provisionerd", @@ -36,16 +38,65 @@ var scanDirs = []string{ // skipPaths lists files that should be excluded from scanning. Their metrics // must be maintained in the static metrics file instead. -// TODO(ssncferreira): Add support for resolving WrapRegistererWithPrefix to -// -// eliminate the need for this skip list. var skipPaths = []string{ - "coderd/aibridged/metrics.go", - "coderd/aibridgedserver/metrics.go", - "enterprise/aibridgeproxyd/metrics.go", "enterprise/scaletest/agentfake/metrics.go", } +// canonicalPrefixConst is the name of the constant a package declares to state +// the Prometheus prefix that its registration wiring applies at runtime. +const canonicalPrefixConst = "PrometheusMetricPrefix" + +// knownMetricNamespaces lists the namespaces that scanned metric names are +// expected to carry. A name outside this set means the metric is registered +// through a prefixing registerer whose prefix the scanner does not know about, +// so validateMetricNamespaces rejects it rather than emitting a name that no +// scrape produces. +var knownMetricNamespaces = []string{ + "agent_", + "coder_", + "coderd_", +} + +// metricPrefixSources maps a file that defines metrics to the file declaring +// the canonical prefix constant applied to them. Keying by the defining file, +// rather than by metric name, lets the scanner bypass the registration wiring, +// and reading the constant rather than repeating its value keeps the generated +// names in step with a prefix rename. Deprecated alias prefixes stay out of the +// generated reference because only the canonical constant is read. +// +// A new file whose metrics are prefixed at registration needs an entry here. +// Two checks catch most missing entries: validateMetricNamespaces rejects the +// resulting name when it carries no known namespace, and +// TestScanAIGatewayFilesAreMapped in scanner_test.go walks the AI Gateway trees +// for unmapped metric files. Neither catches a file outside those trees whose +// options already set a known Namespace, because the scanned name then looks +// valid while the registration wiring adds a further prefix. +var metricPrefixSources = map[string]string{ + "aibridge/keypool/state_collector.go": "aibridge/metrics/metrics.go", + "aibridge/metrics/metrics.go": "aibridge/metrics/metrics.go", + "coderd/aibridged/metrics.go": "aibridge/metrics/metrics.go", + "coderd/aibridgedserver/metrics.go": "aibridge/metrics/metrics.go", + "enterprise/aibridgeproxyd/metrics.go": "enterprise/aibridgeproxyd/metrics.go", +} + +// metricPrefixPaths holds the keys of metricPrefixSources ordered by descending +// length, which is defensive against a future key that could match the same +// file as another key. No current pair of keys is suffix-comparable. +var metricPrefixPaths = sortedMetricPrefixPaths() + +func sortedMetricPrefixPaths() []string { + return slices.SortedFunc(maps.Keys(metricPrefixSources), func(a, b string) int { + if diff := len(b) - len(a); diff != 0 { + return diff + } + return strings.Compare(a, b) + }) +} + +// canonicalPrefixes caches the constant value read from each prefix source +// file, keyed by that file's path. The scanner runs in a single goroutine. +var canonicalPrefixes = map[string]string{} + // MetricType represents the type of Prometheus metric. type MetricType string @@ -109,25 +160,49 @@ func main() { if err != nil { log.Fatalf("Failed to scan directories: %v", err) } + metrics = prepareMetrics(metrics) + if err := validateMetricNamespaces(metrics); err != nil { + log.Fatalf("Failed to validate metric names: %v", err) + } - // Duplicates are not expected since Prometheus enforces unique metric names at registration. - uniqueMetrics := make(map[string]Metric) - for _, m := range metrics { - uniqueMetrics[m.Name] = m + writeMetrics(metrics, os.Stdout) + logf("Successfully parsed %d metrics", len(metrics)) +} + +// validateMetricNamespaces rejects scanned metrics whose names carry no known +// namespace. Such a name means the metrics are prefixed by their registerer and +// the defining file has no metricPrefixSources entry, which would document a +// name that no scrape produces. +func validateMetricNamespaces(metrics []Metric) error { + var unknown []string + for _, metric := range metrics { + if !slices.ContainsFunc(knownMetricNamespaces, func(namespace string) bool { + return strings.HasPrefix(metric.Name, namespace) + }) { + unknown = append(unknown, metric.Name) + } } - metrics = make([]Metric, 0, len(uniqueMetrics)) - for _, m := range uniqueMetrics { - metrics = append(metrics, m) + if len(unknown) == 0 { + return nil } + return xerrors.Errorf("metrics carry no known namespace (%v): set Namespace in the metric options, "+ + "add a metricPrefixSources entry when a registerer applies the prefix, "+ + "or extend knownMetricNamespaces: %s", + knownMetricNamespaces, strings.Join(unknown, ", ")) +} - // Sort metrics by name for consistent output across runs. - sort.Slice(metrics, func(i, j int) bool { - return metrics[i].Name < metrics[j].Name +// prepareMetrics deduplicates and sorts the scanned metrics. Duplicates are not +// expected since Prometheus enforces unique metric names at registration, so the +// deduplication is a safety net rather than a functional requirement. Sorting +// keeps the generated output stable across runs. +func prepareMetrics(metrics []Metric) []Metric { + uniqueMetrics := make(map[string]Metric) + for _, metric := range metrics { + uniqueMetrics[metric.Name] = metric + } + return slices.SortedFunc(maps.Values(uniqueMetrics), func(a, b Metric) int { + return strings.Compare(a.Name, b.Name) }) - - writeMetrics(metrics, os.Stdout) - - logf("Successfully parsed %d metrics", len(metrics)) } // scanAllDirs scans all configured directories for metric definitions. @@ -203,6 +278,11 @@ func scanFile(path string) ([]Metric, error) { // Collect file-local const and var declarations for resolving references. decls := collectDecls(file) + prefix, err := metricPrefixForPath(path) + if err != nil { + return nil, err + } + var metrics []Metric // Walk the AST looking for metric registration calls. @@ -214,6 +294,7 @@ func scanFile(path string) ([]Metric, error) { metric, ok := extractMetricFromCall(call, decls) if ok { + metric.Name = prefix + metric.Name if metric.Help == "" { warnf("metric %q has no HELP description, skipping", metric.Name) // Skip metrics without descriptions, they should be fixed in the source code @@ -229,6 +310,43 @@ func scanFile(path string) ([]Metric, error) { return metrics, nil } +// metricPrefixForPath returns the canonical prefix that the registration wiring +// applies to the metrics defined in path, or an empty string when the file +// defines unprefixed metrics. +func metricPrefixForPath(path string) (string, error) { + cleaned := filepath.ToSlash(filepath.Clean(path)) + for _, definingPath := range metricPrefixPaths { + if cleaned != definingPath && !strings.HasSuffix(cleaned, "/"+definingPath) { + continue + } + prefix, err := canonicalPrefix(metricPrefixSources[definingPath]) + if err != nil { + return "", xerrors.Errorf("resolving metric prefix for %s: %w", path, err) + } + return prefix, nil + } + return "", nil +} + +// canonicalPrefix reads canonicalPrefixConst from the given file. The path is +// relative to the repository root, which is the scanner's working directory. +func canonicalPrefix(sourcePath string) (string, error) { + if prefix, ok := canonicalPrefixes[sourcePath]; ok { + return prefix, nil + } + + file, err := parser.ParseFile(token.NewFileSet(), sourcePath, nil, parser.SkipObjectResolution) + if err != nil { + return "", xerrors.Errorf("parsing %s: %w", sourcePath, err) + } + prefix, ok := collectDecls(file).strings[canonicalPrefixConst] + if !ok { + return "", xerrors.Errorf("%s does not declare %s", sourcePath, canonicalPrefixConst) + } + canonicalPrefixes[sourcePath] = prefix + return prefix, nil +} + // collectPackageConsts collects exported string constants from a file into // the global packageDeclarations map, keyed by package name. func collectPackageConsts(file *ast.File) { @@ -369,22 +487,29 @@ func collectDecls(file *ast.File) declarations { } // extractLabels extracts label names from an expression passed as an argument -// to a metric constructor. Handles both inline []string literals and -// variable references from decls. -// Examples: -// - []string{"label1", "label2"}: ["label1", "label2"] (inline literal) -// - myLabels: resolved value of myLabels variable (variable reference) +// to a metric constructor. It handles inline literals, variable references, +// and append calls that extend a shared label slice. func extractLabels(expr ast.Expr, decls declarations) []string { switch e := expr.(type) { case *ast.CompositeLit: - // []string{"label1", "label2"} return extractStringSlice(e, decls) case *ast.Ident: - // Variable reference like 'labels'. - if labels, ok := decls.stringSlices[e.Name]; ok { - return labels + return slices.Clone(decls.stringSlices[e.Name]) + case *ast.CallExpr: + ident, ok := e.Fun.(*ast.Ident) + if !ok || ident.Name != "append" || len(e.Args) < 2 { + return nil } - return nil + + labels := extractLabels(e.Args[0], decls) + for _, arg := range e.Args[1:] { + if label := resolveStringExpr(arg, decls); label != "" { + labels = append(labels, label) + continue + } + labels = append(labels, extractLabels(arg, decls)...) + } + return labels } return nil } diff --git a/scripts/metricsdocgen/scanner/scanner_test.go b/scripts/metricsdocgen/scanner/scanner_test.go new file mode 100644 index 0000000000000..e6f7c6ed83370 --- /dev/null +++ b/scripts/metricsdocgen/scanner/scanner_test.go @@ -0,0 +1,257 @@ +package main + +import ( + "go/ast" + "go/parser" + "go/token" + "io/fs" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +//nolint:paralleltest // Prefix constants are resolved relative to the process working directory. +func TestMetricPrefixForPath(t *testing.T) { + t.Chdir("../../..") + + testCases := []struct { + name string + path string + want string + }{ + { + name: "common metric", + path: "aibridge/metrics/metrics.go", + want: "coder_ai_gateway_", + }, + { + name: "metric prefixed by another package", + path: "coderd/aibridgedserver/metrics.go", + want: "coder_ai_gateway_", + }, + { + name: "absolute proxy metric", + path: "/repo/enterprise/aibridgeproxyd/metrics.go", + want: "coder_ai_gateway_proxy_", + }, + { + name: "ordinary metric", + path: "coderd/metrics.go", + want: "", + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + prefix, err := metricPrefixForPath(testCase.path) + require.NoError(t, err) + require.Equal(t, testCase.want, prefix) + }) + } +} + +//nolint:paralleltest // Prefix constants are resolved relative to the process working directory. +func TestScanFileAppliesCanonicalPrefixAndAppendLabels(t *testing.T) { + // The metric options come from the fixture below, but the prefix comes from + // the real `aibridge/metrics/metrics.go`, because metricPrefixForPath + // suffix-matches the fixture path and reads the constant from the repository. + source := `package metrics +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) +var baseLabels = []string{"provider", "model"} +func newMetrics(reg prometheus.Registerer) { + promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ + Subsystem: "interceptions", + Name: "total", + Help: "Requests.", + }, append(baseLabels, "status", "route")) +}` + + path := writeSourceFile(t, "aibridge/metrics/metrics.go", source) + t.Chdir("../../..") + + metrics, err := scanFile(path) + require.NoError(t, err) + require.Equal(t, []Metric{{ + Name: "coder_ai_gateway_interceptions_total", + Type: MetricTypeCounter, + Help: "Requests.", + Labels: []string{"provider", "model", "status", "route"}, + }}, metrics) +} + +func TestExtractLabelsAppendDoesNotMutateBaseLabels(t *testing.T) { + t.Parallel() + + // The base slice has three elements so that extractStringSlice leaves spare + // capacity. Without the defensive copy in the ident branch, the second + // append writes over the first result's backing array. + file, err := parser.ParseFile(token.NewFileSet(), "metrics.go", `package metrics +var baseLabels = []string{"provider", "model", "route"} +var first = append(baseLabels, "status") +var second = append(baseLabels, "outcome") +`, parser.SkipObjectResolution) + require.NoError(t, err) + decls := collectDecls(file) + require.Greater(t, cap(decls.stringSlices["baseLabels"]), len(decls.stringSlices["baseLabels"])) + + firstLabels := extractLabels(findValueExpr(t, file, "first"), decls) + secondLabels := extractLabels(findValueExpr(t, file, "second"), decls) + require.Equal(t, []string{"provider", "model", "route", "status"}, firstLabels) + require.Equal(t, []string{"provider", "model", "route", "outcome"}, secondLabels) + require.Equal(t, []string{"provider", "model", "route"}, decls.stringSlices["baseLabels"]) +} + +func TestValidateMetricNamespaces(t *testing.T) { + t.Parallel() + + require.NoError(t, validateMetricNamespaces([]Metric{ + {Name: "coder_ai_gateway_interceptions_total"}, + {Name: "coderd_agents_up"}, + {Name: "agent_boundary_log_proxy_logs_dropped_total"}, + })) + + // A metric defined in a package whose registerer applies the prefix, but with + // no metricPrefixSources entry, reaches this check as a bare local name. + err := validateMetricNamespaces([]Metric{ + {Name: "coder_ai_gateway_interceptions_total"}, + {Name: "outside_counter_total"}, + }) + require.ErrorContains(t, err, "outside_counter_total") + require.ErrorContains(t, err, "metricPrefixSources") +} + +//nolint:paralleltest // The scanner resolves paths relative to the process working directory. +func TestScanAIGatewayFilesAreMapped(t *testing.T) { + t.Chdir("../../..") + + // Metrics defined in these trees are registered through a prefixed + // registerer, so every file that defines metrics must appear in + // metricPrefixSources. A missing entry emits an unprefixed name that no + // documentation section matches. + aiGatewayDirs := []string{ + "aibridge", + "coderd/aibridged", + "coderd/aibridgedserver", + "enterprise/aibridgeproxyd", + } + for _, dir := range aiGatewayDirs { + require.NoError(t, filepath.WalkDir(dir, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { + return nil + } + metrics, err := scanFile(path) + require.NoError(t, err) + if len(metrics) > 0 { + prefix, err := metricPrefixForPath(path) + require.NoError(t, err) + require.NotEmpty(t, prefix, + "%s defines metrics but has no metricPrefixSources entry", path) + } + return nil + })) + } +} + +//nolint:paralleltest // The scanner resolves paths relative to the process working directory. +func TestScanAllDirsCanonicalAIGatewaySets(t *testing.T) { + t.Chdir("../../..") + + metrics, err := scanAllDirs() + require.NoError(t, err) + prepared := prepareMetrics(metrics) + require.NoError(t, validateMetricNamespaces(prepared)) + require.True(t, slices.IsSortedFunc(prepared, func(a, b Metric) int { + return strings.Compare(a.Name, b.Name) + })) + + var common, costControl, proxy []string + for _, metric := range prepared { + switch { + case strings.HasPrefix(metric.Name, "coder_ai_gateway_proxy_"): + proxy = append(proxy, metric.Name) + case strings.HasPrefix(metric.Name, "coder_ai_gateway_cost_control_"): + costControl = append(costControl, metric.Name) + case strings.HasPrefix(metric.Name, "coder_ai_gateway_"): + common = append(common, metric.Name) + } + } + + require.Equal(t, []string{ + "coder_ai_gateway_circuit_breaker_rejects_total", + "coder_ai_gateway_circuit_breaker_state", + "coder_ai_gateway_circuit_breaker_trips_total", + "coder_ai_gateway_injected_tool_invocations_total", + "coder_ai_gateway_interceptions_duration_seconds", + "coder_ai_gateway_interceptions_inflight", + "coder_ai_gateway_interceptions_total", + "coder_ai_gateway_key_pool_exhaustions_total", + "coder_ai_gateway_key_pool_failover_attempts", + "coder_ai_gateway_key_pool_state", + "coder_ai_gateway_key_pool_state_transitions_total", + "coder_ai_gateway_non_injected_tool_selections_total", + "coder_ai_gateway_passthrough_total", + "coder_ai_gateway_prompts_total", + "coder_ai_gateway_provider_info", + "coder_ai_gateway_providers_last_reload_success_timestamp_seconds", + "coder_ai_gateway_providers_last_reload_timestamp_seconds", + "coder_ai_gateway_tokens_total", + }, common) + require.Equal(t, []string{ + "coder_ai_gateway_cost_control_blocked_requests_total", + "coder_ai_gateway_cost_control_blocked_users", + "coder_ai_gateway_cost_control_enforcement_duration_seconds", + "coder_ai_gateway_cost_control_unpriced_token_usage_records_total", + }, costControl) + require.Equal(t, []string{ + "coder_ai_gateway_proxy_connect_sessions_total", + "coder_ai_gateway_proxy_inflight_mitm_requests", + "coder_ai_gateway_proxy_mitm_requests_total", + "coder_ai_gateway_proxy_mitm_responses_total", + "coder_ai_gateway_proxy_provider_info", + "coder_ai_gateway_proxy_providers_last_reload_success_timestamp_seconds", + "coder_ai_gateway_proxy_providers_last_reload_timestamp_seconds", + }, proxy) +} + +func writeSourceFile(t *testing.T, relativePath, source string) string { + t.Helper() + + path := filepath.Join(t.TempDir(), relativePath) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(source), 0o600)) + return path +} + +func findValueExpr(t *testing.T, file *ast.File, name string) ast.Expr { + t.Helper() + + for _, decl := range file.Decls { + genDecl, ok := decl.(*ast.GenDecl) + if !ok { + continue + } + for _, spec := range genDecl.Specs { + valueSpec, ok := spec.(*ast.ValueSpec) + if !ok { + continue + } + for i, ident := range valueSpec.Names { + if ident.Name == name && i < len(valueSpec.Values) { + return valueSpec.Values[i] + } + } + } + } + t.Fatalf("value %q not found", name) + return nil +}