From d52393bd2870ad117ceb60a38a88e1af9d149406 Mon Sep 17 00:00:00 2001 From: Scott Miller Date: Fri, 4 Sep 2026 03:54:08 +0000 Subject: [PATCH] docs(examples/monitoring): add chatd lifecycle Grafana dashboard Add a Grafana dashboard that plots the chatd stage histogram and the turn-end rollups one tree level at a time with a selectable summary statistic, plus summary panels for the chat pipeline and the per-turn partition of turn wall time. The README documents the stage hierarchy by tree level, the dashboard dimensions, each panel group, and setup. Rows that need only the basic stage metrics level are expanded by default; the per-turn distribution panels and the stages observed only at the full level sit in collapsed rows named for the level. A stat panel reports the level from coderd_chatd_stage_metrics_level. The README adds a metric levels section with series cost, example alert rules on the round bucket edges, and a scrape-time relabel snippet for alert-only deployments. --- .../grafana/chatd-lifecycle/README.md | 368 ++ .../grafana/chatd-lifecycle/dashboard.json | 3105 +++++++++++++++++ 2 files changed, 3473 insertions(+) create mode 100644 examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md create mode 100644 examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json diff --git a/examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md b/examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md new file mode 100644 index 00000000000..71bd6776fe6 --- /dev/null +++ b/examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md @@ -0,0 +1,368 @@ +# Chatd Chat Lifecycle Grafana Dashboard + +A Grafana dashboard for diagnosing where time goes in Coder Agents chat +sessions. It plots the stage histogram +`coderd_chatd_stage_duration_seconds{stage,scope,chat_kind,model}` +and the turn-end rollups one tree level at a time with a selectable +summary statistic (mean, p50, p90, p95, p99), plus summary panels for the +whole chat pipeline and a per-turn partition of turn wall time. + +Stage hierarchy, by tree level. `[basic]` marks the stages observed at +`--chat-stage-metrics=basic`; the rest are span-only until the level is +`full` (see [Metric levels](#metric-levels)). The default level is `off`, +which exposes no stage or turn families: + +```text +L0 chat_turn [basic] one sample per turn +L1 ├── queue_wait [basic] queued message insert -> promotion +L1 ├── capacity_wait [basic] concurrent-agent limiter wait +L1 ├── acquisition [basic] trigger message -> worker pickup +L1 └── generation_step [full] one step of a turn (repeats) +L2 ├── prepare [full] prompt build, model resolution, context hydration +L3 │ └── mcp_connect [basic] MCP server connection +L2 ├── retry_backoff [basic] wait between provider attempts +L2 ├── stream [basic] provider stream open -> close +L3 │ ├── time_to_first_token [basic] request open -> first streamed part +L3 │ └── provider_attempt [basic] one provider HTTP round trip, closed on headers (per retry) +L2 ├── thinking [full] reasoning part duration (window inside stream) +L2 ├── tool_call [basic] one local tool call +L2 ├── commit [basic] step persistence transaction +L2 └── compaction [full] auxiliary compaction call +``` + +The tree follows span parentage. Stages overlap in wall time (tool calls +and thinking happen inside the stream; `time_to_first_token` and +`provider_attempt` cover nearly the same window under `stream`), so +per-stage values are a stage-time profile, not a strict decomposition, +and quantile statistics are not additive across stages. + +## Metric levels + +`--chat-stage-metrics` (`CODER_CHAT_STAGE_METRICS`) on `coder server` +selects how much of the instrumentation is exposed as Prometheus series. +Tracing spans are emitted at every level. + +| Level | Families | Stages on `stage_duration_seconds` | +|-----------------|-----------------------------------------------------------------------------------------------|------------------------------------| +| `off` (default) | `coderd_chatd_stage_metrics_level` only | none | +| `basic` | `stage_duration_seconds`, `turn_time_seconds`, `stage_anomalies_total`, `stage_metrics_level` | the `[basic]` stages above | +| `full` | basic plus `turn_stage_seconds`, `turn_stage_count`, `stage_share_of_turn`, `turn_time_share` | all 15 | + +The label schema is the same at every level, so a query written for +`full` returns "No data" rather than an error at `basic`. The dashboard +is laid out accordingly: every row that is expanded by default works at +`basic`; rows whose title ends in `(full level only)` are collapsed and +hold the panels that need the `full` families. The stat panel at the top +shows the level each replica reports on +`coderd_chatd_stage_metrics_level`. + +Series cost per `(chat_kind, model)` pair is roughly 250 at `basic` +(plus a constant ~230 per replica for the model-less wait and connect +stages) and roughly 1,200 at `full`. Multiply by replicas and by the +number of models in use. + +### Alerting without the dashboard + +Alerts only need `coderd_chatd_stage_duration_seconds` for the stages in +question. The buckets are round numbers (`0.05 0.1 0.25 0.5 1 2.5 5 10 +20 30 60 120 300 600 1800 3600`), so a threshold that matches a bucket +edge is exact rather than interpolated: + +```yaml +groups: + - name: chatd-latency + rules: + # More than 10% of first tokens took over 10s in the last 10m, + # with enough samples to mean something. + - alert: ChatdSlowTimeToFirstToken + expr: | + 1 - ( + sum by (model) (rate(coderd_chatd_stage_duration_seconds_bucket{stage="time_to_first_token", scope="turn", le="10"}[10m])) + / + sum by (model) (rate(coderd_chatd_stage_duration_seconds_count{stage="time_to_first_token", scope="turn"}[10m])) + ) > 0.10 + and + sum by (model) (increase(coderd_chatd_stage_duration_seconds_count{stage="time_to_first_token", scope="turn"}[10m])) > 20 + for: 5m + # p95 MCP connect above 5s. mcp_connect carries no model label. + - alert: ChatdSlowMCPConnect + expr: | + histogram_quantile(0.95, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage="mcp_connect"}[10m]))) > 5 + for: 10m +``` + +`time_to_first_token` is only observed when a first token arrived, so a +provider that hangs without streaming does not show up here; pair it with +`stream`, which is observed regardless, or with `provider_attempt` errors +in traces. Saturation paging belongs to the +`coderd_chatd_agents_queued_for_capacity` gauge, not to `capacity_wait`, +which is recorded only after the wait ends. + +An operator who wants only these alerts can keep the two stages and drop +the rest at scrape time. Keep `chat_turn` if the dashboard's `$chat_kind` +and `$model` variables should still populate; they are sourced from it. + +```yaml +metric_relabel_configs: + - source_labels: [__name__, stage] + regex: 'coderd_chatd_stage_duration_seconds.*;(time_to_first_token|mcp_connect|chat_turn)' + action: keep + - source_labels: [__name__] + regex: 'coderd_chatd_(turn_.*|stage_share_of_turn)' + action: drop +``` + +## Dimensions + +The stage histogram carries four labels, exposed as dashboard variables +where noted: + +| Label | Values | Dashboard variable | +|-------------|---------------------------------------------------------------------------------------------------------|-----------------------------| +| `stage` | the 15 stage names above | none (fixed hierarchy) | +| `scope` | `turn` (part of a chat turn) or `background` (detached async work such as title and summary generation) | none (panels pin one scope) | +| `chat_kind` | `root` (a chat a user drives) or `subagent` (a chat spawned by a parent agent) | `$chat_kind` (multi-select) | +| `model` | resolved model ID, empty for stages that are not tied to a model call | `$model` (multi-select) | + +Two more variables apply everywhere: `$datasource` selects the Prometheus +data source and `$stat` selects the summary statistic (mean, p50, p90, +p95, p99) for every stat-aware panel. + +Reasoning effort is deliberately not a metric label: it multiplies series +per model while changing the behavior of only the model-call stages. It +is a span attribute (`reasoning_effort`) on every stage span, so +per-effort analysis is a trace query (for example +`{ name = "time_to_first_token" && span.reasoning_effort = "high" }` in +Tempo). + +`chat_kind` separates root chats, which a user drives, from subagent +chats, which a parent agent spawns and which run as separate chats with +their own turn trees whose wall time overlaps the parent's `tool_call`. +It is a property of the turn, so every turn-scoped stage carries it, +including the stages recorded before a model is resolved. `$chat_kind` +therefore filters every stage panel in full, rather than narrowing part +of the hierarchy the way `$model` does. Background provider calls carry +the kind of the chat that spawned them and are pinned to +`scope="background"` in their own panel. + +Stages that are not tied to a model call (`queue_wait`, `capacity_wait`, +`acquisition`, `mcp_connect`, `commit`, `retry_backoff`) always carry an +empty `model` label on `coderd_chatd_stage_duration_seconds`. Panels +match those stages without the `$model` matcher, since a matcher there +could only subtract, so narrowing the variable keeps the waits populated +and narrows only the model-carrying stages (`chat_turn`, +`generation_step`, `prepare`, `provider_attempt`, `time_to_first_token`, +`stream`, `thinking`, `tool_call`, `compaction`). `chat_turn` is stamped +with the turn's model when the turn ends, so it takes the matcher like +any other model-carrying stage. The exception applies only to the +per-occurrence metric; the turn-end metrics stamp every stage. + +The turn-end metrics behind the collapsed level rows and the Turn time +partition row (`coderd_chatd_turn_stage_seconds`, +`coderd_chatd_turn_stage_count`, `coderd_chatd_stage_share_of_turn`, +`coderd_chatd_turn_time_seconds`, `coderd_chatd_turn_time_share`) are +observed once per turn with the turn's `chat_kind` and `model` already +known, so both variables apply to them without exception. + +## Panels + +### Reading the levels + +The stage tree mixes two units of observation. `chat_turn` is recorded +once per turn, while its descendants are recorded once per occurrence, +and a step-level stage such as `stream` or `tool_call` typically occurs +six to twelve times in a turn. Plotting both on one axis compares a +25-second turn against a 2-second stream and tells you nothing, so the +trend panels are grouped by tree level and every level can be read four +ways: + +| Panel | Metric | Question | Level | +|-------------------------|---------------------------------------|------------------------------------------|-------| +| Duration per occurrence | `coderd_chatd_stage_duration_seconds` | how long does one of these take | basic | +| Seconds per turn | `coderd_chatd_turn_stage_seconds` | how much of a turn does it add up to | full | +| Occurrences per turn | `coderd_chatd_turn_stage_count` | how often does it happen in a turn | full | +| Share of turn | `coderd_chatd_stage_share_of_turn` | what fraction of the turn does it occupy | full | + +The first is per occurrence, the other three are per turn, recorded when +the turn ends. Seconds per turn is roughly occurrences per turn times +duration per occurrence, so the three together separate "each one is +slow" from "it happens too often". At `basic` only the first column is +populated, and the Turn time partition row is the per-turn view; the +other three live in each level's collapsed `(full level only)` row. +Stages still overlap within a level, so shares and seconds at one level +do not sum to the turn; the Turn time partition row is the view that +does add up. + +### Level 0: Turn + +Members: `chat_turn`. + +**Turn duration ($stat)** - wall time of a whole turn, split by model. +This is the denominator every other level is measured against, and the +model split keeps a slow model from hiding inside a blended line. + +**Turns per minute** - completed turns per minute, split by model. Read +it beside turn duration: duration moving with flat throughput is a +latency regression, both moving together is usually a workload change. + +**Turn time mix by model** - the exclusive category partition of a turn +per model, described in the next section. + +### Level 0: Turn time partition + +The stage hierarchy overlaps in wall time, so it can tell you which +stages are slow but not how a turn's seconds divide up. The turn-end +metrics answer that with an exclusive partition of turn wall time, +observed once per turn: + +| Category | Turn time spent | +|-----------------------|-----------------------------------------------------| +| `scheduling` | queueing, capacity admission and worker pickup | +| `time_to_first_token` | provider request open until the first streamed part | +| `streaming` | first part until the stream closes | +| `tool_execution` | local tool calls | +| `provider_error` | attempts that ended in a provider error | +| `retry_backoff` | waiting between provider attempts | +| `compaction` | auxiliary compaction calls | +| `preparation` | prompt build, model resolution and MCP connects | +| `persistence` | the step commit transaction | +| `chatd_overhead` | the step's own work outside every other stage | +| `unattributed` | turn time no category claimed | + +The categories are exclusive and sum to the turn, so these panels do add +up, unlike the stage panels. `unattributed` is the completeness check: +if it grows, real turn time is happening outside every instrumented +stage. The opposite failure, categories that sum to more than the turn, +is emitted as measured with zero `unattributed` and counted in +`coderd_chatd_stage_anomalies_total{reason="overattributed"}`; that +counter also records stage observations dropped for inverted clocks and +turns dropped for a non-positive duration. + +The partition is computed once per turn from the turn's own stages, so +it is exact per turn. The dashboard has no panel that divides aggregate +stage seconds by aggregate `chat_turn` counts: the two observations for +one turn land in different scrapes, and such a ratio mixes turns and can +read well past 100% while long turns are in flight. + +The category partition is available at `basic` because `prepare`, +`commit`, `queue_wait`, and the other stages feed their categories +regardless of whether the stage itself is observed on the histogram. + +**Turn time mix by model** - shown in the Level 0 row above, one +100%-stacked bar per model, each category's total seconds over the range +divided by all categories' total seconds. Dimensions: `$chat_kind` and +`$model` apply; the grouping is fixed to `model`, because Grafana +transformation options do not interpolate dashboard variables and the +matrix transform needs a static row field. How to read: the fastest way +to compare where models spend a turn, for example a model with a large +`time_to_first_token` share against one dominated by `streaming`. + +**Seconds per turn by category** - mean seconds per turn in each +category, stacked, with total turn duration as a line. Category seconds +are divided by the turn count taken from the `unattributed` category's +`_count`, since every category is observed once per turn even when it is +zero. How to read: the stack height is the mean turn duration, so the +line should sit on top of the stack; a gap means the current variable +selection dropped categories. + +**Unattributed turn time** - mean unattributed seconds per turn. How to +read: this is the completeness check for the stage model, so treat a +rising line as an instrumentation gap rather than a workload change. + +**Category share per turn ($stat)** (full level only) - the selected +`$stat` of each category's share of a turn, from +`coderd_chatd_turn_time_share`. How to read: the mix bar shows where +aggregate time goes, this shows how much a category varies per turn, so +a small mean with a large p99 marks a bursty cost such as a slow tool +call or a retry storm in a minority of turns. Quantiles are per category, +so unlike the mean shares they do not sum to 100%. + +### Level 1: Turn children + +Members: `acquisition`, `queue_wait`, `capacity_wait` in the visible +row; `generation_step` in the full-level row. The three scheduling waits +happen once per turn before generation starts; `generation_step` repeats +once per step. + +`capacity_wait` appears in the duration per occurrence panel only. It is +measured by the acquisition loop before the turn exists, so no turn +records it, and its window lies inside `acquisition`, which the turn +does record under the `scheduling` category. + +`capacity_wait` is a lower bound. The refusal history behind it lives +in memory on each replica, while the capacity limit is deployment-wide, +so the wait is measured from the acquiring replica's own first refusal. +A replica that admitted the chat on its first attempt records nothing, +and a restart discards the history. All replicas are woken together +when a chat becomes acquirable, so the undercount is bounded by about +one acquisition interval (30s by default) rather than growing with the +replica count; do not scale the value by replicas. The distortion that +matters is censoring: short waits are the ones most likely to be +missing, which skews the recorded distribution long. For a +deployment-accurate view of time spent waiting on capacity, compare +`acquisition` quantiles while `coderd_chatd_agents_queued_for_capacity` +is above zero against the same quantiles while it is zero. + +Because these are the direct children of the turn, their share panel (in +the full-level row) is the quickest answer to "was this turn slow because +of scheduling or because of generation". At `basic`, the `scheduling` +category in the partition row answers the same question in aggregate. + +### Level 2: Step children + +Members: `retry_backoff`, `stream`, `tool_call`, `commit` in the visible +row; `prepare`, `thinking`, `compaction` in the full-level row. + +These are the stages inside one generation step and they overlap each +other, so read them as a profile of the step. `thinking` and `tool_call` +are reconstructed from timestamps after the fact and attach to the step +even though their windows fall inside `stream` and the tool phase. + +### Level 3: Prepare and stream children + +Members: `mcp_connect` (inside `prepare`), `time_to_first_token` and +`provider_attempt` (inside `stream`). All three are observed at `basic`. + +`provider_attempt` is one HTTP round trip to the provider and closes when +response headers arrive; `time_to_first_token` closes on the first +streamed part, so the two overlap almost entirely. An HTTP retry inside +one stream adds a second `provider_attempt` but no second +`time_to_first_token`. `mcp_connect` is not tied to a model call and is +matched without `$model`; the other two carry the label. + +### Throughput and TTFT + +**Time to first token** - p50/p90/p99/mean of `coderd_chatd_ttft_seconds`, +the pre-existing histogram recorded when the first streamed part +arrives. Dimensions: none of the stage labels; this histogram is +labeled by provider/model internally but the panel aggregates across +them, and `$model` and `$chat_kind` do not apply. The +`time_to_first_token` stage in the profile measures the same interval +and does honor the filters. How to read: the primary user-perceived +responsiveness metric for streaming. + +**Background provider calls ($stat)** - rate and selected `$stat` +duration of background-scope `provider_attempt` samples: detached +title/summary/quickgen requests that are excluded from every other +panel. Dimensions: pinned to the background scope of the +`provider_attempt` stage; `$model` and `$chat_kind` are not applied, +because background work runs outside a turn. How to read: this work +costs provider quota and money but no user-facing turn latency; a spike +here with flat turn panels means background load, not a chat regression. + +## Setup + +1. **Configure a Prometheus data source** that scrapes your coderd + Prometheus endpoint (`--prometheus-enable`). +2. **Import**: in Grafana navigate to **Dashboards** -> **Import** -> + **Upload JSON file** with [`dashboard.json`](./dashboard.json), then map + the Prometheus data source when prompted. +3. **Pick a level**: stage metrics are off by default. Set + `--chat-stage-metrics=basic` on the coderd replicas to fill every + expanded row, or `full` to also populate the collapsed rows. + +Per-session drill-down is available by exporting coderd traces +(`--trace` with standard OTLP environment variables) to a tracing backend +such as Tempo; each chat turn is a `chat_turn` root span whose children +mirror the stage hierarchy above, with `reasoning_effort`, `tool_name`, +and `generation_action` as span attributes. diff --git a/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json b/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json new file mode 100644 index 00000000000..ed90b0551e8 --- /dev/null +++ b/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json @@ -0,0 +1,3105 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Where time goes in Coder Agents chat turns, from the coderd_chatd_stage_duration_seconds stage histogram.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The --chat-stage-metrics level each coderd replica runs at, from coderd_chatd_stage_metrics_level. The visible rows work at basic; the collapsed rows need full. At off, every stage panel is empty.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 107, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "12.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (level) (coderd_chatd_stage_metrics_level)", + "instant": true, + "legendFormat": "stage metrics level: {{level}}", + "refId": "A" + } + ], + "title": "Stage metrics level", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 108, + "panels": [], + "title": "Level 0: Turn", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Wall time of a whole chat turn, from the chat_turn stage, split by model.\n\nOne sample per turn, so this is the top of the tree and the denominator every other level is measured against. Splitting by model keeps a slow model from hiding inside a blended line; $model and $chat_kind still narrow which turns are counted.\n\nMean is rate(_sum)/rate(_count); percentiles are histogram_quantile over the bucket rates. Idle models drop out instead of returning NaN.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 0, + "y": 4 + }, + "id": 15, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n histogram_quantile($stat, sum by (le, model) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"chat_turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", scope=\"turn\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(model) (sum by (model) (rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", scope=\"turn\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (model) (rate(coderd_chatd_stage_duration_seconds_sum{stage=\"chat_turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", scope=\"turn\"}[$__rate_interval])) / (sum by (model) (rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", scope=\"turn\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "legendFormat": "{{model}}", + "range": true, + "refId": "A" + } + ], + "title": "Turn duration (${stat:text})", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Completed chat turns per minute, from the chat_turn sample count, split by model.\n\nRead it next to turn duration: a duration change with flat throughput is a latency regression, while both moving together usually means the workload changed. Only turns that finished are counted.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "opm" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 8, + "y": 4 + }, + "id": 16, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "60 * sum by (model) (rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", scope=\"turn\"}[$__rate_interval]))", + "legendFormat": "{{model}}", + "range": true, + "refId": "A" + } + ], + "title": "Turns per minute", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "How turn wall time splits across the exclusive turn-time categories, per model, over the dashboard time range. Each bar is one model and sums to 100%; the value is that category's total seconds divided by all categories' total seconds, from increase() over the range.\n\nThe categories partition the turn, so unlike the stage panels they do not overlap and do add up. unattributed is the remainder the stage instrumentation could not place; watch it as the completeness check for the model.\n\n$chat_kind and $model apply. Grouping is fixed to model because Grafana transformation options do not interpolate dashboard variables.", + "fieldConfig": { + "defaults": { + "custom": { + "axisPlacement": "auto", + "fillOpacity": 80, + "lineWidth": 1 + }, + "max": 100, + "min": 0, + "unit": "percent" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "scheduling" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "time_to_first_token" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "streaming" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "tool_execution" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "provider_error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "retry_backoff" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "compaction" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "preparation" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "persistence" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "chatd_overhead" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "unattributed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#808080", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 4 + }, + "id": 10, + "options": { + "barRadius": 0, + "barWidth": 0.7, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "never", + "stacking": "normal", + "tooltip": { + "mode": "multi", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0, + "xField": "model\\category" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "100 * sum by (model, category) (increase(coderd_chatd_turn_time_seconds_sum{chat_kind=~\"$chat_kind\", model=~\"$model\"}[$__range]))\n/ on(model) group_left() (sum by (model) (increase(coderd_chatd_turn_time_seconds_sum{chat_kind=~\"$chat_kind\", model=~\"$model\"}[$__range])) > 0)", + "legendFormat": "", + "range": false, + "refId": "A", + "format": "table", + "instant": true + } + ], + "title": "Turn time mix by model", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "category", + "emptyValue": "zero", + "rowField": "model", + "valueField": "Value" + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": { + "model": 0, + "model\\category": 0, + "scheduling": 1, + "time_to_first_token": 2, + "streaming": 3, + "tool_execution": 4, + "provider_error": 5, + "retry_backoff": 6, + "compaction": 7, + "chatd_overhead": 8, + "unattributed": 9 + }, + "renameByName": {} + } + } + ], + "type": "barchart" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 109, + "panels": [], + "title": "Level 0: Turn time partition", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Mean seconds per turn spent in each turn-time category, stacked, plus the total turn duration as a line. Each category's seconds are divided by the turn count, taken from the unattributed category's _count because every category is observed once per turn, including when it is zero.\n\nBecause the categories are exclusive, the stack height equals the mean turn duration and the line should sit on top of the stack. A gap between them means categories were dropped by the $model or $chat_kind selection rather than by the partition.\n\nIntervals with no completed turns are dropped instead of dividing by zero.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 60, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "scheduling" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "time_to_first_token" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "streaming" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "tool_execution" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "provider_error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "retry_backoff" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "compaction" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "preparation" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "persistence" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "chatd_overhead" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "unattributed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#808080", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "total per turn" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": false, + "mode": "none" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineWidth", + "value": 2 + }, + { + "id": "color", + "value": { + "fixedColor": "text", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 11, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (category) (rate(coderd_chatd_turn_time_seconds_sum{chat_kind=~\"$chat_kind\", model=~\"$model\"}[$__rate_interval]))\n/ on() group_left() (sum(rate(coderd_chatd_turn_time_seconds_count{category=\"unattributed\", chat_kind=~\"$chat_kind\", model=~\"$model\"}[$__rate_interval])) > 0)", + "legendFormat": "{{category}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(coderd_chatd_turn_time_seconds_sum{chat_kind=~\"$chat_kind\", model=~\"$model\"}[$__rate_interval]))\n/ on() group_left() (sum(rate(coderd_chatd_turn_time_seconds_count{category=\"unattributed\", chat_kind=~\"$chat_kind\", model=~\"$model\"}[$__rate_interval])) > 0)", + "legendFormat": "total per turn", + "range": true, + "refId": "B" + } + ], + "title": "Seconds per turn by category", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Turn time the stage instrumentation could not place in any category: mean unattributed seconds per turn.\n\nThis is the completeness check for the stage model. Near zero means the categories account for the turn. A rising line means real turn time is happening outside every instrumented stage, so the profile and partition panels are understating something; treat it as a bug in the instrumentation rather than as a workload change.\n\n$chat_kind and $model apply. The per-turn share distribution is in the collapsed row below.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(coderd_chatd_turn_time_seconds_sum{category=\"unattributed\", chat_kind=~\"$chat_kind\", model=~\"$model\"}[$__rate_interval]))\n/ (sum(rate(coderd_chatd_turn_time_seconds_count{category=\"unattributed\", chat_kind=~\"$chat_kind\", model=~\"$model\"}[$__rate_interval])) > 0)", + "legendFormat": "seconds per turn", + "range": true, + "refId": "A" + } + ], + "title": "Unattributed turn time", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 110, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Distribution of each category's share of a turn, from the coderd_chatd_turn_time_share histogram recorded once per turn. $stat selects the statistic, so p99 reads as \"in the worst turns, this category took this fraction of the turn\".\n\nUse it next to the mix bar: the bar shows where aggregate time goes, this shows how much a category varies per turn. A category with a small mean and a large p99 is bursty (a slow tool call or a retry storm in a minority of turns) rather than a steady cost.\n\nQuantiles are per category, so unlike the mean shares they do not sum to 100%. $chat_kind and $model apply.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "scheduling" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "time_to_first_token" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "streaming" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "tool_execution" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "provider_error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "retry_backoff" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "compaction" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "preparation" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "persistence" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "chatd_overhead" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "unattributed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#808080", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n histogram_quantile($stat, sum by (le, category) (rate(coderd_chatd_turn_time_share_bucket{chat_kind=~\"$chat_kind\", model=~\"$model\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(category) (sum by (category) (rate(coderd_chatd_turn_time_share_count{chat_kind=~\"$chat_kind\", model=~\"$model\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (category) (rate(coderd_chatd_turn_time_share_sum{chat_kind=~\"$chat_kind\", model=~\"$model\"}[$__rate_interval])) / (sum by (category) (rate(coderd_chatd_turn_time_share_count{chat_kind=~\"$chat_kind\", model=~\"$model\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "legendFormat": "{{category}}", + "range": true, + "refId": "A" + } + ], + "title": "Category share per turn (${stat:text})", + "type": "timeseries" + } + ], + "title": "Level 0: Turn time partition (full level only: --chat-stage-metrics=full)", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 111, + "panels": [], + "title": "Level 1: Turn children", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Stages that hang directly off chat_turn: the three scheduling waits before generation starts.\n\nMembers: acquisition, queue_wait, capacity_wait. generation_step, which repeats once per step of the turn, is in the collapsed row below because it is observed only at --chat-stage-metrics=full.\n\nOne sample per occurrence of the stage: a stage that runs several times in a turn contributes several samples, so this is \"how long does one of these take\".\n\n$chat_kind applies to every series. These stages are not tied to a model call, so $model does not apply.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 21, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=~\"acquisition|queue_wait|capacity_wait\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"acquisition|queue_wait|capacity_wait\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_stage_duration_seconds_sum{stage=~\"acquisition|queue_wait|capacity_wait\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"acquisition|queue_wait|capacity_wait\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "legendFormat": "{{stage}}", + "range": true, + "refId": "A" + } + ], + "title": "Duration per occurrence (${stat:text})", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 112, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "generation_step: one iteration of the generation loop, which repeats once per step of the turn. Observed only at --chat-stage-metrics=full.\n\nOne sample per occurrence. $chat_kind and $model apply.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 37 + }, + "id": 105, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"generation_step\", model=~\"$model\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", model=~\"$model\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_stage_duration_seconds_sum{stage=\"generation_step\", model=~\"$model\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", model=~\"$model\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "legendFormat": "{{stage}}", + "range": true, + "refId": "B" + } + ], + "title": "Duration per occurrence, full-level stages (${stat:text})", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total seconds each stage occupied within a turn, from coderd_chatd_turn_stage_seconds.\n\nOne sample per turn, recorded when the turn ends, so this is \"how much of a turn does this stage account for in total\" and it already sums the repeats.\n\nCompare with duration per occurrence: a stage can be fast per occurrence and still dominate a turn by repeating. Both variables apply, because the turn-end metrics are stamped with the turn's model even for stages that are not tied to a model call.\n\ncapacity_wait is not in this panel: it is measured before the turn exists and reaches only the per-occurrence profile, and its window lies inside acquisition.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 37 + }, + "id": 22, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_turn_stage_seconds_bucket{stage=~\"acquisition|queue_wait|generation_step\", model=~\"$model\", chat_kind=~\"$chat_kind\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_turn_stage_seconds_count{stage=~\"acquisition|queue_wait|generation_step\", model=~\"$model\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_turn_stage_seconds_sum{stage=~\"acquisition|queue_wait|generation_step\", model=~\"$model\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_turn_stage_seconds_count{stage=~\"acquisition|queue_wait|generation_step\", model=~\"$model\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "legendFormat": "{{stage}}", + "range": true, + "refId": "A" + } + ], + "title": "Seconds per turn (${stat:text})", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "How many times each stage ran in a turn, from coderd_chatd_turn_stage_count.\n\nOne sample per turn, recorded when the turn ends, so this is \"how much of a turn does this stage account for in total\" and it already sums the repeats. A stage that did not occur in a turn is not recorded for that turn, so this reads as the count among turns where the stage happened at all.\n\nThis is the multiplier between the other two panels: seconds per turn is roughly occurrences per turn times duration per occurrence.\n\ncapacity_wait is not in this panel: it is measured before the turn exists and reaches only the per-occurrence profile, and its window lies inside acquisition.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 45 + }, + "id": 23, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_turn_stage_count_bucket{stage=~\"acquisition|queue_wait|generation_step\", model=~\"$model\", chat_kind=~\"$chat_kind\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_turn_stage_count_count{stage=~\"acquisition|queue_wait|generation_step\", model=~\"$model\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_turn_stage_count_sum{stage=~\"acquisition|queue_wait|generation_step\", model=~\"$model\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_turn_stage_count_count{stage=~\"acquisition|queue_wait|generation_step\", model=~\"$model\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "legendFormat": "{{stage}}", + "range": true, + "refId": "A" + } + ], + "title": "Occurrences per turn (${stat:text})", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Fraction of the turn each stage occupied, from coderd_chatd_stage_share_of_turn.\n\nOne sample per turn, recorded when the turn ends, so this is \"how much of a turn does this stage account for in total\" and it already sums the repeats. Stages overlap in wall time, so shares at one level do not sum to 1 and several stages can each approach the whole turn.\n\nUse it to compare levels: a level whose shares are all small means turn time is going somewhere else, which the turn time partition row attributes.\n\ncapacity_wait is not in this panel: it is measured before the turn exists and reaches only the per-occurrence profile, and its window lies inside acquisition.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 45 + }, + "id": 24, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_stage_share_of_turn_bucket{stage=~\"acquisition|queue_wait|generation_step\", model=~\"$model\", chat_kind=~\"$chat_kind\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_share_of_turn_count{stage=~\"acquisition|queue_wait|generation_step\", model=~\"$model\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_stage_share_of_turn_sum{stage=~\"acquisition|queue_wait|generation_step\", model=~\"$model\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_stage_share_of_turn_count{stage=~\"acquisition|queue_wait|generation_step\", model=~\"$model\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "legendFormat": "{{stage}}", + "range": true, + "refId": "A" + } + ], + "title": "Share of turn (${stat:text})", + "type": "timeseries" + } + ], + "title": "Level 1: Turn children (full level only: --chat-stage-metrics=full)", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 113, + "panels": [], + "title": "Level 2: Step children", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Stages inside one generation_step. They overlap each other in wall time, so read them as a profile of the step rather than as a partition of it. mcp_connect (inside prepare) and time_to_first_token and provider_attempt (inside stream) are at level 3.\n\nMembers: retry_backoff, stream, tool_call, commit. prepare, thinking, and compaction are in the collapsed row below because they are observed only at --chat-stage-metrics=full.\n\nOne sample per occurrence of the stage: a stage that runs several times in a turn contributes several samples, so this is \"how long does one of these take\".\n\n$chat_kind applies to every series. $model applies to stream and tool_call; commit and retry_backoff are not tied to a model call and are matched without it.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 38 + }, + "id": 31, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=~\"commit|retry_backoff\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"commit|retry_backoff\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_stage_duration_seconds_sum{stage=~\"commit|retry_backoff\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"commit|retry_backoff\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "legendFormat": "{{stage}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=~\"stream|tool_call\", model=~\"$model\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"stream|tool_call\", model=~\"$model\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_stage_duration_seconds_sum{stage=~\"stream|tool_call\", model=~\"$model\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"stream|tool_call\", model=~\"$model\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "legendFormat": "{{stage}}", + "range": true, + "refId": "B" + } + ], + "title": "Duration per occurrence (${stat:text})", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 47 + }, + "id": 114, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "prepare (generation preparation, including model resolution and tool assembly), thinking (a reasoning block inside the stream), and compaction (a compaction pass). Observed only at --chat-stage-metrics=full.\n\nOne sample per occurrence. $chat_kind and $model apply.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 48 + }, + "id": 106, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=~\"prepare|thinking|compaction\", model=~\"$model\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"prepare|thinking|compaction\", model=~\"$model\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_stage_duration_seconds_sum{stage=~\"prepare|thinking|compaction\", model=~\"$model\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"prepare|thinking|compaction\", model=~\"$model\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "legendFormat": "{{stage}}", + "range": true, + "refId": "A" + } + ], + "title": "Duration per occurrence, full-level stages (${stat:text})", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total seconds each stage occupied within a turn, from coderd_chatd_turn_stage_seconds.\n\nOne sample per turn, recorded when the turn ends, so this is \"how much of a turn does this stage account for in total\" and it already sums the repeats.\n\nCompare with duration per occurrence: a stage can be fast per occurrence and still dominate a turn by repeating. Both variables apply, because the turn-end metrics are stamped with the turn's model even for stages that are not tied to a model call.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 48 + }, + "id": 32, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_turn_stage_seconds_bucket{stage=~\"prepare|stream|thinking|tool_call|commit|compaction|retry_backoff\", model=~\"$model\", chat_kind=~\"$chat_kind\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_turn_stage_seconds_count{stage=~\"prepare|stream|thinking|tool_call|commit|compaction|retry_backoff\", model=~\"$model\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_turn_stage_seconds_sum{stage=~\"prepare|stream|thinking|tool_call|commit|compaction|retry_backoff\", model=~\"$model\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_turn_stage_seconds_count{stage=~\"prepare|stream|thinking|tool_call|commit|compaction|retry_backoff\", model=~\"$model\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "legendFormat": "{{stage}}", + "range": true, + "refId": "A" + } + ], + "title": "Seconds per turn (${stat:text})", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "How many times each stage ran in a turn, from coderd_chatd_turn_stage_count.\n\nOne sample per turn, recorded when the turn ends, so this is \"how much of a turn does this stage account for in total\" and it already sums the repeats. A stage that did not occur in a turn is not recorded for that turn, so this reads as the count among turns where the stage happened at all.\n\nThis is the multiplier between the other two panels: seconds per turn is roughly occurrences per turn times duration per occurrence.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 56 + }, + "id": 33, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_turn_stage_count_bucket{stage=~\"prepare|stream|thinking|tool_call|commit|compaction|retry_backoff\", model=~\"$model\", chat_kind=~\"$chat_kind\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_turn_stage_count_count{stage=~\"prepare|stream|thinking|tool_call|commit|compaction|retry_backoff\", model=~\"$model\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_turn_stage_count_sum{stage=~\"prepare|stream|thinking|tool_call|commit|compaction|retry_backoff\", model=~\"$model\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_turn_stage_count_count{stage=~\"prepare|stream|thinking|tool_call|commit|compaction|retry_backoff\", model=~\"$model\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "legendFormat": "{{stage}}", + "range": true, + "refId": "A" + } + ], + "title": "Occurrences per turn (${stat:text})", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Fraction of the turn each stage occupied, from coderd_chatd_stage_share_of_turn.\n\nOne sample per turn, recorded when the turn ends, so this is \"how much of a turn does this stage account for in total\" and it already sums the repeats. Stages overlap in wall time, so shares at one level do not sum to 1 and several stages can each approach the whole turn.\n\nUse it to compare levels: a level whose shares are all small means turn time is going somewhere else, which the turn time partition row attributes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 56 + }, + "id": 34, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_stage_share_of_turn_bucket{stage=~\"prepare|stream|thinking|tool_call|commit|compaction|retry_backoff\", model=~\"$model\", chat_kind=~\"$chat_kind\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_share_of_turn_count{stage=~\"prepare|stream|thinking|tool_call|commit|compaction|retry_backoff\", model=~\"$model\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_stage_share_of_turn_sum{stage=~\"prepare|stream|thinking|tool_call|commit|compaction|retry_backoff\", model=~\"$model\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_stage_share_of_turn_count{stage=~\"prepare|stream|thinking|tool_call|commit|compaction|retry_backoff\", model=~\"$model\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "legendFormat": "{{stage}}", + "range": true, + "refId": "A" + } + ], + "title": "Share of turn (${stat:text})", + "type": "timeseries" + } + ], + "title": "Level 2: Step children (full level only: --chat-stage-metrics=full)", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 115, + "panels": [], + "title": "Level 3: Prepare and stream children", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Stages nested one level below the step children: mcp_connect inside prepare; time_to_first_token and provider_attempt inside stream. time_to_first_token and provider_attempt overlap each other: the attempt closes on response headers, the first token usually arrives shortly after.\n\nMembers: mcp_connect, time_to_first_token, provider_attempt.\n\nOne sample per occurrence of the stage: a stage that runs several times in a turn contributes several samples, so this is \"how long does one of these take\".\n\n$chat_kind applies to every series. $model applies to time_to_first_token and provider_attempt; mcp_connect is not tied to a model call and is matched without it.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 49 + }, + "id": 41, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"mcp_connect\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_stage_duration_seconds_sum{stage=\"mcp_connect\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "legendFormat": "{{stage}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=~\"time_to_first_token|provider_attempt\", model=~\"$model\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"time_to_first_token|provider_attempt\", model=~\"$model\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_stage_duration_seconds_sum{stage=~\"time_to_first_token|provider_attempt\", model=~\"$model\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"time_to_first_token|provider_attempt\", model=~\"$model\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "legendFormat": "{{stage}}", + "range": true, + "refId": "B" + } + ], + "title": "Duration per occurrence (${stat:text})", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 58 + }, + "id": 116, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total seconds each stage occupied within a turn, from coderd_chatd_turn_stage_seconds.\n\nOne sample per turn, recorded when the turn ends, so this is \"how much of a turn does this stage account for in total\" and it already sums the repeats.\n\nCompare with duration per occurrence: a stage can be fast per occurrence and still dominate a turn by repeating. Both variables apply, because the turn-end metrics are stamped with the turn's model even for stages that are not tied to a model call.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 59 + }, + "id": 42, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_turn_stage_seconds_bucket{stage=~\"mcp_connect|time_to_first_token|provider_attempt\", model=~\"$model\", chat_kind=~\"$chat_kind\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_turn_stage_seconds_count{stage=~\"mcp_connect|time_to_first_token|provider_attempt\", model=~\"$model\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_turn_stage_seconds_sum{stage=~\"mcp_connect|time_to_first_token|provider_attempt\", model=~\"$model\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_turn_stage_seconds_count{stage=~\"mcp_connect|time_to_first_token|provider_attempt\", model=~\"$model\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "legendFormat": "{{stage}}", + "range": true, + "refId": "A" + } + ], + "title": "Seconds per turn (${stat:text})", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "How many times each stage ran in a turn, from coderd_chatd_turn_stage_count.\n\nOne sample per turn, recorded when the turn ends, so this is \"how much of a turn does this stage account for in total\" and it already sums the repeats. A stage that did not occur in a turn is not recorded for that turn, so this reads as the count among turns where the stage happened at all.\n\nThis is the multiplier between the other two panels: seconds per turn is roughly occurrences per turn times duration per occurrence.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 59 + }, + "id": 43, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_turn_stage_count_bucket{stage=~\"mcp_connect|time_to_first_token|provider_attempt\", model=~\"$model\", chat_kind=~\"$chat_kind\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_turn_stage_count_count{stage=~\"mcp_connect|time_to_first_token|provider_attempt\", model=~\"$model\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_turn_stage_count_sum{stage=~\"mcp_connect|time_to_first_token|provider_attempt\", model=~\"$model\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_turn_stage_count_count{stage=~\"mcp_connect|time_to_first_token|provider_attempt\", model=~\"$model\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "legendFormat": "{{stage}}", + "range": true, + "refId": "A" + } + ], + "title": "Occurrences per turn (${stat:text})", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Fraction of the turn each stage occupied, from coderd_chatd_stage_share_of_turn.\n\nOne sample per turn, recorded when the turn ends, so this is \"how much of a turn does this stage account for in total\" and it already sums the repeats. Stages overlap in wall time, so shares at one level do not sum to 1 and several stages can each approach the whole turn.\n\nUse it to compare levels: a level whose shares are all small means turn time is going somewhere else, which the turn time partition row attributes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 67 + }, + "id": 44, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_stage_share_of_turn_bucket{stage=~\"mcp_connect|time_to_first_token|provider_attempt\", model=~\"$model\", chat_kind=~\"$chat_kind\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_share_of_turn_count{stage=~\"mcp_connect|time_to_first_token|provider_attempt\", model=~\"$model\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_stage_share_of_turn_sum{stage=~\"mcp_connect|time_to_first_token|provider_attempt\", model=~\"$model\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_stage_share_of_turn_count{stage=~\"mcp_connect|time_to_first_token|provider_attempt\", model=~\"$model\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "legendFormat": "{{stage}}", + "range": true, + "refId": "A" + } + ], + "title": "Share of turn (${stat:text})", + "type": "timeseries" + } + ], + "title": "Level 3: Prepare and stream children (full level only: --chat-stage-metrics=full)", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 117, + "panels": [], + "title": "Throughput and TTFT", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Time to first token from coderd_chatd_ttft_seconds, the histogram recorded when the first streamed part arrives. The time_to_first_token stage in the profile above measures the same interval scoped to a provider attempt. This histogram carries none of the stage labels, so $model and $chat_kind do not apply to it.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 60 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le) (rate(coderd_chatd_ttft_seconds_bucket[$__rate_interval])))\n and on() (sum(rate(coderd_chatd_ttft_seconds_count[$__rate_interval])) > 0)", + "range": true, + "refId": "A", + "legendFormat": "p50" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le) (rate(coderd_chatd_ttft_seconds_bucket[$__rate_interval])))\n and on() (sum(rate(coderd_chatd_ttft_seconds_count[$__rate_interval])) > 0)", + "range": true, + "refId": "B", + "legendFormat": "p90" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(coderd_chatd_ttft_seconds_bucket[$__rate_interval])))\n and on() (sum(rate(coderd_chatd_ttft_seconds_count[$__rate_interval])) > 0)", + "range": true, + "refId": "C", + "legendFormat": "p99" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(coderd_chatd_ttft_seconds_sum[$__rate_interval])) / (sum(rate(coderd_chatd_ttft_seconds_count[$__rate_interval])) > 0)", + "range": true, + "refId": "D", + "legendFormat": "mean" + } + ], + "title": "Time to first token", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Provider calls made outside a chat turn (scope=\"background\"): detached quickgen and title generation requests. These are excluded from the stage profile and the other stage panels, which are scoped to scope=\"turn\". Rate is on the right axis; duration uses the selected $stat and is guarded so an idle period drops out instead of returning NaN. Background work runs outside a chat turn and carries an empty chat_kind, so $chat_kind is not applied here either.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "calls" + }, + "properties": [ + { + "id": "unit", + "value": "ops" + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 60 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"background\"}[$__rate_interval]))", + "range": true, + "refId": "A", + "legendFormat": "calls" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"provider_attempt\", scope=\"background\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"background\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"provider_attempt\", scope=\"background\"}[$__rate_interval])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"background\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "range": true, + "refId": "B", + "legendFormat": "${stat:text}" + } + ], + "title": "Background provider calls (${stat:text})", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "1m", + "schemaVersion": 39, + "tags": [ + "coder", + "chatd", + "agents" + ], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": true, + "text": "p95", + "value": "0.95" + }, + "description": "Statistic used by the stage profile and stage duration panels. The value 0 selects the mean (rate of _sum over rate of _count); any other value is used as the quantile.", + "hide": 0, + "includeAll": false, + "label": "Statistic", + "multi": false, + "name": "stat", + "options": [ + { + "selected": false, + "text": "mean", + "value": "0" + }, + { + "selected": false, + "text": "p50", + "value": "0.5" + }, + { + "selected": false, + "text": "p90", + "value": "0.9" + }, + { + "selected": true, + "text": "p95", + "value": "0.95" + }, + { + "selected": false, + "text": "p99", + "value": "0.99" + } + ], + "query": "mean : 0,p50 : 0.5,p90 : 0.9,p95 : 0.95,p99 : 0.99", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(coderd_chatd_stage_duration_seconds_count{scope=\"turn\"}, chat_kind)", + "description": "Whether the turn belongs to a root chat or to a subagent chat spawned by a parent agent. Every turn-scoped stage carries this label, so the filter applies to all stage panels. Background provider calls carry an empty value and are not filtered.", + "hide": 0, + "includeAll": true, + "label": "Chat kind", + "multi": true, + "name": "chat_kind", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(coderd_chatd_stage_duration_seconds_count{scope=\"turn\"}, chat_kind)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(coderd_chatd_stage_duration_seconds_count{scope=\"turn\"}, model)", + "description": "Model the stage ran against. Stages that run before a model is resolved carry an empty value and are only included under All, whose value is the regex .* and matches the empty label.", + "hide": 0, + "includeAll": true, + "label": "Model", + "multi": true, + "name": "model", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(coderd_chatd_stage_duration_seconds_count{scope=\"turn\"}, model)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Chatd: chat lifecycle", + "uid": "chatd-lifecycle", + "version": 1, + "weekStart": "" +}