From 4fe2f02fb4f1708d537cc78559903aa996363c1d Mon Sep 17 00:00:00 2001 From: Scott Miller Date: Thu, 27 Aug 2026 23:02:44 +0000 Subject: [PATCH 01/19] feat: add chat lifecycle stage tracing, metrics, and Grafana dashboard Instrument chatd with OpenTelemetry spans and a Prometheus histogram covering the full chat turn lifecycle: queue wait, capacity wait, acquisition, preparation, MCP connect, provider attempts, streaming, time to first token, thinking, tool calls, commit, and compaction. Spans and coderd_chatd_stage_duration_seconds{stage,scope,model,effort} observations are emitted from one shared StageTracer path so traces and metrics cannot drift. Turn-scoped work is separated from detached background quickgen calls via the scope label, and the resolved model and effective reasoning effort are recorded as both span attributes and histogram labels. Add a Grafana dashboard (examples/monitoring/dashboards/grafana/ chatd-lifecycle) with an aggregate stage flamegraph driven by a selectable statistic (mean, p50, p90, p95, p99), stage trends, time share, scheduling waits, TTFT, throughput, and background provider call panels, filterable by model and effort. --- coderd/coderd.go | 1 + coderd/x/chatd/ARCHITECTURE.md | 6 + coderd/x/chatd/capacity.go | 49 + coderd/x/chatd/chatd.go | 27 +- coderd/x/chatd/chatloop/chatloop.go | 55 +- coderd/x/chatd/chatloop/metrics.go | 20 + coderd/x/chatd/chatloop/stage.go | 318 +++++ coderd/x/chatd/chatloop/stage_test.go | 440 +++++++ coderd/x/chatd/chatstate/transitions.go | 10 +- coderd/x/chatd/generation.go | 397 ++++-- coderd/x/chatd/generation_preparer.go | 5 +- coderd/x/chatd/model_routing.go | 5 + coderd/x/chatd/model_routing_aibridge.go | 39 + coderd/x/chatd/modelcall.go | 28 +- coderd/x/chatd/options.go | 1 + coderd/x/chatd/runner.go | 4 + coderd/x/chatd/stage_internal_test.go | 241 ++++ coderd/x/chatd/turn_trace.go | 126 ++ coderd/x/chatd/worker.go | 15 +- docs/admin/integrations/prometheus.md | 1 + .../grafana/chatd-lifecycle/README.md | 143 +++ .../grafana/chatd-lifecycle/dashboard.json | 1073 +++++++++++++++++ scripts/metricsdocgen/generated_metrics | 3 + 23 files changed, 2874 insertions(+), 133 deletions(-) create mode 100644 coderd/x/chatd/chatloop/stage.go create mode 100644 coderd/x/chatd/chatloop/stage_test.go create mode 100644 coderd/x/chatd/stage_internal_test.go create mode 100644 coderd/x/chatd/turn_trace.go create mode 100644 examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md create mode 100644 examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json diff --git a/coderd/coderd.go b/coderd/coderd.go index a6867c793bb..4318e8be934 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -955,6 +955,7 @@ func New(options *Options) *API { HookDispatcher: hookDispatcher, UsageTracker: options.WorkspaceUsageTracker, PrometheusRegistry: options.PrometheusRegistry, + TracerProvider: options.TracerProvider, AgentCapacityUnlock: options.ChatAgentCapacityUnlock, OIDCTokenSource: oidcMCPSrc, NotificationsEnqueuer: options.NotificationsEnqueuer, diff --git a/coderd/x/chatd/ARCHITECTURE.md b/coderd/x/chatd/ARCHITECTURE.md index 42b7caf7c93..9da6e1a4816 100644 --- a/coderd/x/chatd/ARCHITECTURE.md +++ b/coderd/x/chatd/ARCHITECTURE.md @@ -655,6 +655,8 @@ For every matching chat, it locks it, checks if the chat still meets the aforeme When a chat is successfully acquired, the acquisition loop requests the [Runner manager](#runner-manager) to spawn a chat runner for it. + + ### Load balancing The design doesn't attempt to distribute load between workers fairly. Whenever a chat needs an owner, all replicas race to acquire it. If there's a coder replica that has a lower latency to the database, it'll tend to acquire chats more frequently than other replicas. @@ -778,6 +780,8 @@ State updates processed by the loop come from: The runner is responsible for subscribing to the `chat:update:{chat_id}` pubsub channel. During bootstrap, it must first subscribe to the channel and then fetch the initial state of the chat from the database to avoid missing any updates. + + ### Event shape Every event that the runner loop processes has the following shape: @@ -859,6 +863,8 @@ The generation goroutine is responsible for calling the LLM API and executing to It inspects the chat's message history, and decides what's the next step to take. The result of that step is the application of one of the following core state machine transitions: + + - `CommitStep`: applied when an LLM API call returns a response. - `FinishTurn`: applied when the chat processing logic determines that there's no more work to do for the current message history (no pending tool calls, user message is not the last message in the history, etc.). - `FinishError`: applied when the LLM API call fails and the retry limit is reached, determined by the `generation_attempt` value. diff --git a/coderd/x/chatd/capacity.go b/coderd/x/chatd/capacity.go index 1ebcf8669a7..ef9cf1f1ea1 100644 --- a/coderd/x/chatd/capacity.go +++ b/coderd/x/chatd/capacity.go @@ -2,11 +2,14 @@ package chatd import ( "context" + "time" "github.com/google/uuid" "github.com/prometheus/client_golang/prometheus" + "go.opentelemetry.io/otel/attribute" "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/x/chatd/chatloop" ) type capacityMetrics struct { @@ -46,6 +49,52 @@ func (w *chatWorker) capacityMetricsLoop(ctx context.Context) { } } +// noteCapacityRefused remembers when a chat was first refused a +// capacity slot. Only the acquisition loop touches the map, so it +// needs no lock. +func (w *chatWorker) noteCapacityRefused(chatID uuid.UUID) { + if _, ok := w.capacityWaitSince[chatID]; ok { + return + } + w.capacityWaitSince[chatID] = time.Now() +} + +// recordCapacityWait emits the capacity_wait stage for a chat that is +// being acquired after at least one capacity refusal, measured from +// the first refusal this worker saw. Chats admitted on their first +// attempt record nothing. The acquisition pass runs before the turn +// span exists, so the turn scope is stated explicitly. +func (w *chatWorker) recordCapacityWait(ctx context.Context, chat database.Chat) { + since, waited := w.capacityWaitSince[chat.ID] + if !waited { + return + } + delete(w.capacityWaitSince, chat.ID) + w.server.stages.RecordAs(ctx, chatloop.StageCapacityWait, chatloop.ScopeTurn, chatloop.StageModel{}, + since, time.Now(), nil, + attribute.String(chatloop.AttrChatID, chat.ID.String()), + attribute.String(chatloop.AttrChatKind, chatKindAttr(chat)), + ) +} + +// pruneCapacityWaits drops wait starts for chats that are no longer +// acquisition candidates, which happens when they are archived, +// deleted, or picked up by another worker. +func (w *chatWorker) pruneCapacityWaits(candidates []database.GetChatWorkerAcquisitionCandidatesRow) { + if len(w.capacityWaitSince) == 0 { + return + } + stillCandidate := make(map[uuid.UUID]struct{}, len(candidates)) + for _, row := range candidates { + stillCandidate[row.ID] = struct{}{} + } + for chatID := range w.capacityWaitSince { + if _, ok := stillCandidate[chatID]; !ok { + delete(w.capacityWaitSince, chatID) + } + } +} + func (w *chatWorker) refreshCapacityMetrics(ctx context.Context) { active, err := w.opts.Store.CountChatCapacityActiveByPool(ctx, database.CountChatCapacityActiveByPoolParams{ ExcludeChatID: uuid.Nil, diff --git a/coderd/x/chatd/chatd.go b/coderd/x/chatd/chatd.go index 456031add59..7b4f5716132 100644 --- a/coderd/x/chatd/chatd.go +++ b/coderd/x/chatd/chatd.go @@ -21,6 +21,8 @@ import ( "github.com/google/uuid" "github.com/prometheus/client_golang/prometheus" "github.com/sqlc-dev/pqtype" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "golang.org/x/sync/errgroup" "golang.org/x/xerrors" @@ -199,6 +201,7 @@ type Server struct { usageTracker *workspacestats.UsageTracker clock quartz.Clock metrics *chatloop.Metrics + stages *chatloop.StageTracer chatWorker *chatWorker messagePartBuffer *messagepartbuffer.Buffer streamSyncPoller *streamSyncPoller @@ -2143,9 +2146,10 @@ func (p *Server) PromoteQueued( } var ( - result PromoteQueuedResult - refreshChat database.Chat - refreshedOK bool + result PromoteQueuedResult + refreshChat database.Chat + refreshedOK bool + promotedQueuedAt time.Time ) machine := p.newChatMachine(opts.ChatID) updateErr := machine.Update(ctx, func(tx *chatstate.Tx, store database.Store) error { @@ -2165,6 +2169,7 @@ func (p *Server) PromoteQueued( } if promoteResult.InsertedMessage != nil { result.PromotedMessage = *promoteResult.InsertedMessage + promotedQueuedAt = promoteResult.QueuedMessage.CreatedAt } // Capture the chat inside the transaction so the watch event // published below uses the snapshot bump and status change @@ -2184,6 +2189,12 @@ func (p *Server) PromoteQueued( if refreshedOK { p.publishChatPubsubEvent(refreshChat, codersdk.ChatWatchEventKindStatusChange, nil) } + if !promotedQueuedAt.IsZero() { + p.stages.Record(ctx, chatloop.StageQueueWait, chatloop.StageModel{}, + promotedQueuedAt, time.Now(), nil, + attribute.String(chatloop.AttrChatID, opts.ChatID.String()), + ) + } return result, nil } @@ -3061,6 +3072,9 @@ type Config struct { AIBridgeTransportFactory *atomic.Pointer[aibridge.TransportFactory] Experiments codersdk.Experiments PrometheusRegistry prometheus.Registerer + // TracerProvider supplies the tracer used for chat lifecycle + // spans. Nil disables tracing without disabling metrics. + TracerProvider trace.TracerProvider AgentCapacityUnlock AgentCapacityUnlock @@ -3190,6 +3204,7 @@ func New(ps pubsub.Pubsub, cfg Config) *Server { } else { p.metrics = chatloop.NopMetrics() } + p.stages = chatloop.NewStageTracer(cfg.TracerProvider, p.metrics) p.messagePartBuffer = messagepartbuffer.New(messagepartbuffer.Options{Clock: clk}) localStreamPartsDialer := NewLocalStreamPartsDialer(LocalStreamPartsDialerConfig{ Buffer: p.messagePartBuffer, @@ -5081,7 +5096,11 @@ func (p *Server) Close() error { // must be called once the work completes to release the shutdown hook. // The caller is responsible for providing their own timeout. func (p *Server) inflightContext(reqCtx context.Context) (context.Context, func()) { - ctx, cancel := context.WithCancel(context.WithoutCancel(reqCtx)) + // Inflight work outlives the caller, so the caller's span is + // stripped from the context: spans started on this context become + // their own roots instead of children that end after their parent. + detached := trace.ContextWithSpanContext(context.WithoutCancel(reqCtx), trace.SpanContext{}) + ctx, cancel := context.WithCancel(detached) stop := context.AfterFunc(p.ctx, cancel) return ctx, func() { stop() diff --git a/coderd/x/chatd/chatloop/chatloop.go b/coderd/x/chatd/chatloop/chatloop.go index f22cbc8d64b..61570b29da9 100644 --- a/coderd/x/chatd/chatloop/chatloop.go +++ b/coderd/x/chatd/chatloop/chatloop.go @@ -18,6 +18,7 @@ import ( fantasyanthropic "charm.land/fantasy/providers/anthropic" "charm.land/fantasy/schema" "github.com/google/uuid" + "go.opentelemetry.io/otel/attribute" "golang.org/x/xerrors" "cdr.dev/slog/v3" @@ -234,6 +235,12 @@ type GenerateAssistantOptions struct { OnModelStreamStart func() Logger slog.Logger Metrics *Metrics + // Stages records the stream and time_to_first_token stages. A nil + // tracer discards them. + Stages *StageTracer + // StageModel labels the stage spans and durations with the resolved + // model and effective reasoning effort. + StageModel StageModel } // AssistantOutcome is the durable assistant-side result from one model call. @@ -437,8 +444,12 @@ func GenerateAssistant(ctx context.Context, opts GenerateAssistantOptions) (Assi opts.OnModelStreamStart() } stepCtx := chatdebug.ReuseStep(ctx) + streamCtx, streamSpan := opts.Stages.Start(stepCtx, StageStream, + attribute.String(AttrProvider, provider), + ) + streamSpan.SetModel(opts.StageModel) attempt, streamErr := guardedStream( - stepCtx, + streamCtx, provider, modelName, opts.Clock, @@ -447,8 +458,11 @@ func GenerateAssistant(ctx context.Context, opts GenerateAssistantOptions) (Assi return opts.Model.Stream(attemptCtx, call) }, opts.Metrics, + opts.Stages, + opts.StageModel, ) if streamErr != nil { + streamSpan.End(streamErr) wrappedErr := wrapProviderStreamError(errorProvider, streamErr) classified := chaterror.Classify(wrappedErr).WithProvider(errorProvider) if classified.Retryable { @@ -460,6 +474,7 @@ func GenerateAssistant(ctx context.Context, opts GenerateAssistantOptions) (Assi result, processErr := processStepStream(attempt.ctx, attempt.stream, opts.Clock, publishMessagePart) if err := attempt.finish(processErr); err != nil { + streamSpan.End(err) if errors.Is(err, ErrInterrupted) { return AssistantOutcome{}, ErrInterrupted } @@ -472,6 +487,7 @@ func GenerateAssistant(ctx context.Context, opts GenerateAssistantOptions) (Assi } contextLimit := extractContextLimitWithFallback(result.providerMetadata, opts.ContextLimitFallback) + streamSpan.End(nil) result.content = chatsanitize.SanitizeAnthropicProviderToolStepContent( ctx, opts.Logger, provider, modelName, "assistant_helper", 0, result.finishReason, result.content, @@ -874,6 +890,10 @@ func classifyStreamSilenceTimeout( }) } +// errNoFirstToken marks a time_to_first_token window that ended when +// the attempt was released or failed before any part streamed. +var errNoFirstToken = xerrors.New("stream ended before the first token") + func guardedStream( parent context.Context, provider, model string, @@ -881,30 +901,51 @@ func guardedStream( timeout time.Duration, openStream func(context.Context) (fantasy.StreamResponse, error), metrics *Metrics, + stages *StageTracer, + stageModel StageModel, ) (guardedAttempt, error) { attemptCtx, cancelAttempt := context.WithCancelCause(parent) guard := newStreamSilenceGuard(clock, timeout, cancelAttempt) + streamStart := clock.Now() + _, ttftSpan := stages.Start(parent, StageTimeToFirstToken, + attribute.String(AttrProvider, provider), + ) + ttftSpan.SetModel(stageModel) + var ttftOnce sync.Once + // finishTTFT closes the time_to_first_token window exactly once, + // either on the first streamed part or when the attempt is released + // without one. The TTFT histogram only counts windows that a part + // actually closed. + finishTTFT := func(err error) { + ttftOnce.Do(func() { + if err == nil { + metrics.TTFTSeconds.WithLabelValues(provider, model).Observe( + clock.Since(streamStart).Seconds(), + ) + } + ttftSpan.End(err) + }) + } var releaseOnce sync.Once release := func() { releaseOnce.Do(func() { guard.Disarm() cancelAttempt(nil) + finishTTFT(errNoFirstToken) }) } - streamStart := clock.Now() stream, err := openStream(attemptCtx) if err != nil { err = classifyStreamSilenceTimeout(attemptCtx, provider, err) + finishTTFT(err) release() return guardedAttempt{}, err } - recordTTFT := sync.OnceFunc(func() { - metrics.TTFTSeconds.WithLabelValues(provider, model).Observe( - clock.Since(streamStart).Seconds(), - ) - }) + recordTTFT := func() { + finishTTFT(nil) + } return guardedAttempt{ ctx: attemptCtx, stream: fantasy.StreamResponse(func(yield func(fantasy.StreamPart) bool) { diff --git a/coderd/x/chatd/chatloop/metrics.go b/coderd/x/chatd/chatloop/metrics.go index 874a0ca52a0..ef30f6d3d2a 100644 --- a/coderd/x/chatd/chatloop/metrics.go +++ b/coderd/x/chatd/chatloop/metrics.go @@ -3,6 +3,7 @@ package chatloop import ( "context" "errors" + "time" "charm.land/fantasy" "github.com/prometheus/client_golang/prometheus" @@ -34,6 +35,7 @@ type Metrics struct { ToolResultTruncatedTotal *prometheus.CounterVec ToolErrorsTotal *prometheus.CounterVec TTFTSeconds *prometheus.HistogramVec + StageDurationSeconds *prometheus.HistogramVec CompactionTotal *prometheus.CounterVec StepsTotal *prometheus.CounterVec StreamRetriesTotal *prometheus.CounterVec @@ -95,6 +97,14 @@ func NewMetrics(reg prometheus.Registerer) *Metrics { Help: "Time-to-first-token: wall time from LLM request to first streamed chunk.", Buckets: []float64{0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60}, }, []string{"provider", "model"}), + StageDurationSeconds: factory.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "stage_duration_seconds", + Help: "Wall time spent in each chat lifecycle stage. Stages overlap in wall time; this is a stage-time profile, not a partition of the turn. The scope label separates stages that run inside a chat turn from detached background work. The model and effort labels are empty for stages that run before a model is resolved.", + // 10ms .. ~11m, log-spaced. + Buckets: prometheus.ExponentialBuckets(0.01, 2, 17), + }, []string{"stage", "scope", "model", "effort"}), CompactionTotal: factory.NewCounterVec(prometheus.CounterOpts{ Namespace: metricsNamespace, Subsystem: metricsSubsystem, @@ -153,6 +163,16 @@ func NopMetrics() *Metrics { return NewMetrics(prometheus.NewRegistry()) } +// RecordStageDuration observes one chat lifecycle stage duration. +// model and effort are empty when the stage ran before a model was +// resolved. Negative durations are dropped. No-op when m is nil. +func (m *Metrics) RecordStageDuration(stage, scope, model, effort string, elapsed time.Duration) { + if m == nil || elapsed < 0 { + return + } + m.StageDurationSeconds.WithLabelValues(stage, scope, model, effort).Observe(elapsed.Seconds()) +} + // RecordCompaction classifies and records a compaction attempt. // It is a no-op when m is nil. func (m *Metrics) RecordCompaction(provider, model string, compacted bool, err error) { diff --git a/coderd/x/chatd/chatloop/stage.go b/coderd/x/chatd/chatloop/stage.go new file mode 100644 index 00000000000..cfedf1bc4d9 --- /dev/null +++ b/coderd/x/chatd/chatloop/stage.go @@ -0,0 +1,318 @@ +package chatloop + +import ( + "context" + "time" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" + "go.opentelemetry.io/otel/trace/noop" +) + +// Stage names. Every value is both a span name and the `stage` label +// value recorded on stage_duration_seconds. +const ( + StageChatTurn = "chat_turn" + StageQueueWait = "queue_wait" + StageCapacityWait = "capacity_wait" + StageAcquisition = "acquisition" + StageGenerationStep = "generation_step" + StagePrepare = "prepare" + StageMCPConnect = "mcp_connect" + StageProviderAttempt = "provider_attempt" + StageStream = "stream" + StageTimeToFirstToken = "time_to_first_token" + StageThinking = "thinking" + StageToolCall = "tool_call" + StageCommit = "commit" + StageCompaction = "compaction" +) + +// Span attribute keys. Keys are lowercase snake_case and shared by +// every stage that carries the value. +const ( + AttrProvider = "provider" + AttrModel = "model" + AttrReasoningEffort = "reasoning_effort" + AttrChatID = "chat_id" + AttrChatKind = "chat_kind" + AttrGenerationAttempt = "generation_attempt" + AttrGenerationAction = "generation_action" + AttrToolName = "tool_name" + AttrHTTPStatusCode = "http_status_code" + AttrHTTPMethod = "http_method" + AttrHTTPHost = "http_host" + AttrCompactionSource = "compaction_source" + AttrScope = "scope" +) + +// Scope values. A stage is turn scoped when it runs inside a chat +// turn's trace, and background scoped when it runs on work detached +// from the turn, such as title and summary generation that outlives +// the turn that triggered it. +const ( + ScopeTurn = "turn" + ScopeBackground = "background" +) + +// Chat kind attribute values. +const ( + ChatKindRoot = "root" + ChatKindSubagent = "subagent" +) + +// tracerName is the instrumentation scope reported on chatd spans. +const tracerName = "github.com/coder/coder/v2/coderd/x/chatd" + +// StageTracer emits one span and one stage_duration_seconds +// observation per chat lifecycle stage. Both are produced from the +// same call so span and histogram durations cannot diverge. +// +// A nil *StageTracer is usable and discards everything. +type StageTracer struct { + tracer trace.Tracer + metrics *Metrics +} + +// NewStageTracer builds a stage tracer from a tracer provider and the +// chatd metrics. A nil provider falls back to a no-op tracer and nil +// metrics to a discarding registry, so callers without tracing or +// metrics configured still get a usable tracer. +func NewStageTracer(provider trace.TracerProvider, metrics *Metrics) *StageTracer { + if provider == nil { + provider = noop.NewTracerProvider() + } + if metrics == nil { + metrics = NopMetrics() + } + return &StageTracer{ + tracer: provider.Tracer(tracerName), + metrics: metrics, + } +} + +// NopStageTracer returns a stage tracer that discards spans and +// metrics. +func NopStageTracer() *StageTracer { + return NewStageTracer(nil, nil) +} + +func (t *StageTracer) otelTracer() trace.Tracer { + if t == nil || t.tracer == nil { + return noop.NewTracerProvider().Tracer(tracerName) + } + return t.tracer +} + +// StageModel identifies the model a stage ran against. Both fields +// are empty for stages that run before a model is resolved, such as +// the queue and capacity waits. Effort is the effective reasoning +// effort sent to the provider, empty when the model config sets none. +type StageModel struct { + Model string + Effort string +} + +// attributes returns the span attributes for the identity, omitting +// the ones that are unknown. +func (m StageModel) attributes() []attribute.KeyValue { + attrs := make([]attribute.KeyValue, 0, 2) + if m.Model != "" { + attrs = append(attrs, attribute.String(AttrModel, m.Model)) + } + if m.Effort != "" { + attrs = append(attrs, attribute.String(AttrReasoningEffort, m.Effort)) + } + return attrs +} + +// StageSpan is an in-flight stage. End must be called exactly once; +// the duration observation happens there. +type StageSpan struct { + tracer *StageTracer + stage string + scope string + model StageModel + span trace.Span + start time.Time + ended bool +} + +// scopeFromContext classifies work by whether ctx still carries the +// turn's trace. Detached background work has no span in its context, +// so its stages are kept out of the turn profile. +func scopeFromContext(ctx context.Context) string { + if trace.SpanContextFromContext(ctx).IsValid() { + return ScopeTurn + } + return ScopeBackground +} + +// Start begins a stage span as a child of the span in ctx and returns +// a context carrying it. The stage is scoped by the span already in +// ctx, so stages started on a detached context are background scoped. +func (t *StageTracer) Start( + ctx context.Context, + stage string, + attrs ...attribute.KeyValue, +) (context.Context, *StageSpan) { + return t.startSpan(ctx, stage, scopeFromContext(ctx), time.Time{}, + []trace.SpanStartOption{trace.WithAttributes(attrs...)}) +} + +// StartRoot begins a stage span in its own trace, ignoring any span +// in ctx. links records the relationship to the originating span +// context instead of making that span the parent, so the stage's +// trace stays scoped to the chat turn. The span opens a turn, so it +// is turn scoped regardless of what ctx carries. +func (t *StageTracer) StartRoot( + ctx context.Context, + stage string, + links []trace.Link, + attrs ...attribute.KeyValue, +) (context.Context, *StageSpan) { + return t.StartRootAt(ctx, stage, time.Time{}, links, attrs...) +} + +// StartRootAt begins a root stage span that started at an earlier, +// already known instant. The span timestamp and the recorded duration +// both run from start, so stages reconstructed inside the span still +// fall within it. A zero start means the span begins now. +func (t *StageTracer) StartRootAt( + ctx context.Context, + stage string, + start time.Time, + links []trace.Link, + attrs ...attribute.KeyValue, +) (context.Context, *StageSpan) { + return t.startSpan(ctx, stage, ScopeTurn, start, []trace.SpanStartOption{ + trace.WithNewRoot(), + trace.WithLinks(links...), + trace.WithAttributes(attrs...), + }) +} + +func (t *StageTracer) startSpan( + ctx context.Context, + stage string, + scope string, + start time.Time, + opts []trace.SpanStartOption, +) (context.Context, *StageSpan) { + now := time.Now() + if start.IsZero() || start.After(now) { + start = now + } else { + opts = append(opts, trace.WithTimestamp(start)) + } + opts = append(opts, trace.WithAttributes(attribute.String(AttrScope, scope))) + ctx, span := t.otelTracer().Start(ctx, stage, opts...) + return ctx, &StageSpan{ + tracer: t, + stage: stage, + scope: scope, + span: span, + start: start, + } +} + +// SetAttributes adds attributes to the stage span. It is a no-op +// after End. +func (s *StageSpan) SetAttributes(attrs ...attribute.KeyValue) { + if s == nil || s.ended { + return + } + s.span.SetAttributes(attrs...) +} + +// SetModel records the model identity on the span and on the +// duration observation End makes. Stages that only learn the model +// after they start, such as a generation step that resolves it during +// preparation, call this once it is known. +func (s *StageSpan) SetModel(model StageModel) { + if s == nil || s.ended { + return + } + s.model = model + s.span.SetAttributes(model.attributes()...) +} + +// SpanContext returns the span context of the stage span, which is +// invalid when tracing is not configured. +func (s *StageSpan) SpanContext() trace.SpanContext { + if s == nil { + return trace.SpanContext{} + } + return s.span.SpanContext() +} + +// End closes the stage span, records its duration, and marks the span +// as errored when err is non-nil. Calls after the first are ignored so +// a deferred End cannot double-count a stage. +func (s *StageSpan) End(err error) { + if s == nil || s.ended { + return + } + s.ended = true + elapsed := time.Since(s.start) + if err != nil { + s.span.RecordError(err) + s.span.SetStatus(codes.Error, err.Error()) + } + s.span.End() + s.tracer.observe(s.stage, s.scope, s.model, elapsed) +} + +// Record emits an already-finished stage span with explicit start and +// end timestamps. It is for stages whose boundaries are only known +// after the fact, such as durations reconstructed from persisted +// timestamps. The stage is scoped by the span in ctx. Non-positive or +// unset windows are dropped. +func (t *StageTracer) Record( + ctx context.Context, + stage string, + model StageModel, + start, end time.Time, + err error, + attrs ...attribute.KeyValue, +) { + t.RecordAs(ctx, stage, scopeFromContext(ctx), model, start, end, err, attrs...) +} + +// RecordAs is Record with an explicit scope, for stages that belong +// to a turn but are reconstructed outside its trace, such as the +// capacity wait an acquisition pass measures before the turn span +// exists. +func (t *StageTracer) RecordAs( + ctx context.Context, + stage string, + scope string, + model StageModel, + start, end time.Time, + err error, + attrs ...attribute.KeyValue, +) { + if start.IsZero() || end.IsZero() || end.Before(start) { + return + } + _, span := t.otelTracer().Start(ctx, stage, + trace.WithTimestamp(start), + trace.WithAttributes(attrs...), + trace.WithAttributes(model.attributes()...), + trace.WithAttributes(attribute.String(AttrScope, scope)), + ) + if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + } + span.End(trace.WithTimestamp(end)) + t.observe(stage, scope, model, end.Sub(start)) +} + +func (t *StageTracer) observe(stage, scope string, model StageModel, elapsed time.Duration) { + if t == nil || t.metrics == nil { + return + } + t.metrics.RecordStageDuration(stage, scope, model.Model, model.Effort, elapsed) +} diff --git a/coderd/x/chatd/chatloop/stage_test.go b/coderd/x/chatd/chatloop/stage_test.go new file mode 100644 index 00000000000..e0509b21054 --- /dev/null +++ b/coderd/x/chatd/chatloop/stage_test.go @@ -0,0 +1,440 @@ +package chatloop_test + +import ( + "context" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + "go.opentelemetry.io/otel/trace" + "golang.org/x/xerrors" + + "github.com/coder/coder/v2/coderd/x/chatd/chatloop" +) + +// stageFixture wires a stage tracer to an in-memory span recorder and +// a private metrics registry. +type stageFixture struct { + tracer *chatloop.StageTracer + spans *tracetest.SpanRecorder + registry *prometheus.Registry +} + +func newStageFixture(t *testing.T) stageFixture { + t.Helper() + recorder := tracetest.NewSpanRecorder() + provider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder)) + t.Cleanup(func() { + // The test context is already canceled during cleanup, so the + // flush uses a fresh one. + require.NoError(t, provider.Shutdown(context.Background())) + }) + registry := prometheus.NewRegistry() + return stageFixture{ + tracer: chatloop.NewStageTracer(provider, chatloop.NewMetrics(registry)), + spans: recorder, + registry: registry, + } +} + +// stageKey identifies one stage_duration_seconds series. +type stageKey struct { + stage string + scope string + model string + effort string +} + +// stageObservations returns the observation count per stage series +// recorded on coderd_chatd_stage_duration_seconds. +func (f stageFixture) stageObservations(t *testing.T) map[stageKey]uint64 { + t.Helper() + families, err := f.registry.Gather() + require.NoError(t, err) + counts := map[stageKey]uint64{} + for _, family := range families { + if family.GetName() != "coderd_chatd_stage_duration_seconds" { + continue + } + for _, metric := range family.GetMetric() { + key := stageKey{ + stage: labelValue(metric, "stage"), + scope: labelValue(metric, "scope"), + model: labelValue(metric, "model"), + effort: labelValue(metric, "effort"), + } + counts[key] = metric.GetHistogram().GetSampleCount() + } + } + return counts +} + +func (f stageFixture) stageSum(t *testing.T, stage string) float64 { + t.Helper() + families, err := f.registry.Gather() + require.NoError(t, err) + for _, family := range families { + if family.GetName() != "coderd_chatd_stage_duration_seconds" { + continue + } + for _, metric := range family.GetMetric() { + if labelValue(metric, "stage") == stage { + return metric.GetHistogram().GetSampleSum() + } + } + } + t.Fatalf("stage %q was not recorded", stage) + return 0 +} + +func labelValue(metric *dto.Metric, name string) string { + for _, label := range metric.GetLabel() { + if label.GetName() == name { + return label.GetValue() + } + } + return "" +} + +func TestStageTracerStart(t *testing.T) { + t.Parallel() + + t.Run("RecordsSpanAndMetricOnce", func(t *testing.T) { + t.Parallel() + fixture := newStageFixture(t) + + _, span := fixture.tracer.Start(t.Context(), chatloop.StageCommit, + attribute.String(chatloop.AttrProvider, "anthropic"), + ) + span.End(nil) + span.End(nil) + + ended := fixture.spans.Ended() + require.Len(t, ended, 1) + require.Equal(t, chatloop.StageCommit, ended[0].Name()) + require.Equal(t, codes.Unset, ended[0].Status().Code) + require.Contains(t, ended[0].Attributes(), + attribute.String(chatloop.AttrProvider, "anthropic")) + require.Contains(t, ended[0].Attributes(), + attribute.String(chatloop.AttrScope, chatloop.ScopeBackground)) + require.Equal(t, map[stageKey]uint64{ + {stage: chatloop.StageCommit, scope: chatloop.ScopeBackground}: 1, + }, fixture.stageObservations(t)) + }) + + t.Run("MarksErrorStatus", func(t *testing.T) { + t.Parallel() + fixture := newStageFixture(t) + + _, span := fixture.tracer.Start(t.Context(), chatloop.StageStream) + span.End(xerrors.New("stream failed")) + + ended := fixture.spans.Ended() + require.Len(t, ended, 1) + require.Equal(t, codes.Error, ended[0].Status().Code) + require.Equal(t, map[stageKey]uint64{ + {stage: chatloop.StageStream, scope: chatloop.ScopeBackground}: 1, + }, fixture.stageObservations(t)) + }) + + t.Run("NestsUnderParent", func(t *testing.T) { + t.Parallel() + fixture := newStageFixture(t) + + parentCtx, parent := fixture.tracer.Start(t.Context(), chatloop.StageGenerationStep) + _, child := fixture.tracer.Start(parentCtx, chatloop.StagePrepare) + child.End(nil) + parent.End(nil) + + ended := fixture.spans.Ended() + require.Len(t, ended, 2) + require.Equal(t, chatloop.StagePrepare, ended[0].Name()) + require.Equal(t, ended[1].SpanContext().SpanID(), ended[0].Parent().SpanID()) + require.Equal(t, ended[1].SpanContext().TraceID(), ended[0].SpanContext().TraceID()) + }) +} + +func TestStageTracerStartRoot(t *testing.T) { + t.Parallel() + fixture := newStageFixture(t) + + outerCtx, outer := fixture.tracer.Start(t.Context(), chatloop.StageGenerationStep) + _, root := fixture.tracer.StartRoot(outerCtx, chatloop.StageChatTurn, + []trace.Link{{SpanContext: trace.SpanContextFromContext(outerCtx)}}, + ) + root.End(nil) + outer.End(nil) + + ended := fixture.spans.Ended() + require.Len(t, ended, 2) + turn, step := ended[0], ended[1] + require.Equal(t, chatloop.StageChatTurn, turn.Name()) + require.False(t, turn.Parent().IsValid()) + require.NotEqual(t, step.SpanContext().TraceID(), turn.SpanContext().TraceID()) + require.Len(t, turn.Links(), 1) + require.Equal(t, step.SpanContext().SpanID(), turn.Links()[0].SpanContext.SpanID()) +} + +func TestStageTracerStartRootAt(t *testing.T) { + t.Parallel() + fixture := newStageFixture(t) + + start := time.Now().Add(-45 * time.Second) + turnCtx, turn := fixture.tracer.StartRootAt(t.Context(), chatloop.StageChatTurn, start, nil) + fixture.tracer.Record(turnCtx, chatloop.StageAcquisition, chatloop.StageModel{}, start, start.Add(time.Second), nil) + turn.End(nil) + + var chatTurn, acquisition sdktrace.ReadOnlySpan + for _, span := range fixture.spans.Ended() { + switch span.Name() { + case chatloop.StageChatTurn: + chatTurn = span + case chatloop.StageAcquisition: + acquisition = span + } + } + require.NotNil(t, chatTurn) + require.NotNil(t, acquisition) + require.Equal(t, start.UTC(), chatTurn.StartTime().UTC()) + require.False(t, acquisition.StartTime().Before(chatTurn.StartTime())) + require.Equal(t, chatTurn.SpanContext().SpanID(), acquisition.Parent().SpanID()) + // A root turn span opens a turn, and stages recorded inside it + // inherit the turn scope from its context. + require.Contains(t, chatTurn.Attributes(), + attribute.String(chatloop.AttrScope, chatloop.ScopeTurn)) + require.Contains(t, acquisition.Attributes(), + attribute.String(chatloop.AttrScope, chatloop.ScopeTurn)) + require.Equal(t, map[stageKey]uint64{ + {stage: chatloop.StageChatTurn, scope: chatloop.ScopeTurn}: 1, + {stage: chatloop.StageAcquisition, scope: chatloop.ScopeTurn}: 1, + }, fixture.stageObservations(t)) + // The histogram observation runs from the explicit start, so it + // covers the same window the span reports. + require.GreaterOrEqual(t, fixture.stageSum(t, chatloop.StageChatTurn), 45.0) +} + +func TestStageTracerScope(t *testing.T) { + t.Parallel() + + t.Run("TurnWorkStaysInTurnScope", func(t *testing.T) { + t.Parallel() + fixture := newStageFixture(t) + + turnCtx, turn := fixture.tracer.StartRoot(t.Context(), chatloop.StageChatTurn, nil) + stepCtx, step := fixture.tracer.Start(turnCtx, chatloop.StageGenerationStep) + _, attempt := fixture.tracer.Start(stepCtx, chatloop.StageProviderAttempt) + attempt.End(nil) + step.End(nil) + turn.End(nil) + + require.Equal(t, map[stageKey]uint64{ + {stage: chatloop.StageChatTurn, scope: chatloop.ScopeTurn}: 1, + {stage: chatloop.StageGenerationStep, scope: chatloop.ScopeTurn}: 1, + {stage: chatloop.StageProviderAttempt, scope: chatloop.ScopeTurn}: 1, + }, fixture.stageObservations(t)) + }) + + t.Run("DetachedWorkIsBackgroundScope", func(t *testing.T) { + t.Parallel() + fixture := newStageFixture(t) + + turnCtx, turn := fixture.tracer.StartRoot(t.Context(), chatloop.StageChatTurn, nil) + // Background work strips the span from its context, which is + // what detaches its stages from the turn profile. + detachedCtx := trace.ContextWithSpanContext(turnCtx, trace.SpanContext{}) + _, attempt := fixture.tracer.Start(detachedCtx, chatloop.StageProviderAttempt) + attempt.End(nil) + turn.End(nil) + + require.Equal(t, map[stageKey]uint64{ + {stage: chatloop.StageChatTurn, scope: chatloop.ScopeTurn}: 1, + {stage: chatloop.StageProviderAttempt, scope: chatloop.ScopeBackground}: 1, + }, fixture.stageObservations(t)) + + for _, span := range fixture.spans.Ended() { + if span.Name() != chatloop.StageProviderAttempt { + continue + } + require.False(t, span.Parent().IsValid()) + require.Contains(t, span.Attributes(), + attribute.String(chatloop.AttrScope, chatloop.ScopeBackground)) + } + }) + + t.Run("RecordAsOverridesContextScope", func(t *testing.T) { + t.Parallel() + fixture := newStageFixture(t) + + start := time.Now().Add(-5 * time.Second) + fixture.tracer.RecordAs(t.Context(), chatloop.StageCapacityWait, chatloop.ScopeTurn, + chatloop.StageModel{}, start, start.Add(time.Second), nil) + + require.Equal(t, map[stageKey]uint64{ + {stage: chatloop.StageCapacityWait, scope: chatloop.ScopeTurn}: 1, + }, fixture.stageObservations(t)) + ended := fixture.spans.Ended() + require.Len(t, ended, 1) + require.Contains(t, ended[0].Attributes(), + attribute.String(chatloop.AttrScope, chatloop.ScopeTurn)) + }) +} + +func TestStageTracerModelLabels(t *testing.T) { + t.Parallel() + + model := chatloop.StageModel{Model: "claude-sonnet-4-5", Effort: "high"} + + t.Run("SetModelLabelsSpanAndDuration", func(t *testing.T) { + t.Parallel() + fixture := newStageFixture(t) + + turnCtx, turn := fixture.tracer.StartRoot(t.Context(), chatloop.StageChatTurn, nil) + _, step := fixture.tracer.Start(turnCtx, chatloop.StageGenerationStep) + // The step learns its model only after preparation resolves it. + step.SetModel(model) + step.End(nil) + turn.End(nil) + + require.Equal(t, map[stageKey]uint64{ + {stage: chatloop.StageChatTurn, scope: chatloop.ScopeTurn}: 1, + { + stage: chatloop.StageGenerationStep, + scope: chatloop.ScopeTurn, + model: model.Model, + effort: model.Effort, + }: 1, + }, fixture.stageObservations(t)) + + for _, span := range fixture.spans.Ended() { + if span.Name() != chatloop.StageGenerationStep { + continue + } + require.Contains(t, span.Attributes(), + attribute.String(chatloop.AttrModel, model.Model)) + require.Contains(t, span.Attributes(), + attribute.String(chatloop.AttrReasoningEffort, model.Effort)) + } + }) + + t.Run("RecordCarriesModelLabels", func(t *testing.T) { + t.Parallel() + fixture := newStageFixture(t) + + start := time.Now().Add(-time.Second) + fixture.tracer.Record(t.Context(), chatloop.StageThinking, model, start, time.Now(), nil) + + require.Equal(t, map[stageKey]uint64{{ + stage: chatloop.StageThinking, + scope: chatloop.ScopeBackground, + model: model.Model, + effort: model.Effort, + }: 1}, fixture.stageObservations(t)) + + ended := fixture.spans.Ended() + require.Len(t, ended, 1) + require.Contains(t, ended[0].Attributes(), + attribute.String(chatloop.AttrModel, model.Model)) + require.Contains(t, ended[0].Attributes(), + attribute.String(chatloop.AttrReasoningEffort, model.Effort)) + }) + + t.Run("UnknownIdentityUsesEmptyLabels", func(t *testing.T) { + t.Parallel() + fixture := newStageFixture(t) + + start := time.Now().Add(-time.Second) + fixture.tracer.Record(t.Context(), chatloop.StageQueueWait, chatloop.StageModel{}, start, time.Now(), nil) + + require.Equal(t, map[stageKey]uint64{{ + stage: chatloop.StageQueueWait, + scope: chatloop.ScopeBackground, + }: 1}, fixture.stageObservations(t)) + + ended := fixture.spans.Ended() + require.Len(t, ended, 1) + for _, attr := range ended[0].Attributes() { + require.NotEqual(t, chatloop.AttrModel, string(attr.Key)) + require.NotEqual(t, chatloop.AttrReasoningEffort, string(attr.Key)) + } + }) + + t.Run("ModelWithoutEffort", func(t *testing.T) { + t.Parallel() + fixture := newStageFixture(t) + + _, span := fixture.tracer.Start(t.Context(), chatloop.StageStream) + span.SetModel(chatloop.StageModel{Model: "gpt-5"}) + span.End(nil) + + require.Equal(t, map[stageKey]uint64{{ + stage: chatloop.StageStream, + scope: chatloop.ScopeBackground, + model: "gpt-5", + }: 1}, fixture.stageObservations(t)) + }) +} + +func TestStageTracerRecord(t *testing.T) { + t.Parallel() + + t.Run("UsesExplicitTimestamps", func(t *testing.T) { + t.Parallel() + fixture := newStageFixture(t) + + start := time.Now().Add(-90 * time.Second) + end := start.Add(30 * time.Second) + fixture.tracer.Record(t.Context(), chatloop.StageQueueWait, chatloop.StageModel{}, start, end, nil, + attribute.String(chatloop.AttrChatKind, chatloop.ChatKindRoot), + ) + + ended := fixture.spans.Ended() + require.Len(t, ended, 1) + require.Equal(t, chatloop.StageQueueWait, ended[0].Name()) + require.Equal(t, start.UTC(), ended[0].StartTime().UTC()) + require.Equal(t, end.UTC(), ended[0].EndTime().UTC()) + require.Equal(t, map[stageKey]uint64{ + {stage: chatloop.StageQueueWait, scope: chatloop.ScopeBackground}: 1, + }, fixture.stageObservations(t)) + require.InDelta(t, 30, fixture.stageSum(t, chatloop.StageQueueWait), 0.001) + }) + + t.Run("DropsUnusableWindows", func(t *testing.T) { + t.Parallel() + fixture := newStageFixture(t) + + now := time.Now() + fixture.tracer.Record(t.Context(), chatloop.StageAcquisition, chatloop.StageModel{}, time.Time{}, now, nil) + fixture.tracer.Record(t.Context(), chatloop.StageAcquisition, chatloop.StageModel{}, now, time.Time{}, nil) + fixture.tracer.Record(t.Context(), chatloop.StageAcquisition, chatloop.StageModel{}, now, now.Add(-time.Second), nil) + + require.Empty(t, fixture.spans.Ended()) + require.Empty(t, fixture.stageObservations(t)) + }) +} + +func TestStageTracerWithoutProvider(t *testing.T) { + t.Parallel() + + registry := prometheus.NewRegistry() + tracer := chatloop.NewStageTracer(nil, chatloop.NewMetrics(registry)) + _, span := tracer.Start(t.Context(), chatloop.StageToolCall) + span.End(nil) + tracer.Record(t.Context(), chatloop.StageThinking, chatloop.StageModel{}, time.Now().Add(-time.Second), time.Now(), nil) + + fixture := stageFixture{registry: registry} + require.Equal(t, map[stageKey]uint64{ + {stage: chatloop.StageToolCall, scope: chatloop.ScopeBackground}: 1, + {stage: chatloop.StageThinking, scope: chatloop.ScopeBackground}: 1, + }, fixture.stageObservations(t)) + + var nilTracer *chatloop.StageTracer + _, nilSpan := nilTracer.Start(t.Context(), chatloop.StageToolCall) + nilSpan.End(nil) + nilTracer.Record(t.Context(), chatloop.StageThinking, chatloop.StageModel{}, time.Now().Add(-time.Second), time.Now(), nil) +} diff --git a/coderd/x/chatd/chatstate/transitions.go b/coderd/x/chatd/chatstate/transitions.go index 6a607597a6f..19531ac5a22 100644 --- a/coderd/x/chatd/chatstate/transitions.go +++ b/coderd/x/chatd/chatstate/transitions.go @@ -1448,6 +1448,11 @@ type FinishTurnInput struct{} type FinishTurnResult struct { Chat database.Chat PromotedMessage *database.ChatMessage + // PromotedQueuedAt is the queued row's creation time when this + // transition promoted a queue head, and the zero time otherwise. + // The promoted history row carries its insertion time, so this is + // the only record of how long the message sat in the queue. + PromotedQueuedAt time.Time } // FinishTurn completes a running turn. @@ -1509,8 +1514,9 @@ func (tx *Tx) FinishTurn(_ FinishTurnInput) (FinishTurnResult, error) { promoted = &inserted[len(inserted)-1] } return FinishTurnResult{ - Chat: updated, - PromotedMessage: promoted, + Chat: updated, + PromotedMessage: promoted, + PromotedQueuedAt: head.CreatedAt, }, nil } diff --git a/coderd/x/chatd/generation.go b/coderd/x/chatd/generation.go index 5956d93eba1..d8836f41b60 100644 --- a/coderd/x/chatd/generation.go +++ b/coderd/x/chatd/generation.go @@ -5,11 +5,13 @@ import ( "encoding/json" "errors" "strings" + "sync" "time" "charm.land/fantasy" "github.com/google/uuid" "github.com/sqlc-dev/pqtype" + "go.opentelemetry.io/otel/attribute" "golang.org/x/xerrors" "cdr.dev/slog/v3" @@ -68,6 +70,10 @@ type generationPrepared struct { // user-facing errors. See chatloop.GenerateAssistantOptions.ErrorProvider. ResolvedProvider string + // StageModel is the resolved model and effective reasoning effort + // the turn's stages are labeled with. + StageModel chatloop.StageModel + ModelConfigID uuid.UUID CallTemplate fantasy.Call ContextLimitFallback int64 @@ -441,129 +447,167 @@ func (s *taskStarter) StartGeneration(ctx context.Context, input chatWorkerTaskS if err != nil { return xerrors.Errorf("load generation state: %w", err) } - if s.server.hooks.Enabled() { - result, dispatched, err := s.startGenerationSession(ctx, machine, input, chat, messages) - if err != nil { - if errors.Is(err, errTaskExpectedExit) { - return err - } - return s.finishGenerationError(ctx, machine, input, err, generationAttemptNotRequired) - } - if dispatched { - input.HistoryVersion = result.Chat.HistoryVersion - continue - } - } - prepareInput := generationPrepareInput{ - Chat: chat, - Messages: messages, - RecordMCPConnectSummaries: input.DebugTurn.RecordMCPConnectSummaries, + turnCtx := input.Turn.Ensure(ctx, chat, triggerMessageTime(messages)) + var again bool + input, again, err = s.runGenerationStep(turnCtx, machine, input, chat, messages) + if again { + continue } - prepared, err := retryGenerationPhase(ctx, s, "prepare", func() (generationPrepared, error) { - return s.server.prepareGeneration(ctx, prepareInput) - }) + return err + } +} + +// runGenerationStep runs one step of a turn: preparation, the action +// decision, and the action itself. It returns the input to use for +// the next step and whether the caller must reload state and run one. +// The returned input differs from the passed one when a step advances +// the history version. +func (s *taskStarter) runGenerationStep( + ctx context.Context, + machine *chatstate.ChatMachine, + input chatWorkerTaskStartInput, + chat database.Chat, + messages []database.ChatMessage, +) (next chatWorkerTaskStartInput, again bool, err error) { + ctx, stepSpan := s.server.stages.Start(ctx, chatloop.StageGenerationStep, + attribute.String(chatloop.AttrChatID, input.ChatID.String()), + attribute.String(chatloop.AttrChatKind, chatKindAttr(chat)), + attribute.Int64(chatloop.AttrGenerationAttempt, input.GenerationAttempt), + ) + defer func() { stepSpan.End(err) }() + + if s.server.hooks.Enabled() { + result, dispatched, err := s.startGenerationSession(ctx, machine, input, chat, messages) if err != nil { - if errors.Is(err, errTaskExpectedExit) || errors.Is(err, errTaskRetryable) { - return xerrors.Errorf("prepare generation: %w", err) + if errors.Is(err, errTaskExpectedExit) { + return input, false, err } - return s.finishGenerationError(ctx, machine, input, err, generationAttemptNotRequired) - } - cleanup := prepared.Cleanup - var decision generationDecision - if input.StopNudges.consume(stopNudgeKey(prepared.Messages)) { - decision = generationDecision{kind: generationActionGenerateAssistant} - } else { - decision, err = retryGenerationPhase(ctx, s, "decide", func() (generationDecision, error) { - return decideGenerationAction(generationDecisionInput{ - chat: prepared.Chat, - messages: prepared.Messages, - dynamicToolNames: prepared.DynamicToolNames, - exclusiveToolNames: prepared.ExclusiveToolNames, - stopAfterTools: prepared.StopAfterTools, - maxSteps: prepared.MaxSteps, - compactionEnabled: prepared.Compaction != nil, - compactionNeeded: prepared.Compaction != nil && prepared.Compaction.Required, - compactionThresholdPercent: generationCompactionThreshold(prepared.Compaction), - compactionContextLimit: generationCompactionContextLimit(prepared.Compaction), - }) - }) + return input, false, s.finishGenerationError(ctx, machine, input, err, generationAttemptNotRequired) } - if err != nil { - cleanup() - if errors.Is(err, errTaskExpectedExit) || errors.Is(err, errTaskRetryable) { - return xerrors.Errorf("decide generation: %w", err) - } - if errors.Is(err, errCompactionStillOverLimit) && prepared.Compaction != nil { - metricProvider, metricModel := compactionMetricIdentity(prepared.Compaction) - s.server.metrics.RecordCompaction( - metricProvider, - metricModel, - false, - errCompactionStillOverLimit, - ) - } - return s.finishGenerationError(ctx, machine, input, err, generationAttemptNotRequired) - } - - var actionErr error - switch decision.kind { - case generationActionEnterRequiresAction: - cleanup() - return s.enterRequiresAction(ctx, machine, input) - case generationActionFinishTurn: - cleanup() - return s.finishGenerationTurn(ctx, machine, input, decision, generationAttemptNotRequired) - case generationActionGenerateAssistant: - actionErr = s.generateAssistant(ctx, machine, input, prepared) - case generationActionExecuteLocalTools: - actionErr = s.executeLocalTools(ctx, machine, input, prepared, decision) - case generationActionCompact: - actionErr = s.generateCompaction(ctx, machine, input, prepared, compactionSourceForDecision(decision)) - default: - return s.finishGenerationError(ctx, machine, input, xerrors.Errorf("unknown generation action %q", decision.kind), generationAttemptNotRequired) - } - cleanup() - if actionErr == nil { - return nil + if dispatched { + input.HistoryVersion = result.Chat.HistoryVersion + return input, true, nil } - // Task cancellation is handled by the runner, not here. - if ctx.Err() != nil && errors.Is(actionErr, context.Canceled) { - return errors.Join(errTaskExpectedExit, xerrors.Errorf("generation action: %w", actionErr), ctx.Err()) + } + prepareInput := generationPrepareInput{ + Chat: chat, + Messages: messages, + RecordMCPConnectSummaries: input.DebugTurn.RecordMCPConnectSummaries, + } + prepareCtx, prepareSpan := s.server.stages.Start(ctx, chatloop.StagePrepare) + prepared, err := retryGenerationPhase(prepareCtx, s, "prepare", func() (generationPrepared, error) { + return s.server.prepareGeneration(prepareCtx, prepareInput) + }) + if err == nil { + providerAttr := attribute.String(chatloop.AttrProvider, prepared.ResolvedProvider) + prepareSpan.SetAttributes(providerAttr) + prepareSpan.SetModel(prepared.StageModel) + stepSpan.SetAttributes(providerAttr) + stepSpan.SetModel(prepared.StageModel) + } + prepareSpan.End(err) + if err != nil { + if errors.Is(err, errTaskExpectedExit) || errors.Is(err, errTaskRetryable) { + return input, false, xerrors.Errorf("prepare generation: %w", err) } - if errors.Is(actionErr, chatloop.ErrInterrupted) { - return errors.Join(errTaskExpectedExit, xerrors.Errorf("generation action: %w", actionErr)) + return input, false, s.finishGenerationError(ctx, machine, input, err, generationAttemptNotRequired) + } + cleanup := prepared.Cleanup + var decision generationDecision + if input.StopNudges.consume(stopNudgeKey(prepared.Messages)) { + decision = generationDecision{kind: generationActionGenerateAssistant} + } else { + decision, err = retryGenerationPhase(ctx, s, "decide", func() (generationDecision, error) { + return decideGenerationAction(generationDecisionInput{ + chat: prepared.Chat, + messages: prepared.Messages, + dynamicToolNames: prepared.DynamicToolNames, + exclusiveToolNames: prepared.ExclusiveToolNames, + stopAfterTools: prepared.StopAfterTools, + maxSteps: prepared.MaxSteps, + compactionEnabled: prepared.Compaction != nil, + compactionNeeded: prepared.Compaction != nil && prepared.Compaction.Required, + compactionThresholdPercent: generationCompactionThreshold(prepared.Compaction), + compactionContextLimit: generationCompactionContextLimit(prepared.Compaction), + }) + }) + } + if err != nil { + cleanup() + if errors.Is(err, errTaskExpectedExit) || errors.Is(err, errTaskRetryable) { + return input, false, xerrors.Errorf("decide generation: %w", err) + } + if errors.Is(err, errCompactionStillOverLimit) && prepared.Compaction != nil { + metricProvider, metricModel := compactionMetricIdentity(prepared.Compaction) + s.server.metrics.RecordCompaction( + metricProvider, + metricModel, + false, + errCompactionStillOverLimit, + ) } - if errors.Is(actionErr, errTaskExpectedExit) { - return xerrors.Errorf("generation action: %w", actionErr) + return input, false, s.finishGenerationError(ctx, machine, input, err, generationAttemptNotRequired) + } + + stepSpan.SetAttributes(attribute.String(chatloop.AttrGenerationAction, string(decision.kind))) + var actionErr error + switch decision.kind { + case generationActionEnterRequiresAction: + cleanup() + return input, false, s.enterRequiresAction(ctx, machine, input) + case generationActionFinishTurn: + cleanup() + return input, false, s.finishGenerationTurn(ctx, machine, input, decision, generationAttemptNotRequired) + case generationActionGenerateAssistant: + actionErr = s.generateAssistant(ctx, machine, input, prepared) + case generationActionExecuteLocalTools: + actionErr = s.executeLocalTools(ctx, machine, input, prepared, decision) + case generationActionCompact: + actionErr = s.generateCompaction(ctx, machine, input, prepared, compactionSourceForDecision(decision)) + default: + return input, false, s.finishGenerationError(ctx, machine, input, xerrors.Errorf("unknown generation action %q", decision.kind), generationAttemptNotRequired) + } + cleanup() + if actionErr == nil { + return input, false, nil + } + // Task cancellation is handled by the runner, not here. + if ctx.Err() != nil && errors.Is(actionErr, context.Canceled) { + return input, false, errors.Join(errTaskExpectedExit, xerrors.Errorf("generation action: %w", actionErr), ctx.Err()) + } + if errors.Is(actionErr, chatloop.ErrInterrupted) { + return input, false, errors.Join(errTaskExpectedExit, xerrors.Errorf("generation action: %w", actionErr)) + } + if errors.Is(actionErr, errTaskExpectedExit) { + return input, false, xerrors.Errorf("generation action: %w", actionErr) + } + classified := chaterror.Classify(actionErr) + if classified.Retryable { + action := decision.kind + decision, err := s.recordGenerationRetry(ctx, machine, input, classified) + if err != nil { + return input, false, xerrors.Errorf("record generation retry: %w", err) } - classified := chaterror.Classify(actionErr) - if classified.Retryable { - action := decision.kind - decision, err := s.recordGenerationRetry(ctx, machine, input, classified) - if err != nil { - return xerrors.Errorf("record generation retry: %w", err) - } - if decision.retry { - s.opts.Logger.Warn(ctx, "chat generation retrying", - slog.F("chat_id", input.ChatID), - slog.F("worker_id", input.WorkerID), - slog.F("action", action), - slog.F("generation_attempt", decision.generationAttempt), - slog.F("delay", decision.delay), - slog.F("error_kind", classified.Kind), - slog.F("provider", classified.Provider), - slog.F("status_code", classified.StatusCode), - slogError(actionErr), - ) - if err := s.waitGenerationRetry(ctx, decision.delay); err != nil { - return xerrors.Errorf("wait generation retry: %w", err) - } - continue + if decision.retry { + s.opts.Logger.Warn(ctx, "chat generation retrying", + slog.F("chat_id", input.ChatID), + slog.F("worker_id", input.WorkerID), + slog.F("action", action), + slog.F("generation_attempt", decision.generationAttempt), + slog.F("delay", decision.delay), + slog.F("error_kind", classified.Kind), + slog.F("provider", classified.Provider), + slog.F("status_code", classified.StatusCode), + slogError(actionErr), + ) + if err := s.waitGenerationRetry(ctx, decision.delay); err != nil { + return input, false, xerrors.Errorf("wait generation retry: %w", err) } - return s.finishGenerationError(ctx, machine, input, actionErr, requireGenerationAttempt(decision.generationAttempt)) + return input, true, nil } - return s.finishGenerationError(ctx, machine, input, actionErr, generationAttemptNotRequired) + return input, false, s.finishGenerationError(ctx, machine, input, actionErr, requireGenerationAttempt(decision.generationAttempt)) } + return input, false, s.finishGenerationError(ctx, machine, input, actionErr, generationAttemptNotRequired) } func loadGenerationState( @@ -753,10 +797,13 @@ func (s *taskStarter) generateAssistant( Logger: s.opts.Logger, Clock: s.opts.Clock, Metrics: s.server.metrics, + Stages: s.server.stages, + StageModel: prepared.StageModel, }) if err != nil { return xerrors.Errorf("generate assistant: %w", err) } + s.recordThinkingStages(runCtx, prepared, outcome.Step) if len(outcome.Step.Content) == 0 { return s.finishGenerationTurn(ctx, machine, input, generationDecision{kind: generationActionFinishTurn, finishReason: generationFinishReasonComplete}, requireGenerationAttempt(attempt.number)) } @@ -783,6 +830,31 @@ func (s *taskStarter) generateAssistant( return s.commitGenerationStep(ctx, machine, input, attempt.number, generationActionGenerateAssistant, messages, generationCommitHooks{}) } +// recordThinkingStages emits one thinking stage per reasoning part of +// a step, bounded by the part's own start and completion timestamps. +// Parts are paired by index; a part without a completion timestamp +// ends the sweep because later indexes cannot be paired either. +func (s *taskStarter) recordThinkingStages( + ctx context.Context, + prepared generationPrepared, + step chatloop.PersistedStep, +) { + for index, startedAt := range step.ReasoningStartedAt { + if index >= len(step.ReasoningCompletedAt) { + return + } + s.server.stages.Record( + ctx, + chatloop.StageThinking, + prepared.StageModel, + startedAt, + step.ReasoningCompletedAt[index], + nil, + attribute.String(chatloop.AttrProvider, prepared.ResolvedProvider), + ) + } +} + func (s *taskStarter) admitStepToolCalls( ctx context.Context, input chatWorkerTaskStartInput, @@ -861,6 +933,41 @@ func (r *bufferToolBillingRecorder) RecordComplete(dispatchIndex int, completedA r.recordComplete(r.allowedIndexes[dispatchIndex], completedAt) } +// toolCallStageRecorder emits one tool_call stage per local tool call +// while forwarding every callback to the billing recorder it wraps. +// Completions arrive on the tool goroutines, so the pending starts are +// mutex guarded. +type toolCallStageRecorder struct { + inner chatloop.ToolBillingRecorder + record func(dispatchIndex int, startedAt, completedAt time.Time) + + mu sync.Mutex + starts map[int]time.Time +} + +func (r *toolCallStageRecorder) RecordStart(dispatchIndex int, startedAt time.Time) { + if r.inner != nil { + r.inner.RecordStart(dispatchIndex, startedAt) + } + r.mu.Lock() + r.starts[dispatchIndex] = startedAt + r.mu.Unlock() +} + +func (r *toolCallStageRecorder) RecordComplete(dispatchIndex int, completedAt time.Time) { + if r.inner != nil { + r.inner.RecordComplete(dispatchIndex, completedAt) + } + r.mu.Lock() + startedAt, started := r.starts[dispatchIndex] + delete(r.starts, dispatchIndex) + r.mu.Unlock() + if !started { + return + } + r.record(dispatchIndex, startedAt, completedAt) +} + func (s *taskStarter) executeLocalTools( ctx context.Context, machine *chatstate.ChatMachine, @@ -908,6 +1015,20 @@ func (s *taskStarter) executeLocalTools( recordComplete: attempt.recordToolCompletion, } } + toolStages := &toolCallStageRecorder{ + inner: billingRecorder, + starts: make(map[int]time.Time, len(allowed)), + record: func(dispatchIndex int, startedAt, completedAt time.Time) { + toolName := "" + if dispatchIndex >= 0 && dispatchIndex < len(allowed) { + toolName = allowed[dispatchIndex].ToolName + } + s.server.stages.Record(ctx, chatloop.StageToolCall, prepared.StageModel, startedAt, completedAt, nil, + attribute.String(chatloop.AttrToolName, toolName), + attribute.String(chatloop.AttrProvider, provider), + ) + }, + } outcome, err = chatloop.ExecuteLocalTools(ctx, chatloop.ExecuteLocalToolsOptions{ Tools: prepared.Tools, ActiveTools: prepared.ActiveTools, @@ -922,7 +1043,7 @@ func (s *taskStarter) executeLocalTools( ContextLimit: prepared.ContextLimitFallback, ToolNameAliases: subagentToolNameAliases, UnbilledToolNames: unbilledSubagentToolNames, - BillingRecorder: billingRecorder, + BillingRecorder: toolStages, PublishMessagePart: attempt.publish, Logger: s.opts.Logger, Metrics: s.server.metrics, @@ -1052,7 +1173,13 @@ func (s *taskStarter) generateCompaction( // debug run; without it startCompactionDebugRun finds no parent and // skips debug instrumentation entirely. runCtx := input.DebugTurn.Ensure(ctx, prepared.Chat, prepared.Debug) - outcome, err := chatloop.GenerateCompaction(runCtx, compactionOpts) + compactionCtx, compactionSpan := s.server.stages.Start(runCtx, chatloop.StageCompaction, + attribute.String(chatloop.AttrProvider, metricProvider), + attribute.String(chatloop.AttrCompactionSource, string(source)), + ) + compactionSpan.SetModel(compactionStageModel(prepared, metricModel)) + outcome, err := chatloop.GenerateCompaction(compactionCtx, compactionOpts) + compactionSpan.End(err) if err != nil { s.server.metrics.RecordCompaction(metricProvider, metricModel, false, err) return xerrors.Errorf("generate compaction: %w", err) @@ -1108,6 +1235,18 @@ func (s *taskStarter) generateCompaction( return nil } +// compactionStageModel labels the compaction stage with the summary +// model. The effort carries over only when the chat model runs the +// summary; an override resolves its own call options, which this call +// site does not see. +func compactionStageModel(prepared generationPrepared, summaryModel string) chatloop.StageModel { + model := chatloop.StageModel{Model: summaryModel} + if prepared.Compaction != nil && prepared.Compaction.Override == nil { + model.Effort = prepared.StageModel.Effort + } + return model +} + // compactionMetricIdentity returns the provider/model labels for compaction // metrics. Override labels come from prepare-time resolution so events // recorded before the override client is built (still-over-limit) match @@ -1246,7 +1385,11 @@ func (s *taskStarter) commitGenerationStep( } var committed database.Chat insertedMessages := []runnerActionMessage{} - err := machine.Update(ctx, func(tx *chatstate.Tx, store database.Store) error { + commitCtx, commitSpan := s.server.stages.Start(ctx, chatloop.StageCommit, + attribute.String(chatloop.AttrGenerationAction, string(kind)), + attribute.Int64(chatloop.AttrGenerationAttempt, attempt), + ) + err := machine.Update(commitCtx, func(tx *chatstate.Tx, store database.Store) error { if _, err := loadChatForGeneration(ctx, store, input, requireGenerationAttempt(attempt)); err != nil { return xerrors.Errorf("load chat for generation: %w", err) } @@ -1277,6 +1420,7 @@ func (s *taskStarter) commitGenerationStep( committed = loadedChat return nil }) + commitSpan.End(err) if err != nil { return normalizeTaskTransitionError(err, "commit generation step") } @@ -1416,6 +1560,7 @@ func (s *taskStarter) finishGenerationTurnWithoutHook( fence generationAttemptFence, ) error { var committed database.Chat + var promotedQueuedAt time.Time err := machine.Update(ctx, func(tx *chatstate.Tx, store database.Store) error { if _, err := loadChatForGeneration(ctx, store, input, fence); err != nil { return xerrors.Errorf("load chat for generation: %w", err) @@ -1426,6 +1571,7 @@ func (s *taskStarter) finishGenerationTurnWithoutHook( } if finishResult.PromotedMessage != nil { decision.promotedMessageID = finishResult.PromotedMessage.ID + promotedQueuedAt = finishResult.PromotedQueuedAt } committed = finishResult.Chat return nil @@ -1435,9 +1581,27 @@ func (s *taskStarter) finishGenerationTurnWithoutHook( recordGenerationFinishFailure(input.DebugTurn, err) return err } + s.recordQueueWaitStage(ctx, input, promotedQueuedAt) return s.completeGenerationTurn(ctx, input, committed, decision.promotedMessageID) } +// recordQueueWaitStage emits the queue_wait stage for a message just +// promoted out of the queue, measured from the queued row's creation +// to now. It is recorded against the turn span rather than the step +// that observed the promotion, whose window does not contain the +// queue wait. A zero queuedAt means the transition promoted nothing +// and records no stage. +func (s *taskStarter) recordQueueWaitStage( + ctx context.Context, + input chatWorkerTaskStartInput, + queuedAt time.Time, +) { + s.server.stages.Record(input.Turn.Context(ctx), chatloop.StageQueueWait, chatloop.StageModel{}, + queuedAt, time.Now(), nil, + attribute.String(chatloop.AttrChatID, input.ChatID.String()), + ) +} + func (s *taskStarter) finishGenerationTurn( ctx context.Context, machine *chatstate.ChatMachine, @@ -1483,6 +1647,7 @@ func (s *taskStarter) finishGenerationTurn( continueTurn := strings.TrimSpace(response.GetModelContext()) != "" && input.StopNudges.claim(nudgeKey) var committed database.Chat + var promotedQueuedAt time.Time err = machine.Update(ctx, func(tx *chatstate.Tx, store database.Store) error { if _, err := loadChatForGeneration(ctx, store, input, fence); err != nil { return xerrors.Errorf("load chat for generation: %w", err) @@ -1499,6 +1664,7 @@ func (s *taskStarter) finishGenerationTurn( } if finishResult.PromotedMessage != nil { decision.promotedMessageID = finishResult.PromotedMessage.ID + promotedQueuedAt = finishResult.PromotedQueuedAt } committed = finishResult.Chat return nil @@ -1525,6 +1691,7 @@ func (s *taskStarter) finishGenerationTurn( Kind: runnerActionKind(generationActionGenerateAssistant), }) } + s.recordQueueWaitStage(ctx, input, promotedQueuedAt) return s.completeGenerationTurn(ctx, input, committed, decision.promotedMessageID) } diff --git a/coderd/x/chatd/generation_preparer.go b/coderd/x/chatd/generation_preparer.go index ff49843deb6..2e01ebff20f 100644 --- a/coderd/x/chatd/generation_preparer.go +++ b/coderd/x/chatd/generation_preparer.go @@ -369,8 +369,9 @@ func (server *Server) prepareGeneration( logger.Warn(ctx, "failed to load MCP user tokens", slog.Error(tokenErr)) } mcpTokens = server.refreshExpiredMCPTokens(ctx, logger, mcpConnectConfigs, mcpTokens) + connectCtx, connectSpan := server.stages.Start(ctx, chatloop.StageMCPConnect) mcpTools, mcpSummaries, mcpCleanup = mcpclient.ConnectAll( - ctx, + connectCtx, logger, mcpConnectConfigs, mcpTokens, @@ -378,6 +379,7 @@ func (server *Server) prepareGeneration( server.oidcTokenSource, chatprovider.CoderHeaders(chat), ) + connectSpan.End(nil) return nil }) } @@ -738,6 +740,7 @@ func (server *Server) prepareGeneration( ModelRoute: modelRoute, ModelBuildOptions: modelOpts, ResolvedProvider: resolved.resolvedProvider, + StageModel: resolved.stageModel(), ModelConfigID: modelConfig.ID, CallTemplate: resolved.newCall(), ContextLimitFallback: modelConfig.ContextLimit, diff --git a/coderd/x/chatd/model_routing.go b/coderd/x/chatd/model_routing.go index c089570846b..fce53b9e609 100644 --- a/coderd/x/chatd/model_routing.go +++ b/coderd/x/chatd/model_routing.go @@ -8,6 +8,7 @@ import ( "golang.org/x/xerrors" "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/x/chatd/chatloop" "github.com/coder/coder/v2/coderd/x/chatd/chatprovider" "github.com/coder/coder/v2/codersdk" ) @@ -25,6 +26,10 @@ type modelClientRequest struct { type modelBuildOptions struct { ActiveAPIKeyID string RecordHTTP bool + // StageModel labels the provider transport's lifecycle stages. It + // is set by the model call resolver, which knows the resolved model + // and effective reasoning effort before the client is built. + StageModel chatloop.StageModel } func (p *Server) enabledAIProviderByID(ctx context.Context, providerID uuid.UUID) (database.AIProvider, error) { diff --git a/coderd/x/chatd/model_routing_aibridge.go b/coderd/x/chatd/model_routing_aibridge.go index e670fc75041..d7c2e9dcbb6 100644 --- a/coderd/x/chatd/model_routing_aibridge.go +++ b/coderd/x/chatd/model_routing_aibridge.go @@ -11,6 +11,7 @@ import ( fantasyopenai "charm.land/fantasy/providers/openai" fantasyopenaicompat "charm.land/fantasy/providers/openaicompat" "github.com/google/uuid" + "go.opentelemetry.io/otel/attribute" "golang.org/x/xerrors" "github.com/coder/coder/v2/coderd/aibridge" @@ -18,6 +19,7 @@ import ( "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/x/chatd/chatdebug" "github.com/coder/coder/v2/coderd/x/chatd/chaterror" + "github.com/coder/coder/v2/coderd/x/chatd/chatloop" "github.com/coder/coder/v2/coderd/x/chatd/chatprovider" "github.com/coder/coder/v2/codersdk" ) @@ -67,6 +69,42 @@ const ( aiGatewayRequestFormatAnthropic ) +// stageSpanRoundTripper emits one provider_attempt stage per HTTP +// round trip to the model provider, so retried requests each get +// their own span. model labels every attempt with the identity the +// client was built for. +type stageSpanRoundTripper struct { + base http.RoundTripper + stages *chatloop.StageTracer + model chatloop.StageModel +} + +var _ http.RoundTripper = (*stageSpanRoundTripper)(nil) + +func (t *stageSpanRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + ctx, span := t.stages.Start(req.Context(), chatloop.StageProviderAttempt, + attribute.String(chatloop.AttrHTTPMethod, req.Method), + attribute.String(chatloop.AttrHTTPHost, req.URL.Host), + ) + span.SetModel(t.model) + resp, err := t.base.RoundTrip(req.WithContext(ctx)) + if resp != nil { + span.SetAttributes(attribute.Int(chatloop.AttrHTTPStatusCode, resp.StatusCode)) + if err == nil && resp.StatusCode >= http.StatusBadRequest { + err = xerrors.Errorf("provider returned status %d", resp.StatusCode) + // The status error only marks the span; the response and the + // transport's own error are returned untouched. + span.End(err) + return resp, nil + } + } + // The span closes on response headers, not on body completion: the + // streamed body outlives this call and is measured by the stream + // stage. + span.End(err) + return resp, err +} + type aiGatewayRoundTripper struct { base http.RoundTripper apiKeyID string @@ -164,6 +202,7 @@ func (p *Server) newModel( if opts.RecordHTTP { baseRT = &chatdebug.RecordingTransport{Base: baseRT} } + baseRT = &stageSpanRoundTripper{base: baseRT, stages: p.stages, model: opts.StageModel} config := fantasyConfigForAIBridge(route.Provider.Type) extraHeaders := mergeConfigBetaHeaders(req.ExtraHeaders, config.ProviderHint, req.CallConfig) diff --git a/coderd/x/chatd/modelcall.go b/coderd/x/chatd/modelcall.go index e3114867f71..503be06e028 100644 --- a/coderd/x/chatd/modelcall.go +++ b/coderd/x/chatd/modelcall.go @@ -11,6 +11,7 @@ import ( "github.com/coder/coder/v2/coderd/database/dbauthz" "github.com/coder/coder/v2/coderd/util/ptr" "github.com/coder/coder/v2/coderd/x/chatd/chatdebug" + "github.com/coder/coder/v2/coderd/x/chatd/chatloop" "github.com/coder/coder/v2/coderd/x/chatd/chatprovider" "github.com/coder/coder/v2/codersdk" ) @@ -68,8 +69,18 @@ type resolvedModelCall struct { providerOptions fantasy.ProviderOptions resolvedProvider string resolvedModel string - route aiGatewayModelRoute - debugEnabled bool + // resolvedEffort is the reasoning effort actually sent to the + // provider, after the per-turn request is clamped to the config's + // max. Empty when the config configures no reasoning effort. + resolvedEffort string + route aiGatewayModelRoute + debugEnabled bool +} + +// stageModel returns the model identity stage instrumentation labels +// spans and durations with. +func (r resolvedModelCall) stageModel() chatloop.StageModel { + return chatloop.StageModel{Model: r.resolvedModel, Effort: r.resolvedEffort} } // resolveModelCall is the single pipeline from a spec to a ready model @@ -146,8 +157,18 @@ func (p *Server) resolveModelCall(ctx context.Context, spec modelCallSpec) (reso debugSvc := p.debugService() out.debugEnabled = debugSvc != nil && debugSvc.IsEnabled(ctx, spec.chat.ID, spec.chat.OwnerID) + // The effective effort is resolved before the client is built so the + // provider transport can label its spans with it. Passing it back + // into ProviderOptionsForCall is a no-op re-clamp, which keeps the + // label and the call in agreement. + effectiveEffort := chatprovider.ResolveReasoningEffort(spec.requestedEffort, out.callConfig.ReasoningEffort) + if effectiveEffort != nil { + out.resolvedEffort = *effectiveEffort + } + buildOpts := spec.buildOptions buildOpts.RecordHTTP = out.debugEnabled + buildOpts.StageModel = out.stageModel() model, err := p.newModel(ctx, modelClientRequest{ Chat: spec.chat, ModelName: modelName, @@ -169,13 +190,14 @@ func (p *Server) resolveModelCall(ctx context.Context, spec modelCallSpec) (reso } out.model = model - out.providerOptions = chatprovider.ProviderOptionsForCall(out.model, out.callConfig, spec.requestedEffort) + out.providerOptions = chatprovider.ProviderOptionsForCall(out.model, out.callConfig, effectiveEffort) p.logger.Debug(ctx, "resolved model call", slog.F("purpose", spec.purpose), slog.F("chat_id", spec.chat.ID), slog.F("provider", out.resolvedProvider), slog.F("model", out.resolvedModel), + slog.F("reasoning_effort", out.resolvedEffort), slog.F("debug_enabled", out.debugEnabled), ) return out, nil diff --git a/coderd/x/chatd/options.go b/coderd/x/chatd/options.go index b887b928baf..4bc6f130ff9 100644 --- a/coderd/x/chatd/options.go +++ b/coderd/x/chatd/options.go @@ -68,6 +68,7 @@ type chatWorkerTaskStartInput struct { Status database.ChatStatus RequiresActionDeadlineAt sql.NullTime DebugTurn *runnerDebugTurn + Turn *runnerTurnSpan SessionStart *sessionStartTracker StopNudges *stopNudgeTracker } diff --git a/coderd/x/chatd/runner.go b/coderd/x/chatd/runner.go index 72b916a38a6..121452021a9 100644 --- a/coderd/x/chatd/runner.go +++ b/coderd/x/chatd/runner.go @@ -57,6 +57,7 @@ type runner struct { tasksByIndex map[taskIndexKey]taskInstanceID localLocks *localLockSet debugTurn *runnerDebugTurn + turnSpan *runnerTurnSpan sessionStart sessionStartTracker stopNudges stopNudgeTracker } @@ -71,6 +72,7 @@ func newRunner(ctx context.Context, mgr *runnerManager, rec *runnerRecord, opts tasksByIndex: make(map[taskIndexKey]taskInstanceID), localLocks: newLocalLockSet(), debugTurn: newRunnerDebugTurn(ctx, opts.Logger), + turnSpan: newRunnerTurnSpan(mgr.server.stages), } } @@ -86,6 +88,7 @@ func (r *runner) run() { r.cancelActiveTask() r.waitForTasks() r.closeDebugTurn() + r.turnSpan.End(nil) return } } @@ -229,6 +232,7 @@ func (r *runner) spawnTaskIfNeeded(kind taskKind, state runnerStateUpdate) { Status: state.Status, RequiresActionDeadlineAt: state.RequiresActionDeadlineAt, DebugTurn: r.debugTurn, + Turn: r.turnSpan, SessionStart: &r.sessionStart, StopNudges: &r.stopNudges, } diff --git a/coderd/x/chatd/stage_internal_test.go b/coderd/x/chatd/stage_internal_test.go new file mode 100644 index 00000000000..71880523fcb --- /dev/null +++ b/coderd/x/chatd/stage_internal_test.go @@ -0,0 +1,241 @@ +package chatd //nolint:testpackage // Tests unexported stage instrumentation internals. + +import ( + "context" + "io" + "net/http" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + "go.opentelemetry.io/otel/trace" + "golang.org/x/xerrors" + + "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/x/chatd/chatloop" +) + +// newStageTestTracer returns a stage tracer writing into an in-memory +// span recorder. +func newStageTestTracer(t *testing.T) (*chatloop.StageTracer, *tracetest.SpanRecorder) { + t.Helper() + recorder := tracetest.NewSpanRecorder() + provider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder)) + t.Cleanup(func() { + // The test context is already canceled during cleanup, so the + // flush uses a fresh one. + require.NoError(t, provider.Shutdown(context.Background())) + }) + return chatloop.NewStageTracer(provider, chatloop.NopMetrics()), recorder +} + +type stubRoundTripper struct { + status int + err error +} + +func (s stubRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + if s.err != nil { + return nil, s.err + } + return &http.Response{ + StatusCode: s.status, + Body: io.NopCloser(strings.NewReader("")), + Request: req, + }, nil +} + +func TestStageSpanRoundTripper(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + base stubRoundTripper + wantStatusCode codes.Code + wantAttribute bool + wantErr bool + }{ + { + name: "success", + base: stubRoundTripper{status: http.StatusOK}, + wantStatusCode: codes.Unset, + wantAttribute: true, + }, + { + name: "client error", + base: stubRoundTripper{status: http.StatusTooManyRequests}, + wantStatusCode: codes.Error, + wantAttribute: true, + }, + { + name: "server error", + base: stubRoundTripper{status: http.StatusInternalServerError}, + wantStatusCode: codes.Error, + wantAttribute: true, + }, + { + name: "transport error", + base: stubRoundTripper{err: xerrors.New("dial failed")}, + wantStatusCode: codes.Error, + wantErr: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + tracer, recorder := newStageTestTracer(t) + transport := &stageSpanRoundTripper{base: test.base, stages: tracer} + + req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, "https://provider.example/v1/messages", nil) + require.NoError(t, err) + resp, err := transport.RoundTrip(req) + if test.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + require.NotNil(t, resp) + require.Equal(t, test.base.status, resp.StatusCode) + require.NoError(t, resp.Body.Close()) + } + + ended := recorder.Ended() + require.Len(t, ended, 1) + require.Equal(t, chatloop.StageProviderAttempt, ended[0].Name()) + require.Equal(t, test.wantStatusCode, ended[0].Status().Code) + var sawStatusCode bool + for _, attr := range ended[0].Attributes() { + if string(attr.Key) == chatloop.AttrHTTPStatusCode { + sawStatusCode = true + require.Equal(t, int64(test.base.status), attr.Value.AsInt64()) + } + } + require.Equal(t, test.wantAttribute, sawStatusCode) + require.Contains(t, ended[0].Attributes(), + attribute.String(chatloop.AttrScope, chatloop.ScopeBackground)) + }) + } +} + +func TestStageSpanRoundTripperScope(t *testing.T) { + t.Parallel() + tracer, recorder := newStageTestTracer(t) + transport := &stageSpanRoundTripper{base: stubRoundTripper{status: http.StatusOK}, stages: tracer} + + turnCtx, turn := tracer.StartRoot(t.Context(), chatloop.StageChatTurn, nil) + turnReq, err := http.NewRequestWithContext(turnCtx, http.MethodPost, "https://provider.example/v1/messages", nil) + require.NoError(t, err) + turnResp, err := transport.RoundTrip(turnReq) + require.NoError(t, err) + require.NoError(t, turnResp.Body.Close()) + + // Background work runs on a context whose span was stripped, the + // same way inflight chatd tasks detach from the turn. + backgroundCtx := trace.ContextWithSpanContext(turnCtx, trace.SpanContext{}) + backgroundReq, err := http.NewRequestWithContext(backgroundCtx, http.MethodPost, "https://provider.example/v1/messages", nil) + require.NoError(t, err) + backgroundResp, err := transport.RoundTrip(backgroundReq) + require.NoError(t, err) + require.NoError(t, backgroundResp.Body.Close()) + turn.End(nil) + + var scopes []string + for _, span := range recorder.Ended() { + if span.Name() != chatloop.StageProviderAttempt { + continue + } + for _, attr := range span.Attributes() { + if string(attr.Key) == chatloop.AttrScope { + scopes = append(scopes, attr.Value.AsString()) + } + } + } + require.Equal(t, []string{chatloop.ScopeTurn, chatloop.ScopeBackground}, scopes) +} + +func TestStageSpanRoundTripperModel(t *testing.T) { + t.Parallel() + tracer, recorder := newStageTestTracer(t) + model := chatloop.StageModel{Model: "claude-sonnet-4-5", Effort: "medium"} + transport := &stageSpanRoundTripper{ + base: stubRoundTripper{status: http.StatusOK}, + stages: tracer, + model: model, + } + + req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, "https://provider.example/v1/messages", nil) + require.NoError(t, err) + resp, err := transport.RoundTrip(req) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + + ended := recorder.Ended() + require.Len(t, ended, 1) + require.Contains(t, ended[0].Attributes(), + attribute.String(chatloop.AttrModel, model.Model)) + require.Contains(t, ended[0].Attributes(), + attribute.String(chatloop.AttrReasoningEffort, model.Effort)) +} + +func TestRunnerTurnSpanStartsAtTriggerMessage(t *testing.T) { + t.Parallel() + tracer, recorder := newStageTestTracer(t) + turn := newRunnerTurnSpan(tracer) + chat := database.Chat{ID: uuid.New()} + triggerAt := time.Now().Add(-2 * time.Second) + + turnCtx := turn.Ensure(t.Context(), chat, triggerAt) + turn.End(nil) + + ended := recorder.Ended() + require.Len(t, ended, 2) + acquisition, chatTurn := ended[0], ended[1] + require.Equal(t, chatloop.StageAcquisition, acquisition.Name()) + require.Equal(t, chatloop.StageChatTurn, chatTurn.Name()) + require.Equal(t, triggerAt.UTC(), chatTurn.StartTime().UTC()) + require.Equal(t, triggerAt.UTC(), acquisition.StartTime().UTC()) + require.False(t, acquisition.StartTime().Before(chatTurn.StartTime())) + require.False(t, acquisition.EndTime().After(chatTurn.EndTime())) + require.Equal(t, chatTurn.SpanContext().SpanID(), acquisition.Parent().SpanID()) + require.Equal(t, chatTurn.SpanContext().TraceID(), trace.SpanContextFromContext(turnCtx).TraceID()) +} + +func TestRunnerTurnSpanParentsRecordedStages(t *testing.T) { + t.Parallel() + tracer, recorder := newStageTestTracer(t) + turn := newRunnerTurnSpan(tracer) + chat := database.Chat{ID: uuid.New()} + + turnCtx := turn.Ensure(t.Context(), chat, time.Now().Add(-time.Second)) + stepCtx, step := tracer.Start(turnCtx, chatloop.StageGenerationStep) + step.End(nil) + + // The step has already ended, so the queue wait is recorded + // against the turn context the runner keeps. + tracer.Record(turn.Context(stepCtx), chatloop.StageQueueWait, chatloop.StageModel{}, + time.Now().Add(-500*time.Millisecond), time.Now(), nil) + turn.End(nil) + + var queueWait, chatTurn, generationStep sdktrace.ReadOnlySpan + for _, span := range recorder.Ended() { + switch span.Name() { + case chatloop.StageQueueWait: + queueWait = span + case chatloop.StageChatTurn: + chatTurn = span + case chatloop.StageGenerationStep: + generationStep = span + } + } + require.NotNil(t, queueWait) + require.NotNil(t, chatTurn) + require.NotNil(t, generationStep) + require.Equal(t, chatTurn.SpanContext().SpanID(), queueWait.Parent().SpanID()) + require.NotEqual(t, generationStep.SpanContext().SpanID(), queueWait.Parent().SpanID()) +} diff --git a/coderd/x/chatd/turn_trace.go b/coderd/x/chatd/turn_trace.go new file mode 100644 index 00000000000..e6b13a2561c --- /dev/null +++ b/coderd/x/chatd/turn_trace.go @@ -0,0 +1,126 @@ +package chatd + +import ( + "context" + "sync" + "time" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + + "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/x/chatd/chatloop" +) + +// runnerTurnSpan owns the chat_turn span of one runner. A runner is +// created when a chat is acquired and torn down when the chat leaves +// the running states, so the runner's lifetime bounds the turn. +// +// The span is created on the first generation task rather than at +// runner construction: runners are also spawned to abandon or time +// out chats, which run no turn. +type runnerTurnSpan struct { + stages *chatloop.StageTracer + + mu sync.Mutex + span *chatloop.StageSpan + spanCtx trace.SpanContext + started bool + ended bool +} + +func newRunnerTurnSpan(stages *chatloop.StageTracer) *runnerTurnSpan { + return &runnerTurnSpan{stages: stages} +} + +// Ensure starts the chat_turn span on first call and returns a +// context parented to it. triggerAt is the trigger message's +// creation time and becomes the span's start timestamp, so the +// acquisition stage reconstructed from the same instant falls inside +// the turn. The window between triggerAt and now is emitted as the +// acquisition stage, which covers the delay between the message +// landing in history and a worker picking the chat up. +// +// The span is a standalone trace root. The request that triggered the +// turn is handled by a different goroutine, and often a different +// replica, than the worker that runs it, so no inbound span context +// is available here to link. +func (t *runnerTurnSpan) Ensure(ctx context.Context, chat database.Chat, triggerAt time.Time) context.Context { + if t == nil { + return ctx + } + t.mu.Lock() + defer t.mu.Unlock() + if t.ended { + return ctx + } + if t.started { + return t.contextLocked(ctx) + } + t.started = true + + attrs := []attribute.KeyValue{ + attribute.String(chatloop.AttrChatID, chat.ID.String()), + attribute.String(chatloop.AttrChatKind, chatKindAttr(chat)), + } + turnCtx, span := t.stages.StartRootAt(ctx, chatloop.StageChatTurn, triggerAt, nil, attrs...) + t.span = span + t.spanCtx = span.SpanContext() + // The turn's model is not resolved until preparation runs, so the + // acquisition stage carries no model identity. + t.stages.Record(turnCtx, chatloop.StageAcquisition, chatloop.StageModel{}, + triggerAt, time.Now(), nil, attrs...) + return turnCtx +} + +// Context returns ctx parented to the chat_turn span, or ctx +// unchanged while no turn span is open. +func (t *runnerTurnSpan) Context(ctx context.Context) context.Context { + if t == nil { + return ctx + } + t.mu.Lock() + defer t.mu.Unlock() + return t.contextLocked(ctx) +} + +func (t *runnerTurnSpan) contextLocked(ctx context.Context) context.Context { + if !t.started || t.ended || !t.spanCtx.IsValid() { + return ctx + } + return trace.ContextWithSpanContext(ctx, t.spanCtx) +} + +// End closes the chat_turn span. Later calls are ignored. +func (t *runnerTurnSpan) End(err error) { + if t == nil { + return + } + t.mu.Lock() + defer t.mu.Unlock() + if t.ended || !t.started { + t.ended = true + return + } + t.ended = true + t.span.End(err) +} + +// chatKindAttr labels a chat as a subagent or a top-level chat. +func chatKindAttr(chat database.Chat) string { + if chat.ParentChatID.Valid { + return chatloop.ChatKindSubagent + } + return chatloop.ChatKindRoot +} + +// triggerMessageTime returns the creation time of the message that +// triggered the turn, which is the last user prompt in history. It +// returns the zero time when the history holds no user prompt. +func triggerMessageTime(messages []database.ChatMessage) time.Time { + index := lastUserPromptIndex(messages) + if index == -1 { + return time.Time{} + } + return messages[index].CreatedAt +} diff --git a/coderd/x/chatd/worker.go b/coderd/x/chatd/worker.go index 83123f48ccb..53542ae099f 100644 --- a/coderd/x/chatd/worker.go +++ b/coderd/x/chatd/worker.go @@ -28,6 +28,10 @@ type chatWorker struct { unsubscribe func() wakeCh chan struct{} wg sync.WaitGroup + + // capacityWaitSince tracks when each chat was first refused a + // capacity slot. Only the acquisition loop reads or writes it. + capacityWaitSince map[uuid.UUID]time.Time } // newChatWorker constructs a chat worker. The worker is idle until Start is @@ -40,7 +44,11 @@ func newChatWorker(server *Server, opts chatWorkerOptions) (*chatWorker, error) if err != nil { return nil, err } - return &chatWorker{server: server, opts: withDefaults}, nil + return &chatWorker{ + server: server, + opts: withDefaults, + capacityWaitSince: make(map[uuid.UUID]time.Time), + }, nil } // chatWorkerID returns this worker's configured worker ID. @@ -207,6 +215,7 @@ func (w *chatWorker) acquireOnce(ctx context.Context, workerID uuid.UUID, manage acquired := int32(0) rootPoolRefused := false subagentPoolRefused := false + w.pruneCapacityWaits(rows) for _, row := range rows { if acquired >= w.opts.AcquisitionBatchSize { return @@ -220,6 +229,7 @@ func (w *chatWorker) acquireOnce(ctx context.Context, workerID uuid.UUID, manage } candidateAcquired, err := w.acquireCandidateSafely(ctx, workerID, manager, row.ID) if errors.Is(err, errCapacityRefused) { + w.noteCapacityRefused(row.ID) if isSubagent { subagentPoolRefused = true } else { @@ -266,6 +276,7 @@ func (w *chatWorker) acquireCandidate( chatID uuid.UUID, ) (bool, error) { runnerID := uuid.New() + var acquiredChat database.Chat machine := chatstate.NewChatMachine(w.opts.Store, w.opts.Pubsub, chatID) err := machine.Update(ctx, func(tx *chatstate.Tx, store database.Store) error { chat, err := store.GetChatByID(ctx, chatID) @@ -304,6 +315,7 @@ func (w *chatWorker) acquireCandidate( // worker into an immediate retry of this unowned chat. return errCapacityRefused } + acquiredChat = chat _, err = tx.Acquire(chatstate.AcquireInput{WorkerID: workerID, RunnerID: runnerID}) return err }) @@ -316,6 +328,7 @@ func (w *chatWorker) acquireCandidate( if err != nil { return false, err } + w.recordCapacityWait(ctx, acquiredChat) if err := manager.Spawn(ctx, spawnRunnerRequest{ChatID: chatID, WorkerID: workerID, RunnerID: runnerID}); err != nil { if errAbandon := w.abandonAcquiredChat(ctx, workerID, runnerID, chatID); errAbandon != nil { return false, errors.Join(err, errAbandon) diff --git a/docs/admin/integrations/prometheus.md b/docs/admin/integrations/prometheus.md index 7ae8cad8801..f6f2fb35fdd 100644 --- a/docs/admin/integrations/prometheus.md +++ b/docs/admin/integrations/prometheus.md @@ -249,6 +249,7 @@ The `coder_ai_gateway_cost_control_*` metrics are exported only by `coderd`. | `coderd_chatd_hook_input_overrides_total` | counter | Total lifecycle hook input overrides by event. | `event` | | `coderd_chatd_message_count` | histogram | Number of messages in the prompt per LLM request. | `model` `provider` | | `coderd_chatd_prompt_size_bytes` | histogram | Estimated byte size of the prompt per LLM request. | `model` `provider` | +| `coderd_chatd_stage_duration_seconds` | histogram | Wall time spent in each chat lifecycle stage. Stages overlap in wall time; this is a stage-time profile, not a partition of the turn. The scope label separates stages that run inside a chat turn from detached background work. The model and effort labels are empty for stages that run before a model is resolved. | `effort` `model` `scope` `stage` | | `coderd_chatd_steps_total` | counter | Total agentic loop steps across all chats. | `model` `provider` | | `coderd_chatd_stream_buffer_dropped_total` | counter | Number of chat stream buffer events dropped due to the per-chat buffer cap. | | | `coderd_chatd_stream_retries_total` | counter | Total LLM stream retries. | `kind` `model` `provider` | diff --git a/examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md b/examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md new file mode 100644 index 00000000000..27140f54516 --- /dev/null +++ b/examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md @@ -0,0 +1,143 @@ +# Chatd Chat Lifecycle Grafana Dashboard + +A Grafana dashboard for diagnosing where time goes in Coder Agents chat +sessions. It aggregates the `coderd_chatd_stage_duration_seconds{stage,scope}` +histogram into a stage-level flame graph with a selectable summary statistic +(mean, p50, p90, p95, p99), plus summary panels for the whole chat pipeline. + +Stage hierarchy: + +```text +chat_turn +├── queue_wait queued message insert -> promotion +├── capacity_wait concurrent-agent limiter wait +├── acquisition trigger message -> worker pickup +└── generation_step one step of a turn (repeats) + ├── prepare prompt build, model resolution, context hydration + ├── mcp_connect MCP server connection + ├── provider_attempt one provider HTTP round trip (per retry) + │ └── time_to_first_token + ├── stream provider stream open -> close + ├── thinking reasoning part duration + ├── tool_call one local tool call + ├── commit step persistence transaction + └── compaction auxiliary compaction call +``` + +Stages overlap in wall time (tool calls and thinking happen inside the +stream), so the flame graph is a stage-time profile, not a strict +decomposition, and quantile statistics are not additive across stages. + +## Dimensions + +The stage histogram carries four labels, exposed as dashboard variables +where noted: + +| Label | Values | Dashboard variable | +|----------|-----------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------| +| `stage` | the 14 stage names above | none (fixed hierarchy) | +| `scope` | `turn` (part of a chat turn) or `background` (detached async work such as title and summary generation) | none (panels pin one scope) | +| `model` | resolved model ID, empty before a model is resolved | `$model` (multi-select) | +| `effort` | effective reasoning effort sent to the provider (`none`, `minimal`, `low`, `medium`, `high`, `xhigh`, `max`), empty when the model config sets none | `$effort` (multi-select) | + +Two more variables apply everywhere: `$datasource` selects the Prometheus +data source and `$stat` selects the summary statistic (mean, p50, p90, +p95, p99) for every stat-aware panel. + +Stages that run before or outside model resolution (`chat_turn`, +`queue_wait`, `capacity_wait`, `acquisition`, `mcp_connect`, `commit`) +carry empty `model`/`effort` labels. Filtering `$model` or `$effort` to a +specific value therefore zeroes those stages and effectively narrows the +view to the generation stages. + +## Panels + +### Stage profile + +**Stage profile flamegraph ($stat)** - one frame per stage, laid out in +the fixed hierarchy, frame width = the selected `$stat` of that stage's +duration over the dashboard time range. Dimensions: filtered to +`scope="turn"` and the `$model`/`$effort` selections; `$stat` picks the +statistic. How to read: the widest frames under `generation_step` are +where turn time goes; compare `provider_attempt` (request to response +headers) against `stream` (full stream) to separate provider latency +from streaming time. Caveats: stages overlap in wall time (tool calls +and thinking happen inside the stream) and quantiles are not additive, +so a child frame can read wider than its parent at high percentiles; +a stage whose series first appears inside the window reads 0 until its +second sample. + +**Stage profile in hierarchy order ($stat)** - the same query as the +flamegraph drawn as horizontal bars in depth-first order with the tree +indented into the labels. Dimensions: identical to the flamegraph. Use +it to read stages too small to see as frames (prepare, commit, +tool_call are typically milliseconds next to multi-second streams) and +as a numeric check on the flamegraph. + +### Stage trends + +**Stage duration over time ($stat)** - one series per stage, the +selected `$stat` computed over `$__rate_interval`. Dimensions: +`scope="turn"`, `$model`/`$effort` filters, series split by `stage`. +How to read: this is the drill-down for "when did it get slow" - a +regression visible in the profile shows here as a step or trend in the +affected stage. Idle stages drop out rather than plotting NaN. + +**Stage time share of chat_turn** - each stage's total time as a +percentage of total `chat_turn` time, from mean rates of the histogram +sums. Dimensions: numerator is `scope="turn"` with `$model`/`$effort` +applied and split by `stage`; the denominator is all `chat_turn` time +without model/effort filters, because `chat_turn` carries empty +model/effort labels. How to read: this is the "where does the time go" +summary - stages overlap, so series can sum past 100%, but a single +stage rising toward 100% of turn time identifies the dominant cost. + +**Queue, capacity and acquisition wait (p99)** - p99 of the three +pre-generation waits: `queue_wait` (queued message insert to +promotion), `capacity_wait` (concurrent-agent limiter admission) and +`acquisition` (trigger message insert to worker pickup). Dimensions: +`scope="turn"`, fixed to those three stages, split by `stage`; +`$model`/`$effort` apply but these stages carry empty labels, so +non-All selections blank this panel. How to read: these are scheduling +delays before any model work starts - user-visible latency that no +provider-side optimization can fix. + +### Throughput and TTFT + +**Time to first token** - p50/p90/p99/mean of `coderd_chatd_ttft_seconds`, +the pre-existing histogram recorded when the first streamed part +arrives. Dimensions: none of the stage labels; this histogram is +labeled by provider/model internally but the panel aggregates across +them, and `$model`/`$effort` do not apply. The `time_to_first_token` +stage in the profile measures the same interval and does honor the +filters. How to read: the primary user-perceived responsiveness metric +for streaming. + +**Turn rate and stage sample rate** - completed `chat_turn` per second +plus one rate series per other stage. Dimensions: `scope="turn"`, +`$model`/`$effort` applied, split by `stage`. How to read: throughput +and shape - a stage rate above the turn rate means the stage repeats +within a turn (generation steps, provider attempts, tool calls); +`provider_attempt` rising faster than `stream` indicates retries. + +**Background provider calls ($stat)** - rate and selected `$stat` +duration of background-scope `provider_attempt` samples: detached +title/summary/quickgen requests that are excluded from every other +panel. Dimensions: pinned to the background scope of the +`provider_attempt` stage; `$model`/`$effort` are not applied. How to read: +this work costs provider quota and money but no user-facing turn +latency; a spike here with flat turn panels means background load, not +a chat regression. + +## Setup + +1. **Configure a Prometheus data source** that scrapes your coderd + Prometheus endpoint (`--prometheus-enable`). +2. **Import**: in Grafana navigate to **Dashboards** -> **Import** -> + **Upload JSON file** with [`dashboard.json`](./dashboard.json), then map + the Prometheus data source when prompted. + +Per-session drill-down is available by exporting coderd traces +(`--trace` with standard OTLP environment variables) to a tracing backend +such as Tempo; each chat turn is a `chat_turn` root span whose children +mirror the stage hierarchy above. diff --git a/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json b/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json new file mode 100644 index 00000000000..30e93b2cdad --- /dev/null +++ b/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json @@ -0,0 +1,1073 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Where time goes in Coder Agents chat turns, from the coderd_chatd_stage_duration_seconds stage histogram.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 100, + "panels": [], + "title": "Stage profile", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Aggregate profile of chat lifecycle stages over the dashboard time range, using the $stat statistic of coderd_chatd_stage_duration_seconds. Levels come from the fixed stage hierarchy, attached as labels with label_replace and reshaped into Grafana's nested set model by the panel transformations.\n\nStage timings overlap in wall time (stream, thinking and tool_call all run inside a generation_step, and time_to_first_token is part of provider_attempt), and quantiles are not additive across stages. Read this as a stage time profile, not as a strict non-overlapping decomposition of turn latency: child bars can exceed their parent. A stage with no samples in the time range reads as 0. Only scope=\"turn\" samples are included, so detached background provider calls are excluded.\n\nValues come from rate() over counter series, and rate() treats the first sample in the window as its baseline. A stage whose series first appears inside the window reads as 0 until its second observation, so rare stages such as queue_wait, capacity_wait and compaction can render as 0 while real samples exist. Widen the time range to confirm.\n\nThe $model and $effort filters only match stages that ran against a resolved model. Pre-model stages (chat_turn, queue_wait, capacity_wait, acquisition, commit, mcp_connect) carry empty model and effort, so narrowing either variable to a specific value drops them to 0.", + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byNames", + "options": { + "names": [ + "value", + "self" + ] + } + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + } + ] + }, + "gridPos": { + "h": 14, + "w": 14, + "x": 0, + "y": 1 + }, + "id": 1, + "options": {}, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"chat_turn\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"chat_turn\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"chat_turn\", \"\", \"\"),\n\"n\", \"01\", \"\", \"\"),\n\"level\", \"0\", \"\", \"\"),\n\"name\", \"chat_turn\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"queue_wait\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"queue_wait\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"queue_wait\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"queue_wait\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"queue_wait\", \"\", \"\"),\n\"n\", \"02\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ queue_wait\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"capacity_wait\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"capacity_wait\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"capacity_wait\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"capacity_wait\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"capacity_wait\", \"\", \"\"),\n\"n\", \"03\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ capacity_wait\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"acquisition\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"acquisition\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"acquisition\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"acquisition\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"acquisition\", \"\", \"\"),\n\"n\", \"04\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ acquisition\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"generation_step\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"generation_step\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"generation_step\", \"\", \"\"),\n\"n\", \"05\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ generation_step\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"prepare\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"prepare\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"prepare\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"prepare\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"prepare\", \"\", \"\"),\n\"n\", \"06\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ prepare\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"mcp_connect\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"mcp_connect\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"mcp_connect\", \"\", \"\"),\n\"n\", \"07\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ mcp_connect\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"provider_attempt\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"provider_attempt\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"provider_attempt\", \"\", \"\"),\n\"n\", \"08\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ provider_attempt\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"time_to_first_token\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"time_to_first_token\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"time_to_first_token\", \"\", \"\"),\n\"n\", \"09\", \"\", \"\"),\n\"level\", \"3\", \"\", \"\"),\n\"name\", \"│ │ └─ time_to_first_token\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"stream\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"stream\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"stream\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"stream\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"stream\", \"\", \"\"),\n\"n\", \"10\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ stream\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"thinking\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"thinking\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"thinking\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"thinking\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"thinking\", \"\", \"\"),\n\"n\", \"11\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ thinking\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"tool_call\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"tool_call\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"tool_call\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"tool_call\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"tool_call\", \"\", \"\"),\n\"n\", \"12\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ tool_call\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"commit\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"commit\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"commit\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"commit\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"commit\", \"\", \"\"),\n\"n\", \"13\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ commit\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"compaction\", \"\", \"\"),\n\"n\", \"14\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ compaction\", \"\", \"\")\n)", + "range": false, + "refId": "A", + "format": "table", + "instant": true, + "legendFormat": "" + } + ], + "title": "Stage profile flamegraph (${stat:text})", + "transformations": [ + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "field": "n" + } + ] + } + }, + { + "id": "calculateField", + "options": { + "alias": "value", + "binary": { + "left": "Value", + "operator": "*", + "right": "1" + }, + "mode": "binary", + "replaceFields": false + } + }, + { + "id": "calculateField", + "options": { + "alias": "self", + "binary": { + "left": "value", + "operator": "*", + "right": "1" + }, + "mode": "binary", + "replaceFields": false + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "number", + "targetField": "level" + } + ], + "fields": {} + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "n": true, + "name": true, + "Value": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": {} + } + } + ], + "type": "flamegraph" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Same data as the flamegraph, drawn as bars in depth-first hierarchy order with the tree drawn into the labels. Useful when a stage is too small to read in the flamegraph, and as a check on the flamegraph shaping.\n\nStage timings overlap in wall time (stream, thinking and tool_call all run inside a generation_step, and time_to_first_token is part of provider_attempt), and quantiles are not additive across stages. Read this as a stage time profile, not as a strict non-overlapping decomposition of turn latency: child bars can exceed their parent. A stage with no samples in the time range reads as 0. Only scope=\"turn\" samples are included, so detached background provider calls are excluded.\n\nValues come from rate() over counter series, and rate() treats the first sample in the window as its baseline. A stage whose series first appears inside the window reads as 0 until its second observation, so rare stages such as queue_wait, capacity_wait and compaction can render as 0 while real samples exist. Widen the time range to confirm.\n\nThe $model and $effort filters only match stages that ran against a resolved model. Pre-model stages (chat_turn, queue_wait, capacity_wait, acquisition, commit, mcp_connect) carry empty model and effort, so narrowing either variable to a specific value drops them to 0.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-BlPu" + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 10, + "x": 14, + "y": 1 + }, + "id": 2, + "options": { + "barRadius": 0, + "barWidth": 0.8, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom", + "showLegend": false + }, + "orientation": "horizontal", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xField": "name", + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"chat_turn\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"chat_turn\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"chat_turn\", \"\", \"\"),\n\"n\", \"01\", \"\", \"\"),\n\"level\", \"0\", \"\", \"\"),\n\"name\", \"chat_turn\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"queue_wait\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"queue_wait\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"queue_wait\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"queue_wait\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"queue_wait\", \"\", \"\"),\n\"n\", \"02\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ queue_wait\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"capacity_wait\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"capacity_wait\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"capacity_wait\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"capacity_wait\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"capacity_wait\", \"\", \"\"),\n\"n\", \"03\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ capacity_wait\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"acquisition\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"acquisition\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"acquisition\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"acquisition\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"acquisition\", \"\", \"\"),\n\"n\", \"04\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ acquisition\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"generation_step\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"generation_step\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"generation_step\", \"\", \"\"),\n\"n\", \"05\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ generation_step\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"prepare\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"prepare\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"prepare\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"prepare\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"prepare\", \"\", \"\"),\n\"n\", \"06\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ prepare\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"mcp_connect\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"mcp_connect\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"mcp_connect\", \"\", \"\"),\n\"n\", \"07\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ mcp_connect\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"provider_attempt\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"provider_attempt\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"provider_attempt\", \"\", \"\"),\n\"n\", \"08\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ provider_attempt\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"time_to_first_token\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"time_to_first_token\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"time_to_first_token\", \"\", \"\"),\n\"n\", \"09\", \"\", \"\"),\n\"level\", \"3\", \"\", \"\"),\n\"name\", \"│ │ └─ time_to_first_token\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"stream\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"stream\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"stream\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"stream\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"stream\", \"\", \"\"),\n\"n\", \"10\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ stream\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"thinking\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"thinking\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"thinking\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"thinking\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"thinking\", \"\", \"\"),\n\"n\", \"11\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ thinking\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"tool_call\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"tool_call\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"tool_call\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"tool_call\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"tool_call\", \"\", \"\"),\n\"n\", \"12\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ tool_call\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"commit\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"commit\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"commit\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"commit\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"commit\", \"\", \"\"),\n\"n\", \"13\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ commit\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"compaction\", \"\", \"\"),\n\"n\", \"14\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ compaction\", \"\", \"\")\n)", + "range": false, + "refId": "A", + "format": "table", + "instant": true, + "legendFormat": "" + } + ], + "title": "Stage profile in hierarchy order (${stat:text})", + "transformations": [ + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "field": "n" + } + ] + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "n": true, + "level": true, + "label": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Value": "${stat:text}", + "Value #A": "${stat:text}" + } + } + } + ], + "type": "barchart" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 101, + "panels": [], + "title": "Stage trends", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Selected $stat of each lifecycle stage over time, one series per stage. Mean is rate(_sum)/rate(_count); the percentiles are histogram_quantile over the bucket rates. A stage with no samples in an interval has no point rather than NaN.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_stage_duration_seconds_bucket{scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_stage_duration_seconds_sum{scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "range": true, + "refId": "A", + "legendFormat": "{{stage}}" + } + ], + "title": "Stage duration over time (${stat:text})", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total time spent in each stage as a percentage of total chat_turn time, from mean rates (rate of the histogram _sum). Stages overlap, so the series can sum to more than 100%. Intervals with no chat_turn time are dropped instead of dividing by zero. The numerator honours $model and $effort; the chat_turn denominator does not, because chat_turn is recorded before a model is resolved and carries empty labels.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "100 * sum by (stage) (rate(coderd_chatd_stage_duration_seconds_sum{stage!=\"chat_turn\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval]))\n/ on() group_left() (sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"chat_turn\", scope=\"turn\"}[$__rate_interval])) > 0)", + "range": true, + "refId": "A", + "legendFormat": "{{stage}}" + } + ], + "title": "Stage time share of chat_turn", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "p99 of the pre-generation waits: queue_wait (queued message insert to promotion), capacity_wait (capacity limiter acquire) and acquisition (trigger message insert to Acquire applied). These are queueing signals rather than model latency. Intervals with no samples for a stage are dropped instead of returning NaN.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, stage) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=~\"queue_wait|capacity_wait|acquisition\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])))\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"queue_wait|capacity_wait|acquisition\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) > 0)", + "range": true, + "refId": "A", + "legendFormat": "{{stage}} p99" + } + ], + "title": "Queue, capacity and acquisition wait (p99)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 102, + "panels": [], + "title": "Throughput and TTFT", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Time to first token from coderd_chatd_ttft_seconds, the histogram recorded when the first streamed part arrives. The time_to_first_token stage in the profile above measures the same interval scoped to a provider attempt.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 35 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le) (rate(coderd_chatd_ttft_seconds_bucket[$__rate_interval])))\n and on() (sum(rate(coderd_chatd_ttft_seconds_count[$__rate_interval])) > 0)", + "range": true, + "refId": "A", + "legendFormat": "p50" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le) (rate(coderd_chatd_ttft_seconds_bucket[$__rate_interval])))\n and on() (sum(rate(coderd_chatd_ttft_seconds_count[$__rate_interval])) > 0)", + "range": true, + "refId": "B", + "legendFormat": "p90" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(coderd_chatd_ttft_seconds_bucket[$__rate_interval])))\n and on() (sum(rate(coderd_chatd_ttft_seconds_count[$__rate_interval])) > 0)", + "range": true, + "refId": "C", + "legendFormat": "p99" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(coderd_chatd_ttft_seconds_sum[$__rate_interval])) / (sum(rate(coderd_chatd_ttft_seconds_count[$__rate_interval])) > 0)", + "range": true, + "refId": "D", + "legendFormat": "mean" + } + ], + "title": "Time to first token", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Completed chat turns per second and the per-stage sample rate from the histogram _count series. Stage rates above the turn rate mean the stage repeats within a turn (generation steps, tool calls, provider attempts); rates near zero mean the stage rarely fires.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 35 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval]))", + "range": true, + "refId": "A", + "legendFormat": "chat turns" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage!=\"chat_turn\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval]))", + "range": true, + "refId": "B", + "legendFormat": "{{stage}}" + } + ], + "title": "Turn rate and stage sample rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Provider calls made outside a chat turn (scope=\"background\"): detached quickgen and title generation requests. These are excluded from the stage profile and the other stage panels, which are scoped to scope=\"turn\". Rate is on the right axis; duration uses the selected $stat and is guarded so an idle period drops out instead of returning NaN.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "calls" + }, + "properties": [ + { + "id": "unit", + "value": "ops" + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 35 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"background\"}[$__rate_interval]))", + "range": true, + "refId": "A", + "legendFormat": "calls" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"provider_attempt\", scope=\"background\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"background\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"provider_attempt\", scope=\"background\"}[$__rate_interval])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"background\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "range": true, + "refId": "B", + "legendFormat": "${stat:text}" + } + ], + "title": "Background provider calls (${stat:text})", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "1m", + "schemaVersion": 39, + "tags": [ + "coder", + "chatd", + "agents" + ], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": true, + "text": "p95", + "value": "0.95" + }, + "description": "Statistic used by the stage profile and stage duration panels. The value 0 selects the mean (rate of _sum over rate of _count); any other value is used as the quantile.", + "hide": 0, + "includeAll": false, + "label": "Statistic", + "multi": false, + "name": "stat", + "options": [ + { + "selected": false, + "text": "mean", + "value": "0" + }, + { + "selected": false, + "text": "p50", + "value": "0.5" + }, + { + "selected": false, + "text": "p90", + "value": "0.9" + }, + { + "selected": true, + "text": "p95", + "value": "0.95" + }, + { + "selected": false, + "text": "p99", + "value": "0.99" + } + ], + "query": "mean : 0,p50 : 0.5,p90 : 0.9,p95 : 0.95,p99 : 0.99", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(coderd_chatd_stage_duration_seconds_count{scope=\"turn\"}, model)", + "description": "Model the stage ran against. Stages that run before a model is resolved carry an empty value and are only included under All, whose value is the regex .* and matches the empty label.", + "hide": 0, + "includeAll": true, + "label": "Model", + "multi": true, + "name": "model", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(coderd_chatd_stage_duration_seconds_count{scope=\"turn\"}, model)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(coderd_chatd_stage_duration_seconds_count{scope=\"turn\", model=~\"$model\"}, effort)", + "description": "Reasoning effort sent to the provider. Stages that run before a model is resolved carry an empty value and are only included under All, whose value is the regex .* and matches the empty label.", + "hide": 0, + "includeAll": true, + "label": "Effort", + "multi": true, + "name": "effort", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(coderd_chatd_stage_duration_seconds_count{scope=\"turn\", model=~\"$model\"}, effort)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Chatd: chat lifecycle", + "uid": "chatd-lifecycle", + "version": 1, + "weekStart": "" +} diff --git a/scripts/metricsdocgen/generated_metrics b/scripts/metricsdocgen/generated_metrics index 739a4f84f1f..385b8e536da 100644 --- a/scripts/metricsdocgen/generated_metrics +++ b/scripts/metricsdocgen/generated_metrics @@ -319,6 +319,9 @@ coderd_chatd_message_count{provider="",model=""} 0 # HELP coderd_chatd_prompt_size_bytes Estimated byte size of the prompt per LLM request. # TYPE coderd_chatd_prompt_size_bytes histogram coderd_chatd_prompt_size_bytes{provider="",model=""} 0 +# HELP coderd_chatd_stage_duration_seconds Wall time spent in each chat lifecycle stage. Stages overlap in wall time; this is a stage-time profile, not a partition of the turn. The scope label separates stages that run inside a chat turn from detached background work. The model and effort labels are empty for stages that run before a model is resolved. +# TYPE coderd_chatd_stage_duration_seconds histogram +coderd_chatd_stage_duration_seconds{stage="",scope="",model="",effort=""} 0 # HELP coderd_chatd_steps_total Total agentic loop steps across all chats. # TYPE coderd_chatd_steps_total counter coderd_chatd_steps_total{provider="",model=""} 0 From 32d1e57f6d7a816df7c7f8a772cdd5f96f3f05ac Mon Sep 17 00:00:00 2001 From: Scott Miller Date: Tue, 1 Sep 2026 15:50:46 +0000 Subject: [PATCH 02/19] fix: derive stage scope from context marker and correct histogram edge cases Address panel review findings on the chat lifecycle instrumentation: - Carry the stage scope on an explicit context key instead of inferring it from OTel span-context validity, which mislabeled every in-turn stage as background on deployments without tracing enabled. - Extend histogram buckets to ~2.9h so multi-hour turns and waits do not clamp p99 at the previous ~11m top bucket. - Skip the histogram observation for time_to_first_token when the stream fails or ends without a part, so error windows do not pollute the TTFT distribution. - Record queue_wait from PromoteQueued as a standalone turn-scoped span instead of splicing into the promoting HTTP request trace. - Dashboard: drop model/effort matchers from empty-label stages so a concrete model selection no longer zeroes the flamegraph root and wait panels, and report self time as leaf-only instead of duplicating total time. --- coderd/x/chatd/chatd.go | 27 +++- coderd/x/chatd/chatloop/chatloop.go | 9 +- coderd/x/chatd/chatloop/metrics.go | 5 +- coderd/x/chatd/chatloop/stage.go | 56 +++++-- .../x/chatd/chatloop/stage_internal_test.go | 139 ++++++++++++++++++ coderd/x/chatd/chatloop/stage_test.go | 76 +++++++++- coderd/x/chatd/stage_internal_test.go | 64 +++++++- coderd/x/chatd/turn_trace.go | 8 +- .../grafana/chatd-lifecycle/README.md | 79 +++++----- .../grafana/chatd-lifecycle/dashboard.json | 95 ++++++++---- 10 files changed, 465 insertions(+), 93 deletions(-) create mode 100644 coderd/x/chatd/chatloop/stage_internal_test.go diff --git a/coderd/x/chatd/chatd.go b/coderd/x/chatd/chatd.go index 7b4f5716132..9dd8dd86745 100644 --- a/coderd/x/chatd/chatd.go +++ b/coderd/x/chatd/chatd.go @@ -2190,10 +2190,7 @@ func (p *Server) PromoteQueued( p.publishChatPubsubEvent(refreshChat, codersdk.ChatWatchEventKindStatusChange, nil) } if !promotedQueuedAt.IsZero() { - p.stages.Record(ctx, chatloop.StageQueueWait, chatloop.StageModel{}, - promotedQueuedAt, time.Now(), nil, - attribute.String(chatloop.AttrChatID, opts.ChatID.String()), - ) + p.recordQueueWait(ctx, opts.ChatID, promotedQueuedAt, time.Now()) } return result, nil } @@ -5096,10 +5093,12 @@ func (p *Server) Close() error { // must be called once the work completes to release the shutdown hook. // The caller is responsible for providing their own timeout. func (p *Server) inflightContext(reqCtx context.Context) (context.Context, func()) { - // Inflight work outlives the caller, so the caller's span is - // stripped from the context: spans started on this context become - // their own roots instead of children that end after their parent. + // Inflight work outlives the caller, so the caller's span and stage + // scope are stripped from the context: spans started on this context + // become their own roots instead of children that end after their + // parent, and their stages are recorded as background work. detached := trace.ContextWithSpanContext(context.WithoutCancel(reqCtx), trace.SpanContext{}) + detached = chatloop.ContextWithScope(detached, chatloop.ScopeBackground) ctx, cancel := context.WithCancel(detached) stop := context.AfterFunc(p.ctx, cancel) return ctx, func() { @@ -5108,6 +5107,20 @@ func (p *Server) inflightContext(reqCtx context.Context) (context.Context, func( } } +// recordQueueWait emits the queue_wait stage for a message that sat +// queued from queuedAt until promotedAt. The span context is stripped +// from ctx so the stage is a standalone span rather than a child of the +// promoting request's span, which ends before the turn the wait belongs +// to. The scope is set explicitly for the same reason: ctx carries the +// request, not the turn. +func (p *Server) recordQueueWait(ctx context.Context, chatID uuid.UUID, queuedAt, promotedAt time.Time) { + standalone := trace.ContextWithSpanContext(ctx, trace.SpanContext{}) + p.stages.RecordAs(standalone, chatloop.StageQueueWait, chatloop.ScopeTurn, + chatloop.StageModel{}, queuedAt, promotedAt, nil, + attribute.String(chatloop.AttrChatID, chatID.String()), + ) +} + func (p *Server) goInflight(f func()) error { if p.inflightClosed.Load() { return errInflightClosed diff --git a/coderd/x/chatd/chatloop/chatloop.go b/coderd/x/chatd/chatloop/chatloop.go index 61570b29da9..3984695b809 100644 --- a/coderd/x/chatd/chatloop/chatloop.go +++ b/coderd/x/chatd/chatloop/chatloop.go @@ -914,16 +914,19 @@ func guardedStream( var ttftOnce sync.Once // finishTTFT closes the time_to_first_token window exactly once, // either on the first streamed part or when the attempt is released - // without one. The TTFT histogram only counts windows that a part - // actually closed. + // without one. The TTFT histogram and the stage histogram only count + // windows that a part actually closed; a window cut short by a + // failure measures the failure, not the model's latency. finishTTFT := func(err error) { ttftOnce.Do(func() { if err == nil { metrics.TTFTSeconds.WithLabelValues(provider, model).Observe( clock.Since(streamStart).Seconds(), ) + ttftSpan.End(nil) + return } - ttftSpan.End(err) + ttftSpan.EndWithoutObservation(err) }) } var releaseOnce sync.Once diff --git a/coderd/x/chatd/chatloop/metrics.go b/coderd/x/chatd/chatloop/metrics.go index ef30f6d3d2a..2acf13d6a2f 100644 --- a/coderd/x/chatd/chatloop/metrics.go +++ b/coderd/x/chatd/chatloop/metrics.go @@ -102,8 +102,9 @@ func NewMetrics(reg prometheus.Registerer) *Metrics { Subsystem: metricsSubsystem, Name: "stage_duration_seconds", Help: "Wall time spent in each chat lifecycle stage. Stages overlap in wall time; this is a stage-time profile, not a partition of the turn. The scope label separates stages that run inside a chat turn from detached background work. The model and effort labels are empty for stages that run before a model is resolved.", - // 10ms .. ~11m, log-spaced. - Buckets: prometheus.ExponentialBuckets(0.01, 2, 17), + // 10ms .. ~2.9h, log-spaced. The top of the range covers + // long-lived stages such as a chat turn. + Buckets: prometheus.ExponentialBuckets(0.01, 2, 21), }, []string{"stage", "scope", "model", "effort"}), CompactionTotal: factory.NewCounterVec(prometheus.CounterOpts{ Namespace: metricsNamespace, diff --git a/coderd/x/chatd/chatloop/stage.go b/coderd/x/chatd/chatloop/stage.go index cfedf1bc4d9..45e367c99d3 100644 --- a/coderd/x/chatd/chatloop/stage.go +++ b/coderd/x/chatd/chatloop/stage.go @@ -139,19 +139,31 @@ type StageSpan struct { ended bool } -// scopeFromContext classifies work by whether ctx still carries the -// turn's trace. Detached background work has no span in its context, -// so its stages are kept out of the turn profile. +// stageScopeKey keys the stage scope carried by a context. It is +// private so the scope can only be set through ContextWithScope. +type stageScopeKey struct{} + +// ContextWithScope returns ctx carrying scope for the stages started +// on it. The scope is a plain context value rather than a property of +// the span in ctx, so it survives configurations where spans are not +// recorded, such as a no-op tracer provider. +func ContextWithScope(ctx context.Context, scope string) context.Context { + return context.WithValue(ctx, stageScopeKey{}, scope) +} + +// scopeFromContext reads the scope ContextWithScope put on ctx. +// Contexts with no scope are background scoped, so work detached from +// a turn is kept out of the turn profile. func scopeFromContext(ctx context.Context) string { - if trace.SpanContextFromContext(ctx).IsValid() { - return ScopeTurn + if scope, ok := ctx.Value(stageScopeKey{}).(string); ok && scope != "" { + return scope } return ScopeBackground } // Start begins a stage span as a child of the span in ctx and returns -// a context carrying it. The stage is scoped by the span already in -// ctx, so stages started on a detached context are background scoped. +// a context carrying it. The stage takes the scope on ctx, so stages +// started on a context with no scope are background scoped. func (t *StageTracer) Start( ctx context.Context, stage string, @@ -207,7 +219,7 @@ func (t *StageTracer) startSpan( opts = append(opts, trace.WithTimestamp(start)) } opts = append(opts, trace.WithAttributes(attribute.String(AttrScope, scope))) - ctx, span := t.otelTracer().Start(ctx, stage, opts...) + ctx, span := t.otelTracer().Start(ContextWithScope(ctx, scope), stage, opts...) return ctx, &StageSpan{ tracer: t, stage: stage, @@ -251,24 +263,42 @@ func (s *StageSpan) SpanContext() trace.SpanContext { // as errored when err is non-nil. Calls after the first are ignored so // a deferred End cannot double-count a stage. func (s *StageSpan) End(err error) { + if elapsed, ok := s.closeSpan(err); ok { + s.tracer.observe(s.stage, s.scope, s.model, elapsed) + } +} + +// EndWithoutObservation closes the stage span exactly as End does but +// makes no duration observation. It is for stages whose window is only +// comparable across runs when it completed, so a truncated window +// would skew the histogram while the span still needs to report the +// failure. +func (s *StageSpan) EndWithoutObservation(err error) { + s.closeSpan(err) +} + +// closeSpan ends the span and returns the window it covered. ok is +// false for a nil span and for calls after the first, so a deferred +// end cannot double-count a stage. +func (s *StageSpan) closeSpan(err error) (elapsed time.Duration, ok bool) { if s == nil || s.ended { - return + return 0, false } s.ended = true - elapsed := time.Since(s.start) + elapsed = time.Since(s.start) if err != nil { s.span.RecordError(err) s.span.SetStatus(codes.Error, err.Error()) } s.span.End() - s.tracer.observe(s.stage, s.scope, s.model, elapsed) + return elapsed, true } // Record emits an already-finished stage span with explicit start and // end timestamps. It is for stages whose boundaries are only known // after the fact, such as durations reconstructed from persisted -// timestamps. The stage is scoped by the span in ctx. Non-positive or -// unset windows are dropped. +// timestamps. The stage takes the scope on ctx. Non-positive or unset +// windows are dropped. func (t *StageTracer) Record( ctx context.Context, stage string, diff --git a/coderd/x/chatd/chatloop/stage_internal_test.go b/coderd/x/chatd/chatloop/stage_internal_test.go new file mode 100644 index 00000000000..7aaf7d73641 --- /dev/null +++ b/coderd/x/chatd/chatloop/stage_internal_test.go @@ -0,0 +1,139 @@ +package chatloop + +import ( + "context" + "testing" + "time" + + "charm.land/fantasy" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/codes" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + "golang.org/x/xerrors" + + "github.com/coder/quartz" +) + +// ttftStageCount returns the observation count of the +// time_to_first_token series on stage_duration_seconds, and whether the +// series exists at all. +func ttftStageCount(t *testing.T, registry *prometheus.Registry) (uint64, bool) { + t.Helper() + families, err := registry.Gather() + require.NoError(t, err) + for _, family := range families { + if family.GetName() != "coderd_chatd_stage_duration_seconds" { + continue + } + for _, metric := range family.GetMetric() { + if stageLabel(metric) == StageTimeToFirstToken { + return metric.GetHistogram().GetSampleCount(), true + } + } + } + return 0, false +} + +func stageLabel(metric *dto.Metric) string { + for _, label := range metric.GetLabel() { + if label.GetName() == "stage" { + return label.GetValue() + } + } + return "" +} + +func ttftSpanStatus(t *testing.T, spans *tracetest.SpanRecorder) sdktrace.ReadOnlySpan { + t.Helper() + for _, span := range spans.Ended() { + if span.Name() == StageTimeToFirstToken { + return span + } + } + t.Fatalf("no %s span was recorded", StageTimeToFirstToken) + return nil +} + +func TestGuardedStreamTTFTStage(t *testing.T) { + t.Parallel() + + newFixture := func(t *testing.T) (*StageTracer, *tracetest.SpanRecorder, *prometheus.Registry, *Metrics) { + t.Helper() + recorder := tracetest.NewSpanRecorder() + provider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder)) + t.Cleanup(func() { + require.NoError(t, provider.Shutdown(context.Background())) + }) + registry := prometheus.NewRegistry() + metrics := NewMetrics(registry) + return NewStageTracer(provider, metrics), recorder, registry, metrics + } + + t.Run("FirstPartObservesStage", func(t *testing.T) { + t.Parallel() + stages, spans, registry, metrics := newFixture(t) + + attempt, err := guardedStream( + t.Context(), "anthropic", "claude", quartz.NewMock(t), time.Minute, + func(context.Context) (fantasy.StreamResponse, error) { + return fantasy.StreamResponse(func(yield func(fantasy.StreamPart) bool) { + yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeTextDelta, Delta: "hi"}) + }), nil + }, + metrics, stages, StageModel{Model: "claude", Effort: "high"}, + ) + require.NoError(t, err) + parts := 0 + for range attempt.stream { + parts++ + } + require.Equal(t, 1, parts) + attempt.release() + + count, ok := ttftStageCount(t, registry) + require.True(t, ok) + require.Equal(t, uint64(1), count) + require.Equal(t, codes.Unset, ttftSpanStatus(t, spans).Status().Code) + }) + + t.Run("OpenFailureRecordsSpanWithoutObservation", func(t *testing.T) { + t.Parallel() + stages, spans, registry, metrics := newFixture(t) + + openErr := xerrors.New("provider refused the request") + _, err := guardedStream( + t.Context(), "anthropic", "claude", quartz.NewMock(t), time.Minute, + func(context.Context) (fantasy.StreamResponse, error) { + return nil, openErr + }, + metrics, stages, StageModel{Model: "claude"}, + ) + require.ErrorIs(t, err, openErr) + + require.Equal(t, codes.Error, ttftSpanStatus(t, spans).Status().Code) + _, ok := ttftStageCount(t, registry) + require.False(t, ok, "a failed window must not be observed") + }) + + t.Run("ReleaseWithoutPartRecordsSpanWithoutObservation", func(t *testing.T) { + t.Parallel() + stages, spans, registry, metrics := newFixture(t) + + attempt, err := guardedStream( + t.Context(), "anthropic", "claude", quartz.NewMock(t), time.Minute, + func(context.Context) (fantasy.StreamResponse, error) { + return fantasy.StreamResponse(func(func(fantasy.StreamPart) bool) {}), nil + }, + metrics, stages, StageModel{Model: "claude"}, + ) + require.NoError(t, err) + attempt.release() + + require.Equal(t, codes.Error, ttftSpanStatus(t, spans).Status().Code) + _, ok := ttftStageCount(t, registry) + require.False(t, ok, "a window closed without a token must not be observed") + }) +} diff --git a/coderd/x/chatd/chatloop/stage_test.go b/coderd/x/chatd/chatloop/stage_test.go index e0509b21054..c983004a442 100644 --- a/coderd/x/chatd/chatloop/stage_test.go +++ b/coderd/x/chatd/chatloop/stage_test.go @@ -245,9 +245,12 @@ func TestStageTracerScope(t *testing.T) { fixture := newStageFixture(t) turnCtx, turn := fixture.tracer.StartRoot(t.Context(), chatloop.StageChatTurn, nil) - // Background work strips the span from its context, which is - // what detaches its stages from the turn profile. - detachedCtx := trace.ContextWithSpanContext(turnCtx, trace.SpanContext{}) + // Background work detaches from the turn by stripping the span + // and marking the context background scoped. + detachedCtx := chatloop.ContextWithScope( + trace.ContextWithSpanContext(turnCtx, trace.SpanContext{}), + chatloop.ScopeBackground, + ) _, attempt := fixture.tracer.Start(detachedCtx, chatloop.StageProviderAttempt) attempt.End(nil) turn.End(nil) @@ -418,6 +421,48 @@ func TestStageTracerRecord(t *testing.T) { }) } +func TestStageSpanEndWithoutObservation(t *testing.T) { + t.Parallel() + fixture := newStageFixture(t) + + _, span := fixture.tracer.Start(t.Context(), chatloop.StageTimeToFirstToken) + span.EndWithoutObservation(xerrors.New("stream ended before the first token")) + // The first end wins, so a later End cannot revive the + // observation. + span.End(nil) + + ended := fixture.spans.Ended() + require.Len(t, ended, 1) + require.Equal(t, chatloop.StageTimeToFirstToken, ended[0].Name()) + require.Equal(t, codes.Error, ended[0].Status().Code) + require.Empty(t, fixture.stageObservations(t)) +} + +func TestStageDurationBuckets(t *testing.T) { + t.Parallel() + registry := prometheus.NewRegistry() + metrics := chatloop.NewMetrics(registry) + // A turn can run for hours, so the top bucket boundary has to sit + // above the longest plausible stage. + metrics.RecordStageDuration(chatloop.StageChatTurn, chatloop.ScopeTurn, "", "", 2*time.Hour) + + families, err := registry.Gather() + require.NoError(t, err) + var buckets []*dto.Bucket + for _, family := range families { + if family.GetName() != "coderd_chatd_stage_duration_seconds" { + continue + } + require.Len(t, family.GetMetric(), 1) + buckets = family.GetMetric()[0].GetHistogram().GetBucket() + } + require.Len(t, buckets, 21) + top := buckets[len(buckets)-1] + require.Greater(t, top.GetUpperBound(), (3*time.Hour).Seconds()*0.9) + require.Equal(t, uint64(1), top.GetCumulativeCount(), + "a two hour stage must fall inside the buckets") +} + func TestStageTracerWithoutProvider(t *testing.T) { t.Parallel() @@ -438,3 +483,28 @@ func TestStageTracerWithoutProvider(t *testing.T) { nilSpan.End(nil) nilTracer.Record(t.Context(), chatloop.StageThinking, chatloop.StageModel{}, time.Now().Add(-time.Second), time.Now(), nil) } + +// TestStageTracerScopeWithoutProvider covers a metrics-only +// deployment: a no-op tracer produces invalid span contexts, and the +// scope must still follow the turn. +func TestStageTracerScopeWithoutProvider(t *testing.T) { + t.Parallel() + + registry := prometheus.NewRegistry() + tracer := chatloop.NewStageTracer(nil, chatloop.NewMetrics(registry)) + + turnCtx, turn := tracer.StartRoot(t.Context(), chatloop.StageChatTurn, nil) + require.False(t, turn.SpanContext().IsValid()) + stepCtx, step := tracer.Start(turnCtx, chatloop.StageGenerationStep) + start := time.Now().Add(-time.Second) + tracer.Record(stepCtx, chatloop.StageThinking, chatloop.StageModel{}, start, time.Now(), nil) + step.End(nil) + turn.End(nil) + + fixture := stageFixture{registry: registry} + require.Equal(t, map[stageKey]uint64{ + {stage: chatloop.StageChatTurn, scope: chatloop.ScopeTurn}: 1, + {stage: chatloop.StageGenerationStep, scope: chatloop.ScopeTurn}: 1, + {stage: chatloop.StageThinking, scope: chatloop.ScopeTurn}: 1, + }, fixture.stageObservations(t)) +} diff --git a/coderd/x/chatd/stage_internal_test.go b/coderd/x/chatd/stage_internal_test.go index 71880523fcb..d3d760f2af7 100644 --- a/coderd/x/chatd/stage_internal_test.go +++ b/coderd/x/chatd/stage_internal_test.go @@ -135,9 +135,12 @@ func TestStageSpanRoundTripperScope(t *testing.T) { require.NoError(t, err) require.NoError(t, turnResp.Body.Close()) - // Background work runs on a context whose span was stripped, the - // same way inflight chatd tasks detach from the turn. - backgroundCtx := trace.ContextWithSpanContext(turnCtx, trace.SpanContext{}) + // Background work runs on a context detached from the turn, the same + // way inflight chatd tasks are. + backgroundCtx := chatloop.ContextWithScope( + trace.ContextWithSpanContext(turnCtx, trace.SpanContext{}), + chatloop.ScopeBackground, + ) backgroundReq, err := http.NewRequestWithContext(backgroundCtx, http.MethodPost, "https://provider.example/v1/messages", nil) require.NoError(t, err) backgroundResp, err := transport.RoundTrip(backgroundReq) @@ -239,3 +242,58 @@ func TestRunnerTurnSpanParentsRecordedStages(t *testing.T) { require.Equal(t, chatTurn.SpanContext().SpanID(), queueWait.Parent().SpanID()) require.NotEqual(t, generationStep.SpanContext().SpanID(), queueWait.Parent().SpanID()) } + +func TestServerRecordQueueWaitIsStandalone(t *testing.T) { + t.Parallel() + tracer, recorder := newStageTestTracer(t) + server := &Server{stages: tracer} + + // The promoting request has its own span, which the queue wait must + // not join. + requestCtx, requestSpan := tracer.Start(t.Context(), chatloop.StageCommit) + queuedAt := time.Now().Add(-30 * time.Second) + server.recordQueueWait(requestCtx, uuid.New(), queuedAt, queuedAt.Add(20*time.Second)) + requestSpan.End(nil) + + var queueWait, request sdktrace.ReadOnlySpan + for _, span := range recorder.Ended() { + switch span.Name() { + case chatloop.StageQueueWait: + queueWait = span + case chatloop.StageCommit: + request = span + } + } + require.NotNil(t, queueWait) + require.NotNil(t, request) + require.False(t, queueWait.Parent().IsValid()) + require.NotEqual(t, request.SpanContext().TraceID(), queueWait.SpanContext().TraceID()) + require.Contains(t, queueWait.Attributes(), + attribute.String(chatloop.AttrScope, chatloop.ScopeTurn)) +} + +func TestServerInflightContextIsBackgroundScoped(t *testing.T) { + t.Parallel() + tracer, recorder := newStageTestTracer(t) + serverCtx, serverCancel := context.WithCancel(context.Background()) + t.Cleanup(serverCancel) + server := &Server{ctx: serverCtx, stages: tracer} + + turn := newRunnerTurnSpan(tracer) + turnCtx := turn.Ensure(t.Context(), database.Chat{ID: uuid.New()}, time.Now().Add(-time.Second)) + inflightCtx, stop := server.inflightContext(turnCtx) + t.Cleanup(stop) + + _, span := tracer.Start(inflightCtx, chatloop.StageGenerationStep) + span.End(nil) + turn.End(nil) + + for _, ended := range recorder.Ended() { + if ended.Name() != chatloop.StageGenerationStep { + continue + } + require.False(t, ended.Parent().IsValid()) + require.Contains(t, ended.Attributes(), + attribute.String(chatloop.AttrScope, chatloop.ScopeBackground)) + } +} diff --git a/coderd/x/chatd/turn_trace.go b/coderd/x/chatd/turn_trace.go index e6b13a2561c..eecd46a03e2 100644 --- a/coderd/x/chatd/turn_trace.go +++ b/coderd/x/chatd/turn_trace.go @@ -85,7 +85,13 @@ func (t *runnerTurnSpan) Context(ctx context.Context) context.Context { } func (t *runnerTurnSpan) contextLocked(ctx context.Context) context.Context { - if !t.started || t.ended || !t.spanCtx.IsValid() { + if !t.started || t.ended { + return ctx + } + // The scope is set independently of the span context so stages run + // on this context stay turn scoped when tracing is not recording. + ctx = chatloop.ContextWithScope(ctx, chatloop.ScopeTurn) + if !t.spanCtx.IsValid() { return ctx } return trace.ContextWithSpanContext(ctx, t.spanCtx) diff --git a/examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md b/examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md index 27140f54516..3569e175358 100644 --- a/examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md +++ b/examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md @@ -46,9 +46,12 @@ p95, p99) for every stat-aware panel. Stages that run before or outside model resolution (`chat_turn`, `queue_wait`, `capacity_wait`, `acquisition`, `mcp_connect`, `commit`) -carry empty `model`/`effort` labels. Filtering `$model` or `$effort` to a -specific value therefore zeroes those stages and effectively narrows the -view to the generation stages. +always carry empty `model`/`effort` labels. Panels match those stages +without the `$model`/`$effort` matchers, since a matcher there could only +subtract, so narrowing either variable keeps the turn root and the waits +populated and narrows only the model-carrying stages (`generation_step`, +`prepare`, `provider_attempt`, `time_to_first_token`, `stream`, +`thinking`, `tool_call`, `compaction`). ## Panels @@ -56,16 +59,21 @@ view to the generation stages. **Stage profile flamegraph ($stat)** - one frame per stage, laid out in the fixed hierarchy, frame width = the selected `$stat` of that stage's -duration over the dashboard time range. Dimensions: filtered to -`scope="turn"` and the `$model`/`$effort` selections; `$stat` picks the -statistic. How to read: the widest frames under `generation_step` are -where turn time goes; compare `provider_attempt` (request to response -headers) against `stream` (full stream) to separate provider latency -from streaming time. Caveats: stages overlap in wall time (tool calls -and thinking happen inside the stream) and quantiles are not additive, -so a child frame can read wider than its parent at high percentiles; -a stage whose series first appears inside the window reads 0 until its -second sample. +duration over the dashboard time range. Dimensions: filtered to the +turn scope; `$model`/`$effort` apply to the model-carrying stages +only, so a concrete selection narrows the generation stages while the +`chat_turn` root and the pre-model stages keep their full values. +`$stat` picks the statistic. How to read: the widest frames under +`generation_step` are where turn time goes; compare `provider_attempt` +(request to response headers) against `stream` (full stream) to separate +provider latency from streaming time. The `self` column is the stage's +own value for leaf stages and 0 for stages with children; true self time +is not derivable here, because overlapping stages can make a parent +minus its children negative. Caveats: stages overlap in wall time (tool +calls and thinking happen inside the stream) and quantiles are not +additive, so a child frame can read wider than its parent at high +percentiles; a stage whose series first appears inside the window reads 0 +until its second sample. **Stage profile in hierarchy order ($stat)** - the same query as the flamegraph drawn as horizontal bars in depth-first order with the tree @@ -78,29 +86,33 @@ as a numeric check on the flamegraph. **Stage duration over time ($stat)** - one series per stage, the selected `$stat` computed over `$__rate_interval`. Dimensions: -`scope="turn"`, `$model`/`$effort` filters, series split by `stage`. -How to read: this is the drill-down for "when did it get slow" - a -regression visible in the profile shows here as a step or trend in the -affected stage. Idle stages drop out rather than plotting NaN. +`scope="turn"`, series split by `stage`; `$model`/`$effort` apply to the +model-carrying stages only, so the pre-model stages stay plotted under +any selection. How to read: this is the drill-down for "when did it get +slow" - a regression visible in the profile shows here as a step or +trend in the affected stage. Idle stages drop out rather than plotting +NaN. **Stage time share of chat_turn** - each stage's total time as a percentage of total `chat_turn` time, from mean rates of the histogram -sums. Dimensions: numerator is `scope="turn"` with `$model`/`$effort` -applied and split by `stage`; the denominator is all `chat_turn` time -without model/effort filters, because `chat_turn` carries empty -model/effort labels. How to read: this is the "where does the time go" -summary - stages overlap, so series can sum past 100%, but a single -stage rising toward 100% of turn time identifies the dominant cost. +sums. Dimensions: numerator is `scope="turn"` split by `stage`, with +`$model`/`$effort` applied to the model-carrying stages only; the +denominator is all `chat_turn` time without model/effort filters, +because `chat_turn` carries empty model/effort labels. Under a concrete +model the generation stages therefore read as that model's share of all +turn time. How to read: this is the "where does the time go" summary - +stages overlap, so series can sum past 100%, but a single stage rising +toward 100% of turn time identifies the dominant cost. **Queue, capacity and acquisition wait (p99)** - p99 of the three pre-generation waits: `queue_wait` (queued message insert to promotion), `capacity_wait` (concurrent-agent limiter admission) and `acquisition` (trigger message insert to worker pickup). Dimensions: -`scope="turn"`, fixed to those three stages, split by `stage`; -`$model`/`$effort` apply but these stages carry empty labels, so -non-All selections blank this panel. How to read: these are scheduling -delays before any model work starts - user-visible latency that no -provider-side optimization can fix. +`scope="turn"`, fixed to those three stages, split by `stage`; all three +run before a model is resolved, so `$model`/`$effort` are not applied and +the panel stays populated under any selection. How to read: these are +scheduling delays before any model work starts - user-visible latency +that no provider-side optimization can fix. ### Throughput and TTFT @@ -114,11 +126,12 @@ filters. How to read: the primary user-perceived responsiveness metric for streaming. **Turn rate and stage sample rate** - completed `chat_turn` per second -plus one rate series per other stage. Dimensions: `scope="turn"`, -`$model`/`$effort` applied, split by `stage`. How to read: throughput -and shape - a stage rate above the turn rate means the stage repeats -within a turn (generation steps, provider attempts, tool calls); -`provider_attempt` rising faster than `stream` indicates retries. +plus one rate series per other stage. Dimensions: `scope="turn"`, split +by `stage`, with `$model`/`$effort` applied to the model-carrying stages +only. How to read: throughput and shape - a stage rate above the turn +rate means the stage repeats within a turn (generation steps, provider +attempts, tool calls); `provider_attempt` rising faster than `stream` +indicates retries. **Background provider calls ($stat)** - rate and selected `$stat` duration of background-scope `provider_attempt` samples: detached diff --git a/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json b/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json index 30e93b2cdad..3cf703f21cd 100644 --- a/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json +++ b/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json @@ -39,10 +39,10 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Aggregate profile of chat lifecycle stages over the dashboard time range, using the $stat statistic of coderd_chatd_stage_duration_seconds. Levels come from the fixed stage hierarchy, attached as labels with label_replace and reshaped into Grafana's nested set model by the panel transformations.\n\nStage timings overlap in wall time (stream, thinking and tool_call all run inside a generation_step, and time_to_first_token is part of provider_attempt), and quantiles are not additive across stages. Read this as a stage time profile, not as a strict non-overlapping decomposition of turn latency: child bars can exceed their parent. A stage with no samples in the time range reads as 0. Only scope=\"turn\" samples are included, so detached background provider calls are excluded.\n\nValues come from rate() over counter series, and rate() treats the first sample in the window as its baseline. A stage whose series first appears inside the window reads as 0 until its second observation, so rare stages such as queue_wait, capacity_wait and compaction can render as 0 while real samples exist. Widen the time range to confirm.\n\nThe $model and $effort filters only match stages that ran against a resolved model. Pre-model stages (chat_turn, queue_wait, capacity_wait, acquisition, commit, mcp_connect) carry empty model and effort, so narrowing either variable to a specific value drops them to 0.", + "description": "Aggregate profile of chat lifecycle stages over the dashboard time range, using the $stat statistic of coderd_chatd_stage_duration_seconds. Levels come from the fixed stage hierarchy, attached as labels with label_replace and reshaped into Grafana's nested set model by the panel transformations.\n\nStage timings overlap in wall time (stream, thinking and tool_call all run inside a generation_step, and time_to_first_token is part of provider_attempt), and quantiles are not additive across stages. Read this as a stage time profile, not as a strict non-overlapping decomposition of turn latency: child bars can exceed their parent. A stage with no samples in the time range reads as 0. Only scope=\"turn\" samples are included, so detached background provider calls are excluded.\n\nValues come from rate() over counter series, and rate() treats the first sample in the window as its baseline. A stage whose series first appears inside the window reads as 0 until its second observation, so rare stages such as queue_wait, capacity_wait and compaction can render as 0 while real samples exist. Widen the time range to confirm.\n\n$model and $effort apply only to the stages that carry those labels (generation_step, prepare, provider_attempt, time_to_first_token, stream, thinking, tool_call, compaction). The stages recorded before a model is resolved (chat_turn, queue_wait, capacity_wait, acquisition, mcp_connect, commit) are always matched without them, so they stay populated under any selection.\n\nThe self field is the stage's own value for leaf stages and 0 for stages that have children, so the top table's self column reads as leaf time. True self time is not derivable from these histograms: stages overlap in wall time, so a parent minus its children can be negative.\n\nFrame values are seconds. The panel labels them as sample counts and ignores the field unit, because it takes its unit from profile metadata that a Prometheus query cannot set.", "fieldConfig": { "defaults": { - "unit": "s" + "unit": "short" }, "overrides": [ { @@ -79,7 +79,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"chat_turn\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"chat_turn\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"chat_turn\", \"\", \"\"),\n\"n\", \"01\", \"\", \"\"),\n\"level\", \"0\", \"\", \"\"),\n\"name\", \"chat_turn\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"queue_wait\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"queue_wait\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"queue_wait\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"queue_wait\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"queue_wait\", \"\", \"\"),\n\"n\", \"02\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ queue_wait\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"capacity_wait\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"capacity_wait\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"capacity_wait\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"capacity_wait\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"capacity_wait\", \"\", \"\"),\n\"n\", \"03\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ capacity_wait\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"acquisition\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"acquisition\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"acquisition\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"acquisition\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"acquisition\", \"\", \"\"),\n\"n\", \"04\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ acquisition\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"generation_step\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"generation_step\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"generation_step\", \"\", \"\"),\n\"n\", \"05\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ generation_step\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"prepare\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"prepare\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"prepare\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"prepare\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"prepare\", \"\", \"\"),\n\"n\", \"06\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ prepare\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"mcp_connect\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"mcp_connect\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"mcp_connect\", \"\", \"\"),\n\"n\", \"07\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ mcp_connect\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"provider_attempt\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"provider_attempt\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"provider_attempt\", \"\", \"\"),\n\"n\", \"08\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ provider_attempt\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"time_to_first_token\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"time_to_first_token\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"time_to_first_token\", \"\", \"\"),\n\"n\", \"09\", \"\", \"\"),\n\"level\", \"3\", \"\", \"\"),\n\"name\", \"│ │ └─ time_to_first_token\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"stream\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"stream\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"stream\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"stream\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"stream\", \"\", \"\"),\n\"n\", \"10\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ stream\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"thinking\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"thinking\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"thinking\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"thinking\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"thinking\", \"\", \"\"),\n\"n\", \"11\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ thinking\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"tool_call\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"tool_call\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"tool_call\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"tool_call\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"tool_call\", \"\", \"\"),\n\"n\", \"12\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ tool_call\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"commit\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"commit\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"commit\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"commit\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"commit\", \"\", \"\"),\n\"n\", \"13\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ commit\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"compaction\", \"\", \"\"),\n\"n\", \"14\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ compaction\", \"\", \"\")\n)", + "expr": "(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"chat_turn\", scope=\"turn\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"chat_turn\", scope=\"turn\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"chat_turn\", \"\", \"\"),\n\"n\", \"01\", \"\", \"\"),\n\"level\", \"0\", \"\", \"\"),\n\"name\", \"chat_turn\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"queue_wait\", scope=\"turn\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"queue_wait\", scope=\"turn\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"queue_wait\", scope=\"turn\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"queue_wait\", scope=\"turn\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"queue_wait\", \"\", \"\"),\n\"n\", \"02\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ queue_wait\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"capacity_wait\", scope=\"turn\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"capacity_wait\", scope=\"turn\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"capacity_wait\", scope=\"turn\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"capacity_wait\", scope=\"turn\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"capacity_wait\", \"\", \"\"),\n\"n\", \"03\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ capacity_wait\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"acquisition\", scope=\"turn\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"acquisition\", scope=\"turn\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"acquisition\", scope=\"turn\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"acquisition\", scope=\"turn\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"acquisition\", \"\", \"\"),\n\"n\", \"04\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ acquisition\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"generation_step\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"generation_step\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"generation_step\", \"\", \"\"),\n\"n\", \"05\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ generation_step\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"prepare\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"prepare\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"prepare\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"prepare\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"prepare\", \"\", \"\"),\n\"n\", \"06\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ prepare\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"mcp_connect\", scope=\"turn\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", scope=\"turn\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"mcp_connect\", scope=\"turn\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", scope=\"turn\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"mcp_connect\", \"\", \"\"),\n\"n\", \"07\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ mcp_connect\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"provider_attempt\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"provider_attempt\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"provider_attempt\", \"\", \"\"),\n\"n\", \"08\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ provider_attempt\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"time_to_first_token\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"time_to_first_token\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"time_to_first_token\", \"\", \"\"),\n\"n\", \"09\", \"\", \"\"),\n\"level\", \"3\", \"\", \"\"),\n\"name\", \"│ │ └─ time_to_first_token\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"stream\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"stream\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"stream\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"stream\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"stream\", \"\", \"\"),\n\"n\", \"10\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ stream\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"thinking\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"thinking\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"thinking\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"thinking\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"thinking\", \"\", \"\"),\n\"n\", \"11\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ thinking\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"tool_call\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"tool_call\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"tool_call\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"tool_call\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"tool_call\", \"\", \"\"),\n\"n\", \"12\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ tool_call\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"commit\", scope=\"turn\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"commit\", scope=\"turn\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"commit\", scope=\"turn\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"commit\", scope=\"turn\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"commit\", \"\", \"\"),\n\"n\", \"13\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ commit\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"compaction\", \"\", \"\"),\n\"n\", \"14\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ compaction\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)", "range": false, "refId": "A", "format": "table", @@ -113,19 +113,6 @@ "replaceFields": false } }, - { - "id": "calculateField", - "options": { - "alias": "self", - "binary": { - "left": "value", - "operator": "*", - "right": "1" - }, - "mode": "binary", - "replaceFields": false - } - }, { "id": "convertFieldType", "options": { @@ -133,11 +120,28 @@ { "destinationType": "number", "targetField": "level" + }, + { + "destinationType": "number", + "targetField": "leaf" } ], "fields": {} } }, + { + "id": "calculateField", + "options": { + "alias": "self", + "binary": { + "left": "value", + "operator": "*", + "right": "leaf" + }, + "mode": "binary", + "replaceFields": false + } + }, { "id": "organize", "options": { @@ -145,6 +149,7 @@ "Time": true, "n": true, "name": true, + "leaf": true, "Value": true }, "includeByName": {}, @@ -160,7 +165,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Same data as the flamegraph, drawn as bars in depth-first hierarchy order with the tree drawn into the labels. Useful when a stage is too small to read in the flamegraph, and as a check on the flamegraph shaping.\n\nStage timings overlap in wall time (stream, thinking and tool_call all run inside a generation_step, and time_to_first_token is part of provider_attempt), and quantiles are not additive across stages. Read this as a stage time profile, not as a strict non-overlapping decomposition of turn latency: child bars can exceed their parent. A stage with no samples in the time range reads as 0. Only scope=\"turn\" samples are included, so detached background provider calls are excluded.\n\nValues come from rate() over counter series, and rate() treats the first sample in the window as its baseline. A stage whose series first appears inside the window reads as 0 until its second observation, so rare stages such as queue_wait, capacity_wait and compaction can render as 0 while real samples exist. Widen the time range to confirm.\n\nThe $model and $effort filters only match stages that ran against a resolved model. Pre-model stages (chat_turn, queue_wait, capacity_wait, acquisition, commit, mcp_connect) carry empty model and effort, so narrowing either variable to a specific value drops them to 0.", + "description": "Same data as the flamegraph, drawn as bars in depth-first hierarchy order with the tree drawn into the labels. Useful when a stage is too small to read in the flamegraph, and as a check on the flamegraph shaping.\n\nStage timings overlap in wall time (stream, thinking and tool_call all run inside a generation_step, and time_to_first_token is part of provider_attempt), and quantiles are not additive across stages. Read this as a stage time profile, not as a strict non-overlapping decomposition of turn latency: child bars can exceed their parent. A stage with no samples in the time range reads as 0. Only scope=\"turn\" samples are included, so detached background provider calls are excluded.\n\nValues come from rate() over counter series, and rate() treats the first sample in the window as its baseline. A stage whose series first appears inside the window reads as 0 until its second observation, so rare stages such as queue_wait, capacity_wait and compaction can render as 0 while real samples exist. Widen the time range to confirm.\n\n$model and $effort apply only to the stages that carry those labels (generation_step, prepare, provider_attempt, time_to_first_token, stream, thinking, tool_call, compaction). The stages recorded before a model is resolved (chat_turn, queue_wait, capacity_wait, acquisition, mcp_connect, commit) are always matched without them, so they stay populated under any selection.", "fieldConfig": { "defaults": { "color": { @@ -206,7 +211,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"chat_turn\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"chat_turn\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"chat_turn\", \"\", \"\"),\n\"n\", \"01\", \"\", \"\"),\n\"level\", \"0\", \"\", \"\"),\n\"name\", \"chat_turn\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"queue_wait\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"queue_wait\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"queue_wait\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"queue_wait\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"queue_wait\", \"\", \"\"),\n\"n\", \"02\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ queue_wait\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"capacity_wait\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"capacity_wait\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"capacity_wait\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"capacity_wait\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"capacity_wait\", \"\", \"\"),\n\"n\", \"03\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ capacity_wait\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"acquisition\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"acquisition\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"acquisition\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"acquisition\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"acquisition\", \"\", \"\"),\n\"n\", \"04\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ acquisition\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"generation_step\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"generation_step\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"generation_step\", \"\", \"\"),\n\"n\", \"05\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ generation_step\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"prepare\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"prepare\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"prepare\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"prepare\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"prepare\", \"\", \"\"),\n\"n\", \"06\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ prepare\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"mcp_connect\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"mcp_connect\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"mcp_connect\", \"\", \"\"),\n\"n\", \"07\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ mcp_connect\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"provider_attempt\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"provider_attempt\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"provider_attempt\", \"\", \"\"),\n\"n\", \"08\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ provider_attempt\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"time_to_first_token\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"time_to_first_token\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"time_to_first_token\", \"\", \"\"),\n\"n\", \"09\", \"\", \"\"),\n\"level\", \"3\", \"\", \"\"),\n\"name\", \"│ │ └─ time_to_first_token\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"stream\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"stream\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"stream\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"stream\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"stream\", \"\", \"\"),\n\"n\", \"10\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ stream\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"thinking\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"thinking\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"thinking\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"thinking\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"thinking\", \"\", \"\"),\n\"n\", \"11\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ thinking\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"tool_call\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"tool_call\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"tool_call\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"tool_call\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"tool_call\", \"\", \"\"),\n\"n\", \"12\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ tool_call\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"commit\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"commit\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"commit\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"commit\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"commit\", \"\", \"\"),\n\"n\", \"13\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ commit\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"compaction\", \"\", \"\"),\n\"n\", \"14\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ compaction\", \"\", \"\")\n)", + "expr": "(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"chat_turn\", scope=\"turn\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"chat_turn\", scope=\"turn\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"chat_turn\", \"\", \"\"),\n\"n\", \"01\", \"\", \"\"),\n\"level\", \"0\", \"\", \"\"),\n\"name\", \"chat_turn\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"queue_wait\", scope=\"turn\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"queue_wait\", scope=\"turn\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"queue_wait\", scope=\"turn\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"queue_wait\", scope=\"turn\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"queue_wait\", \"\", \"\"),\n\"n\", \"02\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ queue_wait\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"capacity_wait\", scope=\"turn\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"capacity_wait\", scope=\"turn\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"capacity_wait\", scope=\"turn\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"capacity_wait\", scope=\"turn\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"capacity_wait\", \"\", \"\"),\n\"n\", \"03\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ capacity_wait\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"acquisition\", scope=\"turn\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"acquisition\", scope=\"turn\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"acquisition\", scope=\"turn\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"acquisition\", scope=\"turn\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"acquisition\", \"\", \"\"),\n\"n\", \"04\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ acquisition\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"generation_step\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"generation_step\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"generation_step\", \"\", \"\"),\n\"n\", \"05\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ generation_step\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"prepare\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"prepare\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"prepare\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"prepare\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"prepare\", \"\", \"\"),\n\"n\", \"06\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ prepare\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"mcp_connect\", scope=\"turn\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", scope=\"turn\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"mcp_connect\", scope=\"turn\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", scope=\"turn\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"mcp_connect\", \"\", \"\"),\n\"n\", \"07\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ mcp_connect\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"provider_attempt\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"provider_attempt\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"provider_attempt\", \"\", \"\"),\n\"n\", \"08\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ provider_attempt\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"time_to_first_token\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"time_to_first_token\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"time_to_first_token\", \"\", \"\"),\n\"n\", \"09\", \"\", \"\"),\n\"level\", \"3\", \"\", \"\"),\n\"name\", \"│ │ └─ time_to_first_token\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"stream\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"stream\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"stream\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"stream\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"stream\", \"\", \"\"),\n\"n\", \"10\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ stream\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"thinking\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"thinking\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"thinking\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"thinking\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"thinking\", \"\", \"\"),\n\"n\", \"11\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ thinking\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"tool_call\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"tool_call\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"tool_call\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"tool_call\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"tool_call\", \"\", \"\"),\n\"n\", \"12\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ tool_call\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"commit\", scope=\"turn\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"commit\", scope=\"turn\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"commit\", scope=\"turn\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"commit\", scope=\"turn\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"commit\", \"\", \"\"),\n\"n\", \"13\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ commit\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"compaction\", \"\", \"\"),\n\"n\", \"14\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ compaction\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)", "range": false, "refId": "A", "format": "table", @@ -234,7 +239,8 @@ "Time": true, "n": true, "level": true, - "label": true + "label": true, + "leaf": true }, "includeByName": {}, "indexByName": {}, @@ -265,7 +271,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Selected $stat of each lifecycle stage over time, one series per stage. Mean is rate(_sum)/rate(_count); the percentiles are histogram_quantile over the bucket rates. A stage with no samples in an interval has no point rather than NaN.", + "description": "Selected $stat of each lifecycle stage over time, one series per stage. Mean is rate(_sum)/rate(_count); the percentiles are histogram_quantile over the bucket rates. A stage with no samples in an interval has no point rather than NaN.\n\n$model and $effort apply only to the stages that carry those labels (generation_step, prepare, provider_attempt, time_to_first_token, stream, thinking, tool_call, compaction). The stages recorded before a model is resolved (chat_turn, queue_wait, capacity_wait, acquisition, mcp_connect, commit) are always matched without them, so they stay populated under any selection.", "fieldConfig": { "defaults": { "color": { @@ -347,10 +353,21 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_stage_duration_seconds_bucket{scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_stage_duration_seconds_sum{scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=~\"chat_turn|queue_wait|capacity_wait|acquisition|mcp_connect|commit\", scope=\"turn\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"chat_turn|queue_wait|capacity_wait|acquisition|mcp_connect|commit\", scope=\"turn\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_stage_duration_seconds_sum{stage=~\"chat_turn|queue_wait|capacity_wait|acquisition|mcp_connect|commit\", scope=\"turn\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"chat_turn|queue_wait|capacity_wait|acquisition|mcp_connect|commit\", scope=\"turn\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", "range": true, "refId": "A", "legendFormat": "{{stage}}" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=~\"generation_step|prepare|provider_attempt|time_to_first_token|stream|thinking|tool_call|compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"generation_step|prepare|provider_attempt|time_to_first_token|stream|thinking|tool_call|compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_stage_duration_seconds_sum{stage=~\"generation_step|prepare|provider_attempt|time_to_first_token|stream|thinking|tool_call|compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"generation_step|prepare|provider_attempt|time_to_first_token|stream|thinking|tool_call|compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "range": true, + "refId": "B", + "legendFormat": "{{stage}}" } ], "title": "Stage duration over time (${stat:text})", @@ -361,7 +378,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Total time spent in each stage as a percentage of total chat_turn time, from mean rates (rate of the histogram _sum). Stages overlap, so the series can sum to more than 100%. Intervals with no chat_turn time are dropped instead of dividing by zero. The numerator honours $model and $effort; the chat_turn denominator does not, because chat_turn is recorded before a model is resolved and carries empty labels.", + "description": "Total time spent in each stage as a percentage of total chat_turn time, from mean rates (rate of the histogram _sum). Stages overlap, so the series can sum to more than 100%. Intervals with no chat_turn time are dropped instead of dividing by zero. The chat_turn denominator is never filtered by model or effort.\n\n$model and $effort apply only to the stages that carry those labels (generation_step, prepare, provider_attempt, time_to_first_token, stream, thinking, tool_call, compaction). The stages recorded before a model is resolved (chat_turn, queue_wait, capacity_wait, acquisition, mcp_connect, commit) are always matched without them, so they stay populated under any selection.", "fieldConfig": { "defaults": { "color": { @@ -443,10 +460,21 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "100 * sum by (stage) (rate(coderd_chatd_stage_duration_seconds_sum{stage!=\"chat_turn\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval]))\n/ on() group_left() (sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"chat_turn\", scope=\"turn\"}[$__rate_interval])) > 0)", + "expr": "100 * sum by (stage) (rate(coderd_chatd_stage_duration_seconds_sum{stage=~\"queue_wait|capacity_wait|acquisition|mcp_connect|commit\", scope=\"turn\"}[$__rate_interval]))\n/ on() group_left() (sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"chat_turn\", scope=\"turn\"}[$__rate_interval])) > 0)", "range": true, "refId": "A", "legendFormat": "{{stage}}" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "100 * sum by (stage) (rate(coderd_chatd_stage_duration_seconds_sum{stage=~\"generation_step|prepare|provider_attempt|time_to_first_token|stream|thinking|tool_call|compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval]))\n/ on() group_left() (sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"chat_turn\", scope=\"turn\"}[$__rate_interval])) > 0)", + "range": true, + "refId": "B", + "legendFormat": "{{stage}}" } ], "title": "Stage time share of chat_turn", @@ -457,7 +485,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "p99 of the pre-generation waits: queue_wait (queued message insert to promotion), capacity_wait (capacity limiter acquire) and acquisition (trigger message insert to Acquire applied). These are queueing signals rather than model latency. Intervals with no samples for a stage are dropped instead of returning NaN.", + "description": "p99 of the pre-generation waits: queue_wait (queued message insert to promotion), capacity_wait (capacity limiter acquire) and acquisition (trigger message insert to Acquire applied). These are queueing signals rather than model latency. Intervals with no samples for a stage are dropped instead of returning NaN. All three stages run before a model is resolved, so $model and $effort are not applied and the panel stays populated under any selection.", "fieldConfig": { "defaults": { "color": { @@ -539,7 +567,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (le, stage) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=~\"queue_wait|capacity_wait|acquisition\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])))\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"queue_wait|capacity_wait|acquisition\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) > 0)", + "expr": "histogram_quantile(0.99, sum by (le, stage) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=~\"queue_wait|capacity_wait|acquisition\", scope=\"turn\"}[$__rate_interval])))\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"queue_wait|capacity_wait|acquisition\", scope=\"turn\"}[$__rate_interval])) > 0)", "range": true, "refId": "A", "legendFormat": "{{stage}} p99" @@ -695,7 +723,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Completed chat turns per second and the per-stage sample rate from the histogram _count series. Stage rates above the turn rate mean the stage repeats within a turn (generation steps, tool calls, provider attempts); rates near zero mean the stage rarely fires.", + "description": "Completed chat turns per second and the per-stage sample rate from the histogram _count series. Stage rates above the turn rate mean the stage repeats within a turn (generation steps, tool calls, provider attempts); rates near zero mean the stage rarely fires.\n\n$model and $effort apply only to the stages that carry those labels (generation_step, prepare, provider_attempt, time_to_first_token, stream, thinking, tool_call, compaction). The stages recorded before a model is resolved (chat_turn, queue_wait, capacity_wait, acquisition, mcp_connect, commit) are always matched without them, so they stay populated under any selection.", "fieldConfig": { "defaults": { "color": { @@ -777,7 +805,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval]))", + "expr": "sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\"}[$__rate_interval]))", "range": true, "refId": "A", "legendFormat": "chat turns" @@ -788,10 +816,21 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage!=\"chat_turn\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval]))", + "expr": "sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"queue_wait|capacity_wait|acquisition|mcp_connect|commit\", scope=\"turn\"}[$__rate_interval]))", "range": true, "refId": "B", "legendFormat": "{{stage}}" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"generation_step|prepare|provider_attempt|time_to_first_token|stream|thinking|tool_call|compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval]))", + "range": true, + "refId": "C", + "legendFormat": "{{stage}}" } ], "title": "Turn rate and stage sample rate", From 7abe4c1b04c1512e0d239a4d485a906c25faa8e5 Mon Sep 17 00:00:00 2001 From: Scott Miller Date: Tue, 1 Sep 2026 21:28:00 +0000 Subject: [PATCH 03/19] feat: add chat_kind label to chat lifecycle stage metrics Carry the chat kind (root or subagent) on the turn context alongside the stage scope so every turn-scoped stage, including the pre-model stages, records it as a histogram label and span attribute from the shared StageTracer path. Detached background work carries the kind of the chat that triggered it when known. Add a chat_kind dashboard variable applied to every turn-scoped panel, including the time-share denominator and wait panels, and document the root versus subagent distinction. --- coderd/x/chatd/capacity.go | 5 +- coderd/x/chatd/chatd.go | 33 +++-- coderd/x/chatd/chatloop/metrics.go | 11 +- coderd/x/chatd/chatloop/stage.go | 81 ++++++++---- coderd/x/chatd/chatloop/stage_test.go | 67 ++++++++-- coderd/x/chatd/generation.go | 1 - coderd/x/chatd/stage_internal_test.go | 32 ++++- coderd/x/chatd/turn_trace.go | 22 ++-- docs/admin/integrations/prometheus.md | 2 +- .../grafana/chatd-lifecycle/README.md | 115 ++++++++++-------- .../grafana/chatd-lifecycle/dashboard.json | 70 ++++++++--- scripts/metricsdocgen/generated_metrics | 4 +- 12 files changed, 312 insertions(+), 131 deletions(-) diff --git a/coderd/x/chatd/capacity.go b/coderd/x/chatd/capacity.go index ef9cf1f1ea1..55a744d9409 100644 --- a/coderd/x/chatd/capacity.go +++ b/coderd/x/chatd/capacity.go @@ -63,17 +63,18 @@ func (w *chatWorker) noteCapacityRefused(chatID uuid.UUID) { // being acquired after at least one capacity refusal, measured from // the first refusal this worker saw. Chats admitted on their first // attempt record nothing. The acquisition pass runs before the turn -// span exists, so the turn scope is stated explicitly. +// span exists, so the turn scope and the chat kind are stated +// explicitly. func (w *chatWorker) recordCapacityWait(ctx context.Context, chat database.Chat) { since, waited := w.capacityWaitSince[chat.ID] if !waited { return } delete(w.capacityWaitSince, chat.ID) + ctx = chatloop.ContextWithChatKind(ctx, chatKindAttr(chat)) w.server.stages.RecordAs(ctx, chatloop.StageCapacityWait, chatloop.ScopeTurn, chatloop.StageModel{}, since, time.Now(), nil, attribute.String(chatloop.AttrChatID, chat.ID.String()), - attribute.String(chatloop.AttrChatKind, chatKindAttr(chat)), ) } diff --git a/coderd/x/chatd/chatd.go b/coderd/x/chatd/chatd.go index 9dd8dd86745..2afd4355fd7 100644 --- a/coderd/x/chatd/chatd.go +++ b/coderd/x/chatd/chatd.go @@ -2190,7 +2190,11 @@ func (p *Server) PromoteQueued( p.publishChatPubsubEvent(refreshChat, codersdk.ChatWatchEventKindStatusChange, nil) } if !promotedQueuedAt.IsZero() { - p.recordQueueWait(ctx, opts.ChatID, promotedQueuedAt, time.Now()) + var chatKind string + if refreshedOK { + chatKind = chatKindAttr(refreshChat) + } + p.recordQueueWait(ctx, opts.ChatID, chatKind, promotedQueuedAt, time.Now()) } return result, nil } @@ -4615,7 +4619,7 @@ func (p *Server) finalizeSuccessfulTurnStatusLabelWithAfterFunc( logger slog.Logger, afterFinalize func(context.Context, string), ) { - finalizeCtx, stopFinalizeCtx := p.inflightContext(ctx) + finalizeCtx, stopFinalizeCtx := p.inflightChatContext(ctx, chat) if err := p.goInflight(func() { defer stopFinalizeCtx() statusLabel := p.generateFinalTurnStatusLabel(finalizeCtx, chat, status, runResult, logger) @@ -4701,7 +4705,7 @@ func (p *Server) setLastTurnSummaryAsync( if chat.LastTurnSummary.Valid && strings.TrimSpace(chat.LastTurnSummary.String) == summary { return } - updateCtx, stopUpdateCtx := p.inflightContext(ctx) + updateCtx, stopUpdateCtx := p.inflightChatContext(ctx, chat) if err := p.goInflight(func() { defer stopUpdateCtx() p.updateLastTurnSummary(updateCtx, chat, chat.HistoryVersion, summary, logger) @@ -4721,7 +4725,7 @@ func (p *Server) clearLastTurnSummaryAsync( chat database.Chat, logger slog.Logger, ) { - clearCtx, stopClearCtx := p.inflightContext(ctx) + clearCtx, stopClearCtx := p.inflightChatContext(ctx, chat) if err := p.goInflight(func() { defer stopClearCtx() p.updateLastTurnSummary(clearCtx, chat, chat.HistoryVersion, "", logger) @@ -4816,7 +4820,7 @@ func (p *Server) maybeGenerateChatSummaryAsync( if chat.ParentChatID.Valid { return } - ctx, cancel := p.inflightContext(ctx) + ctx, cancel := p.inflightChatContext(ctx, chat) if err := p.goInflight(func() { defer cancel() p.generateAndStoreChatSummary(ctx, logger, chat) @@ -4997,7 +5001,7 @@ func (p *Server) storeSubagentReportSummaryAsync( chat database.Chat, logger slog.Logger, ) { - summaryCtx, stopSummaryCtx := p.inflightContext(ctx) + summaryCtx, stopSummaryCtx := p.inflightChatContext(ctx, chat) if err := p.goInflight(func() { defer stopSummaryCtx() p.storeSubagentReportSummary(summaryCtx, chat, logger) @@ -5111,16 +5115,27 @@ func (p *Server) inflightContext(reqCtx context.Context) (context.Context, func( // queued from queuedAt until promotedAt. The span context is stripped // from ctx so the stage is a standalone span rather than a child of the // promoting request's span, which ends before the turn the wait belongs -// to. The scope is set explicitly for the same reason: ctx carries the -// request, not the turn. -func (p *Server) recordQueueWait(ctx context.Context, chatID uuid.UUID, queuedAt, promotedAt time.Time) { +// to. The scope and chat kind are set explicitly for the same reason: +// ctx carries the request, not the turn. An empty chatKind records the +// stage without one. +func (p *Server) recordQueueWait(ctx context.Context, chatID uuid.UUID, chatKind string, queuedAt, promotedAt time.Time) { standalone := trace.ContextWithSpanContext(ctx, trace.SpanContext{}) + standalone = chatloop.ContextWithChatKind(standalone, chatKind) p.stages.RecordAs(standalone, chatloop.StageQueueWait, chatloop.ScopeTurn, chatloop.StageModel{}, queuedAt, promotedAt, nil, attribute.String(chatloop.AttrChatID, chatID.String()), ) } +// inflightChatContext is inflightContext for work that belongs to a +// known chat. The chat kind is set on the returned context so the +// stages of the detached work carry it even when the caller never ran +// a turn. +func (p *Server) inflightChatContext(reqCtx context.Context, chat database.Chat) (context.Context, func()) { + ctx, stop := p.inflightContext(reqCtx) + return chatloop.ContextWithChatKind(ctx, chatKindAttr(chat)), stop +} + func (p *Server) goInflight(f func()) error { if p.inflightClosed.Load() { return errInflightClosed diff --git a/coderd/x/chatd/chatloop/metrics.go b/coderd/x/chatd/chatloop/metrics.go index 2acf13d6a2f..f6ad211503c 100644 --- a/coderd/x/chatd/chatloop/metrics.go +++ b/coderd/x/chatd/chatloop/metrics.go @@ -101,11 +101,11 @@ func NewMetrics(reg prometheus.Registerer) *Metrics { Namespace: metricsNamespace, Subsystem: metricsSubsystem, Name: "stage_duration_seconds", - Help: "Wall time spent in each chat lifecycle stage. Stages overlap in wall time; this is a stage-time profile, not a partition of the turn. The scope label separates stages that run inside a chat turn from detached background work. The model and effort labels are empty for stages that run before a model is resolved.", + Help: "Wall time spent in each chat lifecycle stage. Stages overlap in wall time; this is a stage-time profile, not a partition of the turn. The scope label separates stages that run inside a chat turn from detached background work. The chat_kind label is empty for stages recorded without a known chat, and the model and effort labels are empty for stages that run before a model is resolved.", // 10ms .. ~2.9h, log-spaced. The top of the range covers // long-lived stages such as a chat turn. Buckets: prometheus.ExponentialBuckets(0.01, 2, 21), - }, []string{"stage", "scope", "model", "effort"}), + }, []string{"stage", "scope", "chat_kind", "model", "effort"}), CompactionTotal: factory.NewCounterVec(prometheus.CounterOpts{ Namespace: metricsNamespace, Subsystem: metricsSubsystem, @@ -165,13 +165,14 @@ func NopMetrics() *Metrics { } // RecordStageDuration observes one chat lifecycle stage duration. -// model and effort are empty when the stage ran before a model was +// chatKind is empty when the stage was recorded without a known chat, +// and model and effort are empty when the stage ran before a model was // resolved. Negative durations are dropped. No-op when m is nil. -func (m *Metrics) RecordStageDuration(stage, scope, model, effort string, elapsed time.Duration) { +func (m *Metrics) RecordStageDuration(stage, scope, chatKind, model, effort string, elapsed time.Duration) { if m == nil || elapsed < 0 { return } - m.StageDurationSeconds.WithLabelValues(stage, scope, model, effort).Observe(elapsed.Seconds()) + m.StageDurationSeconds.WithLabelValues(stage, scope, chatKind, model, effort).Observe(elapsed.Seconds()) } // RecordCompaction classifies and records a compaction attempt. diff --git a/coderd/x/chatd/chatloop/stage.go b/coderd/x/chatd/chatloop/stage.go index 45e367c99d3..b2e26f15aae 100644 --- a/coderd/x/chatd/chatloop/stage.go +++ b/coderd/x/chatd/chatloop/stage.go @@ -130,19 +130,25 @@ func (m StageModel) attributes() []attribute.KeyValue { // StageSpan is an in-flight stage. End must be called exactly once; // the duration observation happens there. type StageSpan struct { - tracer *StageTracer - stage string - scope string - model StageModel - span trace.Span - start time.Time - ended bool + tracer *StageTracer + stage string + scope string + chatKind string + model StageModel + span trace.Span + start time.Time + ended bool } // stageScopeKey keys the stage scope carried by a context. It is // private so the scope can only be set through ContextWithScope. type stageScopeKey struct{} +// stageChatKindKey keys the chat kind carried by a context. It is +// private so the chat kind can only be set through +// ContextWithChatKind. +type stageChatKindKey struct{} + // ContextWithScope returns ctx carrying scope for the stages started // on it. The scope is a plain context value rather than a property of // the span in ctx, so it survives configurations where spans are not @@ -151,6 +157,14 @@ func ContextWithScope(ctx context.Context, scope string) context.Context { return context.WithValue(ctx, stageScopeKey{}, scope) } +// ContextWithChatKind returns ctx carrying kind for the stages started +// on it. Callers that hold the chat row set it once, on the context a +// turn runs on or on the context of a single stage recorded outside a +// turn, and every stage derived from that context carries the value. +func ContextWithChatKind(ctx context.Context, kind string) context.Context { + return context.WithValue(ctx, stageChatKindKey{}, kind) +} + // scopeFromContext reads the scope ContextWithScope put on ctx. // Contexts with no scope are background scoped, so work detached from // a turn is kept out of the turn profile. @@ -161,9 +175,18 @@ func scopeFromContext(ctx context.Context) string { return ScopeBackground } +// chatKindFromContext reads the chat kind ContextWithChatKind put on +// ctx. It is empty when the stage runs without a known chat, which +// keeps the label present but unset rather than guessing a kind. +func chatKindFromContext(ctx context.Context) string { + kind, _ := ctx.Value(stageChatKindKey{}).(string) + return kind +} + // Start begins a stage span as a child of the span in ctx and returns -// a context carrying it. The stage takes the scope on ctx, so stages -// started on a context with no scope are background scoped. +// a context carrying it. The stage takes the scope and chat kind on +// ctx, so stages started on a context with no scope are background +// scoped. func (t *StageTracer) Start( ctx context.Context, stage string, @@ -218,17 +241,30 @@ func (t *StageTracer) startSpan( } else { opts = append(opts, trace.WithTimestamp(start)) } - opts = append(opts, trace.WithAttributes(attribute.String(AttrScope, scope))) + chatKind := chatKindFromContext(ctx) + opts = append(opts, trace.WithAttributes(stageIdentityAttributes(scope, chatKind)...)) ctx, span := t.otelTracer().Start(ContextWithScope(ctx, scope), stage, opts...) return ctx, &StageSpan{ - tracer: t, - stage: stage, - scope: scope, - span: span, - start: start, + tracer: t, + stage: stage, + scope: scope, + chatKind: chatKind, + span: span, + start: start, } } +// stageIdentityAttributes returns the attributes every stage span +// carries. An unknown chat kind is omitted from the span, where an +// absent attribute reads better than an empty one. +func stageIdentityAttributes(scope, chatKind string) []attribute.KeyValue { + attrs := []attribute.KeyValue{attribute.String(AttrScope, scope)} + if chatKind != "" { + attrs = append(attrs, attribute.String(AttrChatKind, chatKind)) + } + return attrs +} + // SetAttributes adds attributes to the stage span. It is a no-op // after End. func (s *StageSpan) SetAttributes(attrs ...attribute.KeyValue) { @@ -264,7 +300,7 @@ func (s *StageSpan) SpanContext() trace.SpanContext { // a deferred End cannot double-count a stage. func (s *StageSpan) End(err error) { if elapsed, ok := s.closeSpan(err); ok { - s.tracer.observe(s.stage, s.scope, s.model, elapsed) + s.tracer.observe(s.stage, s.scope, s.chatKind, s.model, elapsed) } } @@ -297,8 +333,8 @@ func (s *StageSpan) closeSpan(err error) (elapsed time.Duration, ok bool) { // Record emits an already-finished stage span with explicit start and // end timestamps. It is for stages whose boundaries are only known // after the fact, such as durations reconstructed from persisted -// timestamps. The stage takes the scope on ctx. Non-positive or unset -// windows are dropped. +// timestamps. The stage takes the scope and chat kind on ctx. +// Non-positive or unset windows are dropped. func (t *StageTracer) Record( ctx context.Context, stage string, @@ -326,23 +362,24 @@ func (t *StageTracer) RecordAs( if start.IsZero() || end.IsZero() || end.Before(start) { return } + chatKind := chatKindFromContext(ctx) _, span := t.otelTracer().Start(ctx, stage, trace.WithTimestamp(start), trace.WithAttributes(attrs...), trace.WithAttributes(model.attributes()...), - trace.WithAttributes(attribute.String(AttrScope, scope)), + trace.WithAttributes(stageIdentityAttributes(scope, chatKind)...), ) if err != nil { span.RecordError(err) span.SetStatus(codes.Error, err.Error()) } span.End(trace.WithTimestamp(end)) - t.observe(stage, scope, model, end.Sub(start)) + t.observe(stage, scope, chatKind, model, end.Sub(start)) } -func (t *StageTracer) observe(stage, scope string, model StageModel, elapsed time.Duration) { +func (t *StageTracer) observe(stage, scope, chatKind string, model StageModel, elapsed time.Duration) { if t == nil || t.metrics == nil { return } - t.metrics.RecordStageDuration(stage, scope, model.Model, model.Effort, elapsed) + t.metrics.RecordStageDuration(stage, scope, chatKind, model.Model, model.Effort, elapsed) } diff --git a/coderd/x/chatd/chatloop/stage_test.go b/coderd/x/chatd/chatloop/stage_test.go index c983004a442..0bdac30cad3 100644 --- a/coderd/x/chatd/chatloop/stage_test.go +++ b/coderd/x/chatd/chatloop/stage_test.go @@ -45,10 +45,11 @@ func newStageFixture(t *testing.T) stageFixture { // stageKey identifies one stage_duration_seconds series. type stageKey struct { - stage string - scope string - model string - effort string + stage string + scope string + chatKind string + model string + effort string } // stageObservations returns the observation count per stage series @@ -64,10 +65,11 @@ func (f stageFixture) stageObservations(t *testing.T) map[stageKey]uint64 { } for _, metric := range family.GetMetric() { key := stageKey{ - stage: labelValue(metric, "stage"), - scope: labelValue(metric, "scope"), - model: labelValue(metric, "model"), - effort: labelValue(metric, "effort"), + stage: labelValue(metric, "stage"), + scope: labelValue(metric, "scope"), + chatKind: labelValue(metric, "chat_kind"), + model: labelValue(metric, "model"), + effort: labelValue(metric, "effort"), } counts[key] = metric.GetHistogram().GetSampleCount() } @@ -288,6 +290,53 @@ func TestStageTracerScope(t *testing.T) { }) } +func TestStageTracerChatKind(t *testing.T) { + t.Parallel() + + t.Run("InheritedByDerivedStages", func(t *testing.T) { + t.Parallel() + fixture := newStageFixture(t) + + ctx := chatloop.ContextWithChatKind(t.Context(), chatloop.ChatKindSubagent) + turnCtx, turn := fixture.tracer.StartRoot(ctx, chatloop.StageChatTurn, nil) + stepCtx, step := fixture.tracer.Start(turnCtx, chatloop.StageGenerationStep) + start := time.Now().Add(-time.Second) + fixture.tracer.Record(stepCtx, chatloop.StageToolCall, chatloop.StageModel{}, start, time.Now(), nil) + step.End(nil) + turn.End(nil) + + require.Equal(t, map[stageKey]uint64{ + {stage: chatloop.StageChatTurn, scope: chatloop.ScopeTurn, chatKind: chatloop.ChatKindSubagent}: 1, + {stage: chatloop.StageGenerationStep, scope: chatloop.ScopeTurn, chatKind: chatloop.ChatKindSubagent}: 1, + {stage: chatloop.StageToolCall, scope: chatloop.ScopeTurn, chatKind: chatloop.ChatKindSubagent}: 1, + }, fixture.stageObservations(t)) + + for _, span := range fixture.spans.Ended() { + require.Contains(t, span.Attributes(), + attribute.String(chatloop.AttrChatKind, chatloop.ChatKindSubagent)) + } + }) + + t.Run("UnknownChatLeavesLabelEmpty", func(t *testing.T) { + t.Parallel() + fixture := newStageFixture(t) + + start := time.Now().Add(-time.Second) + fixture.tracer.RecordAs(t.Context(), chatloop.StageCapacityWait, chatloop.ScopeTurn, + chatloop.StageModel{}, start, time.Now(), nil) + + require.Equal(t, map[stageKey]uint64{ + {stage: chatloop.StageCapacityWait, scope: chatloop.ScopeTurn}: 1, + }, fixture.stageObservations(t)) + + ended := fixture.spans.Ended() + require.Len(t, ended, 1) + for _, attr := range ended[0].Attributes() { + require.NotEqual(t, chatloop.AttrChatKind, string(attr.Key)) + } + }) +} + func TestStageTracerModelLabels(t *testing.T) { t.Parallel() @@ -444,7 +493,7 @@ func TestStageDurationBuckets(t *testing.T) { metrics := chatloop.NewMetrics(registry) // A turn can run for hours, so the top bucket boundary has to sit // above the longest plausible stage. - metrics.RecordStageDuration(chatloop.StageChatTurn, chatloop.ScopeTurn, "", "", 2*time.Hour) + metrics.RecordStageDuration(chatloop.StageChatTurn, chatloop.ScopeTurn, chatloop.ChatKindRoot, "", "", 2*time.Hour) families, err := registry.Gather() require.NoError(t, err) diff --git a/coderd/x/chatd/generation.go b/coderd/x/chatd/generation.go index d8836f41b60..0c0ebc5ddaa 100644 --- a/coderd/x/chatd/generation.go +++ b/coderd/x/chatd/generation.go @@ -471,7 +471,6 @@ func (s *taskStarter) runGenerationStep( ) (next chatWorkerTaskStartInput, again bool, err error) { ctx, stepSpan := s.server.stages.Start(ctx, chatloop.StageGenerationStep, attribute.String(chatloop.AttrChatID, input.ChatID.String()), - attribute.String(chatloop.AttrChatKind, chatKindAttr(chat)), attribute.Int64(chatloop.AttrGenerationAttempt, input.GenerationAttempt), ) defer func() { stepSpan.End(err) }() diff --git a/coderd/x/chatd/stage_internal_test.go b/coderd/x/chatd/stage_internal_test.go index d3d760f2af7..028be0d37a2 100644 --- a/coderd/x/chatd/stage_internal_test.go +++ b/coderd/x/chatd/stage_internal_test.go @@ -252,7 +252,7 @@ func TestServerRecordQueueWaitIsStandalone(t *testing.T) { // not join. requestCtx, requestSpan := tracer.Start(t.Context(), chatloop.StageCommit) queuedAt := time.Now().Add(-30 * time.Second) - server.recordQueueWait(requestCtx, uuid.New(), queuedAt, queuedAt.Add(20*time.Second)) + server.recordQueueWait(requestCtx, uuid.New(), chatloop.ChatKindSubagent, queuedAt, queuedAt.Add(20*time.Second)) requestSpan.End(nil) var queueWait, request sdktrace.ReadOnlySpan @@ -270,6 +270,8 @@ func TestServerRecordQueueWaitIsStandalone(t *testing.T) { require.NotEqual(t, request.SpanContext().TraceID(), queueWait.SpanContext().TraceID()) require.Contains(t, queueWait.Attributes(), attribute.String(chatloop.AttrScope, chatloop.ScopeTurn)) + require.Contains(t, queueWait.Attributes(), + attribute.String(chatloop.AttrChatKind, chatloop.ChatKindSubagent)) } func TestServerInflightContextIsBackgroundScoped(t *testing.T) { @@ -279,9 +281,10 @@ func TestServerInflightContextIsBackgroundScoped(t *testing.T) { t.Cleanup(serverCancel) server := &Server{ctx: serverCtx, stages: tracer} + chat := database.Chat{ID: uuid.New()} turn := newRunnerTurnSpan(tracer) - turnCtx := turn.Ensure(t.Context(), database.Chat{ID: uuid.New()}, time.Now().Add(-time.Second)) - inflightCtx, stop := server.inflightContext(turnCtx) + turnCtx := turn.Ensure(t.Context(), chat, time.Now().Add(-time.Second)) + inflightCtx, stop := server.inflightChatContext(turnCtx, chat) t.Cleanup(stop) _, span := tracer.Start(inflightCtx, chatloop.StageGenerationStep) @@ -295,5 +298,28 @@ func TestServerInflightContextIsBackgroundScoped(t *testing.T) { require.False(t, ended.Parent().IsValid()) require.Contains(t, ended.Attributes(), attribute.String(chatloop.AttrScope, chatloop.ScopeBackground)) + // Detached work keeps the chat it belongs to, so background + // stages stay attributable to a chat kind. + require.Contains(t, ended.Attributes(), + attribute.String(chatloop.AttrChatKind, chatloop.ChatKindRoot)) + } +} + +func TestRunnerTurnSpanCarriesChatKind(t *testing.T) { + t.Parallel() + tracer, recorder := newStageTestTracer(t) + turn := newRunnerTurnSpan(tracer) + chat := database.Chat{ID: uuid.New(), ParentChatID: uuid.NullUUID{UUID: uuid.New(), Valid: true}} + + turnCtx := turn.Ensure(t.Context(), chat, time.Now().Add(-time.Second)) + _, step := tracer.Start(turnCtx, chatloop.StageGenerationStep) + step.End(nil) + turn.End(nil) + + require.NotEmpty(t, recorder.Ended()) + for _, span := range recorder.Ended() { + require.Contains(t, span.Attributes(), + attribute.String(chatloop.AttrChatKind, chatloop.ChatKindSubagent), + "stage %s must carry the turn's chat kind", span.Name()) } } diff --git a/coderd/x/chatd/turn_trace.go b/coderd/x/chatd/turn_trace.go index eecd46a03e2..a65baad5315 100644 --- a/coderd/x/chatd/turn_trace.go +++ b/coderd/x/chatd/turn_trace.go @@ -22,11 +22,12 @@ import ( type runnerTurnSpan struct { stages *chatloop.StageTracer - mu sync.Mutex - span *chatloop.StageSpan - spanCtx trace.SpanContext - started bool - ended bool + mu sync.Mutex + span *chatloop.StageSpan + spanCtx trace.SpanContext + chatKind string + started bool + ended bool } func newRunnerTurnSpan(stages *chatloop.StageTracer) *runnerTurnSpan { @@ -58,11 +59,14 @@ func (t *runnerTurnSpan) Ensure(ctx context.Context, chat database.Chat, trigger return t.contextLocked(ctx) } t.started = true + t.chatKind = chatKindAttr(chat) attrs := []attribute.KeyValue{ attribute.String(chatloop.AttrChatID, chat.ID.String()), - attribute.String(chatloop.AttrChatKind, chatKindAttr(chat)), } + // The chat kind rides on the context so every stage of the turn + // carries it, on the span and on the duration observation. + ctx = chatloop.ContextWithChatKind(ctx, t.chatKind) turnCtx, span := t.stages.StartRootAt(ctx, chatloop.StageChatTurn, triggerAt, nil, attrs...) t.span = span t.spanCtx = span.SpanContext() @@ -88,9 +92,11 @@ func (t *runnerTurnSpan) contextLocked(ctx context.Context) context.Context { if !t.started || t.ended { return ctx } - // The scope is set independently of the span context so stages run - // on this context stay turn scoped when tracing is not recording. + // The scope and chat kind are set independently of the span context + // so stages run on this context keep them when tracing is not + // recording. ctx = chatloop.ContextWithScope(ctx, chatloop.ScopeTurn) + ctx = chatloop.ContextWithChatKind(ctx, t.chatKind) if !t.spanCtx.IsValid() { return ctx } diff --git a/docs/admin/integrations/prometheus.md b/docs/admin/integrations/prometheus.md index f6f2fb35fdd..cad41e1589e 100644 --- a/docs/admin/integrations/prometheus.md +++ b/docs/admin/integrations/prometheus.md @@ -249,7 +249,7 @@ The `coder_ai_gateway_cost_control_*` metrics are exported only by `coderd`. | `coderd_chatd_hook_input_overrides_total` | counter | Total lifecycle hook input overrides by event. | `event` | | `coderd_chatd_message_count` | histogram | Number of messages in the prompt per LLM request. | `model` `provider` | | `coderd_chatd_prompt_size_bytes` | histogram | Estimated byte size of the prompt per LLM request. | `model` `provider` | -| `coderd_chatd_stage_duration_seconds` | histogram | Wall time spent in each chat lifecycle stage. Stages overlap in wall time; this is a stage-time profile, not a partition of the turn. The scope label separates stages that run inside a chat turn from detached background work. The model and effort labels are empty for stages that run before a model is resolved. | `effort` `model` `scope` `stage` | +| `coderd_chatd_stage_duration_seconds` | histogram | Wall time spent in each chat lifecycle stage. Stages overlap in wall time; this is a stage-time profile, not a partition of the turn. The scope label separates stages that run inside a chat turn from detached background work. The chat_kind label is empty for stages recorded without a known chat, and the model and effort labels are empty for stages that run before a model is resolved. | `chat_kind` `effort` `model` `scope` `stage` | | `coderd_chatd_steps_total` | counter | Total agentic loop steps across all chats. | `model` `provider` | | `coderd_chatd_stream_buffer_dropped_total` | counter | Number of chat stream buffer events dropped due to the per-chat buffer cap. | | | `coderd_chatd_stream_retries_total` | counter | Total LLM stream retries. | `kind` `model` `provider` | diff --git a/examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md b/examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md index 3569e175358..19f1bd91956 100644 --- a/examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md +++ b/examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md @@ -1,8 +1,9 @@ # Chatd Chat Lifecycle Grafana Dashboard A Grafana dashboard for diagnosing where time goes in Coder Agents chat -sessions. It aggregates the `coderd_chatd_stage_duration_seconds{stage,scope}` -histogram into a stage-level flame graph with a selectable summary statistic +sessions. It aggregates the stage histogram +`coderd_chatd_stage_duration_seconds{stage,scope,chat_kind,model,effort}` +into a stage-level flame graph with a selectable summary statistic (mean, p50, p90, p95, p99), plus summary panels for the whole chat pipeline. Stage hierarchy: @@ -30,20 +31,30 @@ decomposition, and quantile statistics are not additive across stages. ## Dimensions -The stage histogram carries four labels, exposed as dashboard variables +The stage histogram carries five labels, exposed as dashboard variables where noted: -| Label | Values | Dashboard variable | -|----------|-----------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------| -| `stage` | the 14 stage names above | none (fixed hierarchy) | -| `scope` | `turn` (part of a chat turn) or `background` (detached async work such as title and summary generation) | none (panels pin one scope) | -| `model` | resolved model ID, empty before a model is resolved | `$model` (multi-select) | -| `effort` | effective reasoning effort sent to the provider (`none`, `minimal`, `low`, `medium`, `high`, `xhigh`, `max`), empty when the model config sets none | `$effort` (multi-select) | +| Label | Values | Dashboard variable | +|-------------|-----------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------| +| `stage` | the 14 stage names above | none (fixed hierarchy) | +| `scope` | `turn` (part of a chat turn) or `background` (detached async work such as title and summary generation) | none (panels pin one scope) | +| `chat_kind` | `root` (a chat a user drives) or `subagent` (a chat spawned by a parent agent), empty for background work | `$chat_kind` (multi-select) | +| `model` | resolved model ID, empty before a model is resolved | `$model` (multi-select) | +| `effort` | effective reasoning effort sent to the provider (`none`, `minimal`, `low`, `medium`, `high`, `xhigh`, `max`), empty when the model config sets none | `$effort` (multi-select) | Two more variables apply everywhere: `$datasource` selects the Prometheus data source and `$stat` selects the summary statistic (mean, p50, p90, p95, p99) for every stat-aware panel. +`chat_kind` separates root chats, which a user drives, from subagent +chats, which a parent agent spawns and which run as separate chats with +their own turn trees. It is a property of the turn, so every turn-scoped +stage carries it, including the stages recorded before a model is +resolved. `$chat_kind` therefore filters every stage panel in full, +rather than narrowing part of the hierarchy the way `$model` and +`$effort` do. Background provider calls run outside a turn and carry an +empty value, so that panel is never filtered by it. + Stages that run before or outside model resolution (`chat_turn`, `queue_wait`, `capacity_wait`, `acquisition`, `mcp_connect`, `commit`) always carry empty `model`/`effort` labels. Panels match those stages @@ -60,20 +71,20 @@ populated and narrows only the model-carrying stages (`generation_step`, **Stage profile flamegraph ($stat)** - one frame per stage, laid out in the fixed hierarchy, frame width = the selected `$stat` of that stage's duration over the dashboard time range. Dimensions: filtered to the -turn scope; `$model`/`$effort` apply to the model-carrying stages -only, so a concrete selection narrows the generation stages while the -`chat_turn` root and the pre-model stages keep their full values. -`$stat` picks the statistic. How to read: the widest frames under -`generation_step` are where turn time goes; compare `provider_attempt` -(request to response headers) against `stream` (full stream) to separate -provider latency from streaming time. The `self` column is the stage's -own value for leaf stages and 0 for stages with children; true self time -is not derivable here, because overlapping stages can make a parent -minus its children negative. Caveats: stages overlap in wall time (tool -calls and thinking happen inside the stream) and quantiles are not -additive, so a child frame can read wider than its parent at high -percentiles; a stage whose series first appears inside the window reads 0 -until its second sample. +turn scope and `$chat_kind`; `$model`/`$effort` apply to the +model-carrying stages only, so a concrete selection narrows the +generation stages while the `chat_turn` root and the pre-model stages +keep their full values. `$stat` picks the statistic. How to read: the +widest frames under `generation_step` are where turn time goes; compare +`provider_attempt` (request to response headers) against `stream` (full +stream) to separate provider latency from streaming time. The `self` +column is the stage's own value for leaf stages and 0 for stages with +children; true self time is not derivable here, because overlapping +stages can make a parent minus its children negative. Caveats: stages +overlap in wall time (tool calls and thinking happen inside the stream) +and quantiles are not additive, so a child frame can read wider than its +parent at high percentiles; a stage whose series first appears inside the +window reads 0 until its second sample. **Stage profile in hierarchy order ($stat)** - the same query as the flamegraph drawn as horizontal bars in depth-first order with the tree @@ -85,22 +96,23 @@ as a numeric check on the flamegraph. ### Stage trends **Stage duration over time ($stat)** - one series per stage, the -selected `$stat` computed over `$__rate_interval`. Dimensions: -`scope="turn"`, series split by `stage`; `$model`/`$effort` apply to the -model-carrying stages only, so the pre-model stages stay plotted under -any selection. How to read: this is the drill-down for "when did it get -slow" - a regression visible in the profile shows here as a step or -trend in the affected stage. Idle stages drop out rather than plotting -NaN. +selected `$stat` computed over `$__rate_interval`. Dimensions: the +turn scope and `$chat_kind`, one series per stage; +`$model`/`$effort` apply to the model-carrying stages only, so the +pre-model stages stay plotted under any model selection. How to read: +this is the drill-down for "when did it get slow" - a regression visible +in the profile shows here as a step or trend in the affected stage. Idle +stages drop out rather than plotting NaN. **Stage time share of chat_turn** - each stage's total time as a percentage of total `chat_turn` time, from mean rates of the histogram sums. Dimensions: numerator is `scope="turn"` split by `stage`, with -`$model`/`$effort` applied to the model-carrying stages only; the -denominator is all `chat_turn` time without model/effort filters, -because `chat_turn` carries empty model/effort labels. Under a concrete -model the generation stages therefore read as that model's share of all -turn time. How to read: this is the "where does the time go" summary - +`$chat_kind` applied and `$model`/`$effort` applied to the +model-carrying stages only; the denominator is `chat_turn` time for the +same `$chat_kind` selection, without model/effort filters, because +`chat_turn` carries empty model/effort labels. Under a concrete model +the generation stages therefore read as that model's share of all turn +time. How to read: this is the "where does the time go" summary - stages overlap, so series can sum past 100%, but a single stage rising toward 100% of turn time identifies the dominant cost. @@ -108,11 +120,11 @@ toward 100% of turn time identifies the dominant cost. pre-generation waits: `queue_wait` (queued message insert to promotion), `capacity_wait` (concurrent-agent limiter admission) and `acquisition` (trigger message insert to worker pickup). Dimensions: -`scope="turn"`, fixed to those three stages, split by `stage`; all three -run before a model is resolved, so `$model`/`$effort` are not applied and -the panel stays populated under any selection. How to read: these are -scheduling delays before any model work starts - user-visible latency -that no provider-side optimization can fix. +`scope="turn"` and `$chat_kind`, fixed to those three stages, split by +`stage`; all three run before a model is resolved, so `$model`/`$effort` +are not applied and the panel stays populated under any model selection. +How to read: these are scheduling delays before any model work starts - +user-visible latency that no provider-side optimization can fix. ### Throughput and TTFT @@ -120,24 +132,25 @@ that no provider-side optimization can fix. the pre-existing histogram recorded when the first streamed part arrives. Dimensions: none of the stage labels; this histogram is labeled by provider/model internally but the panel aggregates across -them, and `$model`/`$effort` do not apply. The `time_to_first_token` -stage in the profile measures the same interval and does honor the -filters. How to read: the primary user-perceived responsiveness metric -for streaming. +them, and `$model`, `$effort` and `$chat_kind` do not apply. The +`time_to_first_token` stage in the profile measures the same interval +and does honor the filters. How to read: the primary user-perceived +responsiveness metric for streaming. **Turn rate and stage sample rate** - completed `chat_turn` per second -plus one rate series per other stage. Dimensions: `scope="turn"`, split -by `stage`, with `$model`/`$effort` applied to the model-carrying stages -only. How to read: throughput and shape - a stage rate above the turn -rate means the stage repeats within a turn (generation steps, provider -attempts, tool calls); `provider_attempt` rising faster than `stream` -indicates retries. +plus one rate series per other stage. Dimensions: `scope="turn"` and +`$chat_kind`, split by `stage`, with `$model`/`$effort` applied to the +model-carrying stages only. How to read: throughput and shape - a stage +rate above the turn rate means the stage repeats within a turn +(generation steps, provider attempts, tool calls); `provider_attempt` +rising faster than `stream` indicates retries. **Background provider calls ($stat)** - rate and selected `$stat` duration of background-scope `provider_attempt` samples: detached title/summary/quickgen requests that are excluded from every other panel. Dimensions: pinned to the background scope of the -`provider_attempt` stage; `$model`/`$effort` are not applied. How to read: +`provider_attempt` stage; `$model`, `$effort` and `$chat_kind` are not +applied, because background work runs outside a turn. How to read: this work costs provider quota and money but no user-facing turn latency; a spike here with flat turn panels means background load, not a chat regression. diff --git a/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json b/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json index 3cf703f21cd..0da8e126ede 100644 --- a/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json +++ b/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json @@ -39,7 +39,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Aggregate profile of chat lifecycle stages over the dashboard time range, using the $stat statistic of coderd_chatd_stage_duration_seconds. Levels come from the fixed stage hierarchy, attached as labels with label_replace and reshaped into Grafana's nested set model by the panel transformations.\n\nStage timings overlap in wall time (stream, thinking and tool_call all run inside a generation_step, and time_to_first_token is part of provider_attempt), and quantiles are not additive across stages. Read this as a stage time profile, not as a strict non-overlapping decomposition of turn latency: child bars can exceed their parent. A stage with no samples in the time range reads as 0. Only scope=\"turn\" samples are included, so detached background provider calls are excluded.\n\nValues come from rate() over counter series, and rate() treats the first sample in the window as its baseline. A stage whose series first appears inside the window reads as 0 until its second observation, so rare stages such as queue_wait, capacity_wait and compaction can render as 0 while real samples exist. Widen the time range to confirm.\n\n$model and $effort apply only to the stages that carry those labels (generation_step, prepare, provider_attempt, time_to_first_token, stream, thinking, tool_call, compaction). The stages recorded before a model is resolved (chat_turn, queue_wait, capacity_wait, acquisition, mcp_connect, commit) are always matched without them, so they stay populated under any selection.\n\nThe self field is the stage's own value for leaf stages and 0 for stages that have children, so the top table's self column reads as leaf time. True self time is not derivable from these histograms: stages overlap in wall time, so a parent minus its children can be negative.\n\nFrame values are seconds. The panel labels them as sample counts and ignores the field unit, because it takes its unit from profile metadata that a Prometheus query cannot set.", + "description": "Aggregate profile of chat lifecycle stages over the dashboard time range, using the $stat statistic of coderd_chatd_stage_duration_seconds. Levels come from the fixed stage hierarchy, attached as labels with label_replace and reshaped into Grafana's nested set model by the panel transformations.\n\nStage timings overlap in wall time (stream, thinking and tool_call all run inside a generation_step, and time_to_first_token is part of provider_attempt), and quantiles are not additive across stages. Read this as a stage time profile, not as a strict non-overlapping decomposition of turn latency: child bars can exceed their parent. A stage with no samples in the time range reads as 0. Only scope=\"turn\" samples are included, so detached background provider calls are excluded.\n\nValues come from rate() over counter series, and rate() treats the first sample in the window as its baseline. A stage whose series first appears inside the window reads as 0 until its second observation, so rare stages such as queue_wait, capacity_wait and compaction can render as 0 while real samples exist. Widen the time range to confirm.\n\n$model and $effort apply only to the stages that carry those labels (generation_step, prepare, provider_attempt, time_to_first_token, stream, thinking, tool_call, compaction). The stages recorded before a model is resolved (chat_turn, queue_wait, capacity_wait, acquisition, mcp_connect, commit) are always matched without them, so they stay populated under any selection.\n\nThe self field is the stage's own value for leaf stages and 0 for stages that have children, so the top table's self column reads as leaf time. True self time is not derivable from these histograms: stages overlap in wall time, so a parent minus its children can be negative.\n\nFrame values are seconds. The panel labels them as sample counts and ignores the field unit, because it takes its unit from profile metadata that a Prometheus query cannot set.\n\n$chat_kind selects root chats, subagent chats spawned by a parent agent, or both. It is a turn property carried by every turn-scoped stage, so unlike $model and $effort it applies to every stage in this panel.", "fieldConfig": { "defaults": { "unit": "short" @@ -79,7 +79,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"chat_turn\", scope=\"turn\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"chat_turn\", scope=\"turn\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"chat_turn\", \"\", \"\"),\n\"n\", \"01\", \"\", \"\"),\n\"level\", \"0\", \"\", \"\"),\n\"name\", \"chat_turn\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"queue_wait\", scope=\"turn\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"queue_wait\", scope=\"turn\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"queue_wait\", scope=\"turn\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"queue_wait\", scope=\"turn\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"queue_wait\", \"\", \"\"),\n\"n\", \"02\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ queue_wait\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"capacity_wait\", scope=\"turn\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"capacity_wait\", scope=\"turn\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"capacity_wait\", scope=\"turn\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"capacity_wait\", scope=\"turn\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"capacity_wait\", \"\", \"\"),\n\"n\", \"03\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ capacity_wait\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"acquisition\", scope=\"turn\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"acquisition\", scope=\"turn\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"acquisition\", scope=\"turn\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"acquisition\", scope=\"turn\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"acquisition\", \"\", \"\"),\n\"n\", \"04\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ acquisition\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"generation_step\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"generation_step\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"generation_step\", \"\", \"\"),\n\"n\", \"05\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ generation_step\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"prepare\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"prepare\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"prepare\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"prepare\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"prepare\", \"\", \"\"),\n\"n\", \"06\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ prepare\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"mcp_connect\", scope=\"turn\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", scope=\"turn\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"mcp_connect\", scope=\"turn\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", scope=\"turn\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"mcp_connect\", \"\", \"\"),\n\"n\", \"07\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ mcp_connect\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"provider_attempt\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"provider_attempt\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"provider_attempt\", \"\", \"\"),\n\"n\", \"08\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ provider_attempt\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"time_to_first_token\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"time_to_first_token\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"time_to_first_token\", \"\", \"\"),\n\"n\", \"09\", \"\", \"\"),\n\"level\", \"3\", \"\", \"\"),\n\"name\", \"│ │ └─ time_to_first_token\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"stream\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"stream\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"stream\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"stream\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"stream\", \"\", \"\"),\n\"n\", \"10\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ stream\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"thinking\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"thinking\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"thinking\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"thinking\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"thinking\", \"\", \"\"),\n\"n\", \"11\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ thinking\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"tool_call\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"tool_call\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"tool_call\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"tool_call\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"tool_call\", \"\", \"\"),\n\"n\", \"12\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ tool_call\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"commit\", scope=\"turn\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"commit\", scope=\"turn\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"commit\", scope=\"turn\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"commit\", scope=\"turn\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"commit\", \"\", \"\"),\n\"n\", \"13\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ commit\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"compaction\", \"\", \"\"),\n\"n\", \"14\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ compaction\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)", + "expr": "(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"chat_turn\", \"\", \"\"),\n\"n\", \"01\", \"\", \"\"),\n\"level\", \"0\", \"\", \"\"),\n\"name\", \"chat_turn\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"queue_wait\", \"\", \"\"),\n\"n\", \"02\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ queue_wait\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"capacity_wait\", \"\", \"\"),\n\"n\", \"03\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ capacity_wait\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"acquisition\", \"\", \"\"),\n\"n\", \"04\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ acquisition\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"generation_step\", \"\", \"\"),\n\"n\", \"05\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ generation_step\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"prepare\", \"\", \"\"),\n\"n\", \"06\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ prepare\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"mcp_connect\", \"\", \"\"),\n\"n\", \"07\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ mcp_connect\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"provider_attempt\", \"\", \"\"),\n\"n\", \"08\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ provider_attempt\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"time_to_first_token\", \"\", \"\"),\n\"n\", \"09\", \"\", \"\"),\n\"level\", \"3\", \"\", \"\"),\n\"name\", \"│ │ └─ time_to_first_token\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"stream\", \"\", \"\"),\n\"n\", \"10\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ stream\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"thinking\", \"\", \"\"),\n\"n\", \"11\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ thinking\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"tool_call\", \"\", \"\"),\n\"n\", \"12\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ tool_call\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"commit\", \"\", \"\"),\n\"n\", \"13\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ commit\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"compaction\", \"\", \"\"),\n\"n\", \"14\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ compaction\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)", "range": false, "refId": "A", "format": "table", @@ -165,7 +165,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Same data as the flamegraph, drawn as bars in depth-first hierarchy order with the tree drawn into the labels. Useful when a stage is too small to read in the flamegraph, and as a check on the flamegraph shaping.\n\nStage timings overlap in wall time (stream, thinking and tool_call all run inside a generation_step, and time_to_first_token is part of provider_attempt), and quantiles are not additive across stages. Read this as a stage time profile, not as a strict non-overlapping decomposition of turn latency: child bars can exceed their parent. A stage with no samples in the time range reads as 0. Only scope=\"turn\" samples are included, so detached background provider calls are excluded.\n\nValues come from rate() over counter series, and rate() treats the first sample in the window as its baseline. A stage whose series first appears inside the window reads as 0 until its second observation, so rare stages such as queue_wait, capacity_wait and compaction can render as 0 while real samples exist. Widen the time range to confirm.\n\n$model and $effort apply only to the stages that carry those labels (generation_step, prepare, provider_attempt, time_to_first_token, stream, thinking, tool_call, compaction). The stages recorded before a model is resolved (chat_turn, queue_wait, capacity_wait, acquisition, mcp_connect, commit) are always matched without them, so they stay populated under any selection.", + "description": "Same data as the flamegraph, drawn as bars in depth-first hierarchy order with the tree drawn into the labels. Useful when a stage is too small to read in the flamegraph, and as a check on the flamegraph shaping.\n\nStage timings overlap in wall time (stream, thinking and tool_call all run inside a generation_step, and time_to_first_token is part of provider_attempt), and quantiles are not additive across stages. Read this as a stage time profile, not as a strict non-overlapping decomposition of turn latency: child bars can exceed their parent. A stage with no samples in the time range reads as 0. Only scope=\"turn\" samples are included, so detached background provider calls are excluded.\n\nValues come from rate() over counter series, and rate() treats the first sample in the window as its baseline. A stage whose series first appears inside the window reads as 0 until its second observation, so rare stages such as queue_wait, capacity_wait and compaction can render as 0 while real samples exist. Widen the time range to confirm.\n\n$model and $effort apply only to the stages that carry those labels (generation_step, prepare, provider_attempt, time_to_first_token, stream, thinking, tool_call, compaction). The stages recorded before a model is resolved (chat_turn, queue_wait, capacity_wait, acquisition, mcp_connect, commit) are always matched without them, so they stay populated under any selection.\n\n$chat_kind selects root chats, subagent chats spawned by a parent agent, or both. It is a turn property carried by every turn-scoped stage, so unlike $model and $effort it applies to every stage in this panel.", "fieldConfig": { "defaults": { "color": { @@ -211,7 +211,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"chat_turn\", scope=\"turn\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"chat_turn\", scope=\"turn\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"chat_turn\", \"\", \"\"),\n\"n\", \"01\", \"\", \"\"),\n\"level\", \"0\", \"\", \"\"),\n\"name\", \"chat_turn\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"queue_wait\", scope=\"turn\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"queue_wait\", scope=\"turn\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"queue_wait\", scope=\"turn\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"queue_wait\", scope=\"turn\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"queue_wait\", \"\", \"\"),\n\"n\", \"02\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ queue_wait\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"capacity_wait\", scope=\"turn\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"capacity_wait\", scope=\"turn\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"capacity_wait\", scope=\"turn\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"capacity_wait\", scope=\"turn\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"capacity_wait\", \"\", \"\"),\n\"n\", \"03\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ capacity_wait\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"acquisition\", scope=\"turn\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"acquisition\", scope=\"turn\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"acquisition\", scope=\"turn\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"acquisition\", scope=\"turn\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"acquisition\", \"\", \"\"),\n\"n\", \"04\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ acquisition\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"generation_step\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"generation_step\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"generation_step\", \"\", \"\"),\n\"n\", \"05\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ generation_step\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"prepare\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"prepare\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"prepare\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"prepare\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"prepare\", \"\", \"\"),\n\"n\", \"06\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ prepare\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"mcp_connect\", scope=\"turn\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", scope=\"turn\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"mcp_connect\", scope=\"turn\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", scope=\"turn\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"mcp_connect\", \"\", \"\"),\n\"n\", \"07\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ mcp_connect\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"provider_attempt\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"provider_attempt\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"provider_attempt\", \"\", \"\"),\n\"n\", \"08\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ provider_attempt\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"time_to_first_token\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"time_to_first_token\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"time_to_first_token\", \"\", \"\"),\n\"n\", \"09\", \"\", \"\"),\n\"level\", \"3\", \"\", \"\"),\n\"name\", \"│ │ └─ time_to_first_token\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"stream\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"stream\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"stream\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"stream\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"stream\", \"\", \"\"),\n\"n\", \"10\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ stream\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"thinking\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"thinking\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"thinking\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"thinking\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"thinking\", \"\", \"\"),\n\"n\", \"11\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ thinking\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"tool_call\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"tool_call\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"tool_call\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"tool_call\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"tool_call\", \"\", \"\"),\n\"n\", \"12\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ tool_call\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"commit\", scope=\"turn\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"commit\", scope=\"turn\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"commit\", scope=\"turn\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"commit\", scope=\"turn\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"commit\", \"\", \"\"),\n\"n\", \"13\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ commit\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"compaction\", \"\", \"\"),\n\"n\", \"14\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ compaction\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)", + "expr": "(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"chat_turn\", \"\", \"\"),\n\"n\", \"01\", \"\", \"\"),\n\"level\", \"0\", \"\", \"\"),\n\"name\", \"chat_turn\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"queue_wait\", \"\", \"\"),\n\"n\", \"02\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ queue_wait\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"capacity_wait\", \"\", \"\"),\n\"n\", \"03\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ capacity_wait\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"acquisition\", \"\", \"\"),\n\"n\", \"04\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ acquisition\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"generation_step\", \"\", \"\"),\n\"n\", \"05\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ generation_step\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"prepare\", \"\", \"\"),\n\"n\", \"06\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ prepare\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"mcp_connect\", \"\", \"\"),\n\"n\", \"07\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ mcp_connect\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"provider_attempt\", \"\", \"\"),\n\"n\", \"08\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ provider_attempt\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"time_to_first_token\", \"\", \"\"),\n\"n\", \"09\", \"\", \"\"),\n\"level\", \"3\", \"\", \"\"),\n\"name\", \"│ │ └─ time_to_first_token\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"stream\", \"\", \"\"),\n\"n\", \"10\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ stream\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"thinking\", \"\", \"\"),\n\"n\", \"11\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ thinking\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"tool_call\", \"\", \"\"),\n\"n\", \"12\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ tool_call\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"commit\", \"\", \"\"),\n\"n\", \"13\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ commit\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"compaction\", \"\", \"\"),\n\"n\", \"14\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ compaction\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)", "range": false, "refId": "A", "format": "table", @@ -271,7 +271,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Selected $stat of each lifecycle stage over time, one series per stage. Mean is rate(_sum)/rate(_count); the percentiles are histogram_quantile over the bucket rates. A stage with no samples in an interval has no point rather than NaN.\n\n$model and $effort apply only to the stages that carry those labels (generation_step, prepare, provider_attempt, time_to_first_token, stream, thinking, tool_call, compaction). The stages recorded before a model is resolved (chat_turn, queue_wait, capacity_wait, acquisition, mcp_connect, commit) are always matched without them, so they stay populated under any selection.", + "description": "Selected $stat of each lifecycle stage over time, one series per stage. Mean is rate(_sum)/rate(_count); the percentiles are histogram_quantile over the bucket rates. A stage with no samples in an interval has no point rather than NaN.\n\n$model and $effort apply only to the stages that carry those labels (generation_step, prepare, provider_attempt, time_to_first_token, stream, thinking, tool_call, compaction). The stages recorded before a model is resolved (chat_turn, queue_wait, capacity_wait, acquisition, mcp_connect, commit) are always matched without them, so they stay populated under any selection.\n\n$chat_kind selects root chats, subagent chats spawned by a parent agent, or both. It is a turn property carried by every turn-scoped stage, so unlike $model and $effort it applies to every stage in this panel.", "fieldConfig": { "defaults": { "color": { @@ -353,7 +353,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=~\"chat_turn|queue_wait|capacity_wait|acquisition|mcp_connect|commit\", scope=\"turn\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"chat_turn|queue_wait|capacity_wait|acquisition|mcp_connect|commit\", scope=\"turn\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_stage_duration_seconds_sum{stage=~\"chat_turn|queue_wait|capacity_wait|acquisition|mcp_connect|commit\", scope=\"turn\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"chat_turn|queue_wait|capacity_wait|acquisition|mcp_connect|commit\", scope=\"turn\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=~\"chat_turn|queue_wait|capacity_wait|acquisition|mcp_connect|commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"chat_turn|queue_wait|capacity_wait|acquisition|mcp_connect|commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_stage_duration_seconds_sum{stage=~\"chat_turn|queue_wait|capacity_wait|acquisition|mcp_connect|commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"chat_turn|queue_wait|capacity_wait|acquisition|mcp_connect|commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", "range": true, "refId": "A", "legendFormat": "{{stage}}" @@ -364,7 +364,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=~\"generation_step|prepare|provider_attempt|time_to_first_token|stream|thinking|tool_call|compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"generation_step|prepare|provider_attempt|time_to_first_token|stream|thinking|tool_call|compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_stage_duration_seconds_sum{stage=~\"generation_step|prepare|provider_attempt|time_to_first_token|stream|thinking|tool_call|compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"generation_step|prepare|provider_attempt|time_to_first_token|stream|thinking|tool_call|compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=~\"generation_step|prepare|provider_attempt|time_to_first_token|stream|thinking|tool_call|compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"generation_step|prepare|provider_attempt|time_to_first_token|stream|thinking|tool_call|compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_stage_duration_seconds_sum{stage=~\"generation_step|prepare|provider_attempt|time_to_first_token|stream|thinking|tool_call|compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"generation_step|prepare|provider_attempt|time_to_first_token|stream|thinking|tool_call|compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", "range": true, "refId": "B", "legendFormat": "{{stage}}" @@ -378,7 +378,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Total time spent in each stage as a percentage of total chat_turn time, from mean rates (rate of the histogram _sum). Stages overlap, so the series can sum to more than 100%. Intervals with no chat_turn time are dropped instead of dividing by zero. The chat_turn denominator is never filtered by model or effort.\n\n$model and $effort apply only to the stages that carry those labels (generation_step, prepare, provider_attempt, time_to_first_token, stream, thinking, tool_call, compaction). The stages recorded before a model is resolved (chat_turn, queue_wait, capacity_wait, acquisition, mcp_connect, commit) are always matched without them, so they stay populated under any selection.", + "description": "Total time spent in each stage as a percentage of total chat_turn time, from mean rates (rate of the histogram _sum). Stages overlap, so the series can sum to more than 100%. Intervals with no chat_turn time are dropped instead of dividing by zero. The chat_turn denominator is never filtered by model or effort.\n\n$model and $effort apply only to the stages that carry those labels (generation_step, prepare, provider_attempt, time_to_first_token, stream, thinking, tool_call, compaction). The stages recorded before a model is resolved (chat_turn, queue_wait, capacity_wait, acquisition, mcp_connect, commit) are always matched without them, so they stay populated under any selection.\n\n$chat_kind selects root chats, subagent chats spawned by a parent agent, or both. It is a turn property carried by every turn-scoped stage, so unlike $model and $effort it applies to every stage in this panel.", "fieldConfig": { "defaults": { "color": { @@ -460,7 +460,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "100 * sum by (stage) (rate(coderd_chatd_stage_duration_seconds_sum{stage=~\"queue_wait|capacity_wait|acquisition|mcp_connect|commit\", scope=\"turn\"}[$__rate_interval]))\n/ on() group_left() (sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"chat_turn\", scope=\"turn\"}[$__rate_interval])) > 0)", + "expr": "100 * sum by (stage) (rate(coderd_chatd_stage_duration_seconds_sum{stage=~\"queue_wait|capacity_wait|acquisition|mcp_connect|commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__rate_interval]))\n/ on() group_left() (sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)", "range": true, "refId": "A", "legendFormat": "{{stage}}" @@ -471,7 +471,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "100 * sum by (stage) (rate(coderd_chatd_stage_duration_seconds_sum{stage=~\"generation_step|prepare|provider_attempt|time_to_first_token|stream|thinking|tool_call|compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval]))\n/ on() group_left() (sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"chat_turn\", scope=\"turn\"}[$__rate_interval])) > 0)", + "expr": "100 * sum by (stage) (rate(coderd_chatd_stage_duration_seconds_sum{stage=~\"generation_step|prepare|provider_attempt|time_to_first_token|stream|thinking|tool_call|compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval]))\n/ on() group_left() (sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)", "range": true, "refId": "B", "legendFormat": "{{stage}}" @@ -485,7 +485,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "p99 of the pre-generation waits: queue_wait (queued message insert to promotion), capacity_wait (capacity limiter acquire) and acquisition (trigger message insert to Acquire applied). These are queueing signals rather than model latency. Intervals with no samples for a stage are dropped instead of returning NaN. All three stages run before a model is resolved, so $model and $effort are not applied and the panel stays populated under any selection.", + "description": "p99 of the pre-generation waits: queue_wait (queued message insert to promotion), capacity_wait (capacity limiter acquire) and acquisition (trigger message insert to Acquire applied). These are queueing signals rather than model latency. Intervals with no samples for a stage are dropped instead of returning NaN. All three stages run before a model is resolved, so $model and $effort are not applied and the panel stays populated under any selection.\n\n$chat_kind selects root chats, subagent chats spawned by a parent agent, or both. It is a turn property carried by every turn-scoped stage, so unlike $model and $effort it applies to every stage in this panel.", "fieldConfig": { "defaults": { "color": { @@ -567,7 +567,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (le, stage) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=~\"queue_wait|capacity_wait|acquisition\", scope=\"turn\"}[$__rate_interval])))\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"queue_wait|capacity_wait|acquisition\", scope=\"turn\"}[$__rate_interval])) > 0)", + "expr": "histogram_quantile(0.99, sum by (le, stage) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=~\"queue_wait|capacity_wait|acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__rate_interval])))\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"queue_wait|capacity_wait|acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)", "range": true, "refId": "A", "legendFormat": "{{stage}} p99" @@ -594,7 +594,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Time to first token from coderd_chatd_ttft_seconds, the histogram recorded when the first streamed part arrives. The time_to_first_token stage in the profile above measures the same interval scoped to a provider attempt.", + "description": "Time to first token from coderd_chatd_ttft_seconds, the histogram recorded when the first streamed part arrives. The time_to_first_token stage in the profile above measures the same interval scoped to a provider attempt. This histogram carries none of the stage labels, so $model, $effort and $chat_kind do not apply to it.", "fieldConfig": { "defaults": { "color": { @@ -723,7 +723,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Completed chat turns per second and the per-stage sample rate from the histogram _count series. Stage rates above the turn rate mean the stage repeats within a turn (generation steps, tool calls, provider attempts); rates near zero mean the stage rarely fires.\n\n$model and $effort apply only to the stages that carry those labels (generation_step, prepare, provider_attempt, time_to_first_token, stream, thinking, tool_call, compaction). The stages recorded before a model is resolved (chat_turn, queue_wait, capacity_wait, acquisition, mcp_connect, commit) are always matched without them, so they stay populated under any selection.", + "description": "Completed chat turns per second and the per-stage sample rate from the histogram _count series. Stage rates above the turn rate mean the stage repeats within a turn (generation steps, tool calls, provider attempts); rates near zero mean the stage rarely fires.\n\n$model and $effort apply only to the stages that carry those labels (generation_step, prepare, provider_attempt, time_to_first_token, stream, thinking, tool_call, compaction). The stages recorded before a model is resolved (chat_turn, queue_wait, capacity_wait, acquisition, mcp_connect, commit) are always matched without them, so they stay populated under any selection.\n\n$chat_kind selects root chats, subagent chats spawned by a parent agent, or both. It is a turn property carried by every turn-scoped stage, so unlike $model and $effort it applies to every stage in this panel.", "fieldConfig": { "defaults": { "color": { @@ -805,7 +805,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\"}[$__rate_interval]))", + "expr": "sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__rate_interval]))", "range": true, "refId": "A", "legendFormat": "chat turns" @@ -816,7 +816,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"queue_wait|capacity_wait|acquisition|mcp_connect|commit\", scope=\"turn\"}[$__rate_interval]))", + "expr": "sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"queue_wait|capacity_wait|acquisition|mcp_connect|commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__rate_interval]))", "range": true, "refId": "B", "legendFormat": "{{stage}}" @@ -827,7 +827,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"generation_step|prepare|provider_attempt|time_to_first_token|stream|thinking|tool_call|compaction\", scope=\"turn\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval]))", + "expr": "sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"generation_step|prepare|provider_attempt|time_to_first_token|stream|thinking|tool_call|compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval]))", "range": true, "refId": "C", "legendFormat": "{{stage}}" @@ -841,7 +841,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Provider calls made outside a chat turn (scope=\"background\"): detached quickgen and title generation requests. These are excluded from the stage profile and the other stage panels, which are scoped to scope=\"turn\". Rate is on the right axis; duration uses the selected $stat and is guarded so an idle period drops out instead of returning NaN.", + "description": "Provider calls made outside a chat turn (scope=\"background\"): detached quickgen and title generation requests. These are excluded from the stage profile and the other stage panels, which are scoped to scope=\"turn\". Rate is on the right axis; duration uses the selected $stat and is guarded so an idle period drops out instead of returning NaN. Background work runs outside a chat turn and carries an empty chat_kind, so $chat_kind is not applied here either.", "fieldConfig": { "defaults": { "color": { @@ -1029,6 +1029,40 @@ "skipUrlSync": false, "type": "custom" }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(coderd_chatd_stage_duration_seconds_count{scope=\"turn\"}, chat_kind)", + "description": "Whether the turn belongs to a root chat or to a subagent chat spawned by a parent agent. Every turn-scoped stage carries this label, so the filter applies to all stage panels. Background provider calls carry an empty value and are not filtered.", + "hide": 0, + "includeAll": true, + "label": "Chat kind", + "multi": true, + "name": "chat_kind", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(coderd_chatd_stage_duration_seconds_count{scope=\"turn\"}, chat_kind)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, { "allValue": ".*", "current": { diff --git a/scripts/metricsdocgen/generated_metrics b/scripts/metricsdocgen/generated_metrics index 385b8e536da..7efd67505e1 100644 --- a/scripts/metricsdocgen/generated_metrics +++ b/scripts/metricsdocgen/generated_metrics @@ -319,9 +319,9 @@ coderd_chatd_message_count{provider="",model=""} 0 # HELP coderd_chatd_prompt_size_bytes Estimated byte size of the prompt per LLM request. # TYPE coderd_chatd_prompt_size_bytes histogram coderd_chatd_prompt_size_bytes{provider="",model=""} 0 -# HELP coderd_chatd_stage_duration_seconds Wall time spent in each chat lifecycle stage. Stages overlap in wall time; this is a stage-time profile, not a partition of the turn. The scope label separates stages that run inside a chat turn from detached background work. The model and effort labels are empty for stages that run before a model is resolved. +# HELP coderd_chatd_stage_duration_seconds Wall time spent in each chat lifecycle stage. Stages overlap in wall time; this is a stage-time profile, not a partition of the turn. The scope label separates stages that run inside a chat turn from detached background work. The chat_kind label is empty for stages recorded without a known chat, and the model and effort labels are empty for stages that run before a model is resolved. # TYPE coderd_chatd_stage_duration_seconds histogram -coderd_chatd_stage_duration_seconds{stage="",scope="",model="",effort=""} 0 +coderd_chatd_stage_duration_seconds{stage="",scope="",chat_kind="",model="",effort=""} 0 # HELP coderd_chatd_steps_total Total agentic loop steps across all chats. # TYPE coderd_chatd_steps_total counter coderd_chatd_steps_total{provider="",model=""} 0 From 807871aef555582a647be0228eaa897a4b7c442e Mon Sep 17 00:00:00 2001 From: Scott Miller Date: Wed, 2 Sep 2026 22:13:18 +0000 Subject: [PATCH 04/19] feat: add turn-end accounting for chat lifecycle stages Accumulate per-turn stage totals on the turn context and emit them when chat_turn ends: coderd_chatd_turn_stage_seconds and coderd_chatd_stage_share_of_turn per stage, and an exclusive partition of turn wall time as coderd_chatd_turn_time_seconds and coderd_chatd_turn_time_share per category (scheduling, time_to_first_token, streaming, provider_error, retry_backoff, tool_execution, compaction, chatd_overhead, unattributed). Children attribute self time to their nearest partitioning ancestor so the categories telescope to the turn duration. Only completed turns emit. Rotate chat_turn when a queued message is promoted so one turn covers one prompt, and stamp the resolved model and effort on the turn root. Dashboard: add a turn time partition row (per-model mix, seconds per turn by category, unattributed time, per-turn category share), per-turn stage share and stage seconds panels, and retire the aggregate-ratio time-share panel whose observations landed at different times. --- coderd/x/chatd/ARCHITECTURE.md | 4 + coderd/x/chatd/chatloop/chatloop.go | 11 +- coderd/x/chatd/chatloop/metrics.go | 73 +- coderd/x/chatd/chatloop/stage.go | 71 +- coderd/x/chatd/chatloop/turnaccounting.go | 386 ++++++ .../chatloop/turnaccounting_internal_test.go | 420 +++++++ coderd/x/chatd/generation.go | 39 +- coderd/x/chatd/stage_internal_test.go | 88 ++ coderd/x/chatd/turn_trace.go | 124 +- docs/admin/integrations/prometheus.md | 4 + .../grafana/chatd-lifecycle/README.md | 107 +- .../grafana/chatd-lifecycle/dashboard.json | 1032 ++++++++++++++++- scripts/metricsdocgen/generated_metrics | 12 + 13 files changed, 2287 insertions(+), 84 deletions(-) create mode 100644 coderd/x/chatd/chatloop/turnaccounting.go create mode 100644 coderd/x/chatd/chatloop/turnaccounting_internal_test.go diff --git a/coderd/x/chatd/ARCHITECTURE.md b/coderd/x/chatd/ARCHITECTURE.md index 9da6e1a4816..f41bc99d307 100644 --- a/coderd/x/chatd/ARCHITECTURE.md +++ b/coderd/x/chatd/ARCHITECTURE.md @@ -782,6 +782,8 @@ The runner is responsible for subscribing to the `chat:update:{chat_id}` pubsub + + ### Event shape Every event that the runner loop processes has the following shape: @@ -865,6 +867,8 @@ It inspects the chat's message history, and decides what's the next step to take + + - `CommitStep`: applied when an LLM API call returns a response. - `FinishTurn`: applied when the chat processing logic determines that there's no more work to do for the current message history (no pending tool calls, user message is not the last message in the history, etc.). - `FinishError`: applied when the LLM API call fails and the retry limit is reached, determined by the `generation_attempt` value. diff --git a/coderd/x/chatd/chatloop/chatloop.go b/coderd/x/chatd/chatloop/chatloop.go index 3984695b809..0a90ab122a3 100644 --- a/coderd/x/chatd/chatloop/chatloop.go +++ b/coderd/x/chatd/chatloop/chatloop.go @@ -471,10 +471,17 @@ func GenerateAssistant(ctx context.Context, opts GenerateAssistantOptions) (Assi return AssistantOutcome{}, wrappedErr } defer attempt.release() + // Releasing the attempt closes the time_to_first_token window, so it + // must happen before the stream stage ends: a window still open then + // would be counted outside the stream that contains it. + endStream := func(err error) { + attempt.release() + streamSpan.End(err) + } result, processErr := processStepStream(attempt.ctx, attempt.stream, opts.Clock, publishMessagePart) if err := attempt.finish(processErr); err != nil { - streamSpan.End(err) + endStream(err) if errors.Is(err, ErrInterrupted) { return AssistantOutcome{}, ErrInterrupted } @@ -487,7 +494,7 @@ func GenerateAssistant(ctx context.Context, opts GenerateAssistantOptions) (Assi } contextLimit := extractContextLimitWithFallback(result.providerMetadata, opts.ContextLimitFallback) - streamSpan.End(nil) + endStream(nil) result.content = chatsanitize.SanitizeAnthropicProviderToolStepContent( ctx, opts.Logger, provider, modelName, "assistant_helper", 0, result.finishReason, result.content, diff --git a/coderd/x/chatd/chatloop/metrics.go b/coderd/x/chatd/chatloop/metrics.go index f6ad211503c..97ff000ee36 100644 --- a/coderd/x/chatd/chatloop/metrics.go +++ b/coderd/x/chatd/chatloop/metrics.go @@ -36,6 +36,10 @@ type Metrics struct { ToolErrorsTotal *prometheus.CounterVec TTFTSeconds *prometheus.HistogramVec StageDurationSeconds *prometheus.HistogramVec + TurnStageSeconds *prometheus.HistogramVec + StageShareOfTurn *prometheus.HistogramVec + TurnTimeSeconds *prometheus.HistogramVec + TurnTimeShare *prometheus.HistogramVec CompactionTotal *prometheus.CounterVec StepsTotal *prometheus.CounterVec StreamRetriesTotal *prometheus.CounterVec @@ -102,10 +106,36 @@ func NewMetrics(reg prometheus.Registerer) *Metrics { Subsystem: metricsSubsystem, Name: "stage_duration_seconds", Help: "Wall time spent in each chat lifecycle stage. Stages overlap in wall time; this is a stage-time profile, not a partition of the turn. The scope label separates stages that run inside a chat turn from detached background work. The chat_kind label is empty for stages recorded without a known chat, and the model and effort labels are empty for stages that run before a model is resolved.", - // 10ms .. ~2.9h, log-spaced. The top of the range covers - // long-lived stages such as a chat turn. - Buckets: prometheus.ExponentialBuckets(0.01, 2, 21), + Buckets: stageDurationBuckets(), }, []string{"stage", "scope", "chat_kind", "model", "effort"}), + TurnStageSeconds: factory.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "turn_stage_seconds", + Help: "Total wall time one chat turn spent in a stage, observed once per turn when the turn ends. Stages overlap, so these do not partition the turn. Only turns that finished normally are counted.", + Buckets: stageDurationBuckets(), + }, []string{"stage", "chat_kind", "model", "effort"}), + StageShareOfTurn: factory.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "stage_share_of_turn", + Help: "Fraction of a chat turn's wall time spent in a stage, observed once per turn when the turn ends. Stages overlap, so shares can exceed 1 and do not sum to 1. Only turns that finished normally are counted.", + Buckets: turnShareBuckets(), + }, []string{"stage", "chat_kind", "model", "effort"}), + TurnTimeSeconds: factory.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "turn_time_seconds", + Help: "Wall time of one chat turn split into disjoint categories that sum to the turn duration, observed once per turn per category when the turn ends. Every category is observed, including the ones with no time. Only turns that finished normally are counted.", + Buckets: stageDurationBuckets(), + }, []string{"category", "chat_kind", "model", "effort"}), + TurnTimeShare: factory.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "turn_time_share", + Help: "Fraction of a chat turn's wall time in each disjoint category, observed once per turn per category when the turn ends. The shares of one turn sum to 1. Only turns that finished normally are counted.", + Buckets: turnShareBuckets(), + }, []string{"category", "chat_kind", "model", "effort"}), CompactionTotal: factory.NewCounterVec(prometheus.CounterOpts{ Namespace: metricsNamespace, Subsystem: metricsSubsystem, @@ -158,6 +188,19 @@ func NewMetrics(reg prometheus.Registerer) *Metrics { } } +// stageDurationBuckets returns the duration buckets shared by the +// stage and turn timing histograms: 10ms to ~2.9h, log-spaced. The top +// of the range covers long-lived stages such as a chat turn. +func stageDurationBuckets() []float64 { + return prometheus.ExponentialBuckets(0.01, 2, 21) +} + +// turnShareBuckets returns the buckets for the share histograms: 0 to +// 1 in twentieths. +func turnShareBuckets() []float64 { + return prometheus.LinearBuckets(0, 0.05, 21) +} + // NopMetrics returns a Metrics instance that discards all data. // Useful for tests and when metrics collection is not desired. func NopMetrics() *Metrics { @@ -175,6 +218,30 @@ func (m *Metrics) RecordStageDuration(stage, scope, chatKind, model, effort stri m.StageDurationSeconds.WithLabelValues(stage, scope, chatKind, model, effort).Observe(elapsed.Seconds()) } +// RecordTurnStage observes the total time one turn spent in a stage +// and that time as a fraction of the turn. Both come from the same +// turn so the pair cannot describe different turns. No-op when m is +// nil. +func (m *Metrics) RecordTurnStage(stage, chatKind, model, effort string, elapsed time.Duration, share float64) { + if m == nil || elapsed < 0 { + return + } + m.TurnStageSeconds.WithLabelValues(stage, chatKind, model, effort).Observe(elapsed.Seconds()) + m.StageShareOfTurn.WithLabelValues(stage, chatKind, model, effort).Observe(share) +} + +// RecordTurnCategory observes one category of a turn's time partition +// and that category as a fraction of the turn. Categories with no time +// are observed as zero so the shares of a turn always sum to 1. No-op +// when m is nil. +func (m *Metrics) RecordTurnCategory(category, chatKind, model, effort string, elapsed time.Duration, share float64) { + if m == nil || elapsed < 0 { + return + } + m.TurnTimeSeconds.WithLabelValues(category, chatKind, model, effort).Observe(elapsed.Seconds()) + m.TurnTimeShare.WithLabelValues(category, chatKind, model, effort).Observe(share) +} + // RecordCompaction classifies and records a compaction attempt. // It is a no-op when m is nil. func (m *Metrics) RecordCompaction(provider, model string, compacted bool, err error) { diff --git a/coderd/x/chatd/chatloop/stage.go b/coderd/x/chatd/chatloop/stage.go index b2e26f15aae..b6f36c6ff63 100644 --- a/coderd/x/chatd/chatloop/stage.go +++ b/coderd/x/chatd/chatloop/stage.go @@ -8,6 +8,8 @@ import ( "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/trace" "go.opentelemetry.io/otel/trace/noop" + + "github.com/coder/quartz" ) // Stage names. Every value is both a span name and the `stage` label @@ -27,8 +29,15 @@ const ( StageToolCall = "tool_call" StageCommit = "commit" StageCompaction = "compaction" + StageRetryBackoff = "retry_backoff" ) +// GenerationActionExecuteLocalTools is the generation_action value of +// a step that runs local tools. Turn accounting reads it to separate +// tool execution from chatd overhead, so it must match the action the +// generation loop reports through SetGenerationAction. +const GenerationActionExecuteLocalTools = "execute_local_tools" + // Span attribute keys. Keys are lowercase snake_case and shared by // every stage that carries the value. const ( @@ -73,6 +82,9 @@ const tracerName = "github.com/coder/coder/v2/coderd/x/chatd" type StageTracer struct { tracer trace.Tracer metrics *Metrics + // clock is the time source for the stage windows measured here. + // Stages recorded from explicit timestamps do not use it. + clock quartz.Clock } // NewStageTracer builds a stage tracer from a tracer provider and the @@ -89,6 +101,7 @@ func NewStageTracer(provider trace.TracerProvider, metrics *Metrics) *StageTrace return &StageTracer{ tracer: provider.Tracer(tracerName), metrics: metrics, + clock: quartz.NewReal(), } } @@ -105,6 +118,13 @@ func (t *StageTracer) otelTracer() trace.Tracer { return t.tracer } +func (t *StageTracer) now() time.Time { + if t == nil || t.clock == nil { + return time.Now() + } + return t.clock.Now() +} + // StageModel identifies the model a stage ran against. Both fields // are empty for stages that run before a model is resolved, such as // the queue and capacity waits. Effort is the effective reasoning @@ -138,6 +158,11 @@ type StageSpan struct { span trace.Span start time.Time ended bool + // acc is the turn the stage runs in, nil outside a turn. + acc *TurnAccumulator + // node is the stage's place in the turn's attribution tree, nil + // for stages that do not partition turn time. + node *stageNode } // stageScopeKey keys the stage scope carried by a context. It is @@ -235,7 +260,7 @@ func (t *StageTracer) startSpan( start time.Time, opts []trace.SpanStartOption, ) (context.Context, *StageSpan) { - now := time.Now() + now := t.now() if start.IsZero() || start.After(now) { start = now } else { @@ -243,6 +268,14 @@ func (t *StageTracer) startSpan( } chatKind := chatKindFromContext(ctx) opts = append(opts, trace.WithAttributes(stageIdentityAttributes(scope, chatKind)...)) + acc := turnAccumulatorFromContext(ctx) + var node *stageNode + if acc != nil { + if _, attributing := attributingStages[stage]; attributing { + node = &stageNode{stage: stage, parent: stageNodeFromContext(ctx)} + ctx = context.WithValue(ctx, stageNodeKey{}, node) + } + } ctx, span := t.otelTracer().Start(ContextWithScope(ctx, scope), stage, opts...) return ctx, &StageSpan{ tracer: t, @@ -251,6 +284,8 @@ func (t *StageTracer) startSpan( chatKind: chatKind, span: span, start: start, + acc: acc, + node: node, } } @@ -283,9 +318,21 @@ func (s *StageSpan) SetModel(model StageModel) { return } s.model = model + s.acc.setModel(model) s.span.SetAttributes(model.attributes()...) } +// SetGenerationAction records the action a generation step took, on +// the span and on the step's turn attribution, where it decides +// whether the step's own time counts as tool execution. +func (s *StageSpan) SetGenerationAction(action string) { + if s == nil || s.ended { + return + } + s.node.setAction(action) + s.span.SetAttributes(attribute.String(AttrGenerationAction, action)) +} + // SpanContext returns the span context of the stage span, which is // invalid when tracing is not configured. func (s *StageSpan) SpanContext() trace.SpanContext { @@ -299,8 +346,11 @@ func (s *StageSpan) SpanContext() trace.SpanContext { // as errored when err is non-nil. Calls after the first are ignored so // a deferred End cannot double-count a stage. func (s *StageSpan) End(err error) { + s.adoptTurnModel() if elapsed, ok := s.closeSpan(err); ok { s.tracer.observe(s.stage, s.scope, s.chatKind, s.model, elapsed) + s.addTurnStageTotal(elapsed) + s.report(elapsed, err) } } @@ -310,7 +360,21 @@ func (s *StageSpan) End(err error) { // would skew the histogram while the span still needs to report the // failure. func (s *StageSpan) EndWithoutObservation(err error) { - s.closeSpan(err) + if elapsed, ok := s.closeSpan(err); ok { + s.report(elapsed, err) + } +} + +// adoptTurnModel gives the turn's root stage the model identity that +// the turn resolved after the root started, so the root is labeled +// like the stages inside it. +func (s *StageSpan) adoptTurnModel() { + if s == nil || s.stage != StageChatTurn || s.model.Model != "" { + return + } + if model := s.acc.Model(); model.Model != "" { + s.SetModel(model) + } } // closeSpan ends the span and returns the window it covered. ok is @@ -321,7 +385,7 @@ func (s *StageSpan) closeSpan(err error) (elapsed time.Duration, ok bool) { return 0, false } s.ended = true - elapsed = time.Since(s.start) + elapsed = s.tracer.now().Sub(s.start) if err != nil { s.span.RecordError(err) s.span.SetStatus(codes.Error, err.Error()) @@ -375,6 +439,7 @@ func (t *StageTracer) RecordAs( } span.End(trace.WithTimestamp(end)) t.observe(stage, scope, chatKind, model, end.Sub(start)) + recordAttribution(ctx, stage, end.Sub(start)) } func (t *StageTracer) observe(stage, scope, chatKind string, model StageModel, elapsed time.Duration) { diff --git a/coderd/x/chatd/chatloop/turnaccounting.go b/coderd/x/chatd/chatloop/turnaccounting.go new file mode 100644 index 00000000000..524d3ff0868 --- /dev/null +++ b/coderd/x/chatd/chatloop/turnaccounting.go @@ -0,0 +1,386 @@ +package chatloop + +import ( + "context" + "sync" + "time" +) + +// Turn time categories. The categories partition a turn's wall time: +// every category is disjoint from the others and unattributed carries +// the remainder, so the categories of one turn sum to its duration. +const ( + CategoryScheduling = "scheduling" + CategoryTimeToFirstToken = "time_to_first_token" + CategoryStreaming = "streaming" + CategoryProviderError = "provider_error" + CategoryRetryBackoff = "retry_backoff" + CategoryToolExecution = "tool_execution" + CategoryCompaction = "compaction" + CategoryChatdOverhead = "chatd_overhead" + CategoryUnattributed = "unattributed" +) + +// TurnTimeCategories lists every category in a fixed order. All of +// them are emitted for each accounted turn, including the ones with no +// time, so a share is comparable across turns. +var TurnTimeCategories = []string{ + CategoryScheduling, + CategoryTimeToFirstToken, + CategoryStreaming, + CategoryProviderError, + CategoryRetryBackoff, + CategoryToolExecution, + CategoryCompaction, + CategoryChatdOverhead, + CategoryUnattributed, +} + +// attributingStages are the stages that take time from the stage they +// run inside and contribute their remaining time to a category. A +// stage outside this set is still summed into the per-stage turn +// totals, but it neither claims time from its parent nor lands in a +// category, because such stages overlap the ones that do: +// provider_attempt overlaps time_to_first_token, and thinking and +// tool_call are reconstructed from timestamps inside a stream that is +// already accounted for. +var attributingStages = map[string]struct{}{ + StageGenerationStep: {}, + StagePrepare: {}, + StageMCPConnect: {}, + StageCommit: {}, + StageStream: {}, + StageTimeToFirstToken: {}, + StageCompaction: {}, + StageRetryBackoff: {}, +} + +// recordedStageCategories maps the stages reconstructed from explicit +// timestamps to their category. They run outside any attributing stage +// and cannot nest, so their full duration is categorized. Recorded +// stages absent from the map contribute to the per-stage totals only. +var recordedStageCategories = map[string]string{ + StageAcquisition: CategoryScheduling, + StageQueueWait: CategoryScheduling, + StageCapacityWait: CategoryScheduling, +} + +// turnAccumulatorKey keys the accumulator of the turn a context runs +// in. It is private so the accumulator can only be attached through +// ContextWithTurnAccumulator. +type turnAccumulatorKey struct{} + +// stageNodeKey keys the innermost attributing stage of a context. +type stageNodeKey struct{} + +// ContextWithTurnAccumulator returns ctx carrying acc, so the stages +// started on it and on its descendants accumulate into the same turn. +func ContextWithTurnAccumulator(ctx context.Context, acc *TurnAccumulator) context.Context { + return context.WithValue(ctx, turnAccumulatorKey{}, acc) +} + +func turnAccumulatorFromContext(ctx context.Context) *TurnAccumulator { + acc, _ := ctx.Value(turnAccumulatorKey{}).(*TurnAccumulator) + return acc +} + +func stageNodeFromContext(ctx context.Context) *stageNode { + node, _ := ctx.Value(stageNodeKey{}).(*stageNode) + return node +} + +// TurnAccumulator sums the stage durations and the category times of +// one chat turn so they can be emitted together when the turn ends, +// instead of arriving spread over the turn as each stage ends. +// +// It is safe for concurrent use: parallel tool calls and the stages +// under them end on different goroutines. +type TurnAccumulator struct { + mu sync.Mutex + stages map[string]time.Duration + categories map[string]time.Duration + model StageModel + completed bool + invalid bool +} + +// NewTurnAccumulator returns an accumulator for one turn. The turn is +// not accounted until MarkCompleted is called, so a turn that never +// reaches its finish transition emits nothing. +func NewTurnAccumulator() *TurnAccumulator { + return &TurnAccumulator{ + stages: map[string]time.Duration{}, + categories: map[string]time.Duration{}, + } +} + +func (a *TurnAccumulator) addStage(stage string, elapsed time.Duration) { + if a == nil || elapsed <= 0 { + return + } + a.mu.Lock() + defer a.mu.Unlock() + a.stages[stage] += elapsed +} + +func (a *TurnAccumulator) addCategory(category string, elapsed time.Duration) { + if a == nil || category == "" || elapsed <= 0 { + return + } + a.mu.Lock() + defer a.mu.Unlock() + a.categories[category] += elapsed +} + +// setModel records the model of the turn on first call. Later calls +// are ignored, so a turn that switches models keeps the identity it +// started with rather than the one it happened to end with. +func (a *TurnAccumulator) setModel(model StageModel) { + if a == nil || model.Model == "" { + return + } + a.mu.Lock() + defer a.mu.Unlock() + if a.model.Model == "" { + a.model = model + } +} + +// Model returns the turn's model identity, empty until a stage +// resolves one. +func (a *TurnAccumulator) Model() StageModel { + if a == nil { + return StageModel{} + } + a.mu.Lock() + defer a.mu.Unlock() + return a.model +} + +// MarkCompleted marks the turn as finished normally, which is what +// makes its accounting emittable. +func (a *TurnAccumulator) MarkCompleted() { + if a == nil { + return + } + a.mu.Lock() + defer a.mu.Unlock() + a.completed = true +} + +// Invalidate drops the turn's accounting. An errored or interrupted +// turn stops partway through its stages, so its category totals do not +// describe a full turn. +func (a *TurnAccumulator) Invalidate() { + if a == nil { + return + } + a.mu.Lock() + defer a.mu.Unlock() + a.invalid = true +} + +// turnAccounting is the emittable state of one turn. +type turnAccounting struct { + stages map[string]time.Duration + categories map[string]time.Duration + model StageModel + emit bool +} + +func (a *TurnAccumulator) snapshot() turnAccounting { + if a == nil { + return turnAccounting{} + } + a.mu.Lock() + defer a.mu.Unlock() + snapshot := turnAccounting{ + stages: make(map[string]time.Duration, len(a.stages)), + categories: make(map[string]time.Duration, len(a.categories)), + model: a.model, + emit: a.completed && !a.invalid, + } + for stage, elapsed := range a.stages { + snapshot.stages[stage] = elapsed + } + for category, elapsed := range a.categories { + snapshot.categories[category] = elapsed + } + return snapshot +} + +// stageNode is the attribution parent of the attributing stages +// started beneath it. A child reports its full duration to its parent +// so the parent's own category only receives the time it did not spend +// inside a child, which keeps the categories disjoint. +type stageNode struct { + stage string + parent *stageNode + + mu sync.Mutex + childTotal time.Duration + // action is the generation action of a generation_step, which + // decides whether its own time is tool execution or overhead. + action string + // failed marks a stream whose first part never arrived, either + // because the stream errored or because it was released empty. + failed bool +} + +func (n *stageNode) setAction(action string) { + if n == nil { + return + } + n.mu.Lock() + defer n.mu.Unlock() + n.action = action +} + +func (n *stageNode) markFailed() { + if n == nil { + return + } + n.mu.Lock() + defer n.mu.Unlock() + n.failed = true +} + +// addChild takes elapsed out of the parent's own time. +func (n *stageNode) addChild(elapsed time.Duration) { + if n == nil || elapsed <= 0 { + return + } + n.mu.Lock() + defer n.mu.Unlock() + n.childTotal += elapsed +} + +func (n *stageNode) state() nodeState { + if n == nil { + return nodeState{} + } + n.mu.Lock() + defer n.mu.Unlock() + return nodeState{childTotal: n.childTotal, action: n.action, failed: n.failed} +} + +// nodeState is a stage's attribution state at the moment it ends. +type nodeState struct { + childTotal time.Duration + action string + failed bool +} + +// category returns the category that the stage's own time belongs to. +// An unknown stage returns an empty category, which drops its time +// into the turn's unattributed remainder. +func (n *stageNode) category(state nodeState, err error) string { + switch n.stage { + case StageGenerationStep: + if state.action == GenerationActionExecuteLocalTools { + return CategoryToolExecution + } + return CategoryChatdOverhead + case StagePrepare, StageMCPConnect, StageCommit: + return CategoryChatdOverhead + case StageCompaction: + return CategoryCompaction + case StageRetryBackoff: + return CategoryRetryBackoff + case StageStream: + if state.failed || err != nil { + return CategoryProviderError + } + return CategoryStreaming + case StageTimeToFirstToken: + if err != nil { + return CategoryProviderError + } + return CategoryTimeToFirstToken + default: + return "" + } +} + +// addTurnStageTotal adds a finished stage to the per-stage totals of +// the turn it ran in. The turn's own stage is skipped: its duration is +// the denominator the other stages are compared against. +func (s *StageSpan) addTurnStageTotal(elapsed time.Duration) { + if s.acc == nil || s.stage == StageChatTurn { + return + } + s.acc.addStage(s.stage, elapsed) +} + +// report folds a finished stage into the turn it ran in. elapsed is +// the stage's full duration; the category receives only the part not +// spent inside a nested attributing stage. +func (s *StageSpan) report(elapsed time.Duration, err error) { + if s.acc == nil { + return + } + if s.stage == StageChatTurn { + s.tracer.emitTurnAccounting(s.acc, s.chatKind, elapsed) + return + } + if s.node == nil { + return + } + state := s.node.state() + own := elapsed - state.childTotal + if own < 0 { + own = 0 + } + s.acc.addCategory(s.node.category(state, err), own) + if s.stage == StageTimeToFirstToken && err != nil { + s.node.parent.markFailed() + } + s.node.parent.addChild(elapsed) +} + +// recordAttribution folds a stage reconstructed from timestamps into +// the turn on ctx. +func recordAttribution(ctx context.Context, stage string, elapsed time.Duration) { + acc := turnAccumulatorFromContext(ctx) + if acc == nil { + return + } + acc.addStage(stage, elapsed) + acc.addCategory(recordedStageCategories[stage], elapsed) +} + +// emitTurnAccounting observes the per-stage totals and the category +// partition of one turn. turnDuration is the turn's own wall time, and +// the categories that did not add up to it become the unattributed +// remainder. Attributed time above the turn duration is clamped, which +// only happens if a stage is counted twice. +func (t *StageTracer) emitTurnAccounting(acc *TurnAccumulator, chatKind string, turnDuration time.Duration) { + if t == nil || t.metrics == nil || turnDuration <= 0 { + return + } + snapshot := acc.snapshot() + if !snapshot.emit { + return + } + turnSeconds := turnDuration.Seconds() + model := snapshot.model + for stage, elapsed := range snapshot.stages { + if elapsed <= 0 { + continue + } + t.metrics.RecordTurnStage(stage, chatKind, model.Model, model.Effort, elapsed, elapsed.Seconds()/turnSeconds) + } + var attributed time.Duration + for _, category := range TurnTimeCategories { + attributed += snapshot.categories[category] + } + unattributed := turnDuration - attributed + if unattributed < 0 { + unattributed = 0 + } + snapshot.categories[CategoryUnattributed] = unattributed + for _, category := range TurnTimeCategories { + elapsed := snapshot.categories[category] + t.metrics.RecordTurnCategory(category, chatKind, model.Model, model.Effort, elapsed, elapsed.Seconds()/turnSeconds) + } +} diff --git a/coderd/x/chatd/chatloop/turnaccounting_internal_test.go b/coderd/x/chatd/chatloop/turnaccounting_internal_test.go new file mode 100644 index 00000000000..19af7c19e93 --- /dev/null +++ b/coderd/x/chatd/chatloop/turnaccounting_internal_test.go @@ -0,0 +1,420 @@ +package chatloop + +import ( + "context" + "testing" + "time" + + "charm.land/fantasy" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/attribute" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + "golang.org/x/xerrors" + + "github.com/coder/coder/v2/coderd/x/chatd/chattest" + "github.com/coder/quartz" +) + +// turnFixture drives a synthetic turn on a mock clock, so every stage +// window is an exact duration. +type turnFixture struct { + tracer *StageTracer + clock *quartz.Mock + spans *tracetest.SpanRecorder + registry *prometheus.Registry +} + +func newTurnFixture(t *testing.T) turnFixture { + t.Helper() + recorder := tracetest.NewSpanRecorder() + provider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder)) + t.Cleanup(func() { + require.NoError(t, provider.Shutdown(context.Background())) + }) + registry := prometheus.NewRegistry() + clock := quartz.NewMock(t) + tracer := NewStageTracer(provider, NewMetrics(registry)) + tracer.clock = clock + return turnFixture{tracer: tracer, clock: clock, spans: recorder, registry: registry} +} + +// sums returns the sample sum of each series of a histogram family, +// keyed by the value of the named label. +func (f turnFixture) sums(t *testing.T, family, label string) map[string]float64 { + t.Helper() + families, err := f.registry.Gather() + require.NoError(t, err) + out := map[string]float64{} + for _, metricFamily := range families { + if metricFamily.GetName() != family { + continue + } + for _, metric := range metricFamily.GetMetric() { + out[metricLabel(metric, label)] = metric.GetHistogram().GetSampleSum() + } + } + return out +} + +func (f turnFixture) labelsOf(t *testing.T, family, label, value string) map[string]string { + t.Helper() + families, err := f.registry.Gather() + require.NoError(t, err) + for _, metricFamily := range families { + if metricFamily.GetName() != family { + continue + } + for _, metric := range metricFamily.GetMetric() { + if metricLabel(metric, label) != value { + continue + } + labels := map[string]string{} + for _, pair := range metric.GetLabel() { + labels[pair.GetName()] = pair.GetValue() + } + return labels + } + } + return nil +} + +func metricLabel(metric *dto.Metric, name string) string { + for _, label := range metric.GetLabel() { + if label.GetName() == name { + return label.GetValue() + } + } + return "" +} + +// syntheticTurn runs one turn made of the stages the categories are +// built from and returns the turn's accumulator and total duration. +func (f turnFixture) syntheticTurn(t *testing.T, model StageModel) (*TurnAccumulator, time.Duration) { + t.Helper() + acc := NewTurnAccumulator() + ctx := ContextWithTurnAccumulator(ContextWithChatKind(t.Context(), ChatKindRoot), acc) + turnStart := f.clock.Now() + turnCtx, turnSpan := f.tracer.StartRootAt(ctx, StageChatTurn, turnStart, nil) + + // Scheduling: a queue wait reconstructed from the queued row. + f.tracer.RecordAs(turnCtx, StageQueueWait, ScopeTurn, StageModel{}, + turnStart.Add(-2*time.Second), turnStart, nil) + + // A generate_assistant step: prepare with an MCP connect, then a + // stream that produced its first part, then the commit. + stepCtx, step := f.tracer.Start(turnCtx, StageGenerationStep) + step.SetGenerationAction("generate_assistant") + step.SetModel(model) + prepareCtx, prepare := f.tracer.Start(stepCtx, StagePrepare) + _, mcp := f.tracer.Start(prepareCtx, StageMCPConnect) + f.clock.Advance(time.Second) + mcp.End(nil) + f.clock.Advance(time.Second) + prepare.End(nil) + streamCtx, stream := f.tracer.Start(stepCtx, StageStream) + _, ttft := f.tracer.Start(streamCtx, StageTimeToFirstToken) + f.clock.Advance(3 * time.Second) + ttft.End(nil) + f.clock.Advance(4 * time.Second) + stream.End(nil) + _, commit := f.tracer.Start(stepCtx, StageCommit) + f.clock.Advance(time.Second) + commit.End(nil) + step.End(nil) + + // An execute_local_tools step: the time between its prepare and + // its commit is the tool execution. + toolStepCtx, toolStep := f.tracer.Start(turnCtx, StageGenerationStep) + toolStep.SetGenerationAction(GenerationActionExecuteLocalTools) + _, toolPrepare := f.tracer.Start(toolStepCtx, StagePrepare) + f.clock.Advance(time.Second) + toolPrepare.End(nil) + f.clock.Advance(5 * time.Second) + _, toolCommit := f.tracer.Start(toolStepCtx, StageCommit) + f.clock.Advance(time.Second) + toolCommit.End(nil) + toolStep.End(nil) + + // A failed step: the stream never produced a first part, and the + // retry delay that follows it is its own category. + failedStepCtx, failedStep := f.tracer.Start(turnCtx, StageGenerationStep) + failedStep.SetGenerationAction("generate_assistant") + failedStreamCtx, failedStream := f.tracer.Start(failedStepCtx, StageStream) + _, failedTTFT := f.tracer.Start(failedStreamCtx, StageTimeToFirstToken) + f.clock.Advance(2 * time.Second) + failedTTFT.EndWithoutObservation(xerrors.New("stream ended before the first token")) + f.clock.Advance(time.Second) + failedStream.End(xerrors.New("provider unavailable")) + _, backoff := f.tracer.Start(failedStepCtx, StageRetryBackoff) + f.clock.Advance(6 * time.Second) + backoff.End(nil) + failedStep.End(nil) + + // A compaction step. + compactionStepCtx, compactionStep := f.tracer.Start(turnCtx, StageGenerationStep) + compactionStep.SetGenerationAction("compact") + _, compaction := f.tracer.Start(compactionStepCtx, StageCompaction) + f.clock.Advance(8 * time.Second) + compaction.End(nil) + compactionStep.End(nil) + + // Time between steps belongs to no stage. + f.clock.Advance(5 * time.Second) + + acc.MarkCompleted() + turnSpan.End(nil) + return acc, f.clock.Now().Sub(turnStart) +} + +func TestTurnAccountingPartition(t *testing.T) { + t.Parallel() + fixture := newTurnFixture(t) + model := StageModel{Model: "claude-sonnet-4-5", Effort: "high"} + + _, turnDuration := fixture.syntheticTurn(t, model) + require.Equal(t, 39*time.Second, turnDuration) + + categories := fixture.sums(t, "coderd_chatd_turn_time_seconds", "category") + require.Equal(t, map[string]float64{ + CategoryScheduling: 2, + CategoryTimeToFirstToken: 3, + CategoryStreaming: 4, + CategoryProviderError: 3, + CategoryRetryBackoff: 6, + CategoryToolExecution: 5, + CategoryCompaction: 8, + CategoryChatdOverhead: 5, + CategoryUnattributed: 3, + }, categories) + + var total float64 + for _, seconds := range categories { + total += seconds + } + require.InDelta(t, turnDuration.Seconds(), total, 0.001, + "the categories must partition the turn") + + shares := fixture.sums(t, "coderd_chatd_turn_time_share", "category") + require.Len(t, shares, len(TurnTimeCategories)) + var shareTotal float64 + for category, share := range shares { + require.InDelta(t, categories[category]/turnDuration.Seconds(), share, 0.001) + shareTotal += share + } + require.InDelta(t, 1, shareTotal, 0.001) +} + +func TestTurnAccountingStageTotals(t *testing.T) { + t.Parallel() + fixture := newTurnFixture(t) + model := StageModel{Model: "claude-sonnet-4-5", Effort: "high"} + + _, turnDuration := fixture.syntheticTurn(t, model) + + stages := fixture.sums(t, "coderd_chatd_turn_stage_seconds", "stage") + require.Equal(t, map[string]float64{ + StageQueueWait: 2, + StageGenerationStep: 10 + 7 + 9 + 8, + StagePrepare: 2 + 1, + StageMCPConnect: 1, + StageStream: 7 + 3, + StageTimeToFirstToken: 3, + StageCommit: 1 + 1, + StageRetryBackoff: 6, + StageCompaction: 8, + }, stages) + require.NotContains(t, stages, StageChatTurn, + "the turn's own duration is the denominator, not a stage of itself") + + shares := fixture.sums(t, "coderd_chatd_stage_share_of_turn", "stage") + for stage, seconds := range stages { + require.InDelta(t, seconds/turnDuration.Seconds(), shares[stage], 0.001, stage) + } + + // The turn's stage rows carry the model the turn resolved. + labels := fixture.labelsOf(t, "coderd_chatd_turn_stage_seconds", "stage", StageStream) + require.Equal(t, model.Model, labels["model"]) + require.Equal(t, model.Effort, labels["effort"]) + require.Equal(t, ChatKindRoot, labels["chat_kind"]) +} + +func TestTurnAccountingStreamWithoutFirstToken(t *testing.T) { + t.Parallel() + fixture := newTurnFixture(t) + + acc := NewTurnAccumulator() + ctx := ContextWithTurnAccumulator(ContextWithChatKind(t.Context(), ChatKindRoot), acc) + turnStart := fixture.clock.Now() + turnCtx, turnSpan := fixture.tracer.StartRootAt(ctx, StageChatTurn, turnStart, nil) + + model := &chattest.FakeModel{ + ProviderName: "google", + ModelName: "test-model", + StreamFn: func(context.Context, fantasy.Call) (fantasy.StreamResponse, error) { + fixture.clock.Advance(2 * time.Second) + return streamFromParts(nil), nil + }, + } + _, err := GenerateAssistant(turnCtx, GenerateAssistantOptions{ + Model: model, + Messages: []fantasy.Message{}, + Clock: fixture.clock, + Metrics: NewMetrics(prometheus.NewRegistry()), + Stages: fixture.tracer, + }) + require.NoError(t, err) + + acc.MarkCompleted() + turnSpan.End(nil) + + // The stream never produced a part, so its whole window is provider + // error time, counted once even though the window closes with the + // attempt rather than with the stream. + categories := fixture.sums(t, "coderd_chatd_turn_time_seconds", "category") + require.InDelta(t, 2, categories[CategoryProviderError], 0.001) + require.Zero(t, categories[CategoryStreaming]) + require.Zero(t, categories[CategoryTimeToFirstToken]) + + var total float64 + for _, seconds := range categories { + total += seconds + } + require.InDelta(t, fixture.clock.Now().Sub(turnStart).Seconds(), total, 0.001) +} + +func TestTurnAccountingRotatedTurnPartition(t *testing.T) { + t.Parallel() + fixture := newTurnFixture(t) + + // A rotated turn is anchored at the moment its message was queued, + // so the queue wait covers the head of the turn and nothing else + // accounts for that window. + acc := NewTurnAccumulator() + ctx := ContextWithTurnAccumulator(ContextWithChatKind(t.Context(), ChatKindRoot), acc) + turnStart := fixture.clock.Now() + turnCtx, turnSpan := fixture.tracer.StartRootAt(ctx, StageChatTurn, turnStart, nil) + + fixture.clock.Advance(2 * time.Second) + fixture.tracer.RecordAs(turnCtx, StageQueueWait, ScopeTurn, StageModel{}, + turnStart, fixture.clock.Now(), nil) + + stepCtx, step := fixture.tracer.Start(turnCtx, StageGenerationStep) + step.SetGenerationAction("generate_assistant") + _, prepare := fixture.tracer.Start(stepCtx, StagePrepare) + fixture.clock.Advance(time.Second) + prepare.End(nil) + step.End(nil) + + acc.MarkCompleted() + turnSpan.End(nil) + + turnDuration := fixture.clock.Now().Sub(turnStart) + require.Equal(t, 3*time.Second, turnDuration) + + categories := fixture.sums(t, "coderd_chatd_turn_time_seconds", "category") + require.InDelta(t, 2, categories[CategoryScheduling], 0.001) + require.InDelta(t, 1, categories[CategoryChatdOverhead], 0.001) + + var total float64 + for _, seconds := range categories { + total += seconds + } + require.InDelta(t, turnDuration.Seconds(), total, 0.001) + + // The turn observed into stage_duration_seconds is the same window + // the categories partition. + stageTurn := fixture.sums(t, "coderd_chatd_stage_duration_seconds", "stage")[StageChatTurn] + require.InDelta(t, total, stageTurn, 0.001) +} + +func TestTurnAccountingSkipsUnfinishedTurns(t *testing.T) { + t.Parallel() + + t.Run("NeverCompleted", func(t *testing.T) { + t.Parallel() + fixture := newTurnFixture(t) + + acc := NewTurnAccumulator() + ctx := ContextWithTurnAccumulator(t.Context(), acc) + turnCtx, turnSpan := fixture.tracer.StartRootAt(ctx, StageChatTurn, fixture.clock.Now(), nil) + _, step := fixture.tracer.Start(turnCtx, StageGenerationStep) + fixture.clock.Advance(time.Second) + step.End(nil) + turnSpan.End(nil) + + require.Empty(t, fixture.sums(t, "coderd_chatd_turn_time_seconds", "category")) + require.Empty(t, fixture.sums(t, "coderd_chatd_turn_stage_seconds", "stage")) + }) + + t.Run("Invalidated", func(t *testing.T) { + t.Parallel() + fixture := newTurnFixture(t) + + acc := NewTurnAccumulator() + ctx := ContextWithTurnAccumulator(t.Context(), acc) + turnCtx, turnSpan := fixture.tracer.StartRootAt(ctx, StageChatTurn, fixture.clock.Now(), nil) + _, step := fixture.tracer.Start(turnCtx, StageGenerationStep) + fixture.clock.Advance(time.Second) + step.End(nil) + acc.MarkCompleted() + acc.Invalidate() + turnSpan.End(nil) + + require.Empty(t, fixture.sums(t, "coderd_chatd_turn_time_seconds", "category")) + }) + + t.Run("EndedWithError", func(t *testing.T) { + t.Parallel() + fixture := newTurnFixture(t) + + acc := NewTurnAccumulator() + ctx := ContextWithTurnAccumulator(t.Context(), acc) + turnCtx, turnSpan := fixture.tracer.StartRootAt(ctx, StageChatTurn, fixture.clock.Now(), nil) + _, step := fixture.tracer.Start(turnCtx, StageGenerationStep) + fixture.clock.Advance(time.Second) + step.End(nil) + acc.MarkCompleted() + acc.Invalidate() + turnSpan.End(xerrors.New("turn failed")) + + require.Empty(t, fixture.sums(t, "coderd_chatd_turn_time_seconds", "category")) + }) +} + +func TestTurnAccountingStampsModelOnRoot(t *testing.T) { + t.Parallel() + fixture := newTurnFixture(t) + model := StageModel{Model: "gpt-5", Effort: "medium"} + + acc := NewTurnAccumulator() + ctx := ContextWithTurnAccumulator(ContextWithChatKind(t.Context(), ChatKindSubagent), acc) + turnCtx, turnSpan := fixture.tracer.StartRootAt(ctx, StageChatTurn, fixture.clock.Now(), nil) + _, step := fixture.tracer.Start(turnCtx, StageGenerationStep) + step.SetModel(model) + fixture.clock.Advance(time.Second) + step.End(nil) + // A later step on another model does not rename the turn. + _, second := fixture.tracer.Start(turnCtx, StageGenerationStep) + second.SetModel(StageModel{Model: "gpt-5-mini"}) + fixture.clock.Advance(time.Second) + second.End(nil) + acc.MarkCompleted() + turnSpan.End(nil) + + var turn sdktrace.ReadOnlySpan + for _, span := range fixture.spans.Ended() { + if span.Name() == StageChatTurn { + turn = span + } + } + require.NotNil(t, turn) + require.Contains(t, turn.Attributes(), attribute.String(AttrModel, model.Model)) + require.Contains(t, turn.Attributes(), attribute.String(AttrReasoningEffort, model.Effort)) + + labels := fixture.labelsOf(t, "coderd_chatd_stage_duration_seconds", "stage", StageChatTurn) + require.Equal(t, model.Model, labels["model"]) + require.Equal(t, model.Effort, labels["effort"]) +} diff --git a/coderd/x/chatd/generation.go b/coderd/x/chatd/generation.go index 0c0ebc5ddaa..3b41c4c268d 100644 --- a/coderd/x/chatd/generation.go +++ b/coderd/x/chatd/generation.go @@ -453,6 +453,11 @@ func (s *taskStarter) StartGeneration(ctx context.Context, input chatWorkerTaskS if again { continue } + if err != nil { + // The turn stops here, so its stage totals cover only part + // of a turn. + input.Turn.Invalidate() + } return err } } @@ -548,7 +553,7 @@ func (s *taskStarter) runGenerationStep( return input, false, s.finishGenerationError(ctx, machine, input, err, generationAttemptNotRequired) } - stepSpan.SetAttributes(attribute.String(chatloop.AttrGenerationAction, string(decision.kind))) + stepSpan.SetGenerationAction(string(decision.kind)) var actionErr error switch decision.kind { case generationActionEnterRequiresAction: @@ -691,13 +696,17 @@ func (*taskStarter) recordGenerationRetry( } func (s *taskStarter) waitGenerationRetry(ctx context.Context, delay time.Duration) error { + _, span := s.server.stages.Start(ctx, chatloop.StageRetryBackoff) timer := s.opts.Clock.NewTimer(delay, "chatworker", "generation-retry") defer timer.Stop() select { case <-timer.C: + span.End(nil) return nil case <-ctx.Done(): - return errors.Join(errTaskExpectedExit, xerrors.Errorf("wait generation retry: %w", ctx.Err())) + err := errors.Join(errTaskExpectedExit, xerrors.Errorf("wait generation retry: %w", ctx.Err())) + span.End(err) + return err } } @@ -1580,10 +1589,29 @@ func (s *taskStarter) finishGenerationTurnWithoutHook( recordGenerationFinishFailure(input.DebugTurn, err) return err } - s.recordQueueWaitStage(ctx, input, promotedQueuedAt) + s.finishTurnAccounting(ctx, input, promotedQueuedAt) return s.completeGenerationTurn(ctx, input, committed, decision.promotedMessageID) } +// finishTurnAccounting closes the turn that just finished and records +// the queue wait of a message the finish transition promoted. A +// promotion opens the next turn here, anchored at the moment the +// message was queued, so the wait it served and the work it causes are +// accounted to the turn it starts rather than the one that released +// it. A zero queuedAt means the transition promoted nothing. +func (s *taskStarter) finishTurnAccounting( + ctx context.Context, + input chatWorkerTaskStartInput, + queuedAt time.Time, +) { + input.Turn.Complete() + if queuedAt.IsZero() { + return + } + input.Turn.Rotate(ctx, queuedAt) + s.recordQueueWaitStage(ctx, input, queuedAt) +} + // recordQueueWaitStage emits the queue_wait stage for a message just // promoted out of the queue, measured from the queued row's creation // to now. It is recorded against the turn span rather than the step @@ -1690,7 +1718,7 @@ func (s *taskStarter) finishGenerationTurn( Kind: runnerActionKind(generationActionGenerateAssistant), }) } - s.recordQueueWaitStage(ctx, input, promotedQueuedAt) + s.finishTurnAccounting(ctx, input, promotedQueuedAt) return s.completeGenerationTurn(ctx, input, committed, decision.promotedMessageID) } @@ -1701,6 +1729,9 @@ func (s *taskStarter) finishGenerationError( cause error, fence generationAttemptFence, ) error { + // The turn ends on an error, so the stages it collected describe a + // partial turn. + input.Turn.Invalidate() classified := chaterror.Classify(cause) // Log the unsanitized cause before persisting so administrators can // diagnose the failure even when the classified user-facing message diff --git a/coderd/x/chatd/stage_internal_test.go b/coderd/x/chatd/stage_internal_test.go index 028be0d37a2..c456f6a3f13 100644 --- a/coderd/x/chatd/stage_internal_test.go +++ b/coderd/x/chatd/stage_internal_test.go @@ -4,6 +4,7 @@ import ( "context" "io" "net/http" + "sort" "strings" "testing" "time" @@ -323,3 +324,90 @@ func TestRunnerTurnSpanCarriesChatKind(t *testing.T) { "stage %s must carry the turn's chat kind", span.Name()) } } + +// turnSpansByStart returns the chat_turn spans the recorder saw, +// ordered by start time. +func turnSpansByStart(t *testing.T, recorder *tracetest.SpanRecorder) []sdktrace.ReadOnlySpan { + t.Helper() + var turns []sdktrace.ReadOnlySpan + for _, span := range recorder.Ended() { + if span.Name() == chatloop.StageChatTurn { + turns = append(turns, span) + } + } + sort.Slice(turns, func(i, j int) bool { + return turns[i].StartTime().Before(turns[j].StartTime()) + }) + return turns +} + +func TestRunnerTurnSpanRotatesOnPromotion(t *testing.T) { + t.Parallel() + tracer, recorder := newStageTestTracer(t) + turn := newRunnerTurnSpan(tracer) + chat := database.Chat{ID: uuid.New()} + + turnCtx := turn.Ensure(t.Context(), chat, time.Now().Add(-time.Minute)) + _, step := tracer.Start(turnCtx, chatloop.StageGenerationStep) + step.End(nil) + + queuedAt := time.Now().Add(-30 * time.Second) + turn.Complete() + turn.Rotate(t.Context(), queuedAt) + // The promoted message's wait belongs to the turn it opens. + tracer.Record(turn.Context(t.Context()), chatloop.StageQueueWait, chatloop.StageModel{}, + queuedAt, time.Now(), nil) + turn.End(nil) + + turns := turnSpansByStart(t, recorder) + require.Len(t, turns, 2) + require.Equal(t, queuedAt.UTC(), turns[1].StartTime().UTC()) + require.NotEqual(t, turns[0].SpanContext().TraceID(), turns[1].SpanContext().TraceID()) + require.Contains(t, turns[1].Attributes(), + attribute.String(chatloop.AttrChatKind, chatloop.ChatKindRoot)) + + var queueWait sdktrace.ReadOnlySpan + for _, span := range recorder.Ended() { + if span.Name() == chatloop.StageQueueWait { + queueWait = span + } + } + require.NotNil(t, queueWait) + require.Equal(t, turns[1].SpanContext().SpanID(), queueWait.Parent().SpanID()) + + // The rotated turn's head is the queue wait, so it records no + // acquisition: both windows start at the same instant and both + // would count as scheduling time. + var acquisitions int + for _, span := range recorder.Ended() { + if span.Name() == chatloop.StageAcquisition { + acquisitions++ + require.Equal(t, turns[0].SpanContext().SpanID(), span.Parent().SpanID()) + } + } + require.Equal(t, 1, acquisitions) +} + +func TestRunnerTurnSpanEnsureOpensTurnPerPrompt(t *testing.T) { + t.Parallel() + tracer, recorder := newStageTestTracer(t) + turn := newRunnerTurnSpan(tracer) + chat := database.Chat{ID: uuid.New()} + + firstTrigger := time.Now().Add(-2 * time.Minute) + turn.Ensure(t.Context(), chat, firstTrigger) + // A second prompt on the same runner reuses the open turn until it + // finishes. + turn.Ensure(t.Context(), chat, firstTrigger) + require.Len(t, turnSpansByStart(t, recorder), 0) + + turn.Complete() + secondTrigger := time.Now().Add(-time.Minute) + turn.Ensure(t.Context(), chat, secondTrigger) + turn.End(nil) + + turns := turnSpansByStart(t, recorder) + require.Len(t, turns, 2) + require.Equal(t, firstTrigger.UTC(), turns[0].StartTime().UTC()) + require.Equal(t, secondTrigger.UTC(), turns[1].StartTime().UTC()) +} diff --git a/coderd/x/chatd/turn_trace.go b/coderd/x/chatd/turn_trace.go index a65baad5315..2c087c5199f 100644 --- a/coderd/x/chatd/turn_trace.go +++ b/coderd/x/chatd/turn_trace.go @@ -14,20 +14,28 @@ import ( // runnerTurnSpan owns the chat_turn span of one runner. A runner is // created when a chat is acquired and torn down when the chat leaves -// the running states, so the runner's lifetime bounds the turn. +// the running states, so the runner's lifetime bounds the turns it +// runs. // // The span is created on the first generation task rather than at // runner construction: runners are also spawned to abandon or time -// out chats, which run no turn. +// out chats, which run no turn. One runner can run several turns in +// sequence: a finished turn is replaced by a new span, either when a +// queued message is promoted or when the next prompt starts a task. type runnerTurnSpan struct { stages *chatloop.StageTracer mu sync.Mutex span *chatloop.StageSpan spanCtx trace.SpanContext + acc *chatloop.TurnAccumulator + chatID string chatKind string started bool ended bool + // finished marks a turn that reached a terminal transition, so a + // further prompt opens a new span instead of extending this one. + finished bool } func newRunnerTurnSpan(stages *chatloop.StageTracer) *runnerTurnSpan { @@ -38,9 +46,12 @@ func newRunnerTurnSpan(stages *chatloop.StageTracer) *runnerTurnSpan { // context parented to it. triggerAt is the trigger message's // creation time and becomes the span's start timestamp, so the // acquisition stage reconstructed from the same instant falls inside -// the turn. The window between triggerAt and now is emitted as the -// acquisition stage, which covers the delay between the message -// landing in history and a worker picking the chat up. +// the turn. The acquisition stage carries no model identity: the +// turn's model is not resolved until preparation runs. +// +// A turn that already reached a terminal transition is replaced: the +// prompt this call runs is a new turn, and folding it into the old +// span would report the two as one. // // The span is a standalone trace root. The request that triggered the // turn is handled by a different goroutine, and often a different @@ -55,25 +66,41 @@ func (t *runnerTurnSpan) Ensure(ctx context.Context, chat database.Chat, trigger if t.ended { return ctx } - if t.started { + if t.started && !t.finished { return t.contextLocked(ctx) } - t.started = true + if t.started { + t.closeLocked(nil) + } + t.chatID = chat.ID.String() t.chatKind = chatKindAttr(chat) + turnCtx := t.startLocked(ctx, triggerAt) + // The window between the trigger message landing in history and a + // worker picking the chat up is the acquisition. A rotated turn has + // no acquisition: its head is the queue wait of the message that + // opened it, and recording both would count that window twice. + t.stages.Record(turnCtx, chatloop.StageAcquisition, chatloop.StageModel{}, + triggerAt, time.Now(), nil, + attribute.String(chatloop.AttrChatID, t.chatID)) + return t.contextLocked(ctx) +} - attrs := []attribute.KeyValue{ - attribute.String(chatloop.AttrChatID, chat.ID.String()), - } - // The chat kind rides on the context so every stage of the turn - // carries it, on the span and on the duration observation. +// startLocked opens a chat_turn span with a fresh accumulator and +// returns the context parented to it. +func (t *runnerTurnSpan) startLocked(ctx context.Context, startAt time.Time) context.Context { + t.started = true + t.finished = false + t.acc = chatloop.NewTurnAccumulator() + + // The chat kind and the accumulator ride on the context so every + // stage of the turn carries the kind and reports its time to the + // turn that contains it. ctx = chatloop.ContextWithChatKind(ctx, t.chatKind) - turnCtx, span := t.stages.StartRootAt(ctx, chatloop.StageChatTurn, triggerAt, nil, attrs...) + ctx = chatloop.ContextWithTurnAccumulator(ctx, t.acc) + turnCtx, span := t.stages.StartRootAt(ctx, chatloop.StageChatTurn, startAt, nil, + attribute.String(chatloop.AttrChatID, t.chatID)) t.span = span t.spanCtx = span.SpanContext() - // The turn's model is not resolved until preparation runs, so the - // acquisition stage carries no model identity. - t.stages.Record(turnCtx, chatloop.StageAcquisition, chatloop.StageModel{}, - triggerAt, time.Now(), nil, attrs...) return turnCtx } @@ -92,17 +119,61 @@ func (t *runnerTurnSpan) contextLocked(ctx context.Context) context.Context { if !t.started || t.ended { return ctx } - // The scope and chat kind are set independently of the span context - // so stages run on this context keep them when tracing is not - // recording. + // The scope, chat kind, and accumulator are set independently of + // the span context so stages run on this context keep them when + // tracing is not recording. ctx = chatloop.ContextWithScope(ctx, chatloop.ScopeTurn) ctx = chatloop.ContextWithChatKind(ctx, t.chatKind) + ctx = chatloop.ContextWithTurnAccumulator(ctx, t.acc) if !t.spanCtx.IsValid() { return ctx } return trace.ContextWithSpanContext(ctx, t.spanCtx) } +// Complete marks the turn as finished normally, which is what makes +// its accounting emittable when the span closes. +func (t *runnerTurnSpan) Complete() { + if t == nil { + return + } + t.mu.Lock() + defer t.mu.Unlock() + t.finished = true + t.acc.MarkCompleted() +} + +// Invalidate drops the turn's accounting. A turn that errored or was +// interrupted stops partway through its stages, so its totals do not +// describe a full turn. +func (t *runnerTurnSpan) Invalidate() { + if t == nil { + return + } + t.mu.Lock() + defer t.mu.Unlock() + t.finished = true + t.acc.Invalidate() +} + +// Rotate closes the current turn and opens the next one, anchored at +// startAt. It is for a queued message promoted by the transition that +// finished the previous turn: the wait that message served and the +// work it causes belong to the turn it opens, not to the one that +// released it. +func (t *runnerTurnSpan) Rotate(ctx context.Context, startAt time.Time) { + if t == nil { + return + } + t.mu.Lock() + defer t.mu.Unlock() + if !t.started || t.ended { + return + } + t.closeLocked(nil) + t.startLocked(ctx, startAt) +} + // End closes the chat_turn span. Later calls are ignored. func (t *runnerTurnSpan) End(err error) { if t == nil { @@ -115,7 +186,20 @@ func (t *runnerTurnSpan) End(err error) { return } t.ended = true + t.closeLocked(err) +} + +// closeLocked ends the open turn span. A non-nil error also drops the +// turn's accounting, because the stages it collected stop where the +// error happened. +func (t *runnerTurnSpan) closeLocked(err error) { + if err != nil { + t.acc.Invalidate() + } t.span.End(err) + t.span = nil + t.spanCtx = trace.SpanContext{} + t.acc = nil } // chatKindAttr labels a chat as a subagent or a top-level chat. diff --git a/docs/admin/integrations/prometheus.md b/docs/admin/integrations/prometheus.md index cad41e1589e..e5662cfac1d 100644 --- a/docs/admin/integrations/prometheus.md +++ b/docs/admin/integrations/prometheus.md @@ -250,6 +250,7 @@ The `coder_ai_gateway_cost_control_*` metrics are exported only by `coderd`. | `coderd_chatd_message_count` | histogram | Number of messages in the prompt per LLM request. | `model` `provider` | | `coderd_chatd_prompt_size_bytes` | histogram | Estimated byte size of the prompt per LLM request. | `model` `provider` | | `coderd_chatd_stage_duration_seconds` | histogram | Wall time spent in each chat lifecycle stage. Stages overlap in wall time; this is a stage-time profile, not a partition of the turn. The scope label separates stages that run inside a chat turn from detached background work. The chat_kind label is empty for stages recorded without a known chat, and the model and effort labels are empty for stages that run before a model is resolved. | `chat_kind` `effort` `model` `scope` `stage` | +| `coderd_chatd_stage_share_of_turn` | histogram | Fraction of a chat turn's wall time spent in a stage, observed once per turn when the turn ends. Stages overlap, so shares can exceed 1 and do not sum to 1. Only turns that finished normally are counted. | `chat_kind` `effort` `model` `stage` | | `coderd_chatd_steps_total` | counter | Total agentic loop steps across all chats. | `model` `provider` | | `coderd_chatd_stream_buffer_dropped_total` | counter | Number of chat stream buffer events dropped due to the per-chat buffer cap. | | | `coderd_chatd_stream_retries_total` | counter | Total LLM stream retries. | `kind` `model` `provider` | @@ -257,6 +258,9 @@ The `coder_ai_gateway_cost_control_*` metrics are exported only by `coderd`. | `coderd_chatd_tool_result_size_bytes` | histogram | Size in bytes of each tool execution result. | `model` `provider` `tool_name` | | `coderd_chatd_tool_result_truncated_total` | counter | Total tool results truncated to fit the model context window. | `model` `provider` `tool_name` | | `coderd_chatd_ttft_seconds` | histogram | Time-to-first-token: wall time from LLM request to first streamed chunk. | `model` `provider` | +| `coderd_chatd_turn_stage_seconds` | histogram | Total wall time one chat turn spent in a stage, observed once per turn when the turn ends. Stages overlap, so these do not partition the turn. Only turns that finished normally are counted. | `chat_kind` `effort` `model` `stage` | +| `coderd_chatd_turn_time_seconds` | histogram | Wall time of one chat turn split into disjoint categories that sum to the turn duration, observed once per turn per category when the turn ends. Every category is observed, including the ones with no time. Only turns that finished normally are counted. | `category` `chat_kind` `effort` `model` | +| `coderd_chatd_turn_time_share` | histogram | Fraction of a chat turn's wall time in each disjoint category, observed once per turn per category when the turn ends. The shares of one turn sum to 1. Only turns that finished normally are counted. | `category` `chat_kind` `effort` `model` | | `coderd_db_query_counts_total` | counter | Total number of queries labelled by HTTP route, method, and query name. | `method` `query` `route` | | `coderd_db_query_latencies_seconds` | histogram | Latency distribution of queries in seconds. | `query` | | `coderd_db_tx_duration_seconds` | histogram | Duration of transactions in seconds. | `success` `tx_id` | diff --git a/examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md b/examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md index 19f1bd91956..3a0a4365107 100644 --- a/examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md +++ b/examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md @@ -4,7 +4,8 @@ A Grafana dashboard for diagnosing where time goes in Coder Agents chat sessions. It aggregates the stage histogram `coderd_chatd_stage_duration_seconds{stage,scope,chat_kind,model,effort}` into a stage-level flame graph with a selectable summary statistic -(mean, p50, p90, p95, p99), plus summary panels for the whole chat pipeline. +(mean, p50, p90, p95, p99), plus summary panels for the whole chat +pipeline and a per-turn partition of turn wall time. Stage hierarchy: @@ -55,14 +56,22 @@ rather than narrowing part of the hierarchy the way `$model` and `$effort` do. Background provider calls run outside a turn and carry an empty value, so that panel is never filtered by it. -Stages that run before or outside model resolution (`chat_turn`, -`queue_wait`, `capacity_wait`, `acquisition`, `mcp_connect`, `commit`) -always carry empty `model`/`effort` labels. Panels match those stages -without the `$model`/`$effort` matchers, since a matcher there could only -subtract, so narrowing either variable keeps the turn root and the waits -populated and narrows only the model-carrying stages (`generation_step`, -`prepare`, `provider_attempt`, `time_to_first_token`, `stream`, -`thinking`, `tool_call`, `compaction`). +Stages that run before a model is resolved (`queue_wait`, +`capacity_wait`, `acquisition`, `mcp_connect`, `commit`) always carry +empty `model`/`effort` labels. Panels match those stages without the +`$model`/`$effort` matchers, since a matcher there could only subtract, +so narrowing either variable keeps the waits populated and narrows only +the model-carrying stages (`chat_turn`, `generation_step`, `prepare`, +`provider_attempt`, `time_to_first_token`, `stream`, `thinking`, +`tool_call`, `compaction`). `chat_turn` is stamped with the turn's model +and effort when the turn ends, so it takes the matchers like any other +model-carrying stage. + +The turn-end metrics behind the Turn time partition row +(`coderd_chatd_turn_time_seconds`, `coderd_chatd_turn_time_share`, +`coderd_chatd_stage_share_of_turn`) are observed once per turn with the +turn's `chat_kind`, `model` and `effort` already known, so all three +variables apply to them without exception. ## Panels @@ -104,17 +113,16 @@ this is the drill-down for "when did it get slow" - a regression visible in the profile shows here as a step or trend in the affected stage. Idle stages drop out rather than plotting NaN. -**Stage time share of chat_turn** - each stage's total time as a -percentage of total `chat_turn` time, from mean rates of the histogram -sums. Dimensions: numerator is `scope="turn"` split by `stage`, with -`$chat_kind` applied and `$model`/`$effort` applied to the -model-carrying stages only; the denominator is `chat_turn` time for the -same `$chat_kind` selection, without model/effort filters, because -`chat_turn` carries empty model/effort labels. Under a concrete model -the generation stages therefore read as that model's share of all turn -time. How to read: this is the "where does the time go" summary - -stages overlap, so series can sum past 100%, but a single stage rising -toward 100% of turn time identifies the dominant cost. +**Stage share of turn ($stat)** - the selected `$stat` of each stage's +per-turn share of its turn, from `coderd_chatd_stage_share_of_turn`, +which records one fraction per stage when a turn ends. Dimensions: one +series per stage, and the chat kind, model and effort variables all +apply, +because the metric is stamped with the turn's identity at turn end. How +to read: this is the exact form of the share question - every sample +comes from one completed turn, so there is no phase skew between +numerator and denominator. Stages overlap, so the shares do not sum to +100%, and at p99 several stages can each approach the whole turn. **Queue, capacity and acquisition wait (p99)** - p99 of the three pre-generation waits: `queue_wait` (queued message insert to @@ -126,6 +134,65 @@ are not applied and the panel stays populated under any model selection. How to read: these are scheduling delays before any model work starts - user-visible latency that no provider-side optimization can fix. +### Turn time partition + +The stage hierarchy overlaps in wall time, so it can tell you which +stages are slow but not how a turn's seconds divide up. The turn-end +metrics answer that with an exclusive partition of turn wall time, +observed once per turn: + +| Category | Turn time spent | +|-----------------------|-----------------------------------------------------| +| `scheduling` | queueing, capacity admission and worker pickup | +| `time_to_first_token` | provider request open until the first streamed part | +| `streaming` | first part until the stream closes | +| `tool_execution` | local tool calls | +| `provider_error` | attempts that ended in a provider error | +| `retry_backoff` | waiting between provider attempts | +| `compaction` | auxiliary compaction calls | +| `chatd_overhead` | prompt build, persistence and other chatd work | +| `unattributed` | turn time no category claimed | + +The categories are exclusive and sum to the turn, so these panels do add +up, unlike the stage panels. `unattributed` is the completeness check: +if it grows, real turn time is happening outside every instrumented +stage. + +These per-turn histograms replaced the earlier time-share panels that +divided aggregated stage seconds by aggregated `chat_turn` seconds: the +two observations for one turn land at different times, so any aggregate +ratio mixes turns and reads well past 100%. + +**Turn time mix by model** - one 100%-stacked bar per model, each +category's total seconds over the range divided by all categories' +total seconds. Dimensions: `$chat_kind`, `$model` and `$effort` all +apply; the grouping is fixed to `model`, because Grafana transformation +options do not interpolate dashboard variables and the matrix transform +needs a static row field. How to read: the fastest way to compare where +models spend a turn, for example a model with a large +`time_to_first_token` share against one dominated by `streaming`. + +**Seconds per turn by category** - mean seconds per turn in each +category, stacked, with total turn duration as a line. Category seconds +are divided by the turn count taken from the `unattributed` category's +`_count`, since every category is observed once per turn even when it is +zero. How to read: the stack height is the mean turn duration, so the +line should sit on top of the stack; a gap means the current variable +selection dropped categories. + +**Unattributed turn time** - mean unattributed seconds per turn and its +mean share of a turn. How to read: this is the completeness check for +the stage model, so treat a rising line as an instrumentation gap rather +than a workload change. + +**Category share per turn ($stat)** - the selected `$stat` of each +category's share of a turn, from `coderd_chatd_turn_time_share`. How to +read: the mix bar shows where aggregate time goes, this shows how much a +category varies per turn, so a small mean with a large p99 marks a +bursty cost such as a slow tool call or a retry storm in a minority of +turns. Quantiles are per category, so unlike the mean shares they do not +sum to 100%. + ### Throughput and TTFT **Time to first token** - p50/p90/p99/mean of `coderd_chatd_ttft_seconds`, diff --git a/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json b/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json index 0da8e126ede..daa2fef3d72 100644 --- a/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json +++ b/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json @@ -79,7 +79,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"chat_turn\", \"\", \"\"),\n\"n\", \"01\", \"\", \"\"),\n\"level\", \"0\", \"\", \"\"),\n\"name\", \"chat_turn\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"queue_wait\", \"\", \"\"),\n\"n\", \"02\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ queue_wait\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"capacity_wait\", \"\", \"\"),\n\"n\", \"03\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ capacity_wait\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"acquisition\", \"\", \"\"),\n\"n\", \"04\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ acquisition\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"generation_step\", \"\", \"\"),\n\"n\", \"05\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ generation_step\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"prepare\", \"\", \"\"),\n\"n\", \"06\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ prepare\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"mcp_connect\", \"\", \"\"),\n\"n\", \"07\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ mcp_connect\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"provider_attempt\", \"\", \"\"),\n\"n\", \"08\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ provider_attempt\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"time_to_first_token\", \"\", \"\"),\n\"n\", \"09\", \"\", \"\"),\n\"level\", \"3\", \"\", \"\"),\n\"name\", \"│ │ └─ time_to_first_token\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"stream\", \"\", \"\"),\n\"n\", \"10\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ stream\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"thinking\", \"\", \"\"),\n\"n\", \"11\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ thinking\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"tool_call\", \"\", \"\"),\n\"n\", \"12\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ tool_call\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"commit\", \"\", \"\"),\n\"n\", \"13\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ commit\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"compaction\", \"\", \"\"),\n\"n\", \"14\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ compaction\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)", + "expr": "(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"chat_turn\", \"\", \"\"),\n\"n\", \"01\", \"\", \"\"),\n\"level\", \"0\", \"\", \"\"),\n\"name\", \"chat_turn\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"queue_wait\", \"\", \"\"),\n\"n\", \"02\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ queue_wait\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"capacity_wait\", \"\", \"\"),\n\"n\", \"03\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ capacity_wait\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"acquisition\", \"\", \"\"),\n\"n\", \"04\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ acquisition\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"generation_step\", \"\", \"\"),\n\"n\", \"05\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ generation_step\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"prepare\", \"\", \"\"),\n\"n\", \"06\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ prepare\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"mcp_connect\", \"\", \"\"),\n\"n\", \"07\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ mcp_connect\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"provider_attempt\", \"\", \"\"),\n\"n\", \"08\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ provider_attempt\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"time_to_first_token\", \"\", \"\"),\n\"n\", \"09\", \"\", \"\"),\n\"level\", \"3\", \"\", \"\"),\n\"name\", \"│ │ └─ time_to_first_token\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"stream\", \"\", \"\"),\n\"n\", \"10\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ stream\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"thinking\", \"\", \"\"),\n\"n\", \"11\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ thinking\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"tool_call\", \"\", \"\"),\n\"n\", \"12\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ tool_call\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"commit\", \"\", \"\"),\n\"n\", \"13\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ commit\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"compaction\", \"\", \"\"),\n\"n\", \"14\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ compaction\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)", "range": false, "refId": "A", "format": "table", @@ -211,7 +211,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"chat_turn\", \"\", \"\"),\n\"n\", \"01\", \"\", \"\"),\n\"level\", \"0\", \"\", \"\"),\n\"name\", \"chat_turn\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"queue_wait\", \"\", \"\"),\n\"n\", \"02\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ queue_wait\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"capacity_wait\", \"\", \"\"),\n\"n\", \"03\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ capacity_wait\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"acquisition\", \"\", \"\"),\n\"n\", \"04\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ acquisition\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"generation_step\", \"\", \"\"),\n\"n\", \"05\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ generation_step\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"prepare\", \"\", \"\"),\n\"n\", \"06\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ prepare\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"mcp_connect\", \"\", \"\"),\n\"n\", \"07\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ mcp_connect\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"provider_attempt\", \"\", \"\"),\n\"n\", \"08\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ provider_attempt\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"time_to_first_token\", \"\", \"\"),\n\"n\", \"09\", \"\", \"\"),\n\"level\", \"3\", \"\", \"\"),\n\"name\", \"│ │ └─ time_to_first_token\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"stream\", \"\", \"\"),\n\"n\", \"10\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ stream\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"thinking\", \"\", \"\"),\n\"n\", \"11\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ thinking\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"tool_call\", \"\", \"\"),\n\"n\", \"12\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ tool_call\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"commit\", \"\", \"\"),\n\"n\", \"13\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ commit\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"compaction\", \"\", \"\"),\n\"n\", \"14\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ compaction\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)", + "expr": "(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"chat_turn\", \"\", \"\"),\n\"n\", \"01\", \"\", \"\"),\n\"level\", \"0\", \"\", \"\"),\n\"name\", \"chat_turn\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"queue_wait\", \"\", \"\"),\n\"n\", \"02\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ queue_wait\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"capacity_wait\", \"\", \"\"),\n\"n\", \"03\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ capacity_wait\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"acquisition\", \"\", \"\"),\n\"n\", \"04\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ acquisition\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"generation_step\", \"\", \"\"),\n\"n\", \"05\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ generation_step\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"prepare\", \"\", \"\"),\n\"n\", \"06\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ prepare\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"mcp_connect\", \"\", \"\"),\n\"n\", \"07\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ mcp_connect\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"provider_attempt\", \"\", \"\"),\n\"n\", \"08\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ provider_attempt\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"time_to_first_token\", \"\", \"\"),\n\"n\", \"09\", \"\", \"\"),\n\"level\", \"3\", \"\", \"\"),\n\"name\", \"│ │ └─ time_to_first_token\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"stream\", \"\", \"\"),\n\"n\", \"10\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ stream\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"thinking\", \"\", \"\"),\n\"n\", \"11\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ thinking\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"tool_call\", \"\", \"\"),\n\"n\", \"12\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ tool_call\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"commit\", \"\", \"\"),\n\"n\", \"13\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ commit\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"compaction\", \"\", \"\"),\n\"n\", \"14\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ compaction\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)", "range": false, "refId": "A", "format": "table", @@ -353,7 +353,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=~\"chat_turn|queue_wait|capacity_wait|acquisition|mcp_connect|commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"chat_turn|queue_wait|capacity_wait|acquisition|mcp_connect|commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_stage_duration_seconds_sum{stage=~\"chat_turn|queue_wait|capacity_wait|acquisition|mcp_connect|commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"chat_turn|queue_wait|capacity_wait|acquisition|mcp_connect|commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=~\"queue_wait|capacity_wait|acquisition|mcp_connect|commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"queue_wait|capacity_wait|acquisition|mcp_connect|commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_stage_duration_seconds_sum{stage=~\"queue_wait|capacity_wait|acquisition|mcp_connect|commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"queue_wait|capacity_wait|acquisition|mcp_connect|commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", "range": true, "refId": "A", "legendFormat": "{{stage}}" @@ -364,7 +364,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=~\"generation_step|prepare|provider_attempt|time_to_first_token|stream|thinking|tool_call|compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"generation_step|prepare|provider_attempt|time_to_first_token|stream|thinking|tool_call|compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_stage_duration_seconds_sum{stage=~\"generation_step|prepare|provider_attempt|time_to_first_token|stream|thinking|tool_call|compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"generation_step|prepare|provider_attempt|time_to_first_token|stream|thinking|tool_call|compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=~\"chat_turn|generation_step|prepare|provider_attempt|time_to_first_token|stream|thinking|tool_call|compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"chat_turn|generation_step|prepare|provider_attempt|time_to_first_token|stream|thinking|tool_call|compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_stage_duration_seconds_sum{stage=~\"chat_turn|generation_step|prepare|provider_attempt|time_to_first_token|stream|thinking|tool_call|compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"chat_turn|generation_step|prepare|provider_attempt|time_to_first_token|stream|thinking|tool_call|compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", "range": true, "refId": "B", "legendFormat": "{{stage}}" @@ -378,7 +378,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Total time spent in each stage as a percentage of total chat_turn time, from mean rates (rate of the histogram _sum). Stages overlap, so the series can sum to more than 100%. Intervals with no chat_turn time are dropped instead of dividing by zero. The chat_turn denominator is never filtered by model or effort.\n\n$model and $effort apply only to the stages that carry those labels (generation_step, prepare, provider_attempt, time_to_first_token, stream, thinking, tool_call, compaction). The stages recorded before a model is resolved (chat_turn, queue_wait, capacity_wait, acquisition, mcp_connect, commit) are always matched without them, so they stay populated under any selection.\n\n$chat_kind selects root chats, subagent chats spawned by a parent agent, or both. It is a turn property carried by every turn-scoped stage, so unlike $model and $effort it applies to every stage in this panel.", + "description": "Total seconds each stage occupied within a turn, from the coderd_chatd_turn_stage_seconds histogram recorded once per turn when chat_turn ends. $stat selects the statistic; an idle stage drops out instead of returning NaN.\n\nThis is the absolute companion to the share panel next to it, and it differs from the stage duration panel above: that one measures single stage occurrences, this one sums every occurrence within a turn, so repeated stages (generation_step, provider_attempt, tool_call) read higher here.\n\nStages overlap in wall time, so these seconds do not add up to the turn duration. $chat_kind, $model and $effort all apply.", "fieldConfig": { "defaults": { "color": { @@ -427,17 +427,718 @@ } ] }, - "unit": "percent" + "unit": "s" }, "overrides": [] }, "gridPos": { - "h": 9, + "h": 9, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 14, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_turn_stage_seconds_bucket{chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_turn_stage_seconds_count{chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_turn_stage_seconds_sum{chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_turn_stage_seconds_count{chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "legendFormat": "{{stage}}", + "range": true, + "refId": "A" + } + ], + "title": "Stage seconds per turn (${stat:text})", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Per-turn fraction of the turn that each stage occupied, from the coderd_chatd_stage_share_of_turn histogram recorded once per turn when chat_turn ends. $stat selects the statistic: mean is rate(_sum)/rate(_count), the percentiles are histogram_quantile over the bucket rates, and an idle stage drops out instead of returning NaN.\n\nUnlike the range-totals bar, every sample here comes from a single completed turn, so there is no phase skew between numerator and denominator. Stages still overlap in wall time, so the shares do not sum to 100% and a p99 share can approach 100% for several stages at once.\n\n$chat_kind, $model and $effort all apply: this metric is recorded at turn end with the turn's model and effort stamped on it.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 25 + }, + "id": 9, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_stage_share_of_turn_bucket{chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_share_of_turn_count{chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_stage_share_of_turn_sum{chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_stage_share_of_turn_count{chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "legendFormat": "{{stage}}", + "range": true, + "refId": "A" + } + ], + "title": "Stage share of turn (${stat:text})", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "p99 of the pre-generation waits: queue_wait (queued message insert to promotion), capacity_wait (capacity limiter acquire) and acquisition (trigger message insert to Acquire applied). These are queueing signals rather than model latency. Intervals with no samples for a stage are dropped instead of returning NaN. All three stages run before a model is resolved, so $model and $effort are not applied and the panel stays populated under any selection.\n\n$chat_kind selects root chats, subagent chats spawned by a parent agent, or both. It is a turn property carried by every turn-scoped stage, so unlike $model and $effort it applies to every stage in this panel.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, stage) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=~\"queue_wait|capacity_wait|acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__rate_interval])))\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"queue_wait|capacity_wait|acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)", + "range": true, + "refId": "A", + "legendFormat": "{{stage}} p99" + } + ], + "title": "Queue, capacity and acquisition wait (p99)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 103, + "panels": [], + "title": "Turn time partition", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "How turn wall time splits across the exclusive turn-time categories, per model, over the dashboard time range. Each bar is one model and sums to 100%; the value is that category's total seconds divided by all categories' total seconds, from increase() over the range.\n\nThe categories partition the turn, so unlike the stage panels they do not overlap and do add up. unattributed is the remainder the stage instrumentation could not place; watch it as the completeness check for the model.\n\n$chat_kind, $model and $effort all apply. Grouping is fixed to model because Grafana transformation options do not interpolate dashboard variables.", + "fieldConfig": { + "defaults": { + "custom": { + "axisPlacement": "auto", + "fillOpacity": 80, + "lineWidth": 1 + }, + "max": 100, + "min": 0, + "unit": "percent" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "scheduling" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "time_to_first_token" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "streaming" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "tool_execution" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "provider_error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "retry_backoff" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "compaction" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "chatd_overhead" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "unattributed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#808080", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 35 + }, + "id": 10, + "options": { + "barRadius": 0, + "barWidth": 0.7, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "never", + "stacking": "normal", + "tooltip": { + "mode": "multi", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0, + "xField": "model\\category" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "100 * sum by (model, category) (increase(coderd_chatd_turn_time_seconds_sum{chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))\n/ on(model) group_left() (sum by (model) (increase(coderd_chatd_turn_time_seconds_sum{chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)", + "legendFormat": "", + "range": false, + "refId": "A", + "format": "table", + "instant": true + } + ], + "title": "Turn time mix by model", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "category", + "emptyValue": "zero", + "rowField": "model", + "valueField": "Value" + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": { + "model": 0, + "model\\category": 0, + "scheduling": 1, + "time_to_first_token": 2, + "streaming": 3, + "tool_execution": 4, + "provider_error": 5, + "retry_backoff": 6, + "compaction": 7, + "chatd_overhead": 8, + "unattributed": 9 + }, + "renameByName": {} + } + } + ], + "type": "barchart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Mean seconds per turn spent in each turn-time category, stacked, plus the total turn duration as a line. Each category's seconds are divided by the turn count, taken from the unattributed category's _count because every category is observed once per turn, including when it is zero.\n\nBecause the categories are exclusive, the stack height equals the mean turn duration and the line should sit on top of the stack. A gap between them means categories were dropped by the $model, $effort or $chat_kind selection rather than by the partition.\n\nIntervals with no completed turns are dropped instead of dividing by zero.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 60, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "scheduling" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "time_to_first_token" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "streaming" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "tool_execution" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "provider_error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "retry_backoff" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "compaction" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "chatd_overhead" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "unattributed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#808080", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "total per turn" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": false, + "mode": "none" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineWidth", + "value": 2 + }, + { + "id": "color", + "value": { + "fixedColor": "text", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, "w": 12, "x": 12, - "y": 16 + "y": 35 }, - "id": 4, + "id": 11, "options": { "legend": { "calcs": [ @@ -460,10 +1161,10 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "100 * sum by (stage) (rate(coderd_chatd_stage_duration_seconds_sum{stage=~\"queue_wait|capacity_wait|acquisition|mcp_connect|commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__rate_interval]))\n/ on() group_left() (sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)", + "expr": "sum by (category) (rate(coderd_chatd_turn_time_seconds_sum{chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval]))\n/ on() group_left() (sum(rate(coderd_chatd_turn_time_seconds_count{category=\"unattributed\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) > 0)", + "legendFormat": "{{category}}", "range": true, - "refId": "A", - "legendFormat": "{{stage}}" + "refId": "A" }, { "datasource": { @@ -471,13 +1172,13 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "100 * sum by (stage) (rate(coderd_chatd_stage_duration_seconds_sum{stage=~\"generation_step|prepare|provider_attempt|time_to_first_token|stream|thinking|tool_call|compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval]))\n/ on() group_left() (sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)", + "expr": "sum(rate(coderd_chatd_turn_time_seconds_sum{chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval]))\n/ on() group_left() (sum(rate(coderd_chatd_turn_time_seconds_count{category=\"unattributed\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) > 0)", + "legendFormat": "total per turn", "range": true, - "refId": "B", - "legendFormat": "{{stage}}" + "refId": "B" } ], - "title": "Stage time share of chat_turn", + "title": "Seconds per turn by category", "type": "timeseries" }, { @@ -485,7 +1186,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "p99 of the pre-generation waits: queue_wait (queued message insert to promotion), capacity_wait (capacity limiter acquire) and acquisition (trigger message insert to Acquire applied). These are queueing signals rather than model latency. Intervals with no samples for a stage are dropped instead of returning NaN. All three stages run before a model is resolved, so $model and $effort are not applied and the panel stays populated under any selection.\n\n$chat_kind selects root chats, subagent chats spawned by a parent agent, or both. It is a turn property carried by every turn-scoped stage, so unlike $model and $effort it applies to every stage in this panel.", + "description": "Turn time the stage instrumentation could not place in any category: mean unattributed seconds per turn on the left axis and the mean unattributed share of a turn on the right.\n\nThis is the completeness check for the stage model. Near zero means the categories account for the turn. A rising line means real turn time is happening outside every instrumented stage, so the profile and partition panels are understating something; treat it as a bug in the instrumentation rather than as a workload change.\n\n$chat_kind, $model and $effort all apply.", "fieldConfig": { "defaults": { "color": { @@ -499,7 +1200,7 @@ "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 10, + "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, @@ -536,15 +1237,39 @@ }, "unit": "s" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "share of turn" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "color", + "value": { + "fixedColor": "#808080", + "mode": "fixed" + } + } + ] + } + ] }, "gridPos": { - "h": 9, - "w": 24, + "h": 8, + "w": 8, "x": 0, - "y": 25 + "y": 45 }, - "id": 5, + "id": 12, "options": { "legend": { "calcs": [ @@ -567,13 +1292,256 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (le, stage) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=~\"queue_wait|capacity_wait|acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__rate_interval])))\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"queue_wait|capacity_wait|acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)", + "expr": "sum(rate(coderd_chatd_turn_time_seconds_sum{category=\"unattributed\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval]))\n/ (sum(rate(coderd_chatd_turn_time_seconds_count{category=\"unattributed\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) > 0)", + "legendFormat": "seconds per turn", "range": true, - "refId": "A", - "legendFormat": "{{stage}} p99" + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(coderd_chatd_turn_time_share_sum{category=\"unattributed\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval]))\n/ (sum(rate(coderd_chatd_turn_time_share_count{category=\"unattributed\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) > 0)", + "legendFormat": "share of turn", + "range": true, + "refId": "B" } ], - "title": "Queue, capacity and acquisition wait (p99)", + "title": "Unattributed turn time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Distribution of each category's share of a turn, from the coderd_chatd_turn_time_share histogram recorded once per turn. $stat selects the statistic, so p99 reads as \"in the worst turns, this category took this fraction of the turn\".\n\nUse it next to the mix bar: the bar shows where aggregate time goes, this shows how much a category varies per turn. A category with a small mean and a large p99 is bursty (a slow tool call or a retry storm in a minority of turns) rather than a steady cost.\n\nQuantiles are per category, so unlike the mean shares they do not sum to 100%. $chat_kind, $model and $effort all apply.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "scheduling" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "time_to_first_token" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "streaming" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "tool_execution" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "provider_error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "retry_backoff" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "compaction" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "chatd_overhead" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "unattributed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#808080", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 16, + "x": 8, + "y": 45 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n histogram_quantile($stat, sum by (le, category) (rate(coderd_chatd_turn_time_share_bucket{chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(category) (sum by (category) (rate(coderd_chatd_turn_time_share_count{chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (category) (rate(coderd_chatd_turn_time_share_sum{chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) / (sum by (category) (rate(coderd_chatd_turn_time_share_count{chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "legendFormat": "{{category}}", + "range": true, + "refId": "A" + } + ], + "title": "Category share per turn (${stat:text})", "type": "timeseries" }, { @@ -582,7 +1550,7 @@ "h": 1, "w": 24, "x": 0, - "y": 34 + "y": 53 }, "id": 102, "panels": [], @@ -651,7 +1619,7 @@ "h": 9, "w": 8, "x": 0, - "y": 35 + "y": 54 }, "id": 6, "options": { @@ -780,7 +1748,7 @@ "h": 9, "w": 8, "x": 8, - "y": 35 + "y": 54 }, "id": 7, "options": { @@ -805,7 +1773,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__rate_interval]))", + "expr": "sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval]))", "range": true, "refId": "A", "legendFormat": "chat turns" @@ -915,7 +1883,7 @@ "h": 9, "w": 8, "x": 16, - "y": 35 + "y": 54 }, "id": 8, "options": { diff --git a/scripts/metricsdocgen/generated_metrics b/scripts/metricsdocgen/generated_metrics index 7efd67505e1..3728d6279a8 100644 --- a/scripts/metricsdocgen/generated_metrics +++ b/scripts/metricsdocgen/generated_metrics @@ -322,6 +322,9 @@ coderd_chatd_prompt_size_bytes{provider="",model=""} 0 # HELP coderd_chatd_stage_duration_seconds Wall time spent in each chat lifecycle stage. Stages overlap in wall time; this is a stage-time profile, not a partition of the turn. The scope label separates stages that run inside a chat turn from detached background work. The chat_kind label is empty for stages recorded without a known chat, and the model and effort labels are empty for stages that run before a model is resolved. # TYPE coderd_chatd_stage_duration_seconds histogram coderd_chatd_stage_duration_seconds{stage="",scope="",chat_kind="",model="",effort=""} 0 +# HELP coderd_chatd_stage_share_of_turn Fraction of a chat turn's wall time spent in a stage, observed once per turn when the turn ends. Stages overlap, so shares can exceed 1 and do not sum to 1. Only turns that finished normally are counted. +# TYPE coderd_chatd_stage_share_of_turn histogram +coderd_chatd_stage_share_of_turn{stage="",chat_kind="",model="",effort=""} 0 # HELP coderd_chatd_steps_total Total agentic loop steps across all chats. # TYPE coderd_chatd_steps_total counter coderd_chatd_steps_total{provider="",model=""} 0 @@ -343,6 +346,15 @@ coderd_chatd_tool_result_truncated_total{provider="",model="",tool_name=""} 0 # HELP coderd_chatd_ttft_seconds Time-to-first-token: wall time from LLM request to first streamed chunk. # TYPE coderd_chatd_ttft_seconds histogram coderd_chatd_ttft_seconds{provider="",model=""} 0 +# HELP coderd_chatd_turn_stage_seconds Total wall time one chat turn spent in a stage, observed once per turn when the turn ends. Stages overlap, so these do not partition the turn. Only turns that finished normally are counted. +# TYPE coderd_chatd_turn_stage_seconds histogram +coderd_chatd_turn_stage_seconds{stage="",chat_kind="",model="",effort=""} 0 +# HELP coderd_chatd_turn_time_seconds Wall time of one chat turn split into disjoint categories that sum to the turn duration, observed once per turn per category when the turn ends. Every category is observed, including the ones with no time. Only turns that finished normally are counted. +# TYPE coderd_chatd_turn_time_seconds histogram +coderd_chatd_turn_time_seconds{category="",chat_kind="",model="",effort=""} 0 +# HELP coderd_chatd_turn_time_share Fraction of a chat turn's wall time in each disjoint category, observed once per turn per category when the turn ends. The shares of one turn sum to 1. Only turns that finished normally are counted. +# TYPE coderd_chatd_turn_time_share histogram +coderd_chatd_turn_time_share{category="",chat_kind="",model="",effort=""} 0 # HELP coderd_db_query_counts_total Total number of queries labelled by HTTP route, method, and query name. # TYPE coderd_db_query_counts_total counter coderd_db_query_counts_total{route="",method="",query=""} 0 From f12789006b27adb27b76f6c33d1a91728e4a8698 Mon Sep 17 00:00:00 2001 From: Scott Miller Date: Thu, 3 Sep 2026 17:36:07 +0000 Subject: [PATCH 05/19] feat: organize chat lifecycle dashboard by stage tree level Add coderd_chatd_turn_stage_count, observed per stage per completed turn with the number of occurrences of that stage in the turn, alongside the per-turn seconds and share. Replace the trend panels that mixed nesting levels and units of observation (one sample per turn beside one sample per occurrence) with one row per tree level: turn, turn children, step children, and stream children, each showing duration per occurrence, seconds per turn, occurrences per turn, and share of turn for that level's stages only. The flamegraph remains the deliberate cross-level view. --- coderd/x/chatd/chatloop/metrics.go | 25 +- coderd/x/chatd/chatloop/turnaccounting.go | 52 +- .../chatloop/turnaccounting_internal_test.go | 18 + docs/admin/integrations/prometheus.md | 1 + .../grafana/chatd-lifecycle/README.md | 172 +- .../grafana/chatd-lifecycle/dashboard.json | 1778 +++++++++++++---- scripts/metricsdocgen/generated_metrics | 3 + 7 files changed, 1577 insertions(+), 472 deletions(-) diff --git a/coderd/x/chatd/chatloop/metrics.go b/coderd/x/chatd/chatloop/metrics.go index 97ff000ee36..7eb3281a28b 100644 --- a/coderd/x/chatd/chatloop/metrics.go +++ b/coderd/x/chatd/chatloop/metrics.go @@ -37,6 +37,7 @@ type Metrics struct { TTFTSeconds *prometheus.HistogramVec StageDurationSeconds *prometheus.HistogramVec TurnStageSeconds *prometheus.HistogramVec + TurnStageCount *prometheus.HistogramVec StageShareOfTurn *prometheus.HistogramVec TurnTimeSeconds *prometheus.HistogramVec TurnTimeShare *prometheus.HistogramVec @@ -115,6 +116,13 @@ func NewMetrics(reg prometheus.Registerer) *Metrics { Help: "Total wall time one chat turn spent in a stage, observed once per turn when the turn ends. Stages overlap, so these do not partition the turn. Only turns that finished normally are counted.", Buckets: stageDurationBuckets(), }, []string{"stage", "chat_kind", "model", "effort"}), + TurnStageCount: factory.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "turn_stage_count", + Help: "Number of times a stage occurred within one chat turn, observed once per turn per stage that occurred. Only turns that finished normally are counted.", + Buckets: turnStageCountBuckets(), + }, []string{"stage", "chat_kind", "model", "effort"}), StageShareOfTurn: factory.NewHistogramVec(prometheus.HistogramOpts{ Namespace: metricsNamespace, Subsystem: metricsSubsystem, @@ -201,6 +209,12 @@ func turnShareBuckets() []float64 { return prometheus.LinearBuckets(0, 0.05, 21) } +// turnStageCountBuckets returns the buckets for per-turn stage counts: +// every small count, widening to 128. +func turnStageCountBuckets() []float64 { + return []float64{1, 2, 3, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128} +} + // NopMetrics returns a Metrics instance that discards all data. // Useful for tests and when metrics collection is not desired. func NopMetrics() *Metrics { @@ -218,16 +232,17 @@ func (m *Metrics) RecordStageDuration(stage, scope, chatKind, model, effort stri m.StageDurationSeconds.WithLabelValues(stage, scope, chatKind, model, effort).Observe(elapsed.Seconds()) } -// RecordTurnStage observes the total time one turn spent in a stage -// and that time as a fraction of the turn. Both come from the same -// turn so the pair cannot describe different turns. No-op when m is -// nil. -func (m *Metrics) RecordTurnStage(stage, chatKind, model, effort string, elapsed time.Duration, share float64) { +// RecordTurnStage observes the total time one turn spent in a stage, +// that time as a fraction of the turn, and how many times the stage +// occurred. All three come from the same turn so they cannot describe +// different turns. No-op when m is nil. +func (m *Metrics) RecordTurnStage(stage, chatKind, model, effort string, elapsed time.Duration, share float64, count int) { if m == nil || elapsed < 0 { return } m.TurnStageSeconds.WithLabelValues(stage, chatKind, model, effort).Observe(elapsed.Seconds()) m.StageShareOfTurn.WithLabelValues(stage, chatKind, model, effort).Observe(share) + m.TurnStageCount.WithLabelValues(stage, chatKind, model, effort).Observe(float64(count)) } // RecordTurnCategory observes one category of a turn's time partition diff --git a/coderd/x/chatd/chatloop/turnaccounting.go b/coderd/x/chatd/chatloop/turnaccounting.go index 524d3ff0868..91ed358ec50 100644 --- a/coderd/x/chatd/chatloop/turnaccounting.go +++ b/coderd/x/chatd/chatloop/turnaccounting.go @@ -96,12 +96,13 @@ func stageNodeFromContext(ctx context.Context) *stageNode { // It is safe for concurrent use: parallel tool calls and the stages // under them end on different goroutines. type TurnAccumulator struct { - mu sync.Mutex - stages map[string]time.Duration - categories map[string]time.Duration - model StageModel - completed bool - invalid bool + mu sync.Mutex + stages map[string]time.Duration + stageCounts map[string]int + categories map[string]time.Duration + model StageModel + completed bool + invalid bool } // NewTurnAccumulator returns an accumulator for one turn. The turn is @@ -109,18 +110,22 @@ type TurnAccumulator struct { // reaches its finish transition emits nothing. func NewTurnAccumulator() *TurnAccumulator { return &TurnAccumulator{ - stages: map[string]time.Duration{}, - categories: map[string]time.Duration{}, + stages: map[string]time.Duration{}, + stageCounts: map[string]int{}, + categories: map[string]time.Duration{}, } } +// addStage records one occurrence of a stage and the time it took. A +// stage that took no measurable time still counts as an occurrence. func (a *TurnAccumulator) addStage(stage string, elapsed time.Duration) { - if a == nil || elapsed <= 0 { + if a == nil || elapsed < 0 { return } a.mu.Lock() defer a.mu.Unlock() a.stages[stage] += elapsed + a.stageCounts[stage]++ } func (a *TurnAccumulator) addCategory(category string, elapsed time.Duration) { @@ -182,10 +187,11 @@ func (a *TurnAccumulator) Invalidate() { // turnAccounting is the emittable state of one turn. type turnAccounting struct { - stages map[string]time.Duration - categories map[string]time.Duration - model StageModel - emit bool + stages map[string]time.Duration + stageCounts map[string]int + categories map[string]time.Duration + model StageModel + emit bool } func (a *TurnAccumulator) snapshot() turnAccounting { @@ -195,14 +201,18 @@ func (a *TurnAccumulator) snapshot() turnAccounting { a.mu.Lock() defer a.mu.Unlock() snapshot := turnAccounting{ - stages: make(map[string]time.Duration, len(a.stages)), - categories: make(map[string]time.Duration, len(a.categories)), - model: a.model, - emit: a.completed && !a.invalid, + stages: make(map[string]time.Duration, len(a.stages)), + stageCounts: make(map[string]int, len(a.stageCounts)), + categories: make(map[string]time.Duration, len(a.categories)), + model: a.model, + emit: a.completed && !a.invalid, } for stage, elapsed := range a.stages { snapshot.stages[stage] = elapsed } + for stage, count := range a.stageCounts { + snapshot.stageCounts[stage] = count + } for category, elapsed := range a.categories { snapshot.categories[category] = elapsed } @@ -364,11 +374,13 @@ func (t *StageTracer) emitTurnAccounting(acc *TurnAccumulator, chatKind string, } turnSeconds := turnDuration.Seconds() model := snapshot.model - for stage, elapsed := range snapshot.stages { - if elapsed <= 0 { + for stage, count := range snapshot.stageCounts { + if count == 0 { continue } - t.metrics.RecordTurnStage(stage, chatKind, model.Model, model.Effort, elapsed, elapsed.Seconds()/turnSeconds) + elapsed := snapshot.stages[stage] + t.metrics.RecordTurnStage(stage, chatKind, model.Model, model.Effort, + elapsed, elapsed.Seconds()/turnSeconds, count) } var attributed time.Duration for _, category := range TurnTimeCategories { diff --git a/coderd/x/chatd/chatloop/turnaccounting_internal_test.go b/coderd/x/chatd/chatloop/turnaccounting_internal_test.go index 19af7c19e93..f3eb19af8f8 100644 --- a/coderd/x/chatd/chatloop/turnaccounting_internal_test.go +++ b/coderd/x/chatd/chatloop/turnaccounting_internal_test.go @@ -234,6 +234,24 @@ func TestTurnAccountingStageTotals(t *testing.T) { require.InDelta(t, seconds/turnDuration.Seconds(), shares[stage], 0.001, stage) } + // One observation per stage per turn, valued at how many times the + // stage occurred. + counts := fixture.sums(t, "coderd_chatd_turn_stage_count", "stage") + require.Equal(t, map[string]float64{ + StageQueueWait: 1, + StageGenerationStep: 4, + StagePrepare: 2, + StageMCPConnect: 1, + StageStream: 2, + StageTimeToFirstToken: 1, + StageCommit: 2, + StageRetryBackoff: 1, + StageCompaction: 1, + }, counts) + for stage := range counts { + require.Contains(t, stages, stage) + } + // The turn's stage rows carry the model the turn resolved. labels := fixture.labelsOf(t, "coderd_chatd_turn_stage_seconds", "stage", StageStream) require.Equal(t, model.Model, labels["model"]) diff --git a/docs/admin/integrations/prometheus.md b/docs/admin/integrations/prometheus.md index e5662cfac1d..fb7ee8928dd 100644 --- a/docs/admin/integrations/prometheus.md +++ b/docs/admin/integrations/prometheus.md @@ -258,6 +258,7 @@ The `coder_ai_gateway_cost_control_*` metrics are exported only by `coderd`. | `coderd_chatd_tool_result_size_bytes` | histogram | Size in bytes of each tool execution result. | `model` `provider` `tool_name` | | `coderd_chatd_tool_result_truncated_total` | counter | Total tool results truncated to fit the model context window. | `model` `provider` `tool_name` | | `coderd_chatd_ttft_seconds` | histogram | Time-to-first-token: wall time from LLM request to first streamed chunk. | `model` `provider` | +| `coderd_chatd_turn_stage_count` | histogram | Number of times a stage occurred within one chat turn, observed once per turn per stage that occurred. Only turns that finished normally are counted. | `chat_kind` `effort` `model` `stage` | | `coderd_chatd_turn_stage_seconds` | histogram | Total wall time one chat turn spent in a stage, observed once per turn when the turn ends. Stages overlap, so these do not partition the turn. Only turns that finished normally are counted. | `chat_kind` `effort` `model` `stage` | | `coderd_chatd_turn_time_seconds` | histogram | Wall time of one chat turn split into disjoint categories that sum to the turn duration, observed once per turn per category when the turn ends. Every category is observed, including the ones with no time. Only turns that finished normally are counted. | `category` `chat_kind` `effort` `model` | | `coderd_chatd_turn_time_share` | histogram | Fraction of a chat turn's wall time in each disjoint category, observed once per turn per category when the turn ends. The shares of one turn sum to 1. Only turns that finished normally are counted. | `category` `chat_kind` `effort` `model` | diff --git a/examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md b/examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md index 3a0a4365107..cd69d37f293 100644 --- a/examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md +++ b/examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md @@ -7,23 +7,24 @@ into a stage-level flame graph with a selectable summary statistic (mean, p50, p90, p95, p99), plus summary panels for the whole chat pipeline and a per-turn partition of turn wall time. -Stage hierarchy: +Stage hierarchy, by tree level: ```text -chat_turn -├── queue_wait queued message insert -> promotion -├── capacity_wait concurrent-agent limiter wait -├── acquisition trigger message -> worker pickup -└── generation_step one step of a turn (repeats) - ├── prepare prompt build, model resolution, context hydration - ├── mcp_connect MCP server connection - ├── provider_attempt one provider HTTP round trip (per retry) - │ └── time_to_first_token - ├── stream provider stream open -> close - ├── thinking reasoning part duration - ├── tool_call one local tool call - ├── commit step persistence transaction - └── compaction auxiliary compaction call +L0 chat_turn one sample per turn +L1 ├── queue_wait queued message insert -> promotion +L1 ├── capacity_wait concurrent-agent limiter wait +L1 ├── acquisition trigger message -> worker pickup +L1 └── generation_step one step of a turn (repeats) +L2 ├── prepare prompt build, model resolution, context hydration +L2 ├── mcp_connect MCP server connection +L2 ├── provider_attempt one provider HTTP round trip (per retry) +L3 │ └── time_to_first_token request open -> first streamed part +L2 ├── stream provider stream open -> close +L2 ├── thinking reasoning part duration +L2 ├── tool_call one local tool call +L2 ├── commit step persistence transaction +L2 ├── compaction auxiliary compaction call +L2 └── retry_backoff wait between provider attempts ``` Stages overlap in wall time (tool calls and thinking happen inside the @@ -37,7 +38,7 @@ where noted: | Label | Values | Dashboard variable | |-------------|-----------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------| -| `stage` | the 14 stage names above | none (fixed hierarchy) | +| `stage` | the 15 stage names above | none (fixed hierarchy) | | `scope` | `turn` (part of a chat turn) or `background` (detached async work such as title and summary generation) | none (panels pin one scope) | | `chat_kind` | `root` (a chat a user drives) or `subagent` (a chat spawned by a parent agent), empty for background work | `$chat_kind` (multi-select) | | `model` | resolved model ID, empty before a model is resolved | `$model` (multi-select) | @@ -57,19 +58,22 @@ rather than narrowing part of the hierarchy the way `$model` and empty value, so that panel is never filtered by it. Stages that run before a model is resolved (`queue_wait`, -`capacity_wait`, `acquisition`, `mcp_connect`, `commit`) always carry -empty `model`/`effort` labels. Panels match those stages without the -`$model`/`$effort` matchers, since a matcher there could only subtract, -so narrowing either variable keeps the waits populated and narrows only -the model-carrying stages (`chat_turn`, `generation_step`, `prepare`, -`provider_attempt`, `time_to_first_token`, `stream`, `thinking`, -`tool_call`, `compaction`). `chat_turn` is stamped with the turn's model -and effort when the turn ends, so it takes the matchers like any other -model-carrying stage. - -The turn-end metrics behind the Turn time partition row -(`coderd_chatd_turn_time_seconds`, `coderd_chatd_turn_time_share`, -`coderd_chatd_stage_share_of_turn`) are observed once per turn with the +`capacity_wait`, `acquisition`, `mcp_connect`, `commit`, +`retry_backoff`) always carry empty `model`/`effort` labels on +`coderd_chatd_stage_duration_seconds`. Panels match those stages without +the `$model`/`$effort` matchers, since a matcher there could only +subtract, so narrowing either variable keeps the waits populated and +narrows only the model-carrying stages (`chat_turn`, `generation_step`, +`prepare`, `provider_attempt`, `time_to_first_token`, `stream`, +`thinking`, `tool_call`, `compaction`). `chat_turn` is stamped with the +turn's model and effort when the turn ends, so it takes the matchers +like any other model-carrying stage. The exception applies only to the +per-occurrence metric; the turn-end metrics stamp every stage. + +The turn-end metrics behind the level rows and the Turn time partition +row (`coderd_chatd_turn_stage_seconds`, `coderd_chatd_turn_stage_count`, +`coderd_chatd_stage_share_of_turn`, `coderd_chatd_turn_time_seconds`, +`coderd_chatd_turn_time_share`) are observed once per turn with the turn's `chat_kind`, `model` and `effort` already known, so all three variables apply to them without exception. @@ -102,39 +106,50 @@ it to read stages too small to see as frames (prepare, commit, tool_call are typically milliseconds next to multi-second streams) and as a numeric check on the flamegraph. -### Stage trends - -**Stage duration over time ($stat)** - one series per stage, the -selected `$stat` computed over `$__rate_interval`. Dimensions: the -turn scope and `$chat_kind`, one series per stage; -`$model`/`$effort` apply to the model-carrying stages only, so the -pre-model stages stay plotted under any model selection. How to read: -this is the drill-down for "when did it get slow" - a regression visible -in the profile shows here as a step or trend in the affected stage. Idle -stages drop out rather than plotting NaN. - -**Stage share of turn ($stat)** - the selected `$stat` of each stage's -per-turn share of its turn, from `coderd_chatd_stage_share_of_turn`, -which records one fraction per stage when a turn ends. Dimensions: one -series per stage, and the chat kind, model and effort variables all -apply, -because the metric is stamped with the turn's identity at turn end. How -to read: this is the exact form of the share question - every sample -comes from one completed turn, so there is no phase skew between -numerator and denominator. Stages overlap, so the shares do not sum to -100%, and at p99 several stages can each approach the whole turn. - -**Queue, capacity and acquisition wait (p99)** - p99 of the three -pre-generation waits: `queue_wait` (queued message insert to -promotion), `capacity_wait` (concurrent-agent limiter admission) and -`acquisition` (trigger message insert to worker pickup). Dimensions: -`scope="turn"` and `$chat_kind`, fixed to those three stages, split by -`stage`; all three run before a model is resolved, so `$model`/`$effort` -are not applied and the panel stays populated under any model selection. -How to read: these are scheduling delays before any model work starts - -user-visible latency that no provider-side optimization can fix. - -### Turn time partition +### Reading the levels + +The stage tree mixes two units of observation. `chat_turn` is recorded +once per turn, while its descendants are recorded once per occurrence, +and a step-level stage such as `stream` or `tool_call` typically occurs +six to twelve times in a turn. Plotting both on one axis compares a +25-second turn against a 2-second stream and tells you nothing, so the +trend panels are grouped by tree level and every level reads the same +four ways: + +| Panel | Metric | Question | +|-------------------------|---------------------------------------|------------------------------------------| +| Duration per occurrence | `coderd_chatd_stage_duration_seconds` | how long does one of these take | +| Seconds per turn | `coderd_chatd_turn_stage_seconds` | how much of a turn does it add up to | +| Occurrences per turn | `coderd_chatd_turn_stage_count` | how often does it happen in a turn | +| Share of turn | `coderd_chatd_stage_share_of_turn` | what fraction of the turn does it occupy | + +The first is per occurrence, the other three are per turn, recorded when +the turn ends. Seconds per turn is roughly occurrences per turn times +duration per occurrence, so the three together separate "each one is +slow" from "it happens too often". Stages still overlap within a level, +so shares and seconds at one level do not sum to the turn; the Turn time +partition row is the view that does add up. + +The Stage profile row above is the one deliberate exception: the flame +graph and its bar chart show all levels together, because a profile is +about relative width across the tree rather than about trends. + +### Level 0: Turn + +Members: `chat_turn`. + +**Turn duration ($stat)** - wall time of a whole turn, split by model. +This is the denominator every other level is measured against, and the +model split keeps a slow model from hiding inside a blended line. + +**Turns per minute** - completed turns per minute, split by model. Read +it beside turn duration: duration moving with flat throughput is a +latency regression, both moving together is usually a workload change. + +**Turn time mix by model** - the exclusive category partition of a turn +per model, described in the next section. + +### Level 0: Turn time partition The stage hierarchy overlaps in wall time, so it can tell you which stages are slow but not how a turn's seconds divide up. The turn-end @@ -163,7 +178,8 @@ divided aggregated stage seconds by aggregated `chat_turn` seconds: the two observations for one turn land at different times, so any aggregate ratio mixes turns and reads well past 100%. -**Turn time mix by model** - one 100%-stacked bar per model, each +**Turn time mix by model** - shown in the Level 0 row above, one +100%-stacked bar per model, each category's total seconds over the range divided by all categories' total seconds. Dimensions: `$chat_kind`, `$model` and `$effort` all apply; the grouping is fixed to `model`, because Grafana transformation @@ -193,6 +209,36 @@ bursty cost such as a slow tool call or a retry storm in a minority of turns. Quantiles are per category, so unlike the mean shares they do not sum to 100%. +### Level 1: Turn children + +Members: `acquisition`, `queue_wait`, `capacity_wait`, +`generation_step`. The three scheduling waits happen once per turn +before generation starts; `generation_step` repeats once per step. + +Because these are the direct children of the turn, their share panel is +the quickest answer to "was this turn slow because of scheduling or +because of generation". + +### Level 2: Step children + +Members: `prepare`, `mcp_connect`, `provider_attempt`, `stream`, +`thinking`, `tool_call`, `commit`, `compaction`, `retry_backoff`. + +These are the stages inside one generation step and they overlap each +other, so read them as a profile of the step. `provider_attempt` is +listed here rather than at level 3: in a trace it wraps the stream, but +it measures one full provider round trip, so it belongs beside `stream` +at step level. It appears at one level only, so the occurrence counts +stay comparable within the row. + +### Level 3: Stream children + +Members: `time_to_first_token`. + +The part of a provider round trip before the first token. It keeps the +same four panels as the other levels so the rows compare directly, even +though the level has a single member. + ### Throughput and TTFT **Time to first token** - p50/p90/p99/mean of `coderd_chatd_ttft_seconds`, diff --git a/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json b/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json index daa2fef3d72..0a2047e6ad3 100644 --- a/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json +++ b/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json @@ -261,9 +261,9 @@ "x": 0, "y": 15 }, - "id": 101, + "id": 104, "panels": [], - "title": "Stage trends", + "title": "Level 0: Turn", "type": "row" }, { @@ -271,7 +271,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Selected $stat of each lifecycle stage over time, one series per stage. Mean is rate(_sum)/rate(_count); the percentiles are histogram_quantile over the bucket rates. A stage with no samples in an interval has no point rather than NaN.\n\n$model and $effort apply only to the stages that carry those labels (generation_step, prepare, provider_attempt, time_to_first_token, stream, thinking, tool_call, compaction). The stages recorded before a model is resolved (chat_turn, queue_wait, capacity_wait, acquisition, mcp_connect, commit) are always matched without them, so they stay populated under any selection.\n\n$chat_kind selects root chats, subagent chats spawned by a parent agent, or both. It is a turn property carried by every turn-scoped stage, so unlike $model and $effort it applies to every stage in this panel.", + "description": "Wall time of a whole chat turn, from the chat_turn stage, split by model.\n\nOne sample per turn, so this is the top of the tree and the denominator every other level is measured against. Splitting by model keeps a slow model from hiding inside a blended line; $model, $effort and $chat_kind still narrow which turns are counted.\n\nMean is rate(_sum)/rate(_count); percentiles are histogram_quantile over the bucket rates. Idle models drop out instead of returning NaN.", "fieldConfig": { "defaults": { "color": { @@ -325,119 +325,12 @@ "overrides": [] }, "gridPos": { - "h": 9, - "w": 12, + "h": 10, + "w": 8, "x": 0, "y": 16 }, - "id": 3, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=~\"queue_wait|capacity_wait|acquisition|mcp_connect|commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"queue_wait|capacity_wait|acquisition|mcp_connect|commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_stage_duration_seconds_sum{stage=~\"queue_wait|capacity_wait|acquisition|mcp_connect|commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"queue_wait|capacity_wait|acquisition|mcp_connect|commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", - "range": true, - "refId": "A", - "legendFormat": "{{stage}}" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=~\"chat_turn|generation_step|prepare|provider_attempt|time_to_first_token|stream|thinking|tool_call|compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"chat_turn|generation_step|prepare|provider_attempt|time_to_first_token|stream|thinking|tool_call|compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_stage_duration_seconds_sum{stage=~\"chat_turn|generation_step|prepare|provider_attempt|time_to_first_token|stream|thinking|tool_call|compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"chat_turn|generation_step|prepare|provider_attempt|time_to_first_token|stream|thinking|tool_call|compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", - "range": true, - "refId": "B", - "legendFormat": "{{stage}}" - } - ], - "title": "Stage duration over time (${stat:text})", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Total seconds each stage occupied within a turn, from the coderd_chatd_turn_stage_seconds histogram recorded once per turn when chat_turn ends. $stat selects the statistic; an idle stage drops out instead of returning NaN.\n\nThis is the absolute companion to the share panel next to it, and it differs from the stage duration panel above: that one measures single stage occurrences, this one sums every occurrence within a turn, so repeated stages (generation_step, provider_attempt, tool_call) read higher here.\n\nStages overlap in wall time, so these seconds do not add up to the turn duration. $chat_kind, $model and $effort all apply.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 14, + "id": 15, "options": { "legend": { "calcs": [ @@ -460,13 +353,13 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_turn_stage_seconds_bucket{chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_turn_stage_seconds_count{chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_turn_stage_seconds_sum{chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_turn_stage_seconds_count{chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", - "legendFormat": "{{stage}}", + "expr": "(\n histogram_quantile($stat, sum by (le, model) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"chat_turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\", scope=\"turn\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(model) (sum by (model) (rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\", scope=\"turn\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (model) (rate(coderd_chatd_stage_duration_seconds_sum{stage=\"chat_turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\", scope=\"turn\"}[$__rate_interval])) / (sum by (model) (rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\", scope=\"turn\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "legendFormat": "{{model}}", "range": true, "refId": "A" } ], - "title": "Stage seconds per turn (${stat:text})", + "title": "Turn duration (${stat:text})", "type": "timeseries" }, { @@ -474,7 +367,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Per-turn fraction of the turn that each stage occupied, from the coderd_chatd_stage_share_of_turn histogram recorded once per turn when chat_turn ends. $stat selects the statistic: mean is rate(_sum)/rate(_count), the percentiles are histogram_quantile over the bucket rates, and an idle stage drops out instead of returning NaN.\n\nUnlike the range-totals bar, every sample here comes from a single completed turn, so there is no phase skew between numerator and denominator. Stages still overlap in wall time, so the shares do not sum to 100% and a p99 share can approach 100% for several stages at once.\n\n$chat_kind, $model and $effort all apply: this metric is recorded at turn end with the turn's model and effort stamped on it.", + "description": "Completed chat turns per minute, from the chat_turn sample count, split by model.\n\nRead it next to turn duration: a duration change with flat throughput is a latency regression, while both moving together usually means the workload changed. Only turns that finished are counted.", "fieldConfig": { "defaults": { "color": { @@ -523,17 +416,17 @@ } ] }, - "unit": "percentunit" + "unit": "opm" }, "overrides": [] }, "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 25 + "h": 10, + "w": 8, + "x": 8, + "y": 16 }, - "id": 9, + "id": 16, "options": { "legend": { "calcs": [ @@ -556,124 +449,15 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_stage_share_of_turn_bucket{chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_share_of_turn_count{chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_stage_share_of_turn_sum{chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_stage_share_of_turn_count{chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", - "legendFormat": "{{stage}}", + "expr": "60 * sum by (model) (rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\", scope=\"turn\"}[$__rate_interval]))", + "legendFormat": "{{model}}", "range": true, "refId": "A" } ], - "title": "Stage share of turn (${stat:text})", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "p99 of the pre-generation waits: queue_wait (queued message insert to promotion), capacity_wait (capacity limiter acquire) and acquisition (trigger message insert to Acquire applied). These are queueing signals rather than model latency. Intervals with no samples for a stage are dropped instead of returning NaN. All three stages run before a model is resolved, so $model and $effort are not applied and the panel stays populated under any selection.\n\n$chat_kind selects root chats, subagent chats spawned by a parent agent, or both. It is a turn property carried by every turn-scoped stage, so unlike $model and $effort it applies to every stage in this panel.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 25 - }, - "id": 5, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (le, stage) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=~\"queue_wait|capacity_wait|acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__rate_interval])))\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"queue_wait|capacity_wait|acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)", - "range": true, - "refId": "A", - "legendFormat": "{{stage}} p99" - } - ], - "title": "Queue, capacity and acquisition wait (p99)", + "title": "Turns per minute", "type": "timeseries" }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 34 - }, - "id": 103, - "panels": [], - "title": "Turn time partition", - "type": "row" - }, { "datasource": { "type": "prometheus", @@ -831,9 +615,9 @@ }, "gridPos": { "h": 10, - "w": 12, - "x": 0, - "y": 35 + "w": 8, + "x": 16, + "y": 16 }, "id": 10, "options": { @@ -908,6 +692,19 @@ ], "type": "barchart" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 103, + "panels": [], + "title": "Level 0: Turn time partition", + "type": "row" + }, { "datasource": { "type": "prometheus", @@ -1135,8 +932,8 @@ "gridPos": { "h": 10, "w": 12, - "x": 12, - "y": 35 + "x": 0, + "y": 27 }, "id": 11, "options": { @@ -1186,7 +983,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Turn time the stage instrumentation could not place in any category: mean unattributed seconds per turn on the left axis and the mean unattributed share of a turn on the right.\n\nThis is the completeness check for the stage model. Near zero means the categories account for the turn. A rising line means real turn time is happening outside every instrumented stage, so the profile and partition panels are understating something; treat it as a bug in the instrumentation rather than as a workload change.\n\n$chat_kind, $model and $effort all apply.", + "description": "Distribution of each category's share of a turn, from the coderd_chatd_turn_time_share histogram recorded once per turn. $stat selects the statistic, so p99 reads as \"in the worst turns, this category took this fraction of the turn\".\n\nUse it next to the mix bar: the bar shows where aggregate time goes, this shows how much a category varies per turn. A category with a small mean and a large p99 is bursty (a slow tool call or a retry storm in a minority of turns) rather than a steady cost.\n\nQuantiles are per category, so unlike the mean shares they do not sum to 100%. $chat_kind, $model and $effort all apply.", "fieldConfig": { "defaults": { "color": { @@ -1200,7 +997,7 @@ "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 20, + "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, @@ -1235,150 +1032,19 @@ } ] }, - "unit": "s" + "unit": "percentunit" }, "overrides": [ { "matcher": { "id": "byName", - "options": "share of turn" + "options": "scheduling" }, "properties": [ - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "custom.axisPlacement", - "value": "right" - }, { "id": "color", "value": { - "fixedColor": "#808080", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 45 - }, - "id": 12, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum(rate(coderd_chatd_turn_time_seconds_sum{category=\"unattributed\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval]))\n/ (sum(rate(coderd_chatd_turn_time_seconds_count{category=\"unattributed\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) > 0)", - "legendFormat": "seconds per turn", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum(rate(coderd_chatd_turn_time_share_sum{category=\"unattributed\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval]))\n/ (sum(rate(coderd_chatd_turn_time_share_count{category=\"unattributed\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) > 0)", - "legendFormat": "share of turn", - "range": true, - "refId": "B" - } - ], - "title": "Unattributed turn time", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Distribution of each category's share of a turn, from the coderd_chatd_turn_time_share histogram recorded once per turn. $stat selects the statistic, so p99 reads as \"in the worst turns, this category took this fraction of the turn\".\n\nUse it next to the mix bar: the bar shows where aggregate time goes, this shows how much a category varies per turn. A category with a small mean and a large p99 is bursty (a slow tool call or a retry storm in a minority of turns) rather than a steady cost.\n\nQuantiles are per category, so unlike the mean shares they do not sum to 100%. $chat_kind, $model and $effort all apply.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "scheduling" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", + "fixedColor": "blue", "mode": "fixed" } } @@ -1507,10 +1173,10 @@ ] }, "gridPos": { - "h": 8, - "w": 16, - "x": 8, - "y": 45 + "h": 10, + "w": 12, + "x": 12, + "y": 27 }, "id": 13, "options": { @@ -1544,13 +1210,1357 @@ "title": "Category share per turn (${stat:text})", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Turn time the stage instrumentation could not place in any category: mean unattributed seconds per turn on the left axis and the mean unattributed share of a turn on the right.\n\nThis is the completeness check for the stage model. Near zero means the categories account for the turn. A rising line means real turn time is happening outside every instrumented stage, so the profile and partition panels are understating something; treat it as a bug in the instrumentation rather than as a workload change.\n\n$chat_kind, $model and $effort all apply.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "share of turn" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "color", + "value": { + "fixedColor": "#808080", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(coderd_chatd_turn_time_seconds_sum{category=\"unattributed\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval]))\n/ (sum(rate(coderd_chatd_turn_time_seconds_count{category=\"unattributed\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) > 0)", + "legendFormat": "seconds per turn", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(coderd_chatd_turn_time_share_sum{category=\"unattributed\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval]))\n/ (sum(rate(coderd_chatd_turn_time_share_count{category=\"unattributed\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval])) > 0)", + "legendFormat": "share of turn", + "range": true, + "refId": "B" + } + ], + "title": "Unattributed turn time", + "type": "timeseries" + }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 53 + "y": 44 + }, + "id": 20, + "panels": [], + "title": "Level 1: Turn children", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Stages that hang directly off chat_turn: the three scheduling waits before generation starts and generation_step, which repeats once per step of the turn.\n\nMembers: acquisition, queue_wait, capacity_wait, generation_step.\n\nOne sample per occurrence of the stage: a stage that runs several times in a turn contributes several samples, so this is \"how long does one of these take\".\n\n$chat_kind applies to every series. $model and $effort apply to generation_step; acquisition, queue_wait, capacity_wait run before a model is resolved and are matched without them.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 45 + }, + "id": 21, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=~\"acquisition|queue_wait|capacity_wait\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"acquisition|queue_wait|capacity_wait\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_stage_duration_seconds_sum{stage=~\"acquisition|queue_wait|capacity_wait\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"acquisition|queue_wait|capacity_wait\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "legendFormat": "{{stage}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"generation_step\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_stage_duration_seconds_sum{stage=\"generation_step\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "legendFormat": "{{stage}}", + "range": true, + "refId": "B" + } + ], + "title": "Duration per occurrence (${stat:text})", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total seconds each stage occupied within a turn, from coderd_chatd_turn_stage_seconds.\n\nOne sample per turn, recorded when the turn ends, so this is \"how much of a turn does this stage account for in total\" and it already sums the repeats.\n\nCompare with duration per occurrence: a stage can be fast per occurrence and still dominate a turn by repeating. All three variables apply, because the turn-end metrics are stamped with the turn's model and effort even for stages that ran before it was resolved.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 45 + }, + "id": 22, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_turn_stage_seconds_bucket{stage=~\"acquisition|queue_wait|capacity_wait|generation_step\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_turn_stage_seconds_count{stage=~\"acquisition|queue_wait|capacity_wait|generation_step\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_turn_stage_seconds_sum{stage=~\"acquisition|queue_wait|capacity_wait|generation_step\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_turn_stage_seconds_count{stage=~\"acquisition|queue_wait|capacity_wait|generation_step\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "legendFormat": "{{stage}}", + "range": true, + "refId": "A" + } + ], + "title": "Seconds per turn (${stat:text})", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "How many times each stage ran in a turn, from coderd_chatd_turn_stage_count.\n\nOne sample per turn, recorded when the turn ends, so this is \"how much of a turn does this stage account for in total\" and it already sums the repeats. A stage that did not occur in a turn is not recorded for that turn, so this reads as the count among turns where the stage happened at all.\n\nThis is the multiplier between the other two panels: seconds per turn is roughly occurrences per turn times duration per occurrence.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 53 + }, + "id": 23, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_turn_stage_count_bucket{stage=~\"acquisition|queue_wait|capacity_wait|generation_step\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_turn_stage_count_count{stage=~\"acquisition|queue_wait|capacity_wait|generation_step\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_turn_stage_count_sum{stage=~\"acquisition|queue_wait|capacity_wait|generation_step\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_turn_stage_count_count{stage=~\"acquisition|queue_wait|capacity_wait|generation_step\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "legendFormat": "{{stage}}", + "range": true, + "refId": "A" + } + ], + "title": "Occurrences per turn (${stat:text})", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Fraction of the turn each stage occupied, from coderd_chatd_stage_share_of_turn.\n\nOne sample per turn, recorded when the turn ends, so this is \"how much of a turn does this stage account for in total\" and it already sums the repeats. Stages overlap in wall time, so shares at one level do not sum to 1 and several stages can each approach the whole turn.\n\nUse it to compare levels: a level whose shares are all small means turn time is going somewhere else, which the turn time partition row attributes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 53 + }, + "id": 24, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_stage_share_of_turn_bucket{stage=~\"acquisition|queue_wait|capacity_wait|generation_step\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_share_of_turn_count{stage=~\"acquisition|queue_wait|capacity_wait|generation_step\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_stage_share_of_turn_sum{stage=~\"acquisition|queue_wait|capacity_wait|generation_step\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_stage_share_of_turn_count{stage=~\"acquisition|queue_wait|capacity_wait|generation_step\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "legendFormat": "{{stage}}", + "range": true, + "refId": "A" + } + ], + "title": "Share of turn (${stat:text})", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 61 + }, + "id": 30, + "panels": [], + "title": "Level 2: Step children", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Stages inside one generation_step. They overlap each other in wall time, so read them as a profile of the step rather than as a partition of it. provider_attempt is listed here rather than at level 3: in a trace it wraps the stream, but it measures one full provider round trip, so it belongs beside stream at step level and appears only once.\n\nMembers: prepare, mcp_connect, provider_attempt, stream, thinking, tool_call, commit, compaction, retry_backoff.\n\nOne sample per occurrence of the stage: a stage that runs several times in a turn contributes several samples, so this is \"how long does one of these take\".\n\n$chat_kind applies to every series. $model and $effort apply to prepare, provider_attempt, stream, thinking, tool_call, compaction; mcp_connect, commit, retry_backoff run before a model is resolved and are matched without them.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 62 + }, + "id": 31, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=~\"mcp_connect|commit|retry_backoff\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"mcp_connect|commit|retry_backoff\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_stage_duration_seconds_sum{stage=~\"mcp_connect|commit|retry_backoff\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"mcp_connect|commit|retry_backoff\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "legendFormat": "{{stage}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=~\"prepare|provider_attempt|stream|thinking|tool_call|compaction\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"prepare|provider_attempt|stream|thinking|tool_call|compaction\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_stage_duration_seconds_sum{stage=~\"prepare|provider_attempt|stream|thinking|tool_call|compaction\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"prepare|provider_attempt|stream|thinking|tool_call|compaction\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "legendFormat": "{{stage}}", + "range": true, + "refId": "B" + } + ], + "title": "Duration per occurrence (${stat:text})", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total seconds each stage occupied within a turn, from coderd_chatd_turn_stage_seconds.\n\nOne sample per turn, recorded when the turn ends, so this is \"how much of a turn does this stage account for in total\" and it already sums the repeats.\n\nCompare with duration per occurrence: a stage can be fast per occurrence and still dominate a turn by repeating. All three variables apply, because the turn-end metrics are stamped with the turn's model and effort even for stages that ran before it was resolved.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 62 + }, + "id": 32, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_turn_stage_seconds_bucket{stage=~\"prepare|mcp_connect|provider_attempt|stream|thinking|tool_call|commit|compaction|retry_backoff\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_turn_stage_seconds_count{stage=~\"prepare|mcp_connect|provider_attempt|stream|thinking|tool_call|commit|compaction|retry_backoff\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_turn_stage_seconds_sum{stage=~\"prepare|mcp_connect|provider_attempt|stream|thinking|tool_call|commit|compaction|retry_backoff\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_turn_stage_seconds_count{stage=~\"prepare|mcp_connect|provider_attempt|stream|thinking|tool_call|commit|compaction|retry_backoff\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "legendFormat": "{{stage}}", + "range": true, + "refId": "A" + } + ], + "title": "Seconds per turn (${stat:text})", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "How many times each stage ran in a turn, from coderd_chatd_turn_stage_count.\n\nOne sample per turn, recorded when the turn ends, so this is \"how much of a turn does this stage account for in total\" and it already sums the repeats. A stage that did not occur in a turn is not recorded for that turn, so this reads as the count among turns where the stage happened at all.\n\nThis is the multiplier between the other two panels: seconds per turn is roughly occurrences per turn times duration per occurrence.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 70 + }, + "id": 33, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_turn_stage_count_bucket{stage=~\"prepare|mcp_connect|provider_attempt|stream|thinking|tool_call|commit|compaction|retry_backoff\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_turn_stage_count_count{stage=~\"prepare|mcp_connect|provider_attempt|stream|thinking|tool_call|commit|compaction|retry_backoff\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_turn_stage_count_sum{stage=~\"prepare|mcp_connect|provider_attempt|stream|thinking|tool_call|commit|compaction|retry_backoff\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_turn_stage_count_count{stage=~\"prepare|mcp_connect|provider_attempt|stream|thinking|tool_call|commit|compaction|retry_backoff\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "legendFormat": "{{stage}}", + "range": true, + "refId": "A" + } + ], + "title": "Occurrences per turn (${stat:text})", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Fraction of the turn each stage occupied, from coderd_chatd_stage_share_of_turn.\n\nOne sample per turn, recorded when the turn ends, so this is \"how much of a turn does this stage account for in total\" and it already sums the repeats. Stages overlap in wall time, so shares at one level do not sum to 1 and several stages can each approach the whole turn.\n\nUse it to compare levels: a level whose shares are all small means turn time is going somewhere else, which the turn time partition row attributes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 70 + }, + "id": 34, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_stage_share_of_turn_bucket{stage=~\"prepare|mcp_connect|provider_attempt|stream|thinking|tool_call|commit|compaction|retry_backoff\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_share_of_turn_count{stage=~\"prepare|mcp_connect|provider_attempt|stream|thinking|tool_call|commit|compaction|retry_backoff\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_stage_share_of_turn_sum{stage=~\"prepare|mcp_connect|provider_attempt|stream|thinking|tool_call|commit|compaction|retry_backoff\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_stage_share_of_turn_count{stage=~\"prepare|mcp_connect|provider_attempt|stream|thinking|tool_call|commit|compaction|retry_backoff\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "legendFormat": "{{stage}}", + "range": true, + "refId": "A" + } + ], + "title": "Share of turn (${stat:text})", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 78 + }, + "id": 40, + "panels": [], + "title": "Level 3: Stream children", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The part of a provider round trip before the first streamed token. It is the only stage nested inside provider_attempt/stream, and it keeps the same three-panel reading as the other levels so the rows compare directly.\n\nMembers: time_to_first_token.\n\nOne sample per occurrence of the stage: a stage that runs several times in a turn contributes several samples, so this is \"how long does one of these take\".\n\n$chat_kind applies to every series. $model and $effort apply to time_to_first_token; no stage here run before a model is resolved and are matched without them.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 79 + }, + "id": 41, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"time_to_first_token\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_stage_duration_seconds_sum{stage=\"time_to_first_token\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\", scope=\"turn\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "legendFormat": "{{stage}}", + "range": true, + "refId": "A" + } + ], + "title": "Duration per occurrence (${stat:text})", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total seconds each stage occupied within a turn, from coderd_chatd_turn_stage_seconds.\n\nOne sample per turn, recorded when the turn ends, so this is \"how much of a turn does this stage account for in total\" and it already sums the repeats.\n\nCompare with duration per occurrence: a stage can be fast per occurrence and still dominate a turn by repeating. All three variables apply, because the turn-end metrics are stamped with the turn's model and effort even for stages that ran before it was resolved.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 79 + }, + "id": 42, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_turn_stage_seconds_bucket{stage=\"time_to_first_token\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_turn_stage_seconds_count{stage=\"time_to_first_token\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_turn_stage_seconds_sum{stage=\"time_to_first_token\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_turn_stage_seconds_count{stage=\"time_to_first_token\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "legendFormat": "{{stage}}", + "range": true, + "refId": "A" + } + ], + "title": "Seconds per turn (${stat:text})", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "How many times each stage ran in a turn, from coderd_chatd_turn_stage_count.\n\nOne sample per turn, recorded when the turn ends, so this is \"how much of a turn does this stage account for in total\" and it already sums the repeats. A stage that did not occur in a turn is not recorded for that turn, so this reads as the count among turns where the stage happened at all.\n\nThis is the multiplier between the other two panels: seconds per turn is roughly occurrences per turn times duration per occurrence.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 87 + }, + "id": 43, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_turn_stage_count_bucket{stage=\"time_to_first_token\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_turn_stage_count_count{stage=\"time_to_first_token\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_turn_stage_count_sum{stage=\"time_to_first_token\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_turn_stage_count_count{stage=\"time_to_first_token\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "legendFormat": "{{stage}}", + "range": true, + "refId": "A" + } + ], + "title": "Occurrences per turn (${stat:text})", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Fraction of the turn each stage occupied, from coderd_chatd_stage_share_of_turn.\n\nOne sample per turn, recorded when the turn ends, so this is \"how much of a turn does this stage account for in total\" and it already sums the repeats. Stages overlap in wall time, so shares at one level do not sum to 1 and several stages can each approach the whole turn.\n\nUse it to compare levels: a level whose shares are all small means turn time is going somewhere else, which the turn time partition row attributes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 87 + }, + "id": 44, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_stage_share_of_turn_bucket{stage=\"time_to_first_token\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_share_of_turn_count{stage=\"time_to_first_token\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_stage_share_of_turn_sum{stage=\"time_to_first_token\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_stage_share_of_turn_count{stage=\"time_to_first_token\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "legendFormat": "{{stage}}", + "range": true, + "refId": "A" + } + ], + "title": "Share of turn (${stat:text})", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 95 }, "id": 102, "panels": [], @@ -1619,7 +2629,7 @@ "h": 9, "w": 8, "x": 0, - "y": 54 + "y": 96 }, "id": 6, "options": { @@ -1748,7 +2758,7 @@ "h": 9, "w": 8, "x": 8, - "y": 54 + "y": 96 }, "id": 7, "options": { @@ -1883,7 +2893,7 @@ "h": 9, "w": 8, "x": 16, - "y": 54 + "y": 96 }, "id": 8, "options": { diff --git a/scripts/metricsdocgen/generated_metrics b/scripts/metricsdocgen/generated_metrics index 3728d6279a8..eec13aa1a67 100644 --- a/scripts/metricsdocgen/generated_metrics +++ b/scripts/metricsdocgen/generated_metrics @@ -346,6 +346,9 @@ coderd_chatd_tool_result_truncated_total{provider="",model="",tool_name=""} 0 # HELP coderd_chatd_ttft_seconds Time-to-first-token: wall time from LLM request to first streamed chunk. # TYPE coderd_chatd_ttft_seconds histogram coderd_chatd_ttft_seconds{provider="",model=""} 0 +# HELP coderd_chatd_turn_stage_count Number of times a stage occurred within one chat turn, observed once per turn per stage that occurred. Only turns that finished normally are counted. +# TYPE coderd_chatd_turn_stage_count histogram +coderd_chatd_turn_stage_count{stage="",chat_kind="",model="",effort=""} 0 # HELP coderd_chatd_turn_stage_seconds Total wall time one chat turn spent in a stage, observed once per turn when the turn ends. Stages overlap, so these do not partition the turn. Only turns that finished normally are counted. # TYPE coderd_chatd_turn_stage_seconds histogram coderd_chatd_turn_stage_seconds{stage="",chat_kind="",model="",effort=""} 0 From fdd7a1af3b251755d1a5bec9a244ec471e4e01a1 Mon Sep 17 00:00:00 2001 From: Scott Miller Date: Thu, 3 Sep 2026 18:26:38 +0000 Subject: [PATCH 06/19] fix(examples/monitoring): render the stage profile hierarchy as a table The bar chart right-anchored its category labels, so the tree indentation was applied at the wrong end and the connectors misaligned in a proportional font. Render the hierarchy as a table with a left-aligned, fixed-width-indented stage column and a gauge cell for the duration, and add the retry_backoff stage to both profile panels. --- .../grafana/chatd-lifecycle/README.md | 19 ++- .../grafana/chatd-lifecycle/dashboard.json | 154 ++++++++++++++---- 2 files changed, 132 insertions(+), 41 deletions(-) diff --git a/examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md b/examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md index cd69d37f293..fa1ca78be0a 100644 --- a/examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md +++ b/examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md @@ -19,12 +19,12 @@ L2 ├── prepare prompt build, model resolution, context hydra L2 ├── mcp_connect MCP server connection L2 ├── provider_attempt one provider HTTP round trip (per retry) L3 │ └── time_to_first_token request open -> first streamed part +L2 ├── retry_backoff wait between provider attempts L2 ├── stream provider stream open -> close L2 ├── thinking reasoning part duration L2 ├── tool_call one local tool call L2 ├── commit step persistence transaction -L2 ├── compaction auxiliary compaction call -L2 └── retry_backoff wait between provider attempts +L2 └── compaction auxiliary compaction call ``` Stages overlap in wall time (tool calls and thinking happen inside the @@ -100,11 +100,14 @@ parent at high percentiles; a stage whose series first appears inside the window reads 0 until its second sample. **Stage profile in hierarchy order ($stat)** - the same query as the -flamegraph drawn as horizontal bars in depth-first order with the tree -indented into the labels. Dimensions: identical to the flamegraph. Use -it to read stages too small to see as frames (prepare, commit, -tool_call are typically milliseconds next to multi-second streams) and -as a numeric check on the flamegraph. +flamegraph, as a table in depth-first order: a Stage column indented by +level, the level itself, and the duration drawn as a gauge bar scaled to +the widest stage, so the bar lengths stay comparable to the flamegraph +frames. Dimensions: identical to the flamegraph. Use it to read stages +too small to see as frames (prepare, commit, tool_call are typically +milliseconds next to multi-second streams) and as a numeric check on the +flamegraph. The indentation uses fixed-width spacing rather than a +drawn tree, so rows at the same depth line up in any font. ### Reading the levels @@ -222,7 +225,7 @@ because of generation". ### Level 2: Step children Members: `prepare`, `mcp_connect`, `provider_attempt`, `stream`, -`thinking`, `tool_call`, `commit`, `compaction`, `retry_backoff`. +`retry_backoff`, `thinking`, `tool_call`, `commit`, `compaction`. These are the stages inside one generation step and they overlap each other, so read them as a profile of the step. `provider_attempt` is diff --git a/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json b/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json index 0a2047e6ad3..7ea4e6f26ad 100644 --- a/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json +++ b/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json @@ -79,7 +79,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"chat_turn\", \"\", \"\"),\n\"n\", \"01\", \"\", \"\"),\n\"level\", \"0\", \"\", \"\"),\n\"name\", \"chat_turn\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"queue_wait\", \"\", \"\"),\n\"n\", \"02\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ queue_wait\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"capacity_wait\", \"\", \"\"),\n\"n\", \"03\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ capacity_wait\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"acquisition\", \"\", \"\"),\n\"n\", \"04\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ acquisition\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"generation_step\", \"\", \"\"),\n\"n\", \"05\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ generation_step\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"prepare\", \"\", \"\"),\n\"n\", \"06\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ prepare\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"mcp_connect\", \"\", \"\"),\n\"n\", \"07\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ mcp_connect\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"provider_attempt\", \"\", \"\"),\n\"n\", \"08\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ provider_attempt\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"time_to_first_token\", \"\", \"\"),\n\"n\", \"09\", \"\", \"\"),\n\"level\", \"3\", \"\", \"\"),\n\"name\", \"│ │ └─ time_to_first_token\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"stream\", \"\", \"\"),\n\"n\", \"10\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ stream\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"thinking\", \"\", \"\"),\n\"n\", \"11\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ thinking\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"tool_call\", \"\", \"\"),\n\"n\", \"12\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ tool_call\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"commit\", \"\", \"\"),\n\"n\", \"13\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ commit\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"compaction\", \"\", \"\"),\n\"n\", \"14\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ compaction\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)", + "expr": "(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"chat_turn\", \"\", \"\"),\n\"n\", \"01\", \"\", \"\"),\n\"level\", \"0\", \"\", \"\"),\n\"name\", \"chat_turn\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"queue_wait\", \"\", \"\"),\n\"n\", \"02\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"  └ queue_wait\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"capacity_wait\", \"\", \"\"),\n\"n\", \"03\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"  └ capacity_wait\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"acquisition\", \"\", \"\"),\n\"n\", \"04\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"  └ acquisition\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"generation_step\", \"\", \"\"),\n\"n\", \"05\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"  └ generation_step\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"prepare\", \"\", \"\"),\n\"n\", \"06\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"    └ prepare\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"mcp_connect\", \"\", \"\"),\n\"n\", \"07\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"    └ mcp_connect\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"provider_attempt\", \"\", \"\"),\n\"n\", \"08\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"    └ provider_attempt\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"time_to_first_token\", \"\", \"\"),\n\"n\", \"09\", \"\", \"\"),\n\"level\", \"3\", \"\", \"\"),\n\"name\", \"      └ time_to_first_token\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"retry_backoff\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"retry_backoff\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"retry_backoff\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"retry_backoff\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"retry_backoff\", \"\", \"\"),\n\"n\", \"10\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"    └ retry_backoff\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"stream\", \"\", \"\"),\n\"n\", \"11\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"    └ stream\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"thinking\", \"\", \"\"),\n\"n\", \"12\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"    └ thinking\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"tool_call\", \"\", \"\"),\n\"n\", \"13\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"    └ tool_call\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"commit\", \"\", \"\"),\n\"n\", \"14\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"    └ commit\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"compaction\", \"\", \"\"),\n\"n\", \"15\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"    └ compaction\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)", "range": false, "refId": "A", "format": "table", @@ -165,15 +165,100 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Same data as the flamegraph, drawn as bars in depth-first hierarchy order with the tree drawn into the labels. Useful when a stage is too small to read in the flamegraph, and as a check on the flamegraph shaping.\n\nStage timings overlap in wall time (stream, thinking and tool_call all run inside a generation_step, and time_to_first_token is part of provider_attempt), and quantiles are not additive across stages. Read this as a stage time profile, not as a strict non-overlapping decomposition of turn latency: child bars can exceed their parent. A stage with no samples in the time range reads as 0. Only scope=\"turn\" samples are included, so detached background provider calls are excluded.\n\nValues come from rate() over counter series, and rate() treats the first sample in the window as its baseline. A stage whose series first appears inside the window reads as 0 until its second observation, so rare stages such as queue_wait, capacity_wait and compaction can render as 0 while real samples exist. Widen the time range to confirm.\n\n$model and $effort apply only to the stages that carry those labels (generation_step, prepare, provider_attempt, time_to_first_token, stream, thinking, tool_call, compaction). The stages recorded before a model is resolved (chat_turn, queue_wait, capacity_wait, acquisition, mcp_connect, commit) are always matched without them, so they stay populated under any selection.\n\n$chat_kind selects root chats, subagent chats spawned by a parent agent, or both. It is a turn property carried by every turn-scoped stage, so unlike $model and $effort it applies to every stage in this panel.", + "description": "Same data as the flamegraph, drawn as bars in depth-first hierarchy order with the tree drawn into the labels. Useful when a stage is too small to read in the flamegraph, and as a check on the flamegraph shaping.\n\nStage timings overlap in wall time (stream, thinking and tool_call all run inside a generation_step, and time_to_first_token is part of provider_attempt), and quantiles are not additive across stages. Read this as a stage time profile, not as a strict non-overlapping decomposition of turn latency: child bars can exceed their parent. A stage with no samples in the time range reads as 0. Only scope=\"turn\" samples are included, so detached background provider calls are excluded.\n\nValues come from rate() over counter series, and rate() treats the first sample in the window as its baseline. A stage whose series first appears inside the window reads as 0 until its second observation, so rare stages such as queue_wait, capacity_wait and compaction can render as 0 while real samples exist. Widen the time range to confirm.\n\n$model and $effort apply only to the stages that carry those labels (generation_step, prepare, provider_attempt, time_to_first_token, stream, thinking, tool_call, compaction). The stages recorded before a model is resolved (chat_turn, queue_wait, capacity_wait, acquisition, mcp_connect, commit) are always matched without them, so they stay populated under any selection.\n\n$chat_kind selects root chats, subagent chats spawned by a parent agent, or both. It is a turn property carried by every turn-scoped stage, so unlike $model and $effort it applies to every stage in this panel.\n\nRows keep the flamegraph's depth-first order and indentation shows the level, two spaces per level with a connector on children, so stages at the same depth line up regardless of font. The duration cell is a gauge scaled to the widest stage in the table, which makes the bar lengths comparable to the flamegraph frames.", "fieldConfig": { "defaults": { - "color": { - "mode": "continuous-BlPu" + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false }, - "unit": "s" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "name" + }, + "properties": [ + { + "id": "displayName", + "value": "Stage" + }, + { + "id": "custom.align", + "value": "left" + }, + { + "id": "custom.width", + "value": 260 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "level" + }, + "properties": [ + { + "id": "displayName", + "value": "Level" + }, + { + "id": "custom.align", + "value": "center" + }, + { + "id": "custom.width", + "value": 70 + }, + { + "id": "unit", + "value": "none" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "displayName", + "value": "Duration (${stat:text})" + }, + { + "id": "unit", + "value": "s" + }, + { + "id": "min", + "value": 0 + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + } + ] + } + ] }, "gridPos": { "h": 14, @@ -183,26 +268,17 @@ }, "id": 2, "options": { - "barRadius": 0, - "barWidth": 0.8, - "fullHighlight": false, - "groupWidth": 0.7, - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom", - "showLegend": false - }, - "orientation": "horizontal", - "showValue": "auto", - "stacking": "none", - "tooltip": { - "mode": "single", - "sort": "none" + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false }, - "xField": "name", - "xTickLabelRotation": 0, - "xTickLabelSpacing": 0 + "showHeader": true, + "sortBy": [] }, "targets": [ { @@ -211,7 +287,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"chat_turn\", \"\", \"\"),\n\"n\", \"01\", \"\", \"\"),\n\"level\", \"0\", \"\", \"\"),\n\"name\", \"chat_turn\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"queue_wait\", \"\", \"\"),\n\"n\", \"02\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ queue_wait\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"capacity_wait\", \"\", \"\"),\n\"n\", \"03\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ capacity_wait\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"acquisition\", \"\", \"\"),\n\"n\", \"04\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ acquisition\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"generation_step\", \"\", \"\"),\n\"n\", \"05\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"├─ generation_step\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"prepare\", \"\", \"\"),\n\"n\", \"06\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ prepare\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"mcp_connect\", \"\", \"\"),\n\"n\", \"07\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ mcp_connect\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"provider_attempt\", \"\", \"\"),\n\"n\", \"08\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ provider_attempt\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"time_to_first_token\", \"\", \"\"),\n\"n\", \"09\", \"\", \"\"),\n\"level\", \"3\", \"\", \"\"),\n\"name\", \"│ │ └─ time_to_first_token\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"stream\", \"\", \"\"),\n\"n\", \"10\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ stream\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"thinking\", \"\", \"\"),\n\"n\", \"11\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ thinking\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"tool_call\", \"\", \"\"),\n\"n\", \"12\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ tool_call\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"commit\", \"\", \"\"),\n\"n\", \"13\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ commit\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"compaction\", \"\", \"\"),\n\"n\", \"14\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"│ ├─ compaction\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)", + "expr": "(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"chat_turn\", \"\", \"\"),\n\"n\", \"01\", \"\", \"\"),\n\"level\", \"0\", \"\", \"\"),\n\"name\", \"chat_turn\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"queue_wait\", \"\", \"\"),\n\"n\", \"02\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"  └ queue_wait\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"capacity_wait\", \"\", \"\"),\n\"n\", \"03\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"  └ capacity_wait\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"acquisition\", \"\", \"\"),\n\"n\", \"04\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"  └ acquisition\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"generation_step\", \"\", \"\"),\n\"n\", \"05\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"  └ generation_step\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"prepare\", \"\", \"\"),\n\"n\", \"06\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"    └ prepare\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"mcp_connect\", \"\", \"\"),\n\"n\", \"07\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"    └ mcp_connect\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"provider_attempt\", \"\", \"\"),\n\"n\", \"08\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"    └ provider_attempt\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"time_to_first_token\", \"\", \"\"),\n\"n\", \"09\", \"\", \"\"),\n\"level\", \"3\", \"\", \"\"),\n\"name\", \"      └ time_to_first_token\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"retry_backoff\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"retry_backoff\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"retry_backoff\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"retry_backoff\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"retry_backoff\", \"\", \"\"),\n\"n\", \"10\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"    └ retry_backoff\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"stream\", \"\", \"\"),\n\"n\", \"11\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"    └ stream\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"thinking\", \"\", \"\"),\n\"n\", \"12\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"    └ thinking\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"tool_call\", \"\", \"\"),\n\"n\", \"13\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"    └ tool_call\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"commit\", \"\", \"\"),\n\"n\", \"14\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"    └ commit\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"compaction\", \"\", \"\"),\n\"n\", \"15\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"    └ compaction\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)", "range": false, "refId": "A", "format": "table", @@ -232,26 +308,38 @@ ] } }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "number", + "targetField": "level" + } + ], + "fields": {} + } + }, { "id": "organize", "options": { "excludeByName": { "Time": true, "n": true, - "level": true, "label": true, "leaf": true }, "includeByName": {}, - "indexByName": {}, - "renameByName": { - "Value": "${stat:text}", - "Value #A": "${stat:text}" - } + "indexByName": { + "name": 0, + "level": 1, + "Value": 2 + }, + "renameByName": {} } } ], - "type": "barchart" + "type": "table" }, { "collapsed": false, From 39fd91993a326ea09b44e16e6a821c71da73b649 Mon Sep 17 00:00:00 2001 From: Scott Miller Date: Thu, 3 Sep 2026 18:35:49 +0000 Subject: [PATCH 07/19] fix(examples/monitoring): place Level 0 legends below the graphs --- .../dashboards/grafana/chatd-lifecycle/dashboard.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json b/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json index 7ea4e6f26ad..0857faee21e 100644 --- a/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json +++ b/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json @@ -425,8 +425,8 @@ "mean", "max" ], - "displayMode": "table", - "placement": "right", + "displayMode": "list", + "placement": "bottom", "showLegend": true }, "tooltip": { @@ -521,8 +521,8 @@ "mean", "max" ], - "displayMode": "table", - "placement": "right", + "displayMode": "list", + "placement": "bottom", "showLegend": true }, "tooltip": { From ae6bfc5e8e7fd9a56f1a2fbf368b61bedb641fd5 Mon Sep 17 00:00:00 2001 From: Scott Miller Date: Thu, 3 Sep 2026 18:42:07 +0000 Subject: [PATCH 08/19] feat(examples/monitoring): show occurrence counts in the stage profile hierarchy Add a Count column to the hierarchy table with the number of occurrences of each stage in the dashboard time range, joined to the duration rows by stage label so depth-first order and zero rows are preserved. --- .../grafana/chatd-lifecycle/README.md | 20 +++--- .../grafana/chatd-lifecycle/dashboard.json | 61 ++++++++++++++++++- 2 files changed, 70 insertions(+), 11 deletions(-) diff --git a/examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md b/examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md index fa1ca78be0a..bb5a9ce03b2 100644 --- a/examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md +++ b/examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md @@ -99,15 +99,19 @@ and quantiles are not additive, so a child frame can read wider than its parent at high percentiles; a stage whose series first appears inside the window reads 0 until its second sample. -**Stage profile in hierarchy order ($stat)** - the same query as the -flamegraph, as a table in depth-first order: a Stage column indented by -level, the level itself, and the duration drawn as a gauge bar scaled to -the widest stage, so the bar lengths stay comparable to the flamegraph -frames. Dimensions: identical to the flamegraph. Use it to read stages +**Stage profile in hierarchy order ($stat)** - the flamegraph query as a +table in depth-first order: a Stage column indented by level, the level +itself, the number of occurrences of the stage in the selected time +range, and the duration drawn as a gauge bar scaled to the widest stage, +so the bar lengths stay comparable to the flamegraph frames. Dimensions: +identical to the flamegraph. The Count column is an exact counter +increase over the range rather than a rate, so a stage that never ran +reads 0 and a stage that ran once reads 1. Use it to read stages too small to see as frames (prepare, commit, tool_call are typically -milliseconds next to multi-second streams) and as a numeric check on the -flamegraph. The indentation uses fixed-width spacing rather than a -drawn tree, so rows at the same depth line up in any font. +milliseconds next to multi-second streams), to tell a rare stage from an +absent one, and as a numeric check on the flamegraph. The indentation +uses fixed-width spacing rather than a drawn tree, so rows at the same +depth line up in any font. ### Reading the levels diff --git a/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json b/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json index 0857faee21e..7c92c307c50 100644 --- a/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json +++ b/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json @@ -165,7 +165,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Same data as the flamegraph, drawn as bars in depth-first hierarchy order with the tree drawn into the labels. Useful when a stage is too small to read in the flamegraph, and as a check on the flamegraph shaping.\n\nStage timings overlap in wall time (stream, thinking and tool_call all run inside a generation_step, and time_to_first_token is part of provider_attempt), and quantiles are not additive across stages. Read this as a stage time profile, not as a strict non-overlapping decomposition of turn latency: child bars can exceed their parent. A stage with no samples in the time range reads as 0. Only scope=\"turn\" samples are included, so detached background provider calls are excluded.\n\nValues come from rate() over counter series, and rate() treats the first sample in the window as its baseline. A stage whose series first appears inside the window reads as 0 until its second observation, so rare stages such as queue_wait, capacity_wait and compaction can render as 0 while real samples exist. Widen the time range to confirm.\n\n$model and $effort apply only to the stages that carry those labels (generation_step, prepare, provider_attempt, time_to_first_token, stream, thinking, tool_call, compaction). The stages recorded before a model is resolved (chat_turn, queue_wait, capacity_wait, acquisition, mcp_connect, commit) are always matched without them, so they stay populated under any selection.\n\n$chat_kind selects root chats, subagent chats spawned by a parent agent, or both. It is a turn property carried by every turn-scoped stage, so unlike $model and $effort it applies to every stage in this panel.\n\nRows keep the flamegraph's depth-first order and indentation shows the level, two spaces per level with a connector on children, so stages at the same depth line up regardless of font. The duration cell is a gauge scaled to the widest stage in the table, which makes the bar lengths comparable to the flamegraph frames.", + "description": "Same data as the flamegraph, drawn as bars in depth-first hierarchy order with the tree drawn into the labels. Useful when a stage is too small to read in the flamegraph, and as a check on the flamegraph shaping.\n\nStage timings overlap in wall time (stream, thinking and tool_call all run inside a generation_step, and time_to_first_token is part of provider_attempt), and quantiles are not additive across stages. Read this as a stage time profile, not as a strict non-overlapping decomposition of turn latency: child bars can exceed their parent. A stage with no samples in the time range reads as 0. Only scope=\"turn\" samples are included, so detached background provider calls are excluded.\n\nValues come from rate() over counter series, and rate() treats the first sample in the window as its baseline. A stage whose series first appears inside the window reads as 0 until its second observation, so rare stages such as queue_wait, capacity_wait and compaction can render as 0 while real samples exist. Widen the time range to confirm.\n\n$model and $effort apply only to the stages that carry those labels (generation_step, prepare, provider_attempt, time_to_first_token, stream, thinking, tool_call, compaction). The stages recorded before a model is resolved (chat_turn, queue_wait, capacity_wait, acquisition, mcp_connect, commit) are always matched without them, so they stay populated under any selection.\n\n$chat_kind selects root chats, subagent chats spawned by a parent agent, or both. It is a turn property carried by every turn-scoped stage, so unlike $model and $effort it applies to every stage in this panel.\n\nThe Count column is the exact counter increase over the selected range, so it is unaffected by the rate() baseline above: a stage with Count 0 never ran in the range, while a nonzero Count next to a 0 duration means the stage ran but its rate() window has only one sample.\n\nRows keep the flamegraph's depth-first order and indentation shows the level, two spaces per level with a connector on children, so stages at the same depth line up regardless of font. The duration cell is a gauge scaled to the widest stage in the table, which makes the bar lengths comparable to the flamegraph frames.", "fieldConfig": { "defaults": { "custom": { @@ -234,7 +234,7 @@ { "matcher": { "id": "byName", - "options": "Value" + "options": "Value #A" }, "properties": [ { @@ -257,6 +257,40 @@ } } ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #B" + }, + "properties": [ + { + "id": "displayName", + "value": "Count" + }, + { + "id": "unit", + "value": "none" + }, + { + "id": "decimals", + "value": 0 + }, + { + "id": "custom.align", + "value": "right" + }, + { + "id": "custom.width", + "value": 90 + }, + { + "id": "custom.cellOptions", + "value": { + "type": "auto" + } + } + ] } ] }, @@ -293,10 +327,30 @@ "format": "table", "instant": true, "legendFormat": "" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n label_replace(label_replace(\n sum(increase(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) or vector(0),\n \"label\", \"chat_turn\", \"\", \"\"),\n \"n\", \"01\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(\n sum(increase(coderd_chatd_stage_duration_seconds_count{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) or vector(0),\n \"label\", \"queue_wait\", \"\", \"\"),\n \"n\", \"02\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(\n sum(increase(coderd_chatd_stage_duration_seconds_count{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) or vector(0),\n \"label\", \"capacity_wait\", \"\", \"\"),\n \"n\", \"03\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(\n sum(increase(coderd_chatd_stage_duration_seconds_count{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) or vector(0),\n \"label\", \"acquisition\", \"\", \"\"),\n \"n\", \"04\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(\n sum(increase(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) or vector(0),\n \"label\", \"generation_step\", \"\", \"\"),\n \"n\", \"05\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(\n sum(increase(coderd_chatd_stage_duration_seconds_count{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) or vector(0),\n \"label\", \"prepare\", \"\", \"\"),\n \"n\", \"06\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(\n sum(increase(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) or vector(0),\n \"label\", \"mcp_connect\", \"\", \"\"),\n \"n\", \"07\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(\n sum(increase(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) or vector(0),\n \"label\", \"provider_attempt\", \"\", \"\"),\n \"n\", \"08\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(\n sum(increase(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) or vector(0),\n \"label\", \"time_to_first_token\", \"\", \"\"),\n \"n\", \"09\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(\n sum(increase(coderd_chatd_stage_duration_seconds_count{stage=\"retry_backoff\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) or vector(0),\n \"label\", \"retry_backoff\", \"\", \"\"),\n \"n\", \"10\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(\n sum(increase(coderd_chatd_stage_duration_seconds_count{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) or vector(0),\n \"label\", \"stream\", \"\", \"\"),\n \"n\", \"11\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(\n sum(increase(coderd_chatd_stage_duration_seconds_count{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) or vector(0),\n \"label\", \"thinking\", \"\", \"\"),\n \"n\", \"12\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(\n sum(increase(coderd_chatd_stage_duration_seconds_count{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) or vector(0),\n \"label\", \"tool_call\", \"\", \"\"),\n \"n\", \"13\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(\n sum(increase(coderd_chatd_stage_duration_seconds_count{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) or vector(0),\n \"label\", \"commit\", \"\", \"\"),\n \"n\", \"14\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(\n sum(increase(coderd_chatd_stage_duration_seconds_count{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) or vector(0),\n \"label\", \"compaction\", \"\", \"\"),\n \"n\", \"15\", \"\", \"\")\n)", + "range": false, + "refId": "B", + "format": "table", + "instant": true, + "legendFormat": "" } ], "title": "Stage profile in hierarchy order (${stat:text})", "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "label", + "mode": "outer" + } + }, { "id": "sortBy", "options": { @@ -333,7 +387,8 @@ "indexByName": { "name": 0, "level": 1, - "Value": 2 + "Value #B": 2, + "Value #A": 3 }, "renameByName": {} } From b84f274e698972a35878cb357eb2d54034715133 Mon Sep 17 00:00:00 2001 From: Scott Miller Date: Thu, 3 Sep 2026 18:50:46 +0000 Subject: [PATCH 09/19] fix(examples/monitoring): scale hierarchy duration bars to the duration column The gauge cell defaulted to the frame-wide max, which the new Count column dominated, so every duration bar rendered as a sliver. --- .../dashboards/grafana/chatd-lifecycle/dashboard.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json b/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json index 7c92c307c50..aba789888af 100644 --- a/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json +++ b/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json @@ -255,6 +255,10 @@ "mode": "gradient", "type": "gauge" } + }, + { + "id": "fieldMinMax", + "value": true } ] }, From d4c1795e282263ef13008203ad793d612568bfb8 Mon Sep 17 00:00:00 2001 From: Scott Miller Date: Thu, 3 Sep 2026 18:51:40 +0000 Subject: [PATCH 10/19] fix(examples/monitoring): stack the hierarchy table below the flamegraph --- .../grafana/chatd-lifecycle/dashboard.json | 66 +++++++++---------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json b/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json index aba789888af..e7e9ad23dad 100644 --- a/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json +++ b/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json @@ -65,8 +65,8 @@ ] }, "gridPos": { - "h": 14, - "w": 14, + "h": 12, + "w": 24, "x": 0, "y": 1 }, @@ -299,10 +299,10 @@ ] }, "gridPos": { - "h": 14, - "w": 10, - "x": 14, - "y": 1 + "h": 17, + "w": 24, + "x": 0, + "y": 13 }, "id": 2, "options": { @@ -406,7 +406,7 @@ "h": 1, "w": 24, "x": 0, - "y": 15 + "y": 30 }, "id": 104, "panels": [], @@ -475,7 +475,7 @@ "h": 10, "w": 8, "x": 0, - "y": 16 + "y": 31 }, "id": 15, "options": { @@ -571,7 +571,7 @@ "h": 10, "w": 8, "x": 8, - "y": 16 + "y": 31 }, "id": 16, "options": { @@ -764,7 +764,7 @@ "h": 10, "w": 8, "x": 16, - "y": 16 + "y": 31 }, "id": 10, "options": { @@ -845,7 +845,7 @@ "h": 1, "w": 24, "x": 0, - "y": 26 + "y": 41 }, "id": 103, "panels": [], @@ -1080,7 +1080,7 @@ "h": 10, "w": 12, "x": 0, - "y": 27 + "y": 42 }, "id": 11, "options": { @@ -1323,7 +1323,7 @@ "h": 10, "w": 12, "x": 12, - "y": 27 + "y": 42 }, "id": 13, "options": { @@ -1443,7 +1443,7 @@ "h": 7, "w": 24, "x": 0, - "y": 37 + "y": 52 }, "id": 12, "options": { @@ -1494,7 +1494,7 @@ "h": 1, "w": 24, "x": 0, - "y": 44 + "y": 59 }, "id": 20, "panels": [], @@ -1563,7 +1563,7 @@ "h": 8, "w": 12, "x": 0, - "y": 45 + "y": 60 }, "id": 21, "options": { @@ -1670,7 +1670,7 @@ "h": 8, "w": 12, "x": 12, - "y": 45 + "y": 60 }, "id": 22, "options": { @@ -1766,7 +1766,7 @@ "h": 8, "w": 12, "x": 0, - "y": 53 + "y": 68 }, "id": 23, "options": { @@ -1862,7 +1862,7 @@ "h": 8, "w": 12, "x": 12, - "y": 53 + "y": 68 }, "id": 24, "options": { @@ -1902,7 +1902,7 @@ "h": 1, "w": 24, "x": 0, - "y": 61 + "y": 76 }, "id": 30, "panels": [], @@ -1971,7 +1971,7 @@ "h": 8, "w": 12, "x": 0, - "y": 62 + "y": 77 }, "id": 31, "options": { @@ -2078,7 +2078,7 @@ "h": 8, "w": 12, "x": 12, - "y": 62 + "y": 77 }, "id": 32, "options": { @@ -2174,7 +2174,7 @@ "h": 8, "w": 12, "x": 0, - "y": 70 + "y": 85 }, "id": 33, "options": { @@ -2270,7 +2270,7 @@ "h": 8, "w": 12, "x": 12, - "y": 70 + "y": 85 }, "id": 34, "options": { @@ -2310,7 +2310,7 @@ "h": 1, "w": 24, "x": 0, - "y": 78 + "y": 93 }, "id": 40, "panels": [], @@ -2379,7 +2379,7 @@ "h": 8, "w": 12, "x": 0, - "y": 79 + "y": 94 }, "id": 41, "options": { @@ -2475,7 +2475,7 @@ "h": 8, "w": 12, "x": 12, - "y": 79 + "y": 94 }, "id": 42, "options": { @@ -2571,7 +2571,7 @@ "h": 8, "w": 12, "x": 0, - "y": 87 + "y": 102 }, "id": 43, "options": { @@ -2667,7 +2667,7 @@ "h": 8, "w": 12, "x": 12, - "y": 87 + "y": 102 }, "id": 44, "options": { @@ -2707,7 +2707,7 @@ "h": 1, "w": 24, "x": 0, - "y": 95 + "y": 110 }, "id": 102, "panels": [], @@ -2776,7 +2776,7 @@ "h": 9, "w": 8, "x": 0, - "y": 96 + "y": 111 }, "id": 6, "options": { @@ -2905,7 +2905,7 @@ "h": 9, "w": 8, "x": 8, - "y": 96 + "y": 111 }, "id": 7, "options": { @@ -3040,7 +3040,7 @@ "h": 9, "w": 8, "x": 16, - "y": 96 + "y": 111 }, "id": 8, "options": { From 5de842ad179b091ae1c914c62340d4d42d30c835 Mon Sep 17 00:00:00 2001 From: Scott Miller Date: Thu, 3 Sep 2026 18:52:35 +0000 Subject: [PATCH 11/19] fix(examples/monitoring): remove the turn rate and stage sample rate panel Turns per minute and occurrences per turn at each level cover what it showed without mixing per-turn and per-occurrence rates. --- .../grafana/chatd-lifecycle/README.md | 8 -- .../grafana/chatd-lifecycle/dashboard.json | 124 +----------------- 2 files changed, 3 insertions(+), 129 deletions(-) diff --git a/examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md b/examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md index bb5a9ce03b2..e11a6548ad3 100644 --- a/examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md +++ b/examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md @@ -257,14 +257,6 @@ them, and `$model`, `$effort` and `$chat_kind` do not apply. The and does honor the filters. How to read: the primary user-perceived responsiveness metric for streaming. -**Turn rate and stage sample rate** - completed `chat_turn` per second -plus one rate series per other stage. Dimensions: `scope="turn"` and -`$chat_kind`, split by `stage`, with `$model`/`$effort` applied to the -model-carrying stages only. How to read: throughput and shape - a stage -rate above the turn rate means the stage repeats within a turn -(generation steps, provider attempts, tool calls); `provider_attempt` -rising faster than `stream` indicates retries. - **Background provider calls ($stat)** - rate and selected `$stat` duration of background-scope `provider_attempt` samples: detached title/summary/quickgen requests that are excluded from every other diff --git a/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json b/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json index e7e9ad23dad..39c9d96492a 100644 --- a/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json +++ b/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json @@ -2774,7 +2774,7 @@ }, "gridPos": { "h": 9, - "w": 8, + "w": 12, "x": 0, "y": 111 }, @@ -2843,124 +2843,6 @@ "title": "Time to first token", "type": "timeseries" }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Completed chat turns per second and the per-stage sample rate from the histogram _count series. Stage rates above the turn rate mean the stage repeats within a turn (generation steps, tool calls, provider attempts); rates near zero mean the stage rarely fires.\n\n$model and $effort apply only to the stages that carry those labels (generation_step, prepare, provider_attempt, time_to_first_token, stream, thinking, tool_call, compaction). The stages recorded before a model is resolved (chat_turn, queue_wait, capacity_wait, acquisition, mcp_connect, commit) are always matched without them, so they stay populated under any selection.\n\n$chat_kind selects root chats, subagent chats spawned by a parent agent, or both. It is a turn property carried by every turn-scoped stage, so unlike $model and $effort it applies to every stage in this panel.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 111 - }, - "id": 7, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval]))", - "range": true, - "refId": "A", - "legendFormat": "chat turns" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"queue_wait|capacity_wait|acquisition|mcp_connect|commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__rate_interval]))", - "range": true, - "refId": "B", - "legendFormat": "{{stage}}" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum by (stage) (rate(coderd_chatd_stage_duration_seconds_count{stage=~\"generation_step|prepare|provider_attempt|time_to_first_token|stream|thinking|tool_call|compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__rate_interval]))", - "range": true, - "refId": "C", - "legendFormat": "{{stage}}" - } - ], - "title": "Turn rate and stage sample rate", - "type": "timeseries" - }, { "datasource": { "type": "prometheus", @@ -3038,8 +2920,8 @@ }, "gridPos": { "h": 9, - "w": 8, - "x": 16, + "w": 12, + "x": 12, "y": 111 }, "id": 8, From da81bb03ae7798fa45d7cf69e9ee5035a96fb71c Mon Sep 17 00:00:00 2001 From: Scott Miller Date: Thu, 3 Sep 2026 19:07:06 +0000 Subject: [PATCH 12/19] fix(examples/monitoring): show the flamegraph without its built-in table The hierarchy table beside it carries the same rows with level and occurrence counts, so the flamegraph takes its full panel width. --- .../grafana/chatd-lifecycle/dashboard.json | 66 ++++++++++--------- 1 file changed, 34 insertions(+), 32 deletions(-) diff --git a/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json b/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json index 39c9d96492a..7812c439109 100644 --- a/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json +++ b/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json @@ -65,13 +65,15 @@ ] }, "gridPos": { - "h": 12, - "w": 24, + "h": 17, + "w": 14, "x": 0, "y": 1 }, "id": 1, - "options": {}, + "options": { + "showFlameGraphOnly": true + }, "targets": [ { "datasource": { @@ -300,9 +302,9 @@ }, "gridPos": { "h": 17, - "w": 24, - "x": 0, - "y": 13 + "w": 10, + "x": 14, + "y": 1 }, "id": 2, "options": { @@ -406,7 +408,7 @@ "h": 1, "w": 24, "x": 0, - "y": 30 + "y": 18 }, "id": 104, "panels": [], @@ -475,7 +477,7 @@ "h": 10, "w": 8, "x": 0, - "y": 31 + "y": 19 }, "id": 15, "options": { @@ -571,7 +573,7 @@ "h": 10, "w": 8, "x": 8, - "y": 31 + "y": 19 }, "id": 16, "options": { @@ -764,7 +766,7 @@ "h": 10, "w": 8, "x": 16, - "y": 31 + "y": 19 }, "id": 10, "options": { @@ -845,7 +847,7 @@ "h": 1, "w": 24, "x": 0, - "y": 41 + "y": 29 }, "id": 103, "panels": [], @@ -1080,7 +1082,7 @@ "h": 10, "w": 12, "x": 0, - "y": 42 + "y": 30 }, "id": 11, "options": { @@ -1323,7 +1325,7 @@ "h": 10, "w": 12, "x": 12, - "y": 42 + "y": 30 }, "id": 13, "options": { @@ -1443,7 +1445,7 @@ "h": 7, "w": 24, "x": 0, - "y": 52 + "y": 40 }, "id": 12, "options": { @@ -1494,7 +1496,7 @@ "h": 1, "w": 24, "x": 0, - "y": 59 + "y": 47 }, "id": 20, "panels": [], @@ -1563,7 +1565,7 @@ "h": 8, "w": 12, "x": 0, - "y": 60 + "y": 48 }, "id": 21, "options": { @@ -1670,7 +1672,7 @@ "h": 8, "w": 12, "x": 12, - "y": 60 + "y": 48 }, "id": 22, "options": { @@ -1766,7 +1768,7 @@ "h": 8, "w": 12, "x": 0, - "y": 68 + "y": 56 }, "id": 23, "options": { @@ -1862,7 +1864,7 @@ "h": 8, "w": 12, "x": 12, - "y": 68 + "y": 56 }, "id": 24, "options": { @@ -1902,7 +1904,7 @@ "h": 1, "w": 24, "x": 0, - "y": 76 + "y": 64 }, "id": 30, "panels": [], @@ -1971,7 +1973,7 @@ "h": 8, "w": 12, "x": 0, - "y": 77 + "y": 65 }, "id": 31, "options": { @@ -2078,7 +2080,7 @@ "h": 8, "w": 12, "x": 12, - "y": 77 + "y": 65 }, "id": 32, "options": { @@ -2174,7 +2176,7 @@ "h": 8, "w": 12, "x": 0, - "y": 85 + "y": 73 }, "id": 33, "options": { @@ -2270,7 +2272,7 @@ "h": 8, "w": 12, "x": 12, - "y": 85 + "y": 73 }, "id": 34, "options": { @@ -2310,7 +2312,7 @@ "h": 1, "w": 24, "x": 0, - "y": 93 + "y": 81 }, "id": 40, "panels": [], @@ -2379,7 +2381,7 @@ "h": 8, "w": 12, "x": 0, - "y": 94 + "y": 82 }, "id": 41, "options": { @@ -2475,7 +2477,7 @@ "h": 8, "w": 12, "x": 12, - "y": 94 + "y": 82 }, "id": 42, "options": { @@ -2571,7 +2573,7 @@ "h": 8, "w": 12, "x": 0, - "y": 102 + "y": 90 }, "id": 43, "options": { @@ -2667,7 +2669,7 @@ "h": 8, "w": 12, "x": 12, - "y": 102 + "y": 90 }, "id": 44, "options": { @@ -2707,7 +2709,7 @@ "h": 1, "w": 24, "x": 0, - "y": 110 + "y": 98 }, "id": 102, "panels": [], @@ -2776,7 +2778,7 @@ "h": 9, "w": 12, "x": 0, - "y": 111 + "y": 99 }, "id": 6, "options": { @@ -2922,7 +2924,7 @@ "h": 9, "w": 12, "x": 12, - "y": 111 + "y": 99 }, "id": 8, "options": { From 5cc93adeb5af198d6e86d510e5777d70da17452b Mon Sep 17 00:00:00 2001 From: Scott Miller Date: Thu, 3 Sep 2026 20:28:34 +0000 Subject: [PATCH 13/19] fix(examples/monitoring): place the hierarchy table before the flamegraph --- .../grafana/chatd-lifecycle/dashboard.json | 258 +++++++++--------- 1 file changed, 129 insertions(+), 129 deletions(-) diff --git a/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json b/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json index 7812c439109..5e1974208fe 100644 --- a/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json +++ b/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json @@ -34,134 +34,6 @@ "title": "Stage profile", "type": "row" }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Aggregate profile of chat lifecycle stages over the dashboard time range, using the $stat statistic of coderd_chatd_stage_duration_seconds. Levels come from the fixed stage hierarchy, attached as labels with label_replace and reshaped into Grafana's nested set model by the panel transformations.\n\nStage timings overlap in wall time (stream, thinking and tool_call all run inside a generation_step, and time_to_first_token is part of provider_attempt), and quantiles are not additive across stages. Read this as a stage time profile, not as a strict non-overlapping decomposition of turn latency: child bars can exceed their parent. A stage with no samples in the time range reads as 0. Only scope=\"turn\" samples are included, so detached background provider calls are excluded.\n\nValues come from rate() over counter series, and rate() treats the first sample in the window as its baseline. A stage whose series first appears inside the window reads as 0 until its second observation, so rare stages such as queue_wait, capacity_wait and compaction can render as 0 while real samples exist. Widen the time range to confirm.\n\n$model and $effort apply only to the stages that carry those labels (generation_step, prepare, provider_attempt, time_to_first_token, stream, thinking, tool_call, compaction). The stages recorded before a model is resolved (chat_turn, queue_wait, capacity_wait, acquisition, mcp_connect, commit) are always matched without them, so they stay populated under any selection.\n\nThe self field is the stage's own value for leaf stages and 0 for stages that have children, so the top table's self column reads as leaf time. True self time is not derivable from these histograms: stages overlap in wall time, so a parent minus its children can be negative.\n\nFrame values are seconds. The panel labels them as sample counts and ignores the field unit, because it takes its unit from profile metadata that a Prometheus query cannot set.\n\n$chat_kind selects root chats, subagent chats spawned by a parent agent, or both. It is a turn property carried by every turn-scoped stage, so unlike $model and $effort it applies to every stage in this panel.", - "fieldConfig": { - "defaults": { - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byNames", - "options": { - "names": [ - "value", - "self" - ] - } - }, - "properties": [ - { - "id": "unit", - "value": "s" - } - ] - } - ] - }, - "gridPos": { - "h": 17, - "w": 14, - "x": 0, - "y": 1 - }, - "id": 1, - "options": { - "showFlameGraphOnly": true - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"chat_turn\", \"\", \"\"),\n\"n\", \"01\", \"\", \"\"),\n\"level\", \"0\", \"\", \"\"),\n\"name\", \"chat_turn\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"queue_wait\", \"\", \"\"),\n\"n\", \"02\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"  └ queue_wait\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"capacity_wait\", \"\", \"\"),\n\"n\", \"03\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"  └ capacity_wait\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"acquisition\", \"\", \"\"),\n\"n\", \"04\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"  └ acquisition\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"generation_step\", \"\", \"\"),\n\"n\", \"05\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"  └ generation_step\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"prepare\", \"\", \"\"),\n\"n\", \"06\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"    └ prepare\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"mcp_connect\", \"\", \"\"),\n\"n\", \"07\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"    └ mcp_connect\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"provider_attempt\", \"\", \"\"),\n\"n\", \"08\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"    └ provider_attempt\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"time_to_first_token\", \"\", \"\"),\n\"n\", \"09\", \"\", \"\"),\n\"level\", \"3\", \"\", \"\"),\n\"name\", \"      └ time_to_first_token\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"retry_backoff\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"retry_backoff\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"retry_backoff\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"retry_backoff\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"retry_backoff\", \"\", \"\"),\n\"n\", \"10\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"    └ retry_backoff\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"stream\", \"\", \"\"),\n\"n\", \"11\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"    └ stream\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"thinking\", \"\", \"\"),\n\"n\", \"12\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"    └ thinking\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"tool_call\", \"\", \"\"),\n\"n\", \"13\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"    └ tool_call\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"commit\", \"\", \"\"),\n\"n\", \"14\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"    └ commit\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"compaction\", \"\", \"\"),\n\"n\", \"15\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"    └ compaction\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)", - "range": false, - "refId": "A", - "format": "table", - "instant": true, - "legendFormat": "" - } - ], - "title": "Stage profile flamegraph (${stat:text})", - "transformations": [ - { - "id": "sortBy", - "options": { - "fields": {}, - "sort": [ - { - "field": "n" - } - ] - } - }, - { - "id": "calculateField", - "options": { - "alias": "value", - "binary": { - "left": "Value", - "operator": "*", - "right": "1" - }, - "mode": "binary", - "replaceFields": false - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "number", - "targetField": "level" - }, - { - "destinationType": "number", - "targetField": "leaf" - } - ], - "fields": {} - } - }, - { - "id": "calculateField", - "options": { - "alias": "self", - "binary": { - "left": "value", - "operator": "*", - "right": "leaf" - }, - "mode": "binary", - "replaceFields": false - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "n": true, - "name": true, - "leaf": true, - "Value": true - }, - "includeByName": {}, - "indexByName": {}, - "renameByName": {} - } - } - ], - "type": "flamegraph" - }, { "datasource": { "type": "prometheus", @@ -303,7 +175,7 @@ "gridPos": { "h": 17, "w": 10, - "x": 14, + "x": 0, "y": 1 }, "id": 2, @@ -402,6 +274,134 @@ ], "type": "table" }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Aggregate profile of chat lifecycle stages over the dashboard time range, using the $stat statistic of coderd_chatd_stage_duration_seconds. Levels come from the fixed stage hierarchy, attached as labels with label_replace and reshaped into Grafana's nested set model by the panel transformations.\n\nStage timings overlap in wall time (stream, thinking and tool_call all run inside a generation_step, and time_to_first_token is part of provider_attempt), and quantiles are not additive across stages. Read this as a stage time profile, not as a strict non-overlapping decomposition of turn latency: child bars can exceed their parent. A stage with no samples in the time range reads as 0. Only scope=\"turn\" samples are included, so detached background provider calls are excluded.\n\nValues come from rate() over counter series, and rate() treats the first sample in the window as its baseline. A stage whose series first appears inside the window reads as 0 until its second observation, so rare stages such as queue_wait, capacity_wait and compaction can render as 0 while real samples exist. Widen the time range to confirm.\n\n$model and $effort apply only to the stages that carry those labels (generation_step, prepare, provider_attempt, time_to_first_token, stream, thinking, tool_call, compaction). The stages recorded before a model is resolved (chat_turn, queue_wait, capacity_wait, acquisition, mcp_connect, commit) are always matched without them, so they stay populated under any selection.\n\nThe self field is the stage's own value for leaf stages and 0 for stages that have children, so the top table's self column reads as leaf time. True self time is not derivable from these histograms: stages overlap in wall time, so a parent minus its children can be negative.\n\nFrame values are seconds. The panel labels them as sample counts and ignores the field unit, because it takes its unit from profile metadata that a Prometheus query cannot set.\n\n$chat_kind selects root chats, subagent chats spawned by a parent agent, or both. It is a turn property carried by every turn-scoped stage, so unlike $model and $effort it applies to every stage in this panel.", + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byNames", + "options": { + "names": [ + "value", + "self" + ] + } + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + } + ] + }, + "gridPos": { + "h": 17, + "w": 14, + "x": 10, + "y": 1 + }, + "id": 1, + "options": { + "showFlameGraphOnly": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"chat_turn\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"chat_turn\", \"\", \"\"),\n\"n\", \"01\", \"\", \"\"),\n\"level\", \"0\", \"\", \"\"),\n\"name\", \"chat_turn\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"queue_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"queue_wait\", \"\", \"\"),\n\"n\", \"02\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"  └ queue_wait\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"capacity_wait\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"capacity_wait\", \"\", \"\"),\n\"n\", \"03\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"  └ capacity_wait\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"acquisition\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"acquisition\", \"\", \"\"),\n\"n\", \"04\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"  └ acquisition\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"generation_step\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"generation_step\", \"\", \"\"),\n\"n\", \"05\", \"\", \"\"),\n\"level\", \"1\", \"\", \"\"),\n\"name\", \"  └ generation_step\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"prepare\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"prepare\", \"\", \"\"),\n\"n\", \"06\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"    └ prepare\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"mcp_connect\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"mcp_connect\", \"\", \"\"),\n\"n\", \"07\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"    └ mcp_connect\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"provider_attempt\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"provider_attempt\", \"\", \"\"),\n\"n\", \"08\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"    └ provider_attempt\", \"\", \"\"),\n\"leaf\", \"0\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"time_to_first_token\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"time_to_first_token\", \"\", \"\"),\n\"n\", \"09\", \"\", \"\"),\n\"level\", \"3\", \"\", \"\"),\n\"name\", \"      └ time_to_first_token\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"retry_backoff\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"retry_backoff\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"retry_backoff\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"retry_backoff\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"retry_backoff\", \"\", \"\"),\n\"n\", \"10\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"    └ retry_backoff\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"stream\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"stream\", \"\", \"\"),\n\"n\", \"11\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"    └ stream\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"thinking\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"thinking\", \"\", \"\"),\n\"n\", \"12\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"    └ thinking\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"tool_call\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"tool_call\", \"\", \"\"),\n\"n\", \"13\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"    └ tool_call\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"commit\", scope=\"turn\", chat_kind=~\"$chat_kind\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"commit\", \"\", \"\"),\n\"n\", \"14\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"    └ commit\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)\nor\n(\n label_replace(label_replace(label_replace(label_replace(label_replace((\n histogram_quantile($stat, sum by (le) (rate(coderd_chatd_stage_duration_seconds_bucket{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range]))) and on() (vector($stat) > bool 0) == 1 and on() (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0)\n )\n or\n (\n sum(rate(coderd_chatd_stage_duration_seconds_sum{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) / (sum(rate(coderd_chatd_stage_duration_seconds_count{stage=\"compaction\", scope=\"turn\", chat_kind=~\"$chat_kind\", model=~\"$model\", effort=~\"$effort\"}[$__range])) > 0) and on() (vector($stat) == bool 0) == 1\n )\n or vector(0),\n\"label\", \"compaction\", \"\", \"\"),\n\"n\", \"15\", \"\", \"\"),\n\"level\", \"2\", \"\", \"\"),\n\"name\", \"    └ compaction\", \"\", \"\"),\n\"leaf\", \"1\", \"\", \"\")\n)", + "range": false, + "refId": "A", + "format": "table", + "instant": true, + "legendFormat": "" + } + ], + "title": "Stage profile flamegraph (${stat:text})", + "transformations": [ + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "field": "n" + } + ] + } + }, + { + "id": "calculateField", + "options": { + "alias": "value", + "binary": { + "left": "Value", + "operator": "*", + "right": "1" + }, + "mode": "binary", + "replaceFields": false + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "number", + "targetField": "level" + }, + { + "destinationType": "number", + "targetField": "leaf" + } + ], + "fields": {} + } + }, + { + "id": "calculateField", + "options": { + "alias": "self", + "binary": { + "left": "value", + "operator": "*", + "right": "leaf" + }, + "mode": "binary", + "replaceFields": false + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "n": true, + "name": true, + "leaf": true, + "Value": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": {} + } + } + ], + "type": "flamegraph" + }, { "collapsed": false, "gridPos": { From 0f706bc2a2f2d5b84d3a2ba3db58b7c4f6e35d8a Mon Sep 17 00:00:00 2001 From: Scott Miller Date: Thu, 3 Sep 2026 22:45:58 +0000 Subject: [PATCH 14/19] fix(coderd/x/chatd): close the chat turn span after its finishing step ends The finishing transition runs inside the last generation step, and the turn's accounting was emitted from there, before the step's own stage had ended. Complete now only marks the turn finished; StartGeneration settles it once runGenerationStep has returned, so the finishing step is counted and the span closes at turn end instead of at runner teardown. Each task carries a turn token so a task that outlives its turn cannot complete or invalidate the turn that replaced it, and Invalidate no longer finishes the turn, so a retried task continues the same turn rather than opening a second one with a duplicate acquisition. --- coderd/x/chatd/ARCHITECTURE.md | 2 + coderd/x/chatd/chatloop/stage.go | 9 +- coderd/x/chatd/generation.go | 50 ++---- coderd/x/chatd/options.go | 7 +- coderd/x/chatd/stage_internal_test.go | 214 ++++++++++++++++++++++++-- coderd/x/chatd/turn_trace.go | 154 ++++++++++++------ 6 files changed, 331 insertions(+), 105 deletions(-) diff --git a/coderd/x/chatd/ARCHITECTURE.md b/coderd/x/chatd/ARCHITECTURE.md index f41bc99d307..4882a0d9498 100644 --- a/coderd/x/chatd/ARCHITECTURE.md +++ b/coderd/x/chatd/ARCHITECTURE.md @@ -784,6 +784,8 @@ The runner is responsible for subscribing to the `chat:update:{chat_id}` pubsub + + ### Event shape Every event that the runner loop processes has the following shape: diff --git a/coderd/x/chatd/chatloop/stage.go b/coderd/x/chatd/chatloop/stage.go index b6f36c6ff63..8c76719cee8 100644 --- a/coderd/x/chatd/chatloop/stage.go +++ b/coderd/x/chatd/chatloop/stage.go @@ -118,7 +118,10 @@ func (t *StageTracer) otelTracer() trace.Tracer { return t.tracer } -func (t *StageTracer) now() time.Time { +// Now returns the current time from the tracer's clock. Callers that +// record stages from explicit timestamps use it so their windows share +// the time source of the stages measured by the tracer. +func (t *StageTracer) Now() time.Time { if t == nil || t.clock == nil { return time.Now() } @@ -260,7 +263,7 @@ func (t *StageTracer) startSpan( start time.Time, opts []trace.SpanStartOption, ) (context.Context, *StageSpan) { - now := t.now() + now := t.Now() if start.IsZero() || start.After(now) { start = now } else { @@ -385,7 +388,7 @@ func (s *StageSpan) closeSpan(err error) (elapsed time.Duration, ok bool) { return 0, false } s.ended = true - elapsed = s.tracer.now().Sub(s.start) + elapsed = s.tracer.Now().Sub(s.start) if err != nil { s.span.RecordError(err) s.span.SetStatus(codes.Error, err.Error()) diff --git a/coderd/x/chatd/generation.go b/coderd/x/chatd/generation.go index 3b41c4c268d..513d06dce86 100644 --- a/coderd/x/chatd/generation.go +++ b/coderd/x/chatd/generation.go @@ -447,7 +447,8 @@ func (s *taskStarter) StartGeneration(ctx context.Context, input chatWorkerTaskS if err != nil { return xerrors.Errorf("load generation state: %w", err) } - turnCtx := input.Turn.Ensure(ctx, chat, triggerMessageTime(messages)) + turnCtx, turnToken := input.Turn.Ensure(ctx, chat, triggerMessageTime(messages)) + input.TurnToken = turnToken var again bool input, again, err = s.runGenerationStep(turnCtx, machine, input, chat, messages) if again { @@ -456,8 +457,11 @@ func (s *taskStarter) StartGeneration(ctx context.Context, input chatWorkerTaskS if err != nil { // The turn stops here, so its stage totals cover only part // of a turn. - input.Turn.Invalidate() + input.Turn.Invalidate(input.TurnToken) } + // The step's stage has ended by now, so a turn the step finished + // closes with that stage counted. + input.Turn.Settle(ctx, input.TurnToken) return err } } @@ -1589,46 +1593,10 @@ func (s *taskStarter) finishGenerationTurnWithoutHook( recordGenerationFinishFailure(input.DebugTurn, err) return err } - s.finishTurnAccounting(ctx, input, promotedQueuedAt) + input.Turn.Complete(input.TurnToken, promotedQueuedAt) return s.completeGenerationTurn(ctx, input, committed, decision.promotedMessageID) } -// finishTurnAccounting closes the turn that just finished and records -// the queue wait of a message the finish transition promoted. A -// promotion opens the next turn here, anchored at the moment the -// message was queued, so the wait it served and the work it causes are -// accounted to the turn it starts rather than the one that released -// it. A zero queuedAt means the transition promoted nothing. -func (s *taskStarter) finishTurnAccounting( - ctx context.Context, - input chatWorkerTaskStartInput, - queuedAt time.Time, -) { - input.Turn.Complete() - if queuedAt.IsZero() { - return - } - input.Turn.Rotate(ctx, queuedAt) - s.recordQueueWaitStage(ctx, input, queuedAt) -} - -// recordQueueWaitStage emits the queue_wait stage for a message just -// promoted out of the queue, measured from the queued row's creation -// to now. It is recorded against the turn span rather than the step -// that observed the promotion, whose window does not contain the -// queue wait. A zero queuedAt means the transition promoted nothing -// and records no stage. -func (s *taskStarter) recordQueueWaitStage( - ctx context.Context, - input chatWorkerTaskStartInput, - queuedAt time.Time, -) { - s.server.stages.Record(input.Turn.Context(ctx), chatloop.StageQueueWait, chatloop.StageModel{}, - queuedAt, time.Now(), nil, - attribute.String(chatloop.AttrChatID, input.ChatID.String()), - ) -} - func (s *taskStarter) finishGenerationTurn( ctx context.Context, machine *chatstate.ChatMachine, @@ -1718,7 +1686,7 @@ func (s *taskStarter) finishGenerationTurn( Kind: runnerActionKind(generationActionGenerateAssistant), }) } - s.finishTurnAccounting(ctx, input, promotedQueuedAt) + input.Turn.Complete(input.TurnToken, promotedQueuedAt) return s.completeGenerationTurn(ctx, input, committed, decision.promotedMessageID) } @@ -1731,7 +1699,7 @@ func (s *taskStarter) finishGenerationError( ) error { // The turn ends on an error, so the stages it collected describe a // partial turn. - input.Turn.Invalidate() + input.Turn.Invalidate(input.TurnToken) classified := chaterror.Classify(cause) // Log the unsanitized cause before persisting so administrators can // diagnose the failure even when the classified user-facing message diff --git a/coderd/x/chatd/options.go b/coderd/x/chatd/options.go index 4bc6f130ff9..edd24f6ab8b 100644 --- a/coderd/x/chatd/options.go +++ b/coderd/x/chatd/options.go @@ -69,8 +69,11 @@ type chatWorkerTaskStartInput struct { RequiresActionDeadlineAt sql.NullTime DebugTurn *runnerDebugTurn Turn *runnerTurnSpan - SessionStart *sessionStartTracker - StopNudges *stopNudgeTracker + // TurnToken identifies the turn this task's steps run in. It is set + // by StartGeneration once Turn has opened or reused a turn span. + TurnToken turnToken + SessionStart *sessionStartTracker + StopNudges *stopNudgeTracker } func (i chatWorkerTaskStartInput) hookTurnID() *uuid.UUID { diff --git a/coderd/x/chatd/stage_internal_test.go b/coderd/x/chatd/stage_internal_test.go index c456f6a3f13..1eff5ddaff2 100644 --- a/coderd/x/chatd/stage_internal_test.go +++ b/coderd/x/chatd/stage_internal_test.go @@ -10,6 +10,7 @@ import ( "time" "github.com/google/uuid" + "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/require" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" @@ -187,6 +188,68 @@ func TestStageSpanRoundTripperModel(t *testing.T) { attribute.String(chatloop.AttrReasoningEffort, model.Effort)) } +// newStageMetricsTracer returns a stage tracer writing spans into an +// in-memory recorder and metrics into a private registry. +func newStageMetricsTracer(t *testing.T) (*chatloop.StageTracer, *tracetest.SpanRecorder, *prometheus.Registry) { + t.Helper() + recorder := tracetest.NewSpanRecorder() + provider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder)) + t.Cleanup(func() { + require.NoError(t, provider.Shutdown(context.Background())) + }) + registry := prometheus.NewRegistry() + return chatloop.NewStageTracer(provider, chatloop.NewMetrics(registry)), recorder, registry +} + +// turnStageCounts returns, per stage, the number of turns that +// observed the stage and the total occurrences they reported. +func turnStageCounts(t *testing.T, registry *prometheus.Registry) map[string]struct{ turns, occurrences uint64 } { + t.Helper() + families, err := registry.Gather() + require.NoError(t, err) + out := map[string]struct{ turns, occurrences uint64 }{} + for _, family := range families { + if family.GetName() != "coderd_chatd_turn_stage_count" { + continue + } + for _, metric := range family.GetMetric() { + var stage string + for _, label := range metric.GetLabel() { + if label.GetName() == "stage" { + stage = label.GetValue() + } + } + hist := metric.GetHistogram() + out[stage] = struct{ turns, occurrences uint64 }{ + turns: hist.GetSampleCount(), + occurrences: uint64(hist.GetSampleSum()), + } + } + } + return out +} + +// emittedTurns returns how many turns reported their category +// partition. +func emittedTurns(t *testing.T, registry *prometheus.Registry) uint64 { + t.Helper() + families, err := registry.Gather() + require.NoError(t, err) + for _, family := range families { + if family.GetName() != "coderd_chatd_turn_time_seconds" { + continue + } + for _, metric := range family.GetMetric() { + for _, label := range metric.GetLabel() { + if label.GetName() == "category" && label.GetValue() == chatloop.CategoryUnattributed { + return metric.GetHistogram().GetSampleCount() + } + } + } + } + return 0 +} + func TestRunnerTurnSpanStartsAtTriggerMessage(t *testing.T) { t.Parallel() tracer, recorder := newStageTestTracer(t) @@ -194,7 +257,7 @@ func TestRunnerTurnSpanStartsAtTriggerMessage(t *testing.T) { chat := database.Chat{ID: uuid.New()} triggerAt := time.Now().Add(-2 * time.Second) - turnCtx := turn.Ensure(t.Context(), chat, triggerAt) + turnCtx, _ := turn.Ensure(t.Context(), chat, triggerAt) turn.End(nil) ended := recorder.Ended() @@ -216,7 +279,7 @@ func TestRunnerTurnSpanParentsRecordedStages(t *testing.T) { turn := newRunnerTurnSpan(tracer) chat := database.Chat{ID: uuid.New()} - turnCtx := turn.Ensure(t.Context(), chat, time.Now().Add(-time.Second)) + turnCtx, _ := turn.Ensure(t.Context(), chat, time.Now().Add(-time.Second)) stepCtx, step := tracer.Start(turnCtx, chatloop.StageGenerationStep) step.End(nil) @@ -284,7 +347,7 @@ func TestServerInflightContextIsBackgroundScoped(t *testing.T) { chat := database.Chat{ID: uuid.New()} turn := newRunnerTurnSpan(tracer) - turnCtx := turn.Ensure(t.Context(), chat, time.Now().Add(-time.Second)) + turnCtx, _ := turn.Ensure(t.Context(), chat, time.Now().Add(-time.Second)) inflightCtx, stop := server.inflightChatContext(turnCtx, chat) t.Cleanup(stop) @@ -312,7 +375,7 @@ func TestRunnerTurnSpanCarriesChatKind(t *testing.T) { turn := newRunnerTurnSpan(tracer) chat := database.Chat{ID: uuid.New(), ParentChatID: uuid.NullUUID{UUID: uuid.New(), Valid: true}} - turnCtx := turn.Ensure(t.Context(), chat, time.Now().Add(-time.Second)) + turnCtx, _ := turn.Ensure(t.Context(), chat, time.Now().Add(-time.Second)) _, step := tracer.Start(turnCtx, chatloop.StageGenerationStep) step.End(nil) turn.End(nil) @@ -341,22 +404,60 @@ func turnSpansByStart(t *testing.T, recorder *tracetest.SpanRecorder) []sdktrace return turns } -func TestRunnerTurnSpanRotatesOnPromotion(t *testing.T) { +func TestRunnerTurnSpanCountsFinishingStep(t *testing.T) { t.Parallel() - tracer, recorder := newStageTestTracer(t) + tracer, recorder, registry := newStageMetricsTracer(t) turn := newRunnerTurnSpan(tracer) chat := database.Chat{ID: uuid.New()} - turnCtx := turn.Ensure(t.Context(), chat, time.Now().Add(-time.Minute)) + turnCtx, token := turn.Ensure(t.Context(), chat, time.Now().Add(-time.Minute)) + // The finishing transition runs inside the step, so Complete + // arrives while the step's stage is still open. _, step := tracer.Start(turnCtx, chatloop.StageGenerationStep) + turn.Complete(token, time.Time{}) + require.EqualValues(t, 0, emittedTurns(t, registry), "the turn must stay open until the step ends") step.End(nil) + turn.Settle(t.Context(), token) + + require.EqualValues(t, 1, emittedTurns(t, registry)) + counts := turnStageCounts(t, registry) + require.EqualValues(t, 1, counts[chatloop.StageGenerationStep].turns) + require.EqualValues(t, 1, counts[chatloop.StageGenerationStep].occurrences) + + turns := turnSpansByStart(t, recorder) + require.Len(t, turns, 1) + for _, span := range recorder.Ended() { + if span.Name() == chatloop.StageGenerationStep { + require.False(t, span.EndTime().After(turns[0].EndTime()), + "the step must end inside its turn span") + } + } + + // Settle closed the turn; the runner's End has nothing left. + turn.End(nil) + require.Len(t, turnSpansByStart(t, recorder), 1) +} + +func TestRunnerTurnSpanSettleRotatesOnPromotion(t *testing.T) { + t.Parallel() + tracer, recorder, registry := newStageMetricsTracer(t) + turn := newRunnerTurnSpan(tracer) + chat := database.Chat{ID: uuid.New()} + turnCtx, token := turn.Ensure(t.Context(), chat, time.Now().Add(-time.Minute)) + _, step := tracer.Start(turnCtx, chatloop.StageGenerationStep) queuedAt := time.Now().Add(-30 * time.Second) - turn.Complete() - turn.Rotate(t.Context(), queuedAt) - // The promoted message's wait belongs to the turn it opens. - tracer.Record(turn.Context(t.Context()), chatloop.StageQueueWait, chatloop.StageModel{}, - queuedAt, time.Now(), nil) + turn.Complete(token, queuedAt) + promotedBy := time.Now() + step.End(nil) + turn.Settle(t.Context(), token) + + require.EqualValues(t, 1, emittedTurns(t, registry)) + require.EqualValues(t, 1, turnStageCounts(t, registry)[chatloop.StageGenerationStep].occurrences) + + // The next turn is open, anchored at the promoted message. + _, nextToken := turn.Ensure(t.Context(), chat, queuedAt) + require.NotEqual(t, token, nextToken) turn.End(nil) turns := turnSpansByStart(t, recorder) @@ -366,6 +467,8 @@ func TestRunnerTurnSpanRotatesOnPromotion(t *testing.T) { require.Contains(t, turns[1].Attributes(), attribute.String(chatloop.AttrChatKind, chatloop.ChatKindRoot)) + // The promoted message's wait belongs to the turn it opens and ends + // when the promotion happened, not when the turn settled. var queueWait sdktrace.ReadOnlySpan for _, span := range recorder.Ended() { if span.Name() == chatloop.StageQueueWait { @@ -374,6 +477,8 @@ func TestRunnerTurnSpanRotatesOnPromotion(t *testing.T) { } require.NotNil(t, queueWait) require.Equal(t, turns[1].SpanContext().SpanID(), queueWait.Parent().SpanID()) + require.Equal(t, queuedAt.UTC(), queueWait.StartTime().UTC()) + require.False(t, queueWait.EndTime().After(promotedBy)) // The rotated turn's head is the queue wait, so it records no // acquisition: both windows start at the same instant and both @@ -388,6 +493,81 @@ func TestRunnerTurnSpanRotatesOnPromotion(t *testing.T) { require.Equal(t, 1, acquisitions) } +func TestRunnerTurnSpanInvalidateAfterPromotionDropsFinishedTurn(t *testing.T) { + t.Parallel() + tracer, recorder, registry := newStageMetricsTracer(t) + turn := newRunnerTurnSpan(tracer) + chat := database.Chat{ID: uuid.New()} + + _, token := turn.Ensure(t.Context(), chat, time.Now().Add(-time.Minute)) + queuedAt := time.Now().Add(-30 * time.Second) + turn.Complete(token, queuedAt) + // Post-commit work of the finished turn failed. The failure belongs + // to the finished turn, not to the one the promotion opens. + turn.Invalidate(token) + turn.Settle(t.Context(), token) + require.EqualValues(t, 0, emittedTurns(t, registry)) + + _, nextToken := turn.Ensure(t.Context(), chat, queuedAt) + turn.Complete(nextToken, time.Time{}) + turn.Settle(t.Context(), nextToken) + require.EqualValues(t, 1, emittedTurns(t, registry)) + require.Len(t, turnSpansByStart(t, recorder), 2) +} + +func TestRunnerTurnSpanIgnoresStaleToken(t *testing.T) { + t.Parallel() + tracer, _, registry := newStageMetricsTracer(t) + turn := newRunnerTurnSpan(tracer) + chat := database.Chat{ID: uuid.New()} + + _, first := turn.Ensure(t.Context(), chat, time.Now().Add(-time.Minute)) + turn.Complete(first, time.Time{}) + // A task for the next prompt arrives before the finishing task has + // settled, and opens the next turn itself. + _, second := turn.Ensure(t.Context(), chat, time.Now().Add(-time.Second)) + require.NotEqual(t, first, second) + require.EqualValues(t, 1, emittedTurns(t, registry)) + + // The finishing task's late calls address a turn that is gone. + turn.Invalidate(first) + turn.Settle(t.Context(), first) + turn.Complete(second, time.Time{}) + turn.Settle(t.Context(), second) + require.EqualValues(t, 2, emittedTurns(t, registry)) +} + +func TestRunnerTurnSpanRetryContinuesTurn(t *testing.T) { + t.Parallel() + tracer, recorder, registry := newStageMetricsTracer(t) + turn := newRunnerTurnSpan(tracer) + chat := database.Chat{ID: uuid.New()} + triggerAt := time.Now().Add(-time.Minute) + + _, token := turn.Ensure(t.Context(), chat, triggerAt) + turn.Invalidate(token) + turn.Settle(t.Context(), token) + + // The retried task runs the same prompt, so it continues the same + // turn and records no second acquisition. + _, retried := turn.Ensure(t.Context(), chat, triggerAt) + require.Equal(t, token, retried) + turn.Complete(retried, time.Time{}) + turn.Settle(t.Context(), retried) + turn.End(nil) + + require.Len(t, turnSpansByStart(t, recorder), 1) + var acquisitions int + for _, span := range recorder.Ended() { + if span.Name() == chatloop.StageAcquisition { + acquisitions++ + } + } + require.Equal(t, 1, acquisitions) + // The turn was invalidated, so its accounting is not emitted. + require.EqualValues(t, 0, emittedTurns(t, registry)) +} + func TestRunnerTurnSpanEnsureOpensTurnPerPrompt(t *testing.T) { t.Parallel() tracer, recorder := newStageTestTracer(t) @@ -395,15 +575,17 @@ func TestRunnerTurnSpanEnsureOpensTurnPerPrompt(t *testing.T) { chat := database.Chat{ID: uuid.New()} firstTrigger := time.Now().Add(-2 * time.Minute) - turn.Ensure(t.Context(), chat, firstTrigger) + _, first := turn.Ensure(t.Context(), chat, firstTrigger) // A second prompt on the same runner reuses the open turn until it // finishes. - turn.Ensure(t.Context(), chat, firstTrigger) + _, again := turn.Ensure(t.Context(), chat, firstTrigger) + require.Equal(t, first, again) require.Len(t, turnSpansByStart(t, recorder), 0) - turn.Complete() + turn.Complete(first, time.Time{}) secondTrigger := time.Now().Add(-time.Minute) - turn.Ensure(t.Context(), chat, secondTrigger) + _, second := turn.Ensure(t.Context(), chat, secondTrigger) + require.NotEqual(t, first, second) turn.End(nil) turns := turnSpansByStart(t, recorder) diff --git a/coderd/x/chatd/turn_trace.go b/coderd/x/chatd/turn_trace.go index 2c087c5199f..49d072bd254 100644 --- a/coderd/x/chatd/turn_trace.go +++ b/coderd/x/chatd/turn_trace.go @@ -12,6 +12,12 @@ import ( "github.com/coder/coder/v2/coderd/x/chatd/chatloop" ) +// turnToken identifies one turn opened by runnerTurnSpan.Ensure. The +// turn methods that take a token act only while that turn is the open +// one, so a task that outlives its turn cannot finish or invalidate the +// turn that replaced it. +type turnToken uint64 + // runnerTurnSpan owns the chat_turn span of one runner. A runner is // created when a chat is acquired and torn down when the chat leaves // the running states, so the runner's lifetime bounds the turns it @@ -22,6 +28,11 @@ import ( // out chats, which run no turn. One runner can run several turns in // sequence: a finished turn is replaced by a new span, either when a // queued message is promoted or when the next prompt starts a task. +// +// A turn closes in two steps. Complete marks it finished while the +// step that finished it is still running; Settle closes the span once +// that step's own stage has ended, so the step is counted in the +// turn's accounting. type runnerTurnSpan struct { stages *chatloop.StageTracer @@ -31,65 +42,84 @@ type runnerTurnSpan struct { acc *chatloop.TurnAccumulator chatID string chatKind string - started bool - ended bool - // finished marks a turn that reached a terminal transition, so a - // further prompt opens a new span instead of extending this one. + // token is the identity of the open turn. It advances every time a + // turn span is started. + token turnToken + // open is true while a turn span exists that has not been closed. + open bool + ended bool + // finished marks an open turn that reached a terminal transition + // and is waiting for Settle to close it. finished bool + // pendingPromotion is the queued message the finishing transition + // promoted, when it promoted one. Settle opens the next turn from + // it. + pendingPromotion *turnPromotion +} + +// turnPromotion records a queued message promoted by the transition +// that finished a turn: when the message was queued and when the +// promotion happened. +type turnPromotion struct { + queuedAt time.Time + promotedAt time.Time } func newRunnerTurnSpan(stages *chatloop.StageTracer) *runnerTurnSpan { return &runnerTurnSpan{stages: stages} } -// Ensure starts the chat_turn span on first call and returns a -// context parented to it. triggerAt is the trigger message's -// creation time and becomes the span's start timestamp, so the -// acquisition stage reconstructed from the same instant falls inside -// the turn. The acquisition stage carries no model identity: the -// turn's model is not resolved until preparation runs. +// Ensure returns a context parented to the open chat_turn span and the +// token of that turn, starting the span when none is open. triggerAt is +// the trigger message's creation time and becomes the span's start +// timestamp, so the acquisition stage reconstructed from the same +// instant falls inside the turn. The acquisition stage carries no model +// identity: the turn's model is not resolved until preparation runs. // -// A turn that already reached a terminal transition is replaced: the -// prompt this call runs is a new turn, and folding it into the old +// A turn that already reached a terminal transition is settled first: +// the prompt this call runs is a new turn, and folding it into the old // span would report the two as one. // // The span is a standalone trace root. The request that triggered the // turn is handled by a different goroutine, and often a different // replica, than the worker that runs it, so no inbound span context // is available here to link. -func (t *runnerTurnSpan) Ensure(ctx context.Context, chat database.Chat, triggerAt time.Time) context.Context { +func (t *runnerTurnSpan) Ensure(ctx context.Context, chat database.Chat, triggerAt time.Time) (context.Context, turnToken) { if t == nil { - return ctx + return ctx, 0 } t.mu.Lock() defer t.mu.Unlock() if t.ended { - return ctx + return ctx, 0 } - if t.started && !t.finished { - return t.contextLocked(ctx) + if t.open && t.finished { + t.settleLocked(ctx) } - if t.started { - t.closeLocked(nil) + if t.open { + return t.contextLocked(ctx), t.token } t.chatID = chat.ID.String() t.chatKind = chatKindAttr(chat) turnCtx := t.startLocked(ctx, triggerAt) // The window between the trigger message landing in history and a - // worker picking the chat up is the acquisition. A rotated turn has - // no acquisition: its head is the queue wait of the message that - // opened it, and recording both would count that window twice. + // worker picking the chat up is the acquisition. A turn opened by a + // promotion has no acquisition: its head is the queue wait of the + // message that opened it, and recording both would count that + // window twice. t.stages.Record(turnCtx, chatloop.StageAcquisition, chatloop.StageModel{}, - triggerAt, time.Now(), nil, + triggerAt, t.stages.Now(), nil, attribute.String(chatloop.AttrChatID, t.chatID)) - return t.contextLocked(ctx) + return t.contextLocked(ctx), t.token } // startLocked opens a chat_turn span with a fresh accumulator and // returns the context parented to it. func (t *runnerTurnSpan) startLocked(ctx context.Context, startAt time.Time) context.Context { - t.started = true + t.token++ + t.open = true t.finished = false + t.pendingPromotion = nil t.acc = chatloop.NewTurnAccumulator() // The chat kind and the accumulator ride on the context so every @@ -116,7 +146,7 @@ func (t *runnerTurnSpan) Context(ctx context.Context) context.Context { } func (t *runnerTurnSpan) contextLocked(ctx context.Context) context.Context { - if !t.started || t.ended { + if !t.open || t.ended { return ctx } // The scope, chat kind, and accumulator are set independently of @@ -131,47 +161,82 @@ func (t *runnerTurnSpan) contextLocked(ctx context.Context) context.Context { return trace.ContextWithSpanContext(ctx, t.spanCtx) } -// Complete marks the turn as finished normally, which is what makes -// its accounting emittable when the span closes. -func (t *runnerTurnSpan) Complete() { +// Complete marks the turn identified by token as finished normally, +// which is what makes its accounting emittable when the span closes. +// The span stays open until Settle so the stage of the step that ran +// the finishing transition is counted once it ends. +// +// A non-zero queuedAt is the creation time of a queued message the +// finishing transition promoted. Settle opens the next turn anchored +// at it, because the wait that message served and the work it causes +// belong to the turn it starts rather than the one that released it. +func (t *runnerTurnSpan) Complete(token turnToken, queuedAt time.Time) { if t == nil { return } t.mu.Lock() defer t.mu.Unlock() + if !t.ownsLocked(token) { + return + } t.finished = true t.acc.MarkCompleted() + if !queuedAt.IsZero() { + t.pendingPromotion = &turnPromotion{queuedAt: queuedAt, promotedAt: t.stages.Now()} + } } -// Invalidate drops the turn's accounting. A turn that errored or was -// interrupted stops partway through its stages, so its totals do not -// describe a full turn. -func (t *runnerTurnSpan) Invalidate() { +// Invalidate drops the accounting of the turn identified by token. A +// turn that errored or was interrupted stops partway through its +// stages, so its totals do not describe a full turn. The span stays +// open: a task retried for the same prompt continues the same turn. +func (t *runnerTurnSpan) Invalidate(token turnToken) { if t == nil { return } t.mu.Lock() defer t.mu.Unlock() - t.finished = true + if !t.ownsLocked(token) { + return + } t.acc.Invalidate() } -// Rotate closes the current turn and opens the next one, anchored at -// startAt. It is for a queued message promoted by the transition that -// finished the previous turn: the wait that message served and the -// work it causes belong to the turn it opens, not to the one that -// released it. -func (t *runnerTurnSpan) Rotate(ctx context.Context, startAt time.Time) { +// Settle closes the turn identified by token if Complete marked it +// finished, and opens the next turn when the finishing transition +// promoted a queued message. It is called after the finishing step's +// own stage has ended. A turn that is not finished is left open. +func (t *runnerTurnSpan) Settle(ctx context.Context, token turnToken) { if t == nil { return } t.mu.Lock() defer t.mu.Unlock() - if !t.started || t.ended { + if !t.ownsLocked(token) || !t.finished { return } + t.settleLocked(ctx) +} + +// ownsLocked reports whether token identifies the open turn. +func (t *runnerTurnSpan) ownsLocked(token turnToken) bool { + return t.open && !t.ended && token == t.token +} + +// settleLocked closes the open, finished turn. When the finishing +// transition promoted a queued message, it opens the next turn anchored +// at the moment the message was queued and records that message's +// queue wait against the new turn. +func (t *runnerTurnSpan) settleLocked(ctx context.Context) { + promotion := t.pendingPromotion t.closeLocked(nil) - t.startLocked(ctx, startAt) + if promotion == nil { + return + } + turnCtx := t.startLocked(ctx, promotion.queuedAt) + t.stages.Record(turnCtx, chatloop.StageQueueWait, chatloop.StageModel{}, + promotion.queuedAt, promotion.promotedAt, nil, + attribute.String(chatloop.AttrChatID, t.chatID)) } // End closes the chat_turn span. Later calls are ignored. @@ -181,7 +246,7 @@ func (t *runnerTurnSpan) End(err error) { } t.mu.Lock() defer t.mu.Unlock() - if t.ended || !t.started { + if t.ended || !t.open { t.ended = true return } @@ -200,6 +265,9 @@ func (t *runnerTurnSpan) closeLocked(err error) { t.span = nil t.spanCtx = trace.SpanContext{} t.acc = nil + t.open = false + t.finished = false + t.pendingPromotion = nil } // chatKindAttr labels a chat as a subagent or a top-level chat. From b284f7e2d8294a57ca835b7f95dce1e5fc2c741d Mon Sep 17 00:00:00 2001 From: Scott Miller Date: Thu, 3 Sep 2026 22:49:26 +0000 Subject: [PATCH 15/19] fix(coderd/x/chatd/chatloop): keep background stages out of the turn accounting Stages started on a context derived from a turn's inherited the turn's accumulator even when they ran in the background scope or under a nil tracer, so detached title, summary, and status label generation was counted as the user's turn. Only turn-scoped stages started by a live tracer now report to the accumulator on their context. --- coderd/x/chatd/chatloop/stage.go | 15 +++- .../chatloop/turnaccounting_internal_test.go | 73 +++++++++++++++++++ 2 files changed, 86 insertions(+), 2 deletions(-) diff --git a/coderd/x/chatd/chatloop/stage.go b/coderd/x/chatd/chatloop/stage.go index 8c76719cee8..7f34696e39b 100644 --- a/coderd/x/chatd/chatloop/stage.go +++ b/coderd/x/chatd/chatloop/stage.go @@ -263,6 +263,9 @@ func (t *StageTracer) startSpan( start time.Time, opts []trace.SpanStartOption, ) (context.Context, *StageSpan) { + if t == nil { + return ctx, nil + } now := t.Now() if start.IsZero() || start.After(now) { start = now @@ -271,7 +274,13 @@ func (t *StageTracer) startSpan( } chatKind := chatKindFromContext(ctx) opts = append(opts, trace.WithAttributes(stageIdentityAttributes(scope, chatKind)...)) - acc := turnAccumulatorFromContext(ctx) + // Only turn-scoped stages report to the turn on ctx. A background + // stage may run on a context derived from a turn's, and its time is + // not the turn's. + var acc *TurnAccumulator + if scope == ScopeTurn { + acc = turnAccumulatorFromContext(ctx) + } var node *stageNode if acc != nil { if _, attributing := attributingStages[stage]; attributing { @@ -442,7 +451,9 @@ func (t *StageTracer) RecordAs( } span.End(trace.WithTimestamp(end)) t.observe(stage, scope, chatKind, model, end.Sub(start)) - recordAttribution(ctx, stage, end.Sub(start)) + if scope == ScopeTurn { + recordAttribution(ctx, stage, end.Sub(start)) + } } func (t *StageTracer) observe(stage, scope, chatKind string, model StageModel, elapsed time.Duration) { diff --git a/coderd/x/chatd/chatloop/turnaccounting_internal_test.go b/coderd/x/chatd/chatloop/turnaccounting_internal_test.go index f3eb19af8f8..c8eda4af22c 100644 --- a/coderd/x/chatd/chatloop/turnaccounting_internal_test.go +++ b/coderd/x/chatd/chatloop/turnaccounting_internal_test.go @@ -402,6 +402,79 @@ func TestTurnAccountingSkipsUnfinishedTurns(t *testing.T) { }) } +func TestTurnAccountingIgnoresWorkOutsideTheTurn(t *testing.T) { + t.Parallel() + + t.Run("BackgroundScope", func(t *testing.T) { + t.Parallel() + fixture := newTurnFixture(t) + + acc := NewTurnAccumulator() + ctx := ContextWithTurnAccumulator(t.Context(), acc) + turnCtx, turnSpan := fixture.tracer.StartRootAt(ctx, StageChatTurn, fixture.clock.Now(), nil) + stepCtx, step := fixture.tracer.Start(turnCtx, StageGenerationStep) + step.SetGenerationAction("generate_assistant") + fixture.clock.Advance(time.Second) + + // Detached work derives its context from the step's but runs + // in the background scope, so the accumulator it inherits must + // not receive its stages. + backgroundCtx := ContextWithScope(stepCtx, ScopeBackground) + bgCtx, bgStream := fixture.tracer.Start(backgroundCtx, StageStream) + _, bgTTFT := fixture.tracer.Start(bgCtx, StageTimeToFirstToken) + fixture.clock.Advance(3 * time.Second) + bgTTFT.End(nil) + fixture.clock.Advance(4 * time.Second) + bgStream.End(nil) + fixture.tracer.Record(backgroundCtx, StageQueueWait, StageModel{}, + fixture.clock.Now().Add(-time.Second), fixture.clock.Now(), nil) + + step.End(nil) + acc.MarkCompleted() + turnSpan.End(nil) + + stages := fixture.sums(t, "coderd_chatd_turn_stage_seconds", "stage") + require.NotContains(t, stages, StageStream) + require.NotContains(t, stages, StageTimeToFirstToken) + require.NotContains(t, stages, StageQueueWait) + categories := fixture.sums(t, "coderd_chatd_turn_time_seconds", "category") + require.Zero(t, categories[CategoryStreaming]) + require.Zero(t, categories[CategoryTimeToFirstToken]) + require.Zero(t, categories[CategoryScheduling]) + // The step's own time is intact: the background stream did not + // report itself as the step's child. + require.Equal(t, 8.0, categories[CategoryChatdOverhead]) + }) + + t.Run("NilTracer", func(t *testing.T) { + t.Parallel() + fixture := newTurnFixture(t) + + acc := NewTurnAccumulator() + ctx := ContextWithTurnAccumulator(t.Context(), acc) + turnCtx, turnSpan := fixture.tracer.StartRootAt(ctx, StageChatTurn, fixture.clock.Now(), nil) + stepCtx, step := fixture.tracer.Start(turnCtx, StageGenerationStep) + step.SetGenerationAction("generate_assistant") + fixture.clock.Advance(time.Second) + + // A caller without a tracer runs on the turn's context. Its + // stages are discarded rather than folded into the turn. + var none *StageTracer + _, stream := none.Start(stepCtx, StageStream) + fixture.clock.Advance(2 * time.Second) + stream.End(nil) + + step.End(nil) + acc.MarkCompleted() + turnSpan.End(nil) + + require.NotContains(t, fixture.sums(t, "coderd_chatd_turn_stage_seconds", "stage"), StageStream) + categories := fixture.sums(t, "coderd_chatd_turn_time_seconds", "category") + require.Zero(t, categories[CategoryStreaming]) + require.Equal(t, 3.0, categories[CategoryChatdOverhead]) + }) +} + func TestTurnAccountingStampsModelOnRoot(t *testing.T) { t.Parallel() fixture := newTurnFixture(t) From 7dbd9b46e6ddb220be22bb9b821cdfc070876f47 Mon Sep 17 00:00:00 2001 From: Scott Miller Date: Thu, 3 Sep 2026 22:55:10 +0000 Subject: [PATCH 16/19] fix(coderd/x/chatd): keep capacity_wait out of the turn partition and fix its bookkeeping capacity_wait is measured by the acquisition loop before a turn exists, so it never reached the turn accumulator, and the three per-turn level 1 panels that matched it stayed empty. Its window also lies inside the acquisition stage the turn records, so categorizing it would double count scheduling time. The stage now feeds the per-occurrence profile only, and the per-turn panels and README say so. The acquisition loop now prunes wait starts only when the candidate batch is complete, since a chat missing from a truncated batch is still waiting, and drops the start when a candidate is skipped for a reason other than capacity. Timestamps come from the worker clock. --- coderd/x/chatd/capacity.go | 21 +++-- coderd/x/chatd/capacity_internal_test.go | 80 +++++++++++++++++++ coderd/x/chatd/chatloop/turnaccounting.go | 8 +- .../chatloop/turnaccounting_internal_test.go | 7 ++ coderd/x/chatd/worker.go | 10 ++- .../grafana/chatd-lifecycle/README.md | 5 ++ .../grafana/chatd-lifecycle/dashboard.json | 12 +-- 7 files changed, 127 insertions(+), 16 deletions(-) create mode 100644 coderd/x/chatd/capacity_internal_test.go diff --git a/coderd/x/chatd/capacity.go b/coderd/x/chatd/capacity.go index 55a744d9409..48b1083d8b9 100644 --- a/coderd/x/chatd/capacity.go +++ b/coderd/x/chatd/capacity.go @@ -2,7 +2,6 @@ package chatd import ( "context" - "time" "github.com/google/uuid" "github.com/prometheus/client_golang/prometheus" @@ -56,7 +55,7 @@ func (w *chatWorker) noteCapacityRefused(chatID uuid.UUID) { if _, ok := w.capacityWaitSince[chatID]; ok { return } - w.capacityWaitSince[chatID] = time.Now() + w.capacityWaitSince[chatID] = w.opts.Clock.Now() } // recordCapacityWait emits the capacity_wait stage for a chat that is @@ -64,7 +63,9 @@ func (w *chatWorker) noteCapacityRefused(chatID uuid.UUID) { // the first refusal this worker saw. Chats admitted on their first // attempt record nothing. The acquisition pass runs before the turn // span exists, so the turn scope and the chat kind are stated -// explicitly. +// explicitly, and the stage reaches the per-occurrence profile only: +// no turn accumulator exists yet to receive it, and its window lies +// inside the acquisition stage the turn records. func (w *chatWorker) recordCapacityWait(ctx context.Context, chat database.Chat) { since, waited := w.capacityWaitSince[chat.ID] if !waited { @@ -73,14 +74,24 @@ func (w *chatWorker) recordCapacityWait(ctx context.Context, chat database.Chat) delete(w.capacityWaitSince, chat.ID) ctx = chatloop.ContextWithChatKind(ctx, chatKindAttr(chat)) w.server.stages.RecordAs(ctx, chatloop.StageCapacityWait, chatloop.ScopeTurn, chatloop.StageModel{}, - since, time.Now(), nil, + since, w.opts.Clock.Now(), nil, attribute.String(chatloop.AttrChatID, chat.ID.String()), ) } +// forgetCapacityWait drops the wait start of a chat this worker will +// not acquire on the current pass for a reason other than capacity: it +// is owned by a live runner, archived, or no longer runnable. A wait +// that resumes later starts from the next refusal. +func (w *chatWorker) forgetCapacityWait(chatID uuid.UUID) { + delete(w.capacityWaitSince, chatID) +} + // pruneCapacityWaits drops wait starts for chats that are no longer // acquisition candidates, which happens when they are archived, -// deleted, or picked up by another worker. +// deleted, or picked up by another worker. candidates must be the +// complete candidate set: a chat missing from a truncated batch is +// still waiting, and dropping it would restart its clock. func (w *chatWorker) pruneCapacityWaits(candidates []database.GetChatWorkerAcquisitionCandidatesRow) { if len(w.capacityWaitSince) == 0 { return diff --git a/coderd/x/chatd/capacity_internal_test.go b/coderd/x/chatd/capacity_internal_test.go new file mode 100644 index 00000000000..7c6e072ae35 --- /dev/null +++ b/coderd/x/chatd/capacity_internal_test.go @@ -0,0 +1,80 @@ +package chatd //nolint:testpackage // Tests the acquisition loop's capacity wait bookkeeping. + +import ( + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + + "github.com/coder/coder/v2/coderd/database" + "github.com/coder/quartz" +) + +func newCapacityWaitWorker(t *testing.T) (*chatWorker, *quartz.Mock) { + t.Helper() + clock := quartz.NewMock(t) + tracer, _ := newStageTestTracer(t) + return &chatWorker{ + server: &Server{stages: tracer}, + opts: chatWorkerOptions{Clock: clock}, + capacityWaitSince: make(map[uuid.UUID]time.Time), + }, clock +} + +func candidateRows(ids ...uuid.UUID) []database.GetChatWorkerAcquisitionCandidatesRow { + rows := make([]database.GetChatWorkerAcquisitionCandidatesRow, 0, len(ids)) + for _, id := range ids { + rows = append(rows, database.GetChatWorkerAcquisitionCandidatesRow{ID: id}) + } + return rows +} + +func TestCapacityWaitBookkeeping(t *testing.T) { + t.Parallel() + + t.Run("FirstRefusalStartsTheClock", func(t *testing.T) { + t.Parallel() + worker, clock := newCapacityWaitWorker(t) + chatID := uuid.New() + + worker.noteCapacityRefused(chatID) + first := worker.capacityWaitSince[chatID] + clock.Advance(time.Second) + worker.noteCapacityRefused(chatID) + require.Equal(t, first, worker.capacityWaitSince[chatID], "a later refusal keeps the first start") + }) + + t.Run("SkippedChatForgetsItsWait", func(t *testing.T) { + t.Parallel() + worker, _ := newCapacityWaitWorker(t) + chatID := uuid.New() + + worker.noteCapacityRefused(chatID) + worker.forgetCapacityWait(chatID) + require.NotContains(t, worker.capacityWaitSince, chatID) + }) + + t.Run("PruneKeepsCandidates", func(t *testing.T) { + t.Parallel() + worker, _ := newCapacityWaitWorker(t) + waiting, gone := uuid.New(), uuid.New() + + worker.noteCapacityRefused(waiting) + worker.noteCapacityRefused(gone) + worker.pruneCapacityWaits(candidateRows(waiting, uuid.New())) + require.Contains(t, worker.capacityWaitSince, waiting) + require.NotContains(t, worker.capacityWaitSince, gone) + }) + + t.Run("RecordClearsTheWait", func(t *testing.T) { + t.Parallel() + worker, clock := newCapacityWaitWorker(t) + chat := database.Chat{ID: uuid.New()} + + worker.noteCapacityRefused(chat.ID) + clock.Advance(time.Second) + worker.recordCapacityWait(t.Context(), chat) + require.NotContains(t, worker.capacityWaitSince, chat.ID) + }) +} diff --git a/coderd/x/chatd/chatloop/turnaccounting.go b/coderd/x/chatd/chatloop/turnaccounting.go index 91ed358ec50..eb8559ad6bc 100644 --- a/coderd/x/chatd/chatloop/turnaccounting.go +++ b/coderd/x/chatd/chatloop/turnaccounting.go @@ -59,10 +59,12 @@ var attributingStages = map[string]struct{}{ // timestamps to their category. They run outside any attributing stage // and cannot nest, so their full duration is categorized. Recorded // stages absent from the map contribute to the per-stage totals only. +// capacity_wait is absent: it is a sub-window of acquisition, measured +// before the turn exists, and categorizing it would count that window +// twice. var recordedStageCategories = map[string]string{ - StageAcquisition: CategoryScheduling, - StageQueueWait: CategoryScheduling, - StageCapacityWait: CategoryScheduling, + StageAcquisition: CategoryScheduling, + StageQueueWait: CategoryScheduling, } // turnAccumulatorKey keys the accumulator of the turn a context runs diff --git a/coderd/x/chatd/chatloop/turnaccounting_internal_test.go b/coderd/x/chatd/chatloop/turnaccounting_internal_test.go index c8eda4af22c..76876367d0d 100644 --- a/coderd/x/chatd/chatloop/turnaccounting_internal_test.go +++ b/coderd/x/chatd/chatloop/turnaccounting_internal_test.go @@ -475,6 +475,13 @@ func TestTurnAccountingIgnoresWorkOutsideTheTurn(t *testing.T) { }) } +func TestTurnAccountingCapacityWaitIsNotCategorized(t *testing.T) { + t.Parallel() + // The capacity wait window lies inside the acquisition window the + // turn records, so categorizing it would count the same time twice. + require.NotContains(t, recordedStageCategories, StageCapacityWait) +} + func TestTurnAccountingStampsModelOnRoot(t *testing.T) { t.Parallel() fixture := newTurnFixture(t) diff --git a/coderd/x/chatd/worker.go b/coderd/x/chatd/worker.go index 53542ae099f..980fc4243a1 100644 --- a/coderd/x/chatd/worker.go +++ b/coderd/x/chatd/worker.go @@ -201,9 +201,10 @@ func (w *chatWorker) acquisitionLoop( func (w *chatWorker) acquireOnce(ctx context.Context, workerID uuid.UUID, manager *runnerManager) { // Fetch twice the budget so one full pool cannot hide candidates in the other. + limit := w.opts.AcquisitionBatchSize * 2 rows, err := w.opts.Store.GetChatWorkerAcquisitionCandidates(ctx, database.GetChatWorkerAcquisitionCandidatesParams{ StaleSeconds: w.opts.HeartbeatStaleSeconds, - LimitCount: w.opts.AcquisitionBatchSize * 2, + LimitCount: limit, }) if err != nil { if ctx.Err() == nil { @@ -215,7 +216,11 @@ func (w *chatWorker) acquireOnce(ctx context.Context, workerID uuid.UUID, manage acquired := int32(0) rootPoolRefused := false subagentPoolRefused := false - w.pruneCapacityWaits(rows) + // A batch shorter than the limit holds every candidate, so a chat + // absent from it has left the candidate set. + if len(rows) < int(limit) { + w.pruneCapacityWaits(rows) + } for _, row := range rows { if acquired >= w.opts.AcquisitionBatchSize { return @@ -323,6 +328,7 @@ func (w *chatWorker) acquireCandidate( return false, errCapacityRefused } if errors.Is(err, errSkipAcquire) || errors.Is(err, chatstate.ErrChatNotFound) { + w.forgetCapacityWait(chatID) return false, nil } if err != nil { diff --git a/examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md b/examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md index e11a6548ad3..0f3ac1860ac 100644 --- a/examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md +++ b/examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md @@ -222,6 +222,11 @@ Members: `acquisition`, `queue_wait`, `capacity_wait`, `generation_step`. The three scheduling waits happen once per turn before generation starts; `generation_step` repeats once per step. +`capacity_wait` appears in the duration per occurrence panel only. It is +measured by the acquisition loop before the turn exists, so no turn +records it, and its window lies inside `acquisition`, which the turn +does record under the `scheduling` category. + Because these are the direct children of the turn, their share panel is the quickest answer to "was this turn slow because of scheduling or because of generation". diff --git a/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json b/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json index 5e1974208fe..ad860b44006 100644 --- a/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json +++ b/examples/monitoring/dashboards/grafana/chatd-lifecycle/dashboard.json @@ -1615,7 +1615,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Total seconds each stage occupied within a turn, from coderd_chatd_turn_stage_seconds.\n\nOne sample per turn, recorded when the turn ends, so this is \"how much of a turn does this stage account for in total\" and it already sums the repeats.\n\nCompare with duration per occurrence: a stage can be fast per occurrence and still dominate a turn by repeating. All three variables apply, because the turn-end metrics are stamped with the turn's model and effort even for stages that ran before it was resolved.", + "description": "Total seconds each stage occupied within a turn, from coderd_chatd_turn_stage_seconds.\n\nOne sample per turn, recorded when the turn ends, so this is \"how much of a turn does this stage account for in total\" and it already sums the repeats.\n\nCompare with duration per occurrence: a stage can be fast per occurrence and still dominate a turn by repeating. All three variables apply, because the turn-end metrics are stamped with the turn's model and effort even for stages that ran before it was resolved.\n\ncapacity_wait is not in this panel: it is measured before the turn exists and reaches only the per-occurrence profile, and its window lies inside acquisition.", "fieldConfig": { "defaults": { "color": { @@ -1697,7 +1697,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_turn_stage_seconds_bucket{stage=~\"acquisition|queue_wait|capacity_wait|generation_step\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_turn_stage_seconds_count{stage=~\"acquisition|queue_wait|capacity_wait|generation_step\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_turn_stage_seconds_sum{stage=~\"acquisition|queue_wait|capacity_wait|generation_step\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_turn_stage_seconds_count{stage=~\"acquisition|queue_wait|capacity_wait|generation_step\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_turn_stage_seconds_bucket{stage=~\"acquisition|queue_wait|generation_step\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_turn_stage_seconds_count{stage=~\"acquisition|queue_wait|generation_step\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_turn_stage_seconds_sum{stage=~\"acquisition|queue_wait|generation_step\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_turn_stage_seconds_count{stage=~\"acquisition|queue_wait|generation_step\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", "legendFormat": "{{stage}}", "range": true, "refId": "A" @@ -1711,7 +1711,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "How many times each stage ran in a turn, from coderd_chatd_turn_stage_count.\n\nOne sample per turn, recorded when the turn ends, so this is \"how much of a turn does this stage account for in total\" and it already sums the repeats. A stage that did not occur in a turn is not recorded for that turn, so this reads as the count among turns where the stage happened at all.\n\nThis is the multiplier between the other two panels: seconds per turn is roughly occurrences per turn times duration per occurrence.", + "description": "How many times each stage ran in a turn, from coderd_chatd_turn_stage_count.\n\nOne sample per turn, recorded when the turn ends, so this is \"how much of a turn does this stage account for in total\" and it already sums the repeats. A stage that did not occur in a turn is not recorded for that turn, so this reads as the count among turns where the stage happened at all.\n\nThis is the multiplier between the other two panels: seconds per turn is roughly occurrences per turn times duration per occurrence.\n\ncapacity_wait is not in this panel: it is measured before the turn exists and reaches only the per-occurrence profile, and its window lies inside acquisition.", "fieldConfig": { "defaults": { "color": { @@ -1793,7 +1793,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_turn_stage_count_bucket{stage=~\"acquisition|queue_wait|capacity_wait|generation_step\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_turn_stage_count_count{stage=~\"acquisition|queue_wait|capacity_wait|generation_step\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_turn_stage_count_sum{stage=~\"acquisition|queue_wait|capacity_wait|generation_step\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_turn_stage_count_count{stage=~\"acquisition|queue_wait|capacity_wait|generation_step\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_turn_stage_count_bucket{stage=~\"acquisition|queue_wait|generation_step\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_turn_stage_count_count{stage=~\"acquisition|queue_wait|generation_step\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_turn_stage_count_sum{stage=~\"acquisition|queue_wait|generation_step\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_turn_stage_count_count{stage=~\"acquisition|queue_wait|generation_step\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", "legendFormat": "{{stage}}", "range": true, "refId": "A" @@ -1807,7 +1807,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Fraction of the turn each stage occupied, from coderd_chatd_stage_share_of_turn.\n\nOne sample per turn, recorded when the turn ends, so this is \"how much of a turn does this stage account for in total\" and it already sums the repeats. Stages overlap in wall time, so shares at one level do not sum to 1 and several stages can each approach the whole turn.\n\nUse it to compare levels: a level whose shares are all small means turn time is going somewhere else, which the turn time partition row attributes.", + "description": "Fraction of the turn each stage occupied, from coderd_chatd_stage_share_of_turn.\n\nOne sample per turn, recorded when the turn ends, so this is \"how much of a turn does this stage account for in total\" and it already sums the repeats. Stages overlap in wall time, so shares at one level do not sum to 1 and several stages can each approach the whole turn.\n\nUse it to compare levels: a level whose shares are all small means turn time is going somewhere else, which the turn time partition row attributes.\n\ncapacity_wait is not in this panel: it is measured before the turn exists and reaches only the per-occurrence profile, and its window lies inside acquisition.", "fieldConfig": { "defaults": { "color": { @@ -1889,7 +1889,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_stage_share_of_turn_bucket{stage=~\"acquisition|queue_wait|capacity_wait|generation_step\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_share_of_turn_count{stage=~\"acquisition|queue_wait|capacity_wait|generation_step\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_stage_share_of_turn_sum{stage=~\"acquisition|queue_wait|capacity_wait|generation_step\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_stage_share_of_turn_count{stage=~\"acquisition|queue_wait|capacity_wait|generation_step\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", + "expr": "(\n histogram_quantile($stat, sum by (le, stage) (rate(coderd_chatd_stage_share_of_turn_bucket{stage=~\"acquisition|queue_wait|generation_step\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])))\n and on() (vector($stat) > bool 0) == 1\n and on(stage) (sum by (stage) (rate(coderd_chatd_stage_share_of_turn_count{stage=~\"acquisition|queue_wait|generation_step\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n)\nor\n(\n sum by (stage) (rate(coderd_chatd_stage_share_of_turn_sum{stage=~\"acquisition|queue_wait|generation_step\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) / (sum by (stage) (rate(coderd_chatd_stage_share_of_turn_count{stage=~\"acquisition|queue_wait|generation_step\", model=~\"$model\", effort=~\"$effort\", chat_kind=~\"$chat_kind\"}[$__rate_interval])) > 0)\n and on() (vector($stat) == bool 0) == 1\n)", "legendFormat": "{{stage}}", "range": true, "refId": "A" From 7651d953b8db6f1bb838701d8dc841db26a873eb Mon Sep 17 00:00:00 2001 From: Scott Miller Date: Thu, 3 Sep 2026 22:59:39 +0000 Subject: [PATCH 17/19] fix(coderd/x/chatd/chatloop): count stage anomalies and recalibrate stage histogram buckets The turn partition was documented as never exceeding the turn duration, but the code only floored the unattributed remainder at zero, so over-attributed turns were emitted with shares summing past 1 and nothing showed it. Those turns are now counted in a new coderd_chatd_stage_anomalies_total counter, alongside stage windows dropped for inverted clocks and turns dropped for a non-positive duration, which were also silent. Buckets now cover the values the metrics describe: durations start at 0.5ms instead of 10ms so the sub-10ms stages resolve, the per-stage share histogram extends past 1 since overlapping and repeating stages exceed the turn, and per-turn stage counts extend past the 1200 step limit instead of stopping at 128. --- coderd/x/chatd/chatloop/metrics.go | 64 +++++++++-- coderd/x/chatd/chatloop/stage.go | 8 ++ coderd/x/chatd/chatloop/stage_test.go | 5 +- coderd/x/chatd/chatloop/turnaccounting.go | 12 +- .../chatloop/turnaccounting_internal_test.go | 107 ++++++++++++++++++ docs/admin/integrations/prometheus.md | 1 + .../grafana/chatd-lifecycle/README.md | 6 +- scripts/metricsdocgen/generated_metrics | 3 + 8 files changed, 192 insertions(+), 14 deletions(-) diff --git a/coderd/x/chatd/chatloop/metrics.go b/coderd/x/chatd/chatloop/metrics.go index 7eb3281a28b..060bd661034 100644 --- a/coderd/x/chatd/chatloop/metrics.go +++ b/coderd/x/chatd/chatloop/metrics.go @@ -24,6 +24,22 @@ const ( CompactionResultSuccess = "success" CompactionResultError = "error" CompactionResultTimeout = "timeout" + + // Label values for StageAnomaliesTotal. + // StageAnomalyNegativeElapsed is a stage whose measured duration + // was negative and was not observed. + StageAnomalyNegativeElapsed = "negative_elapsed" + // StageAnomalyInvertedWindow is a stage reconstructed from + // timestamps whose end preceded its start, or which lacked one of + // them, and was not observed. + StageAnomalyInvertedWindow = "inverted_window" + // StageAnomalyNonPositiveTurn is a finished turn whose duration + // was not positive, so its accounting was not emitted. + StageAnomalyNonPositiveTurn = "nonpositive_turn" + // StageAnomalyOverattributed is a finished turn whose categories + // summed to more than its duration. The categories were emitted as + // measured and the turn's shares sum to more than 1. + StageAnomalyOverattributed = "overattributed" ) // Metrics holds Prometheus metrics for the chatd subsystem. @@ -41,6 +57,7 @@ type Metrics struct { StageShareOfTurn *prometheus.HistogramVec TurnTimeSeconds *prometheus.HistogramVec TurnTimeShare *prometheus.HistogramVec + StageAnomaliesTotal *prometheus.CounterVec CompactionTotal *prometheus.CounterVec StepsTotal *prometheus.CounterVec StreamRetriesTotal *prometheus.CounterVec @@ -128,7 +145,7 @@ func NewMetrics(reg prometheus.Registerer) *Metrics { Subsystem: metricsSubsystem, Name: "stage_share_of_turn", Help: "Fraction of a chat turn's wall time spent in a stage, observed once per turn when the turn ends. Stages overlap, so shares can exceed 1 and do not sum to 1. Only turns that finished normally are counted.", - Buckets: turnShareBuckets(), + Buckets: stageShareBuckets(), }, []string{"stage", "chat_kind", "model", "effort"}), TurnTimeSeconds: factory.NewHistogramVec(prometheus.HistogramOpts{ Namespace: metricsNamespace, @@ -144,6 +161,12 @@ func NewMetrics(reg prometheus.Registerer) *Metrics { Help: "Fraction of a chat turn's wall time in each disjoint category, observed once per turn per category when the turn ends. The shares of one turn sum to 1. Only turns that finished normally are counted.", Buckets: turnShareBuckets(), }, []string{"category", "chat_kind", "model", "effort"}), + StageAnomaliesTotal: factory.NewCounterVec(prometheus.CounterOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "stage_anomalies_total", + Help: "Chat lifecycle stage observations that were dropped or emitted with a known inconsistency, by reason. A steady rate means the stage timings are missing or skewing samples: negative_elapsed and inverted_window are stages whose clocks disagreed, nonpositive_turn is a finished turn whose accounting was not emitted, and overattributed is a turn whose categories summed to more than its duration.", + }, []string{"reason"}), CompactionTotal: factory.NewCounterVec(prometheus.CounterOpts{ Namespace: metricsNamespace, Subsystem: metricsSubsystem, @@ -197,22 +220,32 @@ func NewMetrics(reg prometheus.Registerer) *Metrics { } // stageDurationBuckets returns the duration buckets shared by the -// stage and turn timing histograms: 10ms to ~2.9h, log-spaced. The top -// of the range covers long-lived stages such as a chat turn. +// stage and turn timing histograms: 0.5ms to 3h, log-spaced. The +// bottom of the range resolves the sub-10ms stages (prepare, commit, +// a warm mcp_connect); the top covers long-lived stages such as a +// chat turn. func stageDurationBuckets() []float64 { - return prometheus.ExponentialBuckets(0.01, 2, 21) + return prometheus.ExponentialBucketsRange(0.0005, 3*60*60, 20) } -// turnShareBuckets returns the buckets for the share histograms: 0 to -// 1 in twentieths. +// turnShareBuckets returns the buckets for the category share +// histogram, whose values partition a turn: 0 to 1 in twentieths. func turnShareBuckets() []float64 { return prometheus.LinearBuckets(0, 0.05, 21) } +// stageShareBuckets returns the buckets for the per-stage share +// histogram. Stages overlap in wall time and repeat within a turn, so +// a stage's share can exceed 1; the range extends to 10 so those turns +// are resolved instead of collapsing into the overflow bucket. +func stageShareBuckets() []float64 { + return []float64{0.01, 0.02, 0.05, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1, 1.5, 2, 3, 5, 10} +} + // turnStageCountBuckets returns the buckets for per-turn stage counts: -// every small count, widening to 128. +// every small count, then doubling past the 1200 step limit of a turn. func turnStageCountBuckets() []float64 { - return []float64{1, 2, 3, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128} + return []float64{1, 2, 3, 4, 6, 8, 12, 16, 24, 32, 48, 64, 128, 256, 512, 1024, 2048} } // NopMetrics returns a Metrics instance that discards all data. @@ -226,12 +259,25 @@ func NopMetrics() *Metrics { // and model and effort are empty when the stage ran before a model was // resolved. Negative durations are dropped. No-op when m is nil. func (m *Metrics) RecordStageDuration(stage, scope, chatKind, model, effort string, elapsed time.Duration) { - if m == nil || elapsed < 0 { + if m == nil { + return + } + if elapsed < 0 { + m.RecordStageAnomaly(StageAnomalyNegativeElapsed) return } m.StageDurationSeconds.WithLabelValues(stage, scope, chatKind, model, effort).Observe(elapsed.Seconds()) } +// RecordStageAnomaly counts a stage observation that was dropped or +// emitted inconsistent, by reason. No-op when m is nil. +func (m *Metrics) RecordStageAnomaly(reason string) { + if m == nil { + return + } + m.StageAnomaliesTotal.WithLabelValues(reason).Inc() +} + // RecordTurnStage observes the total time one turn spent in a stage, // that time as a fraction of the turn, and how many times the stage // occurred. All three come from the same turn so they cannot describe diff --git a/coderd/x/chatd/chatloop/stage.go b/coderd/x/chatd/chatloop/stage.go index 7f34696e39b..e3777f5698d 100644 --- a/coderd/x/chatd/chatloop/stage.go +++ b/coderd/x/chatd/chatloop/stage.go @@ -436,6 +436,7 @@ func (t *StageTracer) RecordAs( attrs ...attribute.KeyValue, ) { if start.IsZero() || end.IsZero() || end.Before(start) { + t.recordAnomaly(StageAnomalyInvertedWindow) return } chatKind := chatKindFromContext(ctx) @@ -456,6 +457,13 @@ func (t *StageTracer) RecordAs( } } +func (t *StageTracer) recordAnomaly(reason string) { + if t == nil || t.metrics == nil { + return + } + t.metrics.RecordStageAnomaly(reason) +} + func (t *StageTracer) observe(stage, scope, chatKind string, model StageModel, elapsed time.Duration) { if t == nil || t.metrics == nil { return diff --git a/coderd/x/chatd/chatloop/stage_test.go b/coderd/x/chatd/chatloop/stage_test.go index 0bdac30cad3..5bd6a223fc0 100644 --- a/coderd/x/chatd/chatloop/stage_test.go +++ b/coderd/x/chatd/chatloop/stage_test.go @@ -505,7 +505,10 @@ func TestStageDurationBuckets(t *testing.T) { require.Len(t, family.GetMetric(), 1) buckets = family.GetMetric()[0].GetHistogram().GetBucket() } - require.Len(t, buckets, 21) + require.Len(t, buckets, 20) + // Prepare and commit typically take a few milliseconds, so the + // bottom bucket must resolve below that. + require.Less(t, buckets[0].GetUpperBound(), 0.001) top := buckets[len(buckets)-1] require.Greater(t, top.GetUpperBound(), (3*time.Hour).Seconds()*0.9) require.Equal(t, uint64(1), top.GetCumulativeCount(), diff --git a/coderd/x/chatd/chatloop/turnaccounting.go b/coderd/x/chatd/chatloop/turnaccounting.go index eb8559ad6bc..ea5733beb45 100644 --- a/coderd/x/chatd/chatloop/turnaccounting.go +++ b/coderd/x/chatd/chatloop/turnaccounting.go @@ -364,16 +364,21 @@ func recordAttribution(ctx context.Context, stage string, elapsed time.Duration) // emitTurnAccounting observes the per-stage totals and the category // partition of one turn. turnDuration is the turn's own wall time, and // the categories that did not add up to it become the unattributed -// remainder. Attributed time above the turn duration is clamped, which -// only happens if a stage is counted twice. +// remainder. Categories that add up to more than the turn are emitted +// as measured, with no unattributed time, and counted as an anomaly: +// the turn's shares then sum to more than 1. func (t *StageTracer) emitTurnAccounting(acc *TurnAccumulator, chatKind string, turnDuration time.Duration) { - if t == nil || t.metrics == nil || turnDuration <= 0 { + if t == nil || t.metrics == nil { return } snapshot := acc.snapshot() if !snapshot.emit { return } + if turnDuration <= 0 { + t.recordAnomaly(StageAnomalyNonPositiveTurn) + return + } turnSeconds := turnDuration.Seconds() model := snapshot.model for stage, count := range snapshot.stageCounts { @@ -391,6 +396,7 @@ func (t *StageTracer) emitTurnAccounting(acc *TurnAccumulator, chatKind string, unattributed := turnDuration - attributed if unattributed < 0 { unattributed = 0 + t.recordAnomaly(StageAnomalyOverattributed) } snapshot.categories[CategoryUnattributed] = unattributed for _, category := range TurnTimeCategories { diff --git a/coderd/x/chatd/chatloop/turnaccounting_internal_test.go b/coderd/x/chatd/chatloop/turnaccounting_internal_test.go index 76876367d0d..c3f4e898e6b 100644 --- a/coderd/x/chatd/chatloop/turnaccounting_internal_test.go +++ b/coderd/x/chatd/chatloop/turnaccounting_internal_test.go @@ -482,6 +482,113 @@ func TestTurnAccountingCapacityWaitIsNotCategorized(t *testing.T) { require.NotContains(t, recordedStageCategories, StageCapacityWait) } +// anomalies returns the anomaly counter values keyed by reason. +func (f turnFixture) anomalies(t *testing.T) map[string]float64 { + t.Helper() + families, err := f.registry.Gather() + require.NoError(t, err) + out := map[string]float64{} + for _, family := range families { + if family.GetName() != "coderd_chatd_stage_anomalies_total" { + continue + } + for _, metric := range family.GetMetric() { + out[metricLabel(metric, "reason")] = metric.GetCounter().GetValue() + } + } + return out +} + +func TestTurnAccountingAnomalies(t *testing.T) { + t.Parallel() + + t.Run("Overattributed", func(t *testing.T) { + t.Parallel() + fixture := newTurnFixture(t) + + acc := NewTurnAccumulator() + ctx := ContextWithTurnAccumulator(t.Context(), acc) + turnStart := fixture.clock.Now() + turnCtx, turnSpan := fixture.tracer.StartRootAt(ctx, StageChatTurn, turnStart, nil) + // Two steps overlapping in wall time each report their full + // duration, so the categories exceed the turn. + _, first := fixture.tracer.Start(turnCtx, StageGenerationStep) + _, second := fixture.tracer.Start(turnCtx, StageGenerationStep) + fixture.clock.Advance(4 * time.Second) + first.End(nil) + second.End(nil) + acc.MarkCompleted() + turnSpan.End(nil) + + categories := fixture.sums(t, "coderd_chatd_turn_time_seconds", "category") + require.Equal(t, 8.0, categories[CategoryChatdOverhead], "categories are emitted as measured") + require.Equal(t, 0.0, categories[CategoryUnattributed]) + require.Equal(t, 1.0, fixture.anomalies(t)[StageAnomalyOverattributed]) + }) + + t.Run("NonPositiveTurn", func(t *testing.T) { + t.Parallel() + fixture := newTurnFixture(t) + + acc := NewTurnAccumulator() + ctx := ContextWithTurnAccumulator(t.Context(), acc) + _, turnSpan := fixture.tracer.StartRootAt(ctx, StageChatTurn, fixture.clock.Now(), nil) + acc.MarkCompleted() + turnSpan.End(nil) + + require.Empty(t, fixture.sums(t, "coderd_chatd_turn_time_seconds", "category")) + require.Equal(t, 1.0, fixture.anomalies(t)[StageAnomalyNonPositiveTurn]) + }) + + t.Run("InvertedWindow", func(t *testing.T) { + t.Parallel() + fixture := newTurnFixture(t) + + now := fixture.clock.Now() + fixture.tracer.Record(t.Context(), StageQueueWait, StageModel{}, now, now.Add(-time.Second), nil) + fixture.tracer.Record(t.Context(), StageQueueWait, StageModel{}, time.Time{}, now, nil) + + require.Empty(t, fixture.sums(t, "coderd_chatd_stage_duration_seconds", "stage")) + require.Equal(t, 2.0, fixture.anomalies(t)[StageAnomalyInvertedWindow]) + }) + + t.Run("CleanTurnCountsNothing", func(t *testing.T) { + t.Parallel() + fixture := newTurnFixture(t) + fixture.syntheticTurn(t, StageModel{}) + require.Empty(t, fixture.anomalies(t)) + }) +} + +func TestTurnMetricBuckets(t *testing.T) { + t.Parallel() + registry := prometheus.NewRegistry() + metrics := NewMetrics(registry) + // A stage that repeats can occupy more than the whole turn, and a + // turn can run up to 1200 steps. + metrics.RecordTurnStage(StageStream, ChatKindRoot, "", "", time.Minute, 3.5, 1200) + + families, err := registry.Gather() + require.NoError(t, err) + topBucket := func(name string) *dto.Bucket { + for _, family := range families { + if family.GetName() != name { + continue + } + buckets := family.GetMetric()[0].GetHistogram().GetBucket() + return buckets[len(buckets)-1] + } + t.Fatalf("metric %s not found", name) + return nil + } + share := topBucket("coderd_chatd_stage_share_of_turn") + require.Greater(t, share.GetUpperBound(), 3.5) + require.Equal(t, uint64(1), share.GetCumulativeCount(), "a share above 1 must fall inside the buckets") + count := topBucket("coderd_chatd_turn_stage_count") + require.Greater(t, count.GetUpperBound(), 1200.0) + require.Equal(t, uint64(1), count.GetCumulativeCount(), "a 1200 step turn must fall inside the buckets") +} + func TestTurnAccountingStampsModelOnRoot(t *testing.T) { t.Parallel() fixture := newTurnFixture(t) diff --git a/docs/admin/integrations/prometheus.md b/docs/admin/integrations/prometheus.md index fb7ee8928dd..8978626e91f 100644 --- a/docs/admin/integrations/prometheus.md +++ b/docs/admin/integrations/prometheus.md @@ -249,6 +249,7 @@ The `coder_ai_gateway_cost_control_*` metrics are exported only by `coderd`. | `coderd_chatd_hook_input_overrides_total` | counter | Total lifecycle hook input overrides by event. | `event` | | `coderd_chatd_message_count` | histogram | Number of messages in the prompt per LLM request. | `model` `provider` | | `coderd_chatd_prompt_size_bytes` | histogram | Estimated byte size of the prompt per LLM request. | `model` `provider` | +| `coderd_chatd_stage_anomalies_total` | counter | Chat lifecycle stage observations that were dropped or emitted with a known inconsistency, by reason. A steady rate means the stage timings are missing or skewing samples: negative_elapsed and inverted_window are stages whose clocks disagreed, nonpositive_turn is a finished turn whose accounting was not emitted, and overattributed is a turn whose categories summed to more than its duration. | `reason` | | `coderd_chatd_stage_duration_seconds` | histogram | Wall time spent in each chat lifecycle stage. Stages overlap in wall time; this is a stage-time profile, not a partition of the turn. The scope label separates stages that run inside a chat turn from detached background work. The chat_kind label is empty for stages recorded without a known chat, and the model and effort labels are empty for stages that run before a model is resolved. | `chat_kind` `effort` `model` `scope` `stage` | | `coderd_chatd_stage_share_of_turn` | histogram | Fraction of a chat turn's wall time spent in a stage, observed once per turn when the turn ends. Stages overlap, so shares can exceed 1 and do not sum to 1. Only turns that finished normally are counted. | `chat_kind` `effort` `model` `stage` | | `coderd_chatd_steps_total` | counter | Total agentic loop steps across all chats. | `model` `provider` | diff --git a/examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md b/examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md index 0f3ac1860ac..caba955c2a8 100644 --- a/examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md +++ b/examples/monitoring/dashboards/grafana/chatd-lifecycle/README.md @@ -178,7 +178,11 @@ observed once per turn: The categories are exclusive and sum to the turn, so these panels do add up, unlike the stage panels. `unattributed` is the completeness check: if it grows, real turn time is happening outside every instrumented -stage. +stage. The opposite failure, categories that sum to more than the turn, +is emitted as measured with zero `unattributed` and counted in +`coderd_chatd_stage_anomalies_total{reason="overattributed"}`; that +counter also records stage observations dropped for inverted clocks and +turns dropped for a non-positive duration. These per-turn histograms replaced the earlier time-share panels that divided aggregated stage seconds by aggregated `chat_turn` seconds: the diff --git a/scripts/metricsdocgen/generated_metrics b/scripts/metricsdocgen/generated_metrics index eec13aa1a67..40ed02f1cf8 100644 --- a/scripts/metricsdocgen/generated_metrics +++ b/scripts/metricsdocgen/generated_metrics @@ -319,6 +319,9 @@ coderd_chatd_message_count{provider="",model=""} 0 # HELP coderd_chatd_prompt_size_bytes Estimated byte size of the prompt per LLM request. # TYPE coderd_chatd_prompt_size_bytes histogram coderd_chatd_prompt_size_bytes{provider="",model=""} 0 +# HELP coderd_chatd_stage_anomalies_total Chat lifecycle stage observations that were dropped or emitted with a known inconsistency, by reason. A steady rate means the stage timings are missing or skewing samples: negative_elapsed and inverted_window are stages whose clocks disagreed, nonpositive_turn is a finished turn whose accounting was not emitted, and overattributed is a turn whose categories summed to more than its duration. +# TYPE coderd_chatd_stage_anomalies_total counter +coderd_chatd_stage_anomalies_total{reason=""} 0 # HELP coderd_chatd_stage_duration_seconds Wall time spent in each chat lifecycle stage. Stages overlap in wall time; this is a stage-time profile, not a partition of the turn. The scope label separates stages that run inside a chat turn from detached background work. The chat_kind label is empty for stages recorded without a known chat, and the model and effort labels are empty for stages that run before a model is resolved. # TYPE coderd_chatd_stage_duration_seconds histogram coderd_chatd_stage_duration_seconds{stage="",scope="",chat_kind="",model="",effort=""} 0 From dd5ba02f8943d2f237dbcdf95372ad52451d6d30 Mon Sep 17 00:00:00 2001 From: Scott Miller Date: Thu, 3 Sep 2026 23:01:23 +0000 Subject: [PATCH 18/19] fix(coderd/x/chatd): take the promotion time from the stage tracer clock The queue wait recorded on promotion ended at the wall clock while every other stage window ends at the tracer's clock. --- coderd/x/chatd/chatd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coderd/x/chatd/chatd.go b/coderd/x/chatd/chatd.go index 2afd4355fd7..bb395e5662a 100644 --- a/coderd/x/chatd/chatd.go +++ b/coderd/x/chatd/chatd.go @@ -2194,7 +2194,7 @@ func (p *Server) PromoteQueued( if refreshedOK { chatKind = chatKindAttr(refreshChat) } - p.recordQueueWait(ctx, opts.ChatID, chatKind, promotedQueuedAt, time.Now()) + p.recordQueueWait(ctx, opts.ChatID, chatKind, promotedQueuedAt, p.stages.Now()) } return result, nil } From abffa575d290c7fe7285b62d56c075cd6352f7d1 Mon Sep 17 00:00:00 2001 From: Scott Miller Date: Thu, 3 Sep 2026 23:43:55 +0000 Subject: [PATCH 19/19] chore(coderd/x/chatd): trim lifecycle stage comments to local behavior Drop comment text that restated the code, described a specific caller or callee, or explained the change rather than the mechanism. --- coderd/x/chatd/capacity.go | 7 ++----- coderd/x/chatd/chatd.go | 10 ++++------ coderd/x/chatd/chatloop/stage.go | 25 ++++++++++-------------- coderd/x/chatd/chatstate/transitions.go | 2 -- coderd/x/chatd/model_routing.go | 4 +--- coderd/x/chatd/model_routing_aibridge.go | 5 ++--- coderd/x/chatd/modelcall.go | 6 ++---- coderd/x/chatd/options.go | 4 ++-- coderd/x/chatd/turn_trace.go | 23 ++++++++++------------ 9 files changed, 33 insertions(+), 53 deletions(-) diff --git a/coderd/x/chatd/capacity.go b/coderd/x/chatd/capacity.go index 48b1083d8b9..b619178d8cd 100644 --- a/coderd/x/chatd/capacity.go +++ b/coderd/x/chatd/capacity.go @@ -61,11 +61,8 @@ func (w *chatWorker) noteCapacityRefused(chatID uuid.UUID) { // recordCapacityWait emits the capacity_wait stage for a chat that is // being acquired after at least one capacity refusal, measured from // the first refusal this worker saw. Chats admitted on their first -// attempt record nothing. The acquisition pass runs before the turn -// span exists, so the turn scope and the chat kind are stated -// explicitly, and the stage reaches the per-occurrence profile only: -// no turn accumulator exists yet to receive it, and its window lies -// inside the acquisition stage the turn records. +// attempt record nothing. No turn span exists at this point, so the +// turn scope and the chat kind are stated explicitly. func (w *chatWorker) recordCapacityWait(ctx context.Context, chat database.Chat) { since, waited := w.capacityWaitSince[chat.ID] if !waited { diff --git a/coderd/x/chatd/chatd.go b/coderd/x/chatd/chatd.go index bb395e5662a..c66f9649e36 100644 --- a/coderd/x/chatd/chatd.go +++ b/coderd/x/chatd/chatd.go @@ -5113,10 +5113,9 @@ func (p *Server) inflightContext(reqCtx context.Context) (context.Context, func( // recordQueueWait emits the queue_wait stage for a message that sat // queued from queuedAt until promotedAt. The span context is stripped -// from ctx so the stage is a standalone span rather than a child of the -// promoting request's span, which ends before the turn the wait belongs -// to. The scope and chat kind are set explicitly for the same reason: -// ctx carries the request, not the turn. An empty chatKind records the +// from ctx so the stage is a standalone span rather than a child of +// the span in ctx, and the scope and chat kind are set explicitly +// because ctx does not carry the turn's. An empty chatKind records the // stage without one. func (p *Server) recordQueueWait(ctx context.Context, chatID uuid.UUID, chatKind string, queuedAt, promotedAt time.Time) { standalone := trace.ContextWithSpanContext(ctx, trace.SpanContext{}) @@ -5129,8 +5128,7 @@ func (p *Server) recordQueueWait(ctx context.Context, chatID uuid.UUID, chatKind // inflightChatContext is inflightContext for work that belongs to a // known chat. The chat kind is set on the returned context so the -// stages of the detached work carry it even when the caller never ran -// a turn. +// stages of the detached work carry it. func (p *Server) inflightChatContext(reqCtx context.Context, chat database.Chat) (context.Context, func()) { ctx, stop := p.inflightContext(reqCtx) return chatloop.ContextWithChatKind(ctx, chatKindAttr(chat)), stop diff --git a/coderd/x/chatd/chatloop/stage.go b/coderd/x/chatd/chatloop/stage.go index e3777f5698d..66185c42d47 100644 --- a/coderd/x/chatd/chatloop/stage.go +++ b/coderd/x/chatd/chatloop/stage.go @@ -33,9 +33,9 @@ const ( ) // GenerationActionExecuteLocalTools is the generation_action value of -// a step that runs local tools. Turn accounting reads it to separate -// tool execution from chatd overhead, so it must match the action the -// generation loop reports through SetGenerationAction. +// a step that runs local tools. Turn accounting compares the value +// passed to SetGenerationAction against it to separate tool execution +// from chatd overhead. const GenerationActionExecuteLocalTools = "execute_local_tools" // Span attribute keys. Keys are lowercase snake_case and shared by @@ -58,8 +58,7 @@ const ( // Scope values. A stage is turn scoped when it runs inside a chat // turn's trace, and background scoped when it runs on work detached -// from the turn, such as title and summary generation that outlives -// the turn that triggered it. +// from the turn. const ( ScopeTurn = "turn" ScopeBackground = "background" @@ -118,9 +117,8 @@ func (t *StageTracer) otelTracer() trace.Tracer { return t.tracer } -// Now returns the current time from the tracer's clock. Callers that -// record stages from explicit timestamps use it so their windows share -// the time source of the stages measured by the tracer. +// Now returns the current time from the tracer's clock, the time +// source for the stage windows it measures. func (t *StageTracer) Now() time.Time { if t == nil || t.clock == nil { return time.Now() @@ -322,9 +320,8 @@ func (s *StageSpan) SetAttributes(attrs ...attribute.KeyValue) { } // SetModel records the model identity on the span and on the -// duration observation End makes. Stages that only learn the model -// after they start, such as a generation step that resolves it during -// preparation, call this once it is known. +// duration observation End makes, for stages that learn the model +// after they start. func (s *StageSpan) SetModel(model StageModel) { if s == nil || s.ended { return @@ -422,10 +419,8 @@ func (t *StageTracer) Record( t.RecordAs(ctx, stage, scopeFromContext(ctx), model, start, end, err, attrs...) } -// RecordAs is Record with an explicit scope, for stages that belong -// to a turn but are reconstructed outside its trace, such as the -// capacity wait an acquisition pass measures before the turn span -// exists. +// RecordAs is Record with an explicit scope, for stages recorded on a +// context that does not carry the scope they belong to. func (t *StageTracer) RecordAs( ctx context.Context, stage string, diff --git a/coderd/x/chatd/chatstate/transitions.go b/coderd/x/chatd/chatstate/transitions.go index 19531ac5a22..a0ed1e40cc4 100644 --- a/coderd/x/chatd/chatstate/transitions.go +++ b/coderd/x/chatd/chatstate/transitions.go @@ -1450,8 +1450,6 @@ type FinishTurnResult struct { PromotedMessage *database.ChatMessage // PromotedQueuedAt is the queued row's creation time when this // transition promoted a queue head, and the zero time otherwise. - // The promoted history row carries its insertion time, so this is - // the only record of how long the message sat in the queue. PromotedQueuedAt time.Time } diff --git a/coderd/x/chatd/model_routing.go b/coderd/x/chatd/model_routing.go index fce53b9e609..a6804b8c145 100644 --- a/coderd/x/chatd/model_routing.go +++ b/coderd/x/chatd/model_routing.go @@ -26,9 +26,7 @@ type modelClientRequest struct { type modelBuildOptions struct { ActiveAPIKeyID string RecordHTTP bool - // StageModel labels the provider transport's lifecycle stages. It - // is set by the model call resolver, which knows the resolved model - // and effective reasoning effort before the client is built. + // StageModel labels the provider transport's lifecycle stages. StageModel chatloop.StageModel } diff --git a/coderd/x/chatd/model_routing_aibridge.go b/coderd/x/chatd/model_routing_aibridge.go index d7c2e9dcbb6..f01301547f2 100644 --- a/coderd/x/chatd/model_routing_aibridge.go +++ b/coderd/x/chatd/model_routing_aibridge.go @@ -98,9 +98,8 @@ func (t *stageSpanRoundTripper) RoundTrip(req *http.Request) (*http.Response, er return resp, nil } } - // The span closes on response headers, not on body completion: the - // streamed body outlives this call and is measured by the stream - // stage. + // The span closes on response headers, not on body completion; the + // streamed body outlives this call. span.End(err) return resp, err } diff --git a/coderd/x/chatd/modelcall.go b/coderd/x/chatd/modelcall.go index 503be06e028..a5bacde4031 100644 --- a/coderd/x/chatd/modelcall.go +++ b/coderd/x/chatd/modelcall.go @@ -157,10 +157,8 @@ func (p *Server) resolveModelCall(ctx context.Context, spec modelCallSpec) (reso debugSvc := p.debugService() out.debugEnabled = debugSvc != nil && debugSvc.IsEnabled(ctx, spec.chat.ID, spec.chat.OwnerID) - // The effective effort is resolved before the client is built so the - // provider transport can label its spans with it. Passing it back - // into ProviderOptionsForCall is a no-op re-clamp, which keeps the - // label and the call in agreement. + // The effort is resolved once here so the transport's stage labels + // and the provider call options carry the same value. effectiveEffort := chatprovider.ResolveReasoningEffort(spec.requestedEffort, out.callConfig.ReasoningEffort) if effectiveEffort != nil { out.resolvedEffort = *effectiveEffort diff --git a/coderd/x/chatd/options.go b/coderd/x/chatd/options.go index edd24f6ab8b..c15acfcb6a3 100644 --- a/coderd/x/chatd/options.go +++ b/coderd/x/chatd/options.go @@ -69,8 +69,8 @@ type chatWorkerTaskStartInput struct { RequiresActionDeadlineAt sql.NullTime DebugTurn *runnerDebugTurn Turn *runnerTurnSpan - // TurnToken identifies the turn this task's steps run in. It is set - // by StartGeneration once Turn has opened or reused a turn span. + // TurnToken identifies the turn this task's steps run in. The zero + // token identifies no turn. TurnToken turnToken SessionStart *sessionStartTracker StopNudges *stopNudgeTracker diff --git a/coderd/x/chatd/turn_trace.go b/coderd/x/chatd/turn_trace.go index 49d072bd254..026fe7243d2 100644 --- a/coderd/x/chatd/turn_trace.go +++ b/coderd/x/chatd/turn_trace.go @@ -12,10 +12,10 @@ import ( "github.com/coder/coder/v2/coderd/x/chatd/chatloop" ) -// turnToken identifies one turn opened by runnerTurnSpan.Ensure. The -// turn methods that take a token act only while that turn is the open -// one, so a task that outlives its turn cannot finish or invalidate the -// turn that replaced it. +// turnToken identifies one turn. Methods that take a token act only +// while that turn is the open one, so a holder of a token for a turn +// that has since been replaced cannot finish or invalidate the +// replacement. type turnToken uint64 // runnerTurnSpan owns the chat_turn span of one runner. A runner is @@ -29,10 +29,9 @@ type turnToken uint64 // sequence: a finished turn is replaced by a new span, either when a // queued message is promoted or when the next prompt starts a task. // -// A turn closes in two steps. Complete marks it finished while the -// step that finished it is still running; Settle closes the span once -// that step's own stage has ended, so the step is counted in the -// turn's accounting. +// A turn closes in two steps: Complete marks it finished and Settle +// closes the span, so stages still open at Complete are counted in +// the turn's accounting if they end before Settle. type runnerTurnSpan struct { stages *chatloop.StageTracer @@ -163,8 +162,7 @@ func (t *runnerTurnSpan) contextLocked(ctx context.Context) context.Context { // Complete marks the turn identified by token as finished normally, // which is what makes its accounting emittable when the span closes. -// The span stays open until Settle so the stage of the step that ran -// the finishing transition is counted once it ends. +// The span stays open until Settle. // // A non-zero queuedAt is the creation time of a queued message the // finishing transition promoted. Settle opens the next turn anchored @@ -189,7 +187,7 @@ func (t *runnerTurnSpan) Complete(token turnToken, queuedAt time.Time) { // Invalidate drops the accounting of the turn identified by token. A // turn that errored or was interrupted stops partway through its // stages, so its totals do not describe a full turn. The span stays -// open: a task retried for the same prompt continues the same turn. +// open and a later Ensure continues it. func (t *runnerTurnSpan) Invalidate(token turnToken) { if t == nil { return @@ -204,8 +202,7 @@ func (t *runnerTurnSpan) Invalidate(token turnToken) { // Settle closes the turn identified by token if Complete marked it // finished, and opens the next turn when the finishing transition -// promoted a queued message. It is called after the finishing step's -// own stage has ended. A turn that is not finished is left open. +// promoted a queued message. A turn that is not finished is left open. func (t *runnerTurnSpan) Settle(ctx context.Context, token turnToken) { if t == nil { return