Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 422889e

Browse files
jscottmillerjscottmiller
committed
feat: account chatd turn time by stage and category
Each chat_turn carries a TurnAccumulator that turn-scoped stages report to as they end. Stages that partition the turn form an attribution tree so a step's own time is split into disjoint categories, with tool execution separated from chatd overhead by the step's generation action. When a turn that Complete marked finished settles, its per-stage totals, counts, shares, and category partition are observed once on the turn histograms; Invalidate drops the accounting of a turn that stopped partway through on an error or interruption. Turns whose duration is not positive or whose categories overrun it are counted as anomalies.
1 parent 773324f commit 422889e

10 files changed

Lines changed: 1440 additions & 23 deletions

File tree

‎coderd/x/chatd/ARCHITECTURE.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -869,6 +869,8 @@ It inspects the chat's message history, and decides what's the next step to take
869869

870870
<!-- TODO: document the generation goroutine's lifecycle stages (`generation_step`, `prepare`, `mcp_connect`, `provider_attempt`, `stream`, `time_to_first_token`, `thinking`, `tool_call`, `commit`, `compaction`, `queue_wait`) and the `coderd_chatd_stage_duration_seconds` histogram they feed. -->
871871

872+
<!-- TODO: document the `retry_backoff` stage and the per-turn accounting emitted when a turn that finished normally ends: `coderd_chatd_turn_stage_seconds`, `coderd_chatd_stage_share_of_turn`, `coderd_chatd_turn_time_seconds`, and `coderd_chatd_turn_time_share`, including which stage each turn time category is built from. -->
873+
872874
- `CommitStep`: applied when an LLM API call returns a response.
873875
- `FinishTurn`: applied when the chat processing logic determines that there's no more work to do for the current message history (no pending tool calls, user message is not the last message in the history, etc.).
874876
- `FinishError`: applied when the LLM API call fails and the retry limit is reached, determined by the `generation_attempt` value.

‎coderd/x/chatd/chatloop/metrics.go‎

Lines changed: 100 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,13 @@ const (
3333
// timestamps whose end preceded its start, or which lacked one of
3434
// them, and was not observed.
3535
StageAnomalyInvertedWindow = "inverted_window"
36+
// StageAnomalyNonPositiveTurn is a finished turn whose duration
37+
// was not positive, so its accounting was not emitted.
38+
StageAnomalyNonPositiveTurn = "nonpositive_turn"
39+
// StageAnomalyOverattributed is a finished turn whose categories
40+
// summed to more than its duration. The categories were emitted as
41+
// measured and the turn's shares sum to more than 1.
42+
StageAnomalyOverattributed = "overattributed"
3643
)
3744

3845
// Metrics holds Prometheus metrics for the chatd subsystem.
@@ -45,6 +52,11 @@ type Metrics struct {
4552
ToolErrorsTotal *prometheus.CounterVec
4653
TTFTSeconds *prometheus.HistogramVec
4754
StageDurationSeconds *prometheus.HistogramVec
55+
TurnStageSeconds *prometheus.HistogramVec
56+
TurnStageCount *prometheus.HistogramVec
57+
StageShareOfTurn *prometheus.HistogramVec
58+
TurnTimeSeconds *prometheus.HistogramVec
59+
TurnTimeShare *prometheus.HistogramVec
4860
StageAnomaliesTotal *prometheus.CounterVec
4961
CompactionTotal *prometheus.CounterVec
5062
StepsTotal *prometheus.CounterVec
@@ -114,11 +126,46 @@ func NewMetrics(reg prometheus.Registerer) *Metrics {
114126
Help: "Wall time spent in each chat lifecycle stage. Stages overlap in wall time; this is a stage-time profile, not a partition of the turn. The scope label separates stages that run inside a chat turn from detached background work. The chat_kind label is empty for stages recorded without a known chat, and the model and effort labels are empty for stages that run before a model is resolved.",
115127
Buckets: stageDurationBuckets(),
116128
}, []string{"stage", "scope", "chat_kind", "model", "effort"}),
129+
TurnStageSeconds: factory.NewHistogramVec(prometheus.HistogramOpts{
130+
Namespace: metricsNamespace,
131+
Subsystem: metricsSubsystem,
132+
Name: "turn_stage_seconds",
133+
Help: "Total wall time one chat turn spent in a stage, observed once per turn when the turn ends. Stages overlap, so these do not partition the turn. Only turns that finished normally are counted.",
134+
Buckets: stageDurationBuckets(),
135+
}, []string{"stage", "chat_kind", "model", "effort"}),
136+
TurnStageCount: factory.NewHistogramVec(prometheus.HistogramOpts{
137+
Namespace: metricsNamespace,
138+
Subsystem: metricsSubsystem,
139+
Name: "turn_stage_count",
140+
Help: "Number of times a stage occurred within one chat turn, observed once per turn per stage that occurred. Only turns that finished normally are counted.",
141+
Buckets: turnStageCountBuckets(),
142+
}, []string{"stage", "chat_kind", "model", "effort"}),
143+
StageShareOfTurn: factory.NewHistogramVec(prometheus.HistogramOpts{
144+
Namespace: metricsNamespace,
145+
Subsystem: metricsSubsystem,
146+
Name: "stage_share_of_turn",
147+
Help: "Fraction of a chat turn's wall time spent in a stage, observed once per turn when the turn ends. Stages overlap, so shares can exceed 1 and do not sum to 1. Only turns that finished normally are counted.",
148+
Buckets: stageShareBuckets(),
149+
}, []string{"stage", "chat_kind", "model", "effort"}),
150+
TurnTimeSeconds: factory.NewHistogramVec(prometheus.HistogramOpts{
151+
Namespace: metricsNamespace,
152+
Subsystem: metricsSubsystem,
153+
Name: "turn_time_seconds",
154+
Help: "Wall time of one chat turn split into disjoint categories that sum to the turn duration, observed once per turn per category when the turn ends. Every category is observed, including the ones with no time. Only turns that finished normally are counted.",
155+
Buckets: stageDurationBuckets(),
156+
}, []string{"category", "chat_kind", "model", "effort"}),
157+
TurnTimeShare: factory.NewHistogramVec(prometheus.HistogramOpts{
158+
Namespace: metricsNamespace,
159+
Subsystem: metricsSubsystem,
160+
Name: "turn_time_share",
161+
Help: "Fraction of a chat turn's wall time in each disjoint category, observed once per turn per category when the turn ends. The shares of one turn sum to 1. Only turns that finished normally are counted.",
162+
Buckets: turnShareBuckets(),
163+
}, []string{"category", "chat_kind", "model", "effort"}),
117164
StageAnomaliesTotal: factory.NewCounterVec(prometheus.CounterOpts{
118165
Namespace: metricsNamespace,
119166
Subsystem: metricsSubsystem,
120167
Name: "stage_anomalies_total",
121-
Help: "Chat lifecycle stage observations that were dropped, by reason. A steady rate means the stage timings are missing samples: negative_elapsed and inverted_window are stages whose clocks disagreed.",
168+
Help: "Chat lifecycle stage observations that were dropped or emitted with a known inconsistency, by reason. A steady rate means the stage timings are missing or skewing samples: negative_elapsed and inverted_window are stages whose clocks disagreed, nonpositive_turn is a finished turn whose accounting was not emitted, and overattributed is a turn whose categories summed to more than its duration.",
122169
}, []string{"reason"}),
123170
CompactionTotal: factory.NewCounterVec(prometheus.CounterOpts{
124171
Namespace: metricsNamespace,
@@ -172,14 +219,35 @@ func NewMetrics(reg prometheus.Registerer) *Metrics {
172219
}
173220
}
174221

175-
// stageDurationBuckets returns the duration buckets for the stage
176-
// timing histogram: 0.5ms to 3h, log-spaced. The bottom of the range
177-
// resolves the sub-10ms stages (prepare, commit, a warm mcp_connect);
178-
// the top covers long-lived stages such as a chat turn.
222+
// stageDurationBuckets returns the duration buckets shared by the
223+
// stage and turn timing histograms: 0.5ms to 3h, log-spaced. The
224+
// bottom of the range resolves the sub-10ms stages (prepare, commit,
225+
// a warm mcp_connect); the top covers long-lived stages such as a
226+
// chat turn.
179227
func stageDurationBuckets() []float64 {
180228
return prometheus.ExponentialBucketsRange(0.0005, 3*60*60, 20)
181229
}
182230

231+
// turnShareBuckets returns the buckets for the category share
232+
// histogram, whose values partition a turn: 0 to 1 in twentieths.
233+
func turnShareBuckets() []float64 {
234+
return prometheus.LinearBuckets(0, 0.05, 21)
235+
}
236+
237+
// stageShareBuckets returns the buckets for the per-stage share
238+
// histogram. Stages overlap in wall time and repeat within a turn, so
239+
// a stage's share can exceed 1; the range extends to 10 so those turns
240+
// are resolved instead of collapsing into the overflow bucket.
241+
func stageShareBuckets() []float64 {
242+
return []float64{0.01, 0.02, 0.05, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1, 1.5, 2, 3, 5, 10}
243+
}
244+
245+
// turnStageCountBuckets returns the buckets for per-turn stage counts:
246+
// every small count, then doubling past the 1200 step limit of a turn.
247+
func turnStageCountBuckets() []float64 {
248+
return []float64{1, 2, 3, 4, 6, 8, 12, 16, 24, 32, 48, 64, 128, 256, 512, 1024, 2048}
249+
}
250+
183251
// NopMetrics returns a Metrics instance that discards all data.
184252
// Useful for tests and when metrics collection is not desired.
185253
func NopMetrics() *Metrics {
@@ -201,15 +269,40 @@ func (m *Metrics) RecordStageDuration(stage, scope, chatKind, model, effort stri
201269
m.StageDurationSeconds.WithLabelValues(stage, scope, chatKind, model, effort).Observe(elapsed.Seconds())
202270
}
203271

204-
// RecordStageAnomaly counts a stage observation that was dropped, by
205-
// reason. No-op when m is nil.
272+
// RecordStageAnomaly counts a stage observation that was dropped or
273+
// emitted inconsistent, by reason. No-op when m is nil.
206274
func (m *Metrics) RecordStageAnomaly(reason string) {
207275
if m == nil {
208276
return
209277
}
210278
m.StageAnomaliesTotal.WithLabelValues(reason).Inc()
211279
}
212280

281+
// RecordTurnStage observes the total time one turn spent in a stage,
282+
// that time as a fraction of the turn, and how many times the stage
283+
// occurred. All three come from the same turn so they cannot describe
284+
// different turns. No-op when m is nil.
285+
func (m *Metrics) RecordTurnStage(stage, chatKind, model, effort string, elapsed time.Duration, share float64, count int) {
286+
if m == nil || elapsed < 0 {
287+
return
288+
}
289+
m.TurnStageSeconds.WithLabelValues(stage, chatKind, model, effort).Observe(elapsed.Seconds())
290+
m.StageShareOfTurn.WithLabelValues(stage, chatKind, model, effort).Observe(share)
291+
m.TurnStageCount.WithLabelValues(stage, chatKind, model, effort).Observe(float64(count))
292+
}
293+
294+
// RecordTurnCategory observes one category of a turn's time partition
295+
// and that category as a fraction of the turn. Categories with no time
296+
// are observed as zero so the shares of a turn always sum to 1. No-op
297+
// when m is nil.
298+
func (m *Metrics) RecordTurnCategory(category, chatKind, model, effort string, elapsed time.Duration, share float64) {
299+
if m == nil || elapsed < 0 {
300+
return
301+
}
302+
m.TurnTimeSeconds.WithLabelValues(category, chatKind, model, effort).Observe(elapsed.Seconds())
303+
m.TurnTimeShare.WithLabelValues(category, chatKind, model, effort).Observe(share)
304+
}
305+
213306
// RecordCompaction classifies and records a compaction attempt.
214307
// It is a no-op when m is nil.
215308
func (m *Metrics) RecordCompaction(provider, model string, compacted bool, err error) {

‎coderd/x/chatd/chatloop/stage.go‎

Lines changed: 60 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,12 @@ const (
3232
StageRetryBackoff = "retry_backoff"
3333
)
3434

35+
// GenerationActionExecuteLocalTools is the generation_action value of
36+
// a step that runs local tools. Turn accounting compares the value
37+
// passed to SetGenerationAction against it to separate tool execution
38+
// from chatd overhead.
39+
const GenerationActionExecuteLocalTools = "execute_local_tools"
40+
3541
// Span attribute keys. Keys are lowercase snake_case and shared by
3642
// every stage that carries the value.
3743
const (
@@ -153,6 +159,11 @@ type StageSpan struct {
153159
span trace.Span
154160
start time.Time
155161
ended bool
162+
// acc is the turn the stage runs in, nil outside a turn.
163+
acc *TurnAccumulator
164+
// node is the stage's place in the turn's attribution tree, nil
165+
// for stages that do not partition turn time.
166+
node *stageNode
156167
}
157168

158169
// stageScopeKey keys the stage scope carried by a context. It is
@@ -261,6 +272,20 @@ func (t *StageTracer) startSpan(
261272
}
262273
chatKind := chatKindFromContext(ctx)
263274
opts = append(opts, trace.WithAttributes(stageIdentityAttributes(scope, chatKind)...))
275+
// Only turn-scoped stages report to the turn on ctx. A background
276+
// stage may run on a context derived from a turn's, and its time is
277+
// not the turn's.
278+
var acc *TurnAccumulator
279+
if scope == ScopeTurn {
280+
acc = turnAccumulatorFromContext(ctx)
281+
}
282+
var node *stageNode
283+
if acc != nil {
284+
if _, attributing := attributingStages[stage]; attributing {
285+
node = &stageNode{stage: stage, parent: stageNodeFromContext(ctx)}
286+
ctx = context.WithValue(ctx, stageNodeKey{}, node)
287+
}
288+
}
264289
ctx, span := t.otelTracer().Start(ContextWithScope(ctx, scope), stage, opts...)
265290
return ctx, &StageSpan{
266291
tracer: t,
@@ -269,6 +294,8 @@ func (t *StageTracer) startSpan(
269294
chatKind: chatKind,
270295
span: span,
271296
start: start,
297+
acc: acc,
298+
node: node,
272299
}
273300
}
274301

@@ -300,9 +327,21 @@ func (s *StageSpan) SetModel(model StageModel) {
300327
return
301328
}
302329
s.model = model
330+
s.acc.setModel(model)
303331
s.span.SetAttributes(model.attributes()...)
304332
}
305333

334+
// SetGenerationAction records the action a generation step took, on
335+
// the span and on the step's turn attribution, where it decides
336+
// whether the step's own time counts as tool execution.
337+
func (s *StageSpan) SetGenerationAction(action string) {
338+
if s == nil || s.ended {
339+
return
340+
}
341+
s.node.setAction(action)
342+
s.span.SetAttributes(attribute.String(AttrGenerationAction, action))
343+
}
344+
306345
// SpanContext returns the span context of the stage span, which is
307346
// invalid when tracing is not configured.
308347
func (s *StageSpan) SpanContext() trace.SpanContext {
@@ -316,8 +355,11 @@ func (s *StageSpan) SpanContext() trace.SpanContext {
316355
// as errored when err is non-nil. Calls after the first are ignored so
317356
// a deferred End cannot double-count a stage.
318357
func (s *StageSpan) End(err error) {
358+
s.adoptTurnModel()
319359
if elapsed, ok := s.closeSpan(err); ok {
320360
s.tracer.observe(s.stage, s.scope, s.chatKind, s.model, elapsed)
361+
s.addTurnStageTotal(elapsed)
362+
s.report(elapsed, err)
321363
}
322364
}
323365

@@ -327,7 +369,21 @@ func (s *StageSpan) End(err error) {
327369
// would skew the histogram while the span still needs to report the
328370
// failure.
329371
func (s *StageSpan) EndWithoutObservation(err error) {
330-
s.closeSpan(err)
372+
if elapsed, ok := s.closeSpan(err); ok {
373+
s.report(elapsed, err)
374+
}
375+
}
376+
377+
// adoptTurnModel gives the turn's root stage the model identity that
378+
// the turn resolved after the root started, so the root is labeled
379+
// like the stages inside it.
380+
func (s *StageSpan) adoptTurnModel() {
381+
if s == nil || s.stage != StageChatTurn || s.model.Model != "" {
382+
return
383+
}
384+
if model := s.acc.Model(); model.Model != "" {
385+
s.SetModel(model)
386+
}
331387
}
332388

333389
// closeSpan ends the span and returns the window it covered. ok is
@@ -391,6 +447,9 @@ func (t *StageTracer) RecordAs(
391447
}
392448
span.End(trace.WithTimestamp(end))
393449
t.observe(stage, scope, chatKind, model, end.Sub(start))
450+
if scope == ScopeTurn {
451+
recordAttribution(ctx, stage, end.Sub(start))
452+
}
394453
}
395454

396455
func (t *StageTracer) recordAnomaly(reason string) {

0 commit comments

Comments
 (0)