coder · jscottmiller · Aug 27, 2026 · Sep 1, 2026 · Sep 1, 2026 · Sep 2, 2026
diff --git a/coderd/coderd.go b/coderd/coderd.go
@@ -955,6 +955,7 @@ func New(options *Options) *API {
 				HookDispatcher:                 hookDispatcher,
 				UsageTracker:                   options.WorkspaceUsageTracker,
 				PrometheusRegistry:             options.PrometheusRegistry,
+				TracerProvider:                 options.TracerProvider,
 				AgentCapacityUnlock:            options.ChatAgentCapacityUnlock,
 				OIDCTokenSource:                oidcMCPSrc,
 				NotificationsEnqueuer:          options.NotificationsEnqueuer,

diff --git a/coderd/x/chatd/ARCHITECTURE.md b/coderd/x/chatd/ARCHITECTURE.md
@@ -655,6 +655,8 @@ For every matching chat, it locks it, checks if the chat still meets the aforeme
 
 When a chat is successfully acquired, the acquisition loop requests the [Runner manager](#runner-manager) to spawn a chat runner for it.
 
+<!-- TODO: document the acquisition loop's per-chat capacity refusal tracking, which now emits the `capacity_wait` lifecycle stage on the acquisition that follows a refusal. -->
+
 ### Load balancing
 
 The design doesn't attempt to distribute load between workers fairly. Whenever a chat needs an owner, all replicas race to acquire it. If there's a coder replica that has a lower latency to the database, it'll tend to acquire chats more frequently than other replicas.
@@ -778,6 +780,12 @@ State updates processed by the loop come from:
 
 The runner is responsible for subscribing to the `chat:update:{chat_id}` pubsub channel. During bootstrap, it must first subscribe to the channel and then fetch the initial state of the chat from the database to avoid missing any updates.
 
+<!-- TODO: document that the runner owns the turn-scoped `chat_turn` trace span, started by the first generation task and ended when the runner exits, and that the generation goroutine's stages hang off it. -->
+
+<!-- TODO: document that the runner now opens one `chat_turn` span per prompt rather than one per runner: the span is replaced when a finish transition promotes a queued message (anchored at the moment that message was queued) and when a new prompt starts a task after the previous turn finished. -->
+
+<!-- TODO: document that a turn closes in two steps: the finishing transition marks it complete from inside the generation step, and the span closes (settles) after that step's stage has ended so the step is counted; and that each task holds a turn token so a task that outlives its turn cannot complete or invalidate the turn that replaced it. -->
+
 ### Event shape
 
 Every event that the runner loop processes has the following shape:
@@ -859,6 +867,10 @@ The generation goroutine is responsible for calling the LLM API and executing to
 
 It inspects the chat's message history, and decides what's the next step to take. The result of that step is the application of one of the following core state machine transitions:
 
+<!-- TODO: document the generation goroutine's lifecycle stages (`generation_step`, `prepare`, `mcp_connect`, `provider_attempt`, `stream`, `time_to_first_token`, `thinking`, `tool_call`, `commit`, `compaction`, `queue_wait`) and the `coderd_chatd_stage_duration_seconds` histogram they feed. -->
+
+<!-- TODO: document the `retry_backoff` stage and the per-turn accounting emitted when a turn that finished normally ends: `coderd_chatd_turn_stage_seconds`, `coderd_chatd_stage_share_of_turn`, `coderd_chatd_turn_time_seconds`, and `coderd_chatd_turn_time_share`, including which stage each turn time category is built from. -->
+
 - `CommitStep`: applied when an LLM API call returns a response.
 - `FinishTurn`: applied when the chat processing logic determines that there's no more work to do for the current message history (no pending tool calls, user message is not the last message in the history, etc.).
 - `FinishError`: applied when the LLM API call fails and the retry limit is reached, determined by the `generation_attempt` value.

diff --git a/coderd/x/chatd/capacity.go b/coderd/x/chatd/capacity.go
@@ -5,8 +5,10 @@ import (
 
 	"github.com/google/uuid"
 	"github.com/prometheus/client_golang/prometheus"
+	"go.opentelemetry.io/otel/attribute"
 
 	"github.com/coder/coder/v2/coderd/database"
+	"github.com/coder/coder/v2/coderd/x/chatd/chatloop"
 )
 
 type capacityMetrics struct {
@@ -46,6 +48,62 @@ func (w *chatWorker) capacityMetricsLoop(ctx context.Context) {
 	}
 }
 
+// noteCapacityRefused remembers when a chat was first refused a
+// capacity slot. Only the acquisition loop touches the map, so it
+// needs no lock.
+func (w *chatWorker) noteCapacityRefused(chatID uuid.UUID) {
+	if _, ok := w.capacityWaitSince[chatID]; ok {
+		return
+	}
+	w.capacityWaitSince[chatID] = w.opts.Clock.Now()
+}
+
+// recordCapacityWait emits the capacity_wait stage for a chat that is
+// being acquired after at least one capacity refusal, measured from
+// the first refusal this worker saw. Chats admitted on their first
+// attempt record nothing. No turn span exists at this point, so the
+// turn scope and the chat kind are stated explicitly.
+func (w *chatWorker) recordCapacityWait(ctx context.Context, chat database.Chat) {
+	since, waited := w.capacityWaitSince[chat.ID]
+	if !waited {
+		return
+	}
+	delete(w.capacityWaitSince, chat.ID)
+	ctx = chatloop.ContextWithChatKind(ctx, chatKindAttr(chat))
+	w.server.stages.RecordAs(ctx, chatloop.StageCapacityWait, chatloop.ScopeTurn, chatloop.StageModel{},
+		since, w.opts.Clock.Now(), nil,
+		attribute.String(chatloop.AttrChatID, chat.ID.String()),
+	)
+}
+
+// forgetCapacityWait drops the wait start of a chat this worker will
+// not acquire on the current pass for a reason other than capacity: it
+// is owned by a live runner, archived, or no longer runnable. A wait
+// that resumes later starts from the next refusal.
+func (w *chatWorker) forgetCapacityWait(chatID uuid.UUID) {
+	delete(w.capacityWaitSince, chatID)
+}
+
+// pruneCapacityWaits drops wait starts for chats that are no longer
+// acquisition candidates, which happens when they are archived,
+// deleted, or picked up by another worker. candidates must be the
+// complete candidate set: a chat missing from a truncated batch is
+// still waiting, and dropping it would restart its clock.
+func (w *chatWorker) pruneCapacityWaits(candidates []database.GetChatWorkerAcquisitionCandidatesRow) {
+	if len(w.capacityWaitSince) == 0 {
+		return
+	}
+	stillCandidate := make(map[uuid.UUID]struct{}, len(candidates))
+	for _, row := range candidates {
+		stillCandidate[row.ID] = struct{}{}
+	}
+	for chatID := range w.capacityWaitSince {
+		if _, ok := stillCandidate[chatID]; !ok {
+			delete(w.capacityWaitSince, chatID)
+		}
+	}
+}
+
 func (w *chatWorker) refreshCapacityMetrics(ctx context.Context) {
 	active, err := w.opts.Store.CountChatCapacityActiveByPool(ctx, database.CountChatCapacityActiveByPoolParams{
 		ExcludeChatID: uuid.Nil,

diff --git a/coderd/x/chatd/capacity_internal_test.go b/coderd/x/chatd/capacity_internal_test.go
@@ -0,0 +1,80 @@
+package chatd //nolint:testpackage // Tests the acquisition loop's capacity wait bookkeeping.
+
+import (
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/require"
+
+	"github.com/coder/coder/v2/coderd/database"
+	"github.com/coder/quartz"
+)
+
+func newCapacityWaitWorker(t *testing.T) (*chatWorker, *quartz.Mock) {
+	t.Helper()
+	clock := quartz.NewMock(t)
+	tracer, _ := newStageTestTracer(t)
+	return &chatWorker{
+		server:            &Server{stages: tracer},
+		opts:              chatWorkerOptions{Clock: clock},
+		capacityWaitSince: make(map[uuid.UUID]time.Time),
+	}, clock
+}
+
+func candidateRows(ids ...uuid.UUID) []database.GetChatWorkerAcquisitionCandidatesRow {
+	rows := make([]database.GetChatWorkerAcquisitionCandidatesRow, 0, len(ids))
+	for _, id := range ids {
+		rows = append(rows, database.GetChatWorkerAcquisitionCandidatesRow{ID: id})
+	}
+	return rows
+}
+
+func TestCapacityWaitBookkeeping(t *testing.T) {
+	t.Parallel()
+
+	t.Run("FirstRefusalStartsTheClock", func(t *testing.T) {
+		t.Parallel()
+		worker, clock := newCapacityWaitWorker(t)
+		chatID := uuid.New()
+
+		worker.noteCapacityRefused(chatID)
+		first := worker.capacityWaitSince[chatID]
+		clock.Advance(time.Second)
+		worker.noteCapacityRefused(chatID)
+		require.Equal(t, first, worker.capacityWaitSince[chatID], "a later refusal keeps the first start")
+	})
+
+	t.Run("SkippedChatForgetsItsWait", func(t *testing.T) {
+		t.Parallel()
+		worker, _ := newCapacityWaitWorker(t)
+		chatID := uuid.New()
+
+		worker.noteCapacityRefused(chatID)
+		worker.forgetCapacityWait(chatID)
+		require.NotContains(t, worker.capacityWaitSince, chatID)
+	})
+
+	t.Run("PruneKeepsCandidates", func(t *testing.T) {
+		t.Parallel()
+		worker, _ := newCapacityWaitWorker(t)
+		waiting, gone := uuid.New(), uuid.New()
+
+		worker.noteCapacityRefused(waiting)
+		worker.noteCapacityRefused(gone)
+		worker.pruneCapacityWaits(candidateRows(waiting, uuid.New()))
+		require.Contains(t, worker.capacityWaitSince, waiting)
+		require.NotContains(t, worker.capacityWaitSince, gone)
+	})
+
+	t.Run("RecordClearsTheWait", func(t *testing.T) {
+		t.Parallel()
+		worker, clock := newCapacityWaitWorker(t)
+		chat := database.Chat{ID: uuid.New()}
+
+		worker.noteCapacityRefused(chat.ID)
+		clock.Advance(time.Second)
+		worker.recordCapacityWait(t.Context(), chat)
+		require.NotContains(t, worker.capacityWaitSince, chat.ID)
+	})
+}
diff --git a/coderd/x/chatd/chatd.go b/coderd/x/chatd/chatd.go
@@ -21,6 +21,8 @@ import (
 	"github.com/google/uuid"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sqlc-dev/pqtype"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 	"golang.org/x/sync/errgroup"
 	"golang.org/x/xerrors"
 
@@ -199,6 +201,7 @@ type Server struct {
 	usageTracker         *workspacestats.UsageTracker
 	clock                quartz.Clock
 	metrics              *chatloop.Metrics
+	stages               *chatloop.StageTracer
 	chatWorker           *chatWorker
 	messagePartBuffer    *messagepartbuffer.Buffer
 	streamSyncPoller     *streamSyncPoller
@@ -2143,9 +2146,10 @@ func (p *Server) PromoteQueued(
 	}
 
 	var (
-		result      PromoteQueuedResult
-		refreshChat database.Chat
-		refreshedOK bool
+		result           PromoteQueuedResult
+		refreshChat      database.Chat
+		refreshedOK      bool
+		promotedQueuedAt time.Time
 	)
 	machine := p.newChatMachine(opts.ChatID)
 	updateErr := machine.Update(ctx, func(tx *chatstate.Tx, store database.Store) error {
@@ -2165,6 +2169,7 @@ func (p *Server) PromoteQueued(
 		}
 		if promoteResult.InsertedMessage != nil {
 			result.PromotedMessage = *promoteResult.InsertedMessage
+			promotedQueuedAt = promoteResult.QueuedMessage.CreatedAt
 		}
 		// Capture the chat inside the transaction so the watch event
 		// published below uses the snapshot bump and status change
@@ -2184,6 +2189,13 @@ func (p *Server) PromoteQueued(
 	if refreshedOK {
 		p.publishChatPubsubEvent(refreshChat, codersdk.ChatWatchEventKindStatusChange, nil)
 	}
+	if !promotedQueuedAt.IsZero() {
+		var chatKind string
+		if refreshedOK {
+			chatKind = chatKindAttr(refreshChat)
+		}
+		p.recordQueueWait(ctx, opts.ChatID, chatKind, promotedQueuedAt, p.stages.Now())
+	}
 	return result, nil
 }
 
@@ -3061,6 +3073,9 @@ type Config struct {
 	AIBridgeTransportFactory       *atomic.Pointer[aibridge.TransportFactory]
 	Experiments                    codersdk.Experiments
 	PrometheusRegistry             prometheus.Registerer
+	// TracerProvider supplies the tracer used for chat lifecycle
+	// spans. Nil disables tracing without disabling metrics.
+	TracerProvider trace.TracerProvider
 
 	AgentCapacityUnlock AgentCapacityUnlock
 
@@ -3190,6 +3205,7 @@ func New(ps pubsub.Pubsub, cfg Config) *Server {
 	} else {
 		p.metrics = chatloop.NopMetrics()
 	}
+	p.stages = chatloop.NewStageTracer(cfg.TracerProvider, p.metrics)
 	p.messagePartBuffer = messagepartbuffer.New(messagepartbuffer.Options{Clock: clk})
 	localStreamPartsDialer := NewLocalStreamPartsDialer(LocalStreamPartsDialerConfig{
 		Buffer: p.messagePartBuffer,
@@ -4603,7 +4619,7 @@ func (p *Server) finalizeSuccessfulTurnStatusLabelWithAfterFunc(
 	logger slog.Logger,
 	afterFinalize func(context.Context, string),
 ) {
-	finalizeCtx, stopFinalizeCtx := p.inflightContext(ctx)
+	finalizeCtx, stopFinalizeCtx := p.inflightChatContext(ctx, chat)
 	if err := p.goInflight(func() {
 		defer stopFinalizeCtx()
 		statusLabel := p.generateFinalTurnStatusLabel(finalizeCtx, chat, status, runResult, logger)
@@ -4689,7 +4705,7 @@ func (p *Server) setLastTurnSummaryAsync(
 	if chat.LastTurnSummary.Valid && strings.TrimSpace(chat.LastTurnSummary.String) == summary {
 		return
 	}
-	updateCtx, stopUpdateCtx := p.inflightContext(ctx)
+	updateCtx, stopUpdateCtx := p.inflightChatContext(ctx, chat)
 	if err := p.goInflight(func() {
 		defer stopUpdateCtx()
 		p.updateLastTurnSummary(updateCtx, chat, chat.HistoryVersion, summary, logger)
@@ -4709,7 +4725,7 @@ func (p *Server) clearLastTurnSummaryAsync(
 	chat database.Chat,
 	logger slog.Logger,
 ) {
-	clearCtx, stopClearCtx := p.inflightContext(ctx)
+	clearCtx, stopClearCtx := p.inflightChatContext(ctx, chat)
 	if err := p.goInflight(func() {
 		defer stopClearCtx()
 		p.updateLastTurnSummary(clearCtx, chat, chat.HistoryVersion, "", logger)
@@ -4804,7 +4820,7 @@ func (p *Server) maybeGenerateChatSummaryAsync(
 	if chat.ParentChatID.Valid {
 		return
 	}
-	ctx, cancel := p.inflightContext(ctx)
+	ctx, cancel := p.inflightChatContext(ctx, chat)
 	if err := p.goInflight(func() {
 		defer cancel()
 		p.generateAndStoreChatSummary(ctx, logger, chat)
@@ -4985,7 +5001,7 @@ func (p *Server) storeSubagentReportSummaryAsync(
 	chat database.Chat,
 	logger slog.Logger,
 ) {
-	summaryCtx, stopSummaryCtx := p.inflightContext(ctx)
+	summaryCtx, stopSummaryCtx := p.inflightChatContext(ctx, chat)
 	if err := p.goInflight(func() {
 		defer stopSummaryCtx()
 		p.storeSubagentReportSummary(summaryCtx, chat, logger)
@@ -5081,14 +5097,43 @@ func (p *Server) Close() error {
 // must be called once the work completes to release the shutdown hook.
 // The caller is responsible for providing their own timeout.
 func (p *Server) inflightContext(reqCtx context.Context) (context.Context, func()) {
-	ctx, cancel := context.WithCancel(context.WithoutCancel(reqCtx))
+	// Inflight work outlives the caller, so the caller's span and stage
+	// scope are stripped from the context: spans started on this context
+	// become their own roots instead of children that end after their
+	// parent, and their stages are recorded as background work.
+	detached := trace.ContextWithSpanContext(context.WithoutCancel(reqCtx), trace.SpanContext{})
+	detached = chatloop.ContextWithScope(detached, chatloop.ScopeBackground)
+	ctx, cancel := context.WithCancel(detached)
 	stop := context.AfterFunc(p.ctx, cancel)
 	return ctx, func() {
 		stop()
 		cancel()
 	}
 }
 
+// recordQueueWait emits the queue_wait stage for a message that sat
+// queued from queuedAt until promotedAt. The span context is stripped
+// from ctx so the stage is a standalone span rather than a child of
+// the span in ctx, and the scope and chat kind are set explicitly
+// because ctx does not carry the turn's. An empty chatKind records the
+// stage without one.
+func (p *Server) recordQueueWait(ctx context.Context, chatID uuid.UUID, chatKind string, queuedAt, promotedAt time.Time) {
+	standalone := trace.ContextWithSpanContext(ctx, trace.SpanContext{})
+	standalone = chatloop.ContextWithChatKind(standalone, chatKind)
+	p.stages.RecordAs(standalone, chatloop.StageQueueWait, chatloop.ScopeTurn,
+		chatloop.StageModel{}, queuedAt, promotedAt, nil,
+		attribute.String(chatloop.AttrChatID, chatID.String()),
+	)
+}
+
+// inflightChatContext is inflightContext for work that belongs to a
+// known chat. The chat kind is set on the returned context so the
+// stages of the detached work carry it.
+func (p *Server) inflightChatContext(reqCtx context.Context, chat database.Chat) (context.Context, func()) {
+	ctx, stop := p.inflightContext(reqCtx)
+	return chatloop.ContextWithChatKind(ctx, chatKindAttr(chat)), stop
+}
+
 func (p *Server) goInflight(f func()) error {
 	if p.inflightClosed.Load() {
 		return errInflightClosed