coder · jscottmiller · Sep 4, 2026
diff --git a/coderd/coderd.go b/coderd/coderd.go
@@ -956,6 +956,7 @@ func New(options *Options) *API {
 				UsageTracker:                   options.WorkspaceUsageTracker,
 				PrometheusRegistry:             options.PrometheusRegistry,
 				StageMetrics:                   codersdk.NewChatStageMetricsLevelFromString(options.DeploymentValues.AI.Chat.StageMetrics),
+				TracerProvider:                 options.TracerProvider,
 				AgentCapacityUnlock:            options.ChatAgentCapacityUnlock,
 				OIDCTokenSource:                oidcMCPSrc,
 				NotificationsEnqueuer:          options.NotificationsEnqueuer,

diff --git a/coderd/x/chatd/chatd.go b/coderd/x/chatd/chatd.go
@@ -21,6 +21,7 @@ import (
 	"github.com/google/uuid"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sqlc-dev/pqtype"
+	"go.opentelemetry.io/otel/trace"
 	"golang.org/x/sync/errgroup"
 	"golang.org/x/xerrors"
 
@@ -199,6 +200,7 @@ type Server struct {
 	usageTracker         *workspacestats.UsageTracker
 	clock                quartz.Clock
 	metrics              *chatloop.Metrics
+	stages               *chatloop.StageTracer
 	chatWorker           *chatWorker
 	messagePartBuffer    *messagepartbuffer.Buffer
 	streamSyncPoller     *streamSyncPoller
@@ -3063,7 +3065,8 @@ type Config struct {
 	PrometheusRegistry             prometheus.Registerer
 	// StageMetrics selects which chat lifecycle stage metric families
 	// are registered. The zero value means codersdk.ChatStageMetricsLevelOff.
-	StageMetrics codersdk.ChatStageMetricsLevel
+	StageMetrics   codersdk.ChatStageMetricsLevel
+	TracerProvider trace.TracerProvider
 
 	AgentCapacityUnlock AgentCapacityUnlock
 
@@ -3195,6 +3198,7 @@ func New(ps pubsub.Pubsub, cfg Config) *Server {
 	} else {
 		p.metrics = chatloop.NopMetrics()
 	}
+	p.stages = chatloop.NewStageTracer(cfg.TracerProvider, p.metrics)
 	p.messagePartBuffer = messagepartbuffer.New(messagepartbuffer.Options{Clock: clk})
 	localStreamPartsDialer := NewLocalStreamPartsDialer(LocalStreamPartsDialerConfig{
 		Buffer: p.messagePartBuffer,

diff --git a/coderd/x/chatd/model_routing.go b/coderd/x/chatd/model_routing.go
@@ -8,6 +8,7 @@ import (
 	"golang.org/x/xerrors"
 
 	"github.com/coder/coder/v2/coderd/database"
+	"github.com/coder/coder/v2/coderd/x/chatd/chatloop"
 	"github.com/coder/coder/v2/coderd/x/chatd/chatprovider"
 	"github.com/coder/coder/v2/codersdk"
 )
@@ -25,6 +26,7 @@ type modelClientRequest struct {
 type modelBuildOptions struct {
 	ActiveAPIKeyID string
 	RecordHTTP     bool
+	StageModel     chatloop.StageModel
 }
 
 func (p *Server) enabledAIProviderByID(ctx context.Context, providerID uuid.UUID) (database.AIProvider, error) {

diff --git a/coderd/x/chatd/model_routing_aibridge.go b/coderd/x/chatd/model_routing_aibridge.go
@@ -11,13 +11,15 @@ import (
 	fantasyopenai "charm.land/fantasy/providers/openai"
 	fantasyopenaicompat "charm.land/fantasy/providers/openaicompat"
 	"github.com/google/uuid"
+	"go.opentelemetry.io/otel/attribute"
 	"golang.org/x/xerrors"
 
 	"github.com/coder/coder/v2/coderd/aibridge"
 	"github.com/coder/coder/v2/coderd/aibridged"
 	"github.com/coder/coder/v2/coderd/database"
 	"github.com/coder/coder/v2/coderd/x/chatd/chatdebug"
 	"github.com/coder/coder/v2/coderd/x/chatd/chaterror"
+	"github.com/coder/coder/v2/coderd/x/chatd/chatloop"
 	"github.com/coder/coder/v2/coderd/x/chatd/chatprovider"
 	"github.com/coder/coder/v2/codersdk"
 )
@@ -67,6 +69,41 @@ const (
 	aiGatewayRequestFormatAnthropic
 )
 
+// stageSpanRoundTripper emits one provider_attempt stage per HTTP
+// round trip to the model provider, so retried requests each get
+// their own span. model labels every attempt with the identity the
+// client was built for.
+type stageSpanRoundTripper struct {
+	base   http.RoundTripper
+	stages *chatloop.StageTracer
+	model  chatloop.StageModel
+}
+
+var _ http.RoundTripper = (*stageSpanRoundTripper)(nil)
+
+func (t *stageSpanRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	ctx, span := t.stages.Start(req.Context(), chatloop.StageProviderAttempt,
+		attribute.String(chatloop.AttrHTTPMethod, req.Method),
+		attribute.String(chatloop.AttrHTTPHost, req.URL.Host),
+	)
+	span.SetModel(t.model)
+	resp, err := t.base.RoundTrip(req.WithContext(ctx))
+	if resp != nil {
+		span.SetAttributes(attribute.Int(chatloop.AttrHTTPStatusCode, resp.StatusCode))
+		if err == nil && resp.StatusCode >= http.StatusBadRequest {
+			err = xerrors.Errorf("provider returned status %d", resp.StatusCode)
+			// The status error only marks the span; the response and the
+			// transport's own error are returned untouched.
+			span.End(err)
+			return resp, nil
+		}
+	}
+	// The span closes on response headers, not on body completion; the
+	// streamed body outlives this call.
+	span.End(err)
+	return resp, err
+}
+
 type aiGatewayRoundTripper struct {
 	base         http.RoundTripper
 	apiKeyID     string
@@ -164,6 +201,7 @@ func (p *Server) newModel(
 	if opts.RecordHTTP {
 		baseRT = &chatdebug.RecordingTransport{Base: baseRT}
 	}
+	baseRT = &stageSpanRoundTripper{base: baseRT, stages: p.stages, model: opts.StageModel}
 
 	config := fantasyConfigForAIBridge(route.Provider.Type)
 	extraHeaders := mergeConfigBetaHeaders(req.ExtraHeaders, config.ProviderHint, req.CallConfig)

diff --git a/coderd/x/chatd/modelcall.go b/coderd/x/chatd/modelcall.go
@@ -11,6 +11,7 @@ import (
 	"github.com/coder/coder/v2/coderd/database/dbauthz"
 	"github.com/coder/coder/v2/coderd/util/ptr"
 	"github.com/coder/coder/v2/coderd/x/chatd/chatdebug"
+	"github.com/coder/coder/v2/coderd/x/chatd/chatloop"
 	"github.com/coder/coder/v2/coderd/x/chatd/chatprovider"
 	"github.com/coder/coder/v2/codersdk"
 )
@@ -68,8 +69,18 @@ type resolvedModelCall struct {
 	providerOptions  fantasy.ProviderOptions
 	resolvedProvider string
 	resolvedModel    string
-	route            aiGatewayModelRoute
-	debugEnabled     bool
+	// resolvedEffort is the reasoning effort actually sent to the
+	// provider, after the per-turn request is clamped to the config's
+	// max. Empty when the config configures no reasoning effort.
+	resolvedEffort string
+	route          aiGatewayModelRoute
+	debugEnabled   bool
+}
+
+// stageModel returns the model identity stage instrumentation labels
+// spans and durations with.
+func (r resolvedModelCall) stageModel() chatloop.StageModel {
+	return chatloop.StageModel{Model: r.resolvedModel, Effort: r.resolvedEffort}
 }
 
 // resolveModelCall is the single pipeline from a spec to a ready model
@@ -146,8 +157,16 @@ func (p *Server) resolveModelCall(ctx context.Context, spec modelCallSpec) (reso
 	debugSvc := p.debugService()
 	out.debugEnabled = debugSvc != nil && debugSvc.IsEnabled(ctx, spec.chat.ID, spec.chat.OwnerID)
 
+	// The effort is resolved once here so the transport's stage labels
+	// and the provider call options carry the same value.
+	effectiveEffort := chatprovider.ResolveReasoningEffort(spec.requestedEffort, out.callConfig.ReasoningEffort)
+	if effectiveEffort != nil {
+		out.resolvedEffort = *effectiveEffort
+	}
+
 	buildOpts := spec.buildOptions
 	buildOpts.RecordHTTP = out.debugEnabled
+	buildOpts.StageModel = out.stageModel()
 	model, err := p.newModel(ctx, modelClientRequest{
 		Chat:         spec.chat,
 		ModelName:    modelName,
@@ -169,13 +188,14 @@ func (p *Server) resolveModelCall(ctx context.Context, spec modelCallSpec) (reso
 	}
 	out.model = model
 
-	out.providerOptions = chatprovider.ProviderOptionsForCall(out.model, out.callConfig, spec.requestedEffort)
+	out.providerOptions = chatprovider.ProviderOptionsForCall(out.model, out.callConfig, effectiveEffort)
 
 	p.logger.Debug(ctx, "resolved model call",
 		slog.F("purpose", spec.purpose),
 		slog.F("chat_id", spec.chat.ID),
 		slog.F("provider", out.resolvedProvider),
 		slog.F("model", out.resolvedModel),
+		slog.F("reasoning_effort", out.resolvedEffort),
 		slog.F("debug_enabled", out.debugEnabled),
 	)
 	return out, nil

diff --git a/coderd/x/chatd/stage_internal_test.go b/coderd/x/chatd/stage_internal_test.go
@@ -0,0 +1,184 @@
+package chatd //nolint:testpackage // Tests unexported stage instrumentation internals.
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	"go.opentelemetry.io/otel/sdk/trace/tracetest"
+	"go.opentelemetry.io/otel/trace"
+	"golang.org/x/xerrors"
+
+	"github.com/coder/coder/v2/coderd/x/chatd/chatloop"
+)
+
+// newStageTestTracer returns a stage tracer writing into an in-memory
+// span recorder.
+func newStageTestTracer(t *testing.T) (*chatloop.StageTracer, *tracetest.SpanRecorder) {
+	t.Helper()
+	recorder := tracetest.NewSpanRecorder()
+	provider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder))
+	t.Cleanup(func() {
+		// The test context is already canceled during cleanup, so the
+		// flush uses a fresh one.
+		require.NoError(t, provider.Shutdown(context.Background()))
+	})
+	return chatloop.NewStageTracer(provider, chatloop.NopMetrics()), recorder
+}
+
+type stubRoundTripper struct {
+	status int
+	err    error
+}
+
+func (s stubRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	if s.err != nil {
+		return nil, s.err
+	}
+	return &http.Response{
+		StatusCode: s.status,
+		Body:       io.NopCloser(strings.NewReader("")),
+		Request:    req,
+	}, nil
+}
+
+func TestStageSpanRoundTripper(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name           string
+		base           stubRoundTripper
+		wantStatusCode codes.Code
+		wantAttribute  bool
+		wantErr        bool
+	}{
+		{
+			name:           "success",
+			base:           stubRoundTripper{status: http.StatusOK},
+			wantStatusCode: codes.Unset,
+			wantAttribute:  true,
+		},
+		{
+			name:           "client error",
+			base:           stubRoundTripper{status: http.StatusTooManyRequests},
+			wantStatusCode: codes.Error,
+			wantAttribute:  true,
+		},
+		{
+			name:           "server error",
+			base:           stubRoundTripper{status: http.StatusInternalServerError},
+			wantStatusCode: codes.Error,
+			wantAttribute:  true,
+		},
+		{
+			name:           "transport error",
+			base:           stubRoundTripper{err: xerrors.New("dial failed")},
+			wantStatusCode: codes.Error,
+			wantErr:        true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			t.Parallel()
+			tracer, recorder := newStageTestTracer(t)
+			transport := &stageSpanRoundTripper{base: test.base, stages: tracer}
+
+			req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, "https://provider.example/v1/messages", nil)
+			require.NoError(t, err)
+			resp, err := transport.RoundTrip(req)
+			if test.wantErr {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+				require.NotNil(t, resp)
+				require.Equal(t, test.base.status, resp.StatusCode)
+				require.NoError(t, resp.Body.Close())
+			}
+
+			ended := recorder.Ended()
+			require.Len(t, ended, 1)
+			require.Equal(t, chatloop.StageProviderAttempt, ended[0].Name())
+			require.Equal(t, test.wantStatusCode, ended[0].Status().Code)
+			var sawStatusCode bool
+			for _, attr := range ended[0].Attributes() {
+				if string(attr.Key) == chatloop.AttrHTTPStatusCode {
+					sawStatusCode = true
+					require.Equal(t, int64(test.base.status), attr.Value.AsInt64())
+				}
+			}
+			require.Equal(t, test.wantAttribute, sawStatusCode)
+			require.Contains(t, ended[0].Attributes(),
+				attribute.String(chatloop.AttrScope, chatloop.ScopeBackground))
+		})
+	}
+}
+
+func TestStageSpanRoundTripperScope(t *testing.T) {
+	t.Parallel()
+	tracer, recorder := newStageTestTracer(t)
+	transport := &stageSpanRoundTripper{base: stubRoundTripper{status: http.StatusOK}, stages: tracer}
+
+	turnCtx, turn := tracer.StartRoot(t.Context(), chatloop.StageChatTurn, nil)
+	turnReq, err := http.NewRequestWithContext(turnCtx, http.MethodPost, "https://provider.example/v1/messages", nil)
+	require.NoError(t, err)
+	turnResp, err := transport.RoundTrip(turnReq)
+	require.NoError(t, err)
+	require.NoError(t, turnResp.Body.Close())
+
+	// Background work runs on a context detached from the turn, the same
+	// way inflight chatd tasks are.
+	backgroundCtx := chatloop.ContextWithScope(
+		trace.ContextWithSpanContext(turnCtx, trace.SpanContext{}),
+		chatloop.ScopeBackground,
+	)
+	backgroundReq, err := http.NewRequestWithContext(backgroundCtx, http.MethodPost, "https://provider.example/v1/messages", nil)
+	require.NoError(t, err)
+	backgroundResp, err := transport.RoundTrip(backgroundReq)
+	require.NoError(t, err)
+	require.NoError(t, backgroundResp.Body.Close())
+	turn.End(nil)
+
+	var scopes []string
+	for _, span := range recorder.Ended() {
+		if span.Name() != chatloop.StageProviderAttempt {
+			continue
+		}
+		for _, attr := range span.Attributes() {
+			if string(attr.Key) == chatloop.AttrScope {
+				scopes = append(scopes, attr.Value.AsString())
+			}
+		}
+	}
+	require.Equal(t, []string{chatloop.ScopeTurn, chatloop.ScopeBackground}, scopes)
+}
+
+func TestStageSpanRoundTripperModel(t *testing.T) {
+	t.Parallel()
+	tracer, recorder := newStageTestTracer(t)
+	model := chatloop.StageModel{Model: "claude-sonnet-4-5", Effort: "medium"}
+	transport := &stageSpanRoundTripper{
+		base:   stubRoundTripper{status: http.StatusOK},
+		stages: tracer,
+		model:  model,
+	}
+
+	req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, "https://provider.example/v1/messages", nil)
+	require.NoError(t, err)
+	resp, err := transport.RoundTrip(req)
+	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
+
+	ended := recorder.Ended()
+	require.Len(t, ended, 1)
+	require.Contains(t, ended[0].Attributes(),
+		attribute.String(chatloop.AttrModel, model.Model))
+	require.Contains(t, ended[0].Attributes(),
+		attribute.String(chatloop.AttrReasoningEffort, model.Effort))
+}