Thanks to visit codestin.com
Credit goes to github.com

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 82 additions & 0 deletions coderd/x/chatd/active_turn_debug.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"cdr.dev/slog/v3"
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/x/chatd/chatdebug"
"github.com/coder/coder/v2/coderd/x/chatd/mcpclient"
)

type runnerDebugTurn struct {
Expand Down Expand Up @@ -49,7 +50,14 @@ func (d *runnerDebugTurn) Ensure(

d.mu.Lock()
defer d.mu.Unlock()
return d.ensureLocked(ctx, chat, debug)
}

func (d *runnerDebugTurn) ensureLocked(
ctx context.Context,
chat database.Chat,
debug *generationDebug,
) context.Context {
// Check finalized/disabled before created: once the turn is
// finalized, new contexts must not be attributed to the
// finalized run, even if it was created earlier.
Expand All @@ -68,6 +76,17 @@ func (d *runnerDebugTurn) Ensure(
seedSummary := chatdebug.SeedSummary(
chatdebug.TruncateLabel(debug.TriggerLabel, chatdebug.MaxLabelLength),
)
// Carry per-server MCP connect outcomes (and their dropped
// count) stashed by RecordMCPConnectSummaries before the run
// existed, so slow or failing servers appear in the run instead
// of as a silent gap before the first step. Seeded keys survive
// FinalizeRun's summary aggregation.
for key, stashed := range d.seedSummary {
if seedSummary == nil {
seedSummary = make(map[string]any, len(d.seedSummary))
}
seedSummary[key] = stashed
}
rootChatID := uuid.Nil
if chat.RootChatID.Valid {
rootChatID = chat.RootChatID.UUID
Expand Down Expand Up @@ -121,6 +140,69 @@ func (d *runnerDebugTurn) Context(ctx context.Context) context.Context {
return d.contextLocked(ctx)
}

// RecordMCPConnectSummaries merges one preparation's per-server MCP
// connect outcomes into the mcp_connect summary key, creating the
// debug run if it does not exist yet. Preparation invokes it as soon
// as its MCP connect phase completes, so every attempt is recorded:
// preparations that fail after connecting, decision errors, and
// actions that never reach Ensure (local tool execution,
// requires-action, turn finishing). Creating the run here matters
// when the first preparation connects and then fails: no model or
// compaction action ever runs Ensure, and Finalize discards the
// stash of a never-created run.
func (d *runnerDebugTurn) RecordMCPConnectSummaries(
ctx context.Context,
chat database.Chat,
debug *generationDebug,
summaries []mcpclient.ConnectSummary,
) {
if d == nil || len(summaries) == 0 {
return
}
d.mu.Lock()
defer d.mu.Unlock()
if d.disabled || d.finalized {
return
}
// Merge before ensuring so the outcomes ride the seed summary
// when this call is the one that creates the run.
d.mergeMCPConnectSummariesLocked(summaries)
d.ensureLocked(ctx, chat, debug)
}

// maxMCPConnectSummaryEntries bounds the retained per-preparation MCP
// connect outcomes. A turn may run up to 1,200 generation steps, each
// reconnecting to every selected server, so unbounded retention could
// grow one run summary to megabytes of database and API payload. The
// newest entries win because the tail of the history is what shows a
// server that degraded mid-turn; the mcp_connect_dropped count keeps
// the truncation visible.
const maxMCPConnectSummaryEntries = 100

// mergeMCPConnectSummariesLocked appends a preparation's per-server
// MCP connect outcomes to the mcp_connect summary key. chatd
// reconnects to every configured MCP server on each generation step
// while the run is created only once, so without this merge only
// one preparation's outcomes would survive to the finalized run and
// a server that degrades mid-turn would still be reported as
// connected.
func (d *runnerDebugTurn) mergeMCPConnectSummariesLocked(summaries []mcpclient.ConnectSummary) {
if len(summaries) == 0 {
return
}
if d.seedSummary == nil {
d.seedSummary = make(map[string]any, 1)
}
existing, _ := d.seedSummary["mcp_connect"].([]mcpclient.ConnectSummary)
existing = append(existing, summaries...)
if over := len(existing) - maxMCPConnectSummaryEntries; over > 0 {
existing = existing[over:]
dropped, _ := d.seedSummary["mcp_connect_dropped"].(int)
d.seedSummary["mcp_connect_dropped"] = dropped + over
}
d.seedSummary["mcp_connect"] = existing
}

func (d *runnerDebugTurn) contextLocked(ctx context.Context) context.Context {
if !d.created || d.runContext.RunID == uuid.Nil {
return ctx
Expand Down
179 changes: 179 additions & 0 deletions coderd/x/chatd/active_turn_debug_internal_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ package chatd
import (
"context"
"database/sql"
"encoding/json"
"fmt"
"testing"

"github.com/google/uuid"
Expand All @@ -12,6 +14,7 @@ import (
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/database/dbmock"
"github.com/coder/coder/v2/coderd/x/chatd/chatdebug"
"github.com/coder/coder/v2/coderd/x/chatd/mcpclient"
"github.com/coder/coder/v2/testutil"
)

Expand Down Expand Up @@ -153,3 +156,179 @@ func TestRunnerDebugTurnFinalizeOnce(t *testing.T) {
turn.Finalize(ctx)
turn.Finalize(ctx)
}

func TestRunnerDebugTurnRecordMCPConnectSummaries(t *testing.T) {
t.Parallel()

ctx := testutil.Context(t, testutil.WaitLong)
runnerCtx, cancel := context.WithCancel(ctx)
defer cancel()
ctrl := gomock.NewController(t)
db := dbmock.NewMockStore(ctrl)
chatID := uuid.New()
runID := uuid.New()
configID := uuid.New()
svc := chatdebug.NewService(db, testutil.Logger(t), nil)
turn := newRunnerDebugTurn(runnerCtx, testutil.Logger(t))

var seededSummary []byte
db.EXPECT().InsertChatDebugRun(gomock.Any(), gomock.Any()).
DoAndReturn(func(_ context.Context, params database.InsertChatDebugRunParams) (database.ChatDebugRun, error) {
require.True(t, params.Summary.Valid)
seededSummary = params.Summary.RawMessage
return database.ChatDebugRun{
ID: runID,
ChatID: chatID,
Kind: string(chatdebug.KindChatTurn),
Status: string(chatdebug.StatusInProgress),
}, nil
}).
Times(1)
db.EXPECT().GetChatDebugStepsByRunID(gomock.Any(), runID).Return(nil, nil).Times(1)
var finalSummary []byte
db.EXPECT().UpdateChatDebugRun(gomock.Any(), gomock.Any()).
DoAndReturn(func(_ context.Context, params database.UpdateChatDebugRunParams) (database.ChatDebugRun, error) {
require.True(t, params.Summary.Valid)
finalSummary = params.Summary.RawMessage
return database.ChatDebugRun{ID: runID, ChatID: chatID}, nil
}).Times(1)

debug := generationDebug{
Enabled: true,
Service: svc,
TriggerMessageID: 1,
ModelConfig: database.ChatModelConfig{ID: uuid.New()},
}
first := []mcpclient.ConnectSummary{{
ConfigID: configID,
Slug: "registry",
Outcome: mcpclient.ConnectOutcomeConnected,
DurationMS: 17,
ToolCount: 1,
}}
second := []mcpclient.ConnectSummary{{
ConfigID: configID,
Slug: "registry",
Outcome: mcpclient.ConnectOutcomeTimeout,
DurationMS: 10000,
Error: "connect: context deadline exceeded",
}}

// Preparation records each attempt as soon as its connect phase
// completes; the first record creates the run with the outcome
// already seeded, and a later Ensure must not create a second
// run.
turn.RecordMCPConnectSummaries(ctx, database.Chat{ID: chatID}, &debug, first)
turn.Ensure(ctx, database.Chat{ID: chatID}, &debug)
// A later generation step reconnects and reports a degraded
// outcome for the same server; its action may never reach
// Ensure, and the outcome must still survive to the finalized
// summary.
turn.RecordMCPConnectSummaries(ctx, database.Chat{ID: chatID}, &debug, second)
turn.RecordOutcome(chatdebug.StatusCompleted)
turn.Finalize(ctx)

var seeded struct {
MCPConnect []mcpclient.ConnectSummary `json:"mcp_connect"`
}
require.NoError(t, json.Unmarshal(seededSummary, &seeded))
require.Len(t, seeded.MCPConnect, 1)
require.Equal(t, mcpclient.ConnectOutcomeConnected, seeded.MCPConnect[0].Outcome)

var summary struct {
MCPConnect []mcpclient.ConnectSummary `json:"mcp_connect"`
}
require.NoError(t, json.Unmarshal(finalSummary, &summary))
require.Len(t, summary.MCPConnect, 2)
require.Equal(t, mcpclient.ConnectOutcomeConnected, summary.MCPConnect[0].Outcome)
require.Equal(t, mcpclient.ConnectOutcomeTimeout, summary.MCPConnect[1].Outcome)
}

func TestRunnerDebugTurnBoundsMCPConnectSummaries(t *testing.T) {
t.Parallel()

ctx := testutil.Context(t, testutil.WaitLong)
runnerCtx, cancel := context.WithCancel(ctx)
defer cancel()
ctrl := gomock.NewController(t)
db := dbmock.NewMockStore(ctrl)
chatID := uuid.New()
runID := uuid.New()
svc := chatdebug.NewService(db, testutil.Logger(t), nil)
turn := newRunnerDebugTurn(runnerCtx, testutil.Logger(t))

var seededSummary []byte
db.EXPECT().InsertChatDebugRun(gomock.Any(), gomock.Any()).
DoAndReturn(func(_ context.Context, params database.InsertChatDebugRunParams) (database.ChatDebugRun, error) {
require.True(t, params.Summary.Valid)
seededSummary = params.Summary.RawMessage
return database.ChatDebugRun{
ID: runID,
ChatID: chatID,
Kind: string(chatdebug.KindChatTurn),
Status: string(chatdebug.StatusInProgress),
}, nil
}).
Times(1)
db.EXPECT().GetChatDebugStepsByRunID(gomock.Any(), runID).Return(nil, nil).Times(1)
var finalSummary []byte
db.EXPECT().UpdateChatDebugRun(gomock.Any(), gomock.Any()).
DoAndReturn(func(_ context.Context, params database.UpdateChatDebugRunParams) (database.ChatDebugRun, error) {
require.True(t, params.Summary.Valid)
finalSummary = params.Summary.RawMessage
return database.ChatDebugRun{ID: runID, ChatID: chatID}, nil
}).Times(1)

debug := generationDebug{
Enabled: true,
Service: svc,
TriggerMessageID: 1,
ModelConfig: database.ChatModelConfig{ID: uuid.New()},
}
// 25 preparations against 5 servers produce 125 outcomes,
// overflowing the cap. The first record creates the run.
for prep := 0; prep < 25; prep++ {
batch := make([]mcpclient.ConnectSummary, 5)
for server := range batch {
batch[server] = mcpclient.ConnectSummary{
ConfigID: uuid.New(),
Slug: fmt.Sprintf("server-%d", server),
Outcome: mcpclient.ConnectOutcomeConnected,
DurationMS: int64(prep),
}
}
turn.RecordMCPConnectSummaries(ctx, database.Chat{ID: chatID}, &debug, batch)
}

type boundedSummary struct {
MCPConnect []mcpclient.ConnectSummary `json:"mcp_connect"`
MCPConnectDropped int `json:"mcp_connect_dropped"`
}
// The run was created by the first record, so the seed carries
// only that preparation's outcomes.
var seeded boundedSummary
require.NoError(t, json.Unmarshal(seededSummary, &seeded))
require.Len(t, seeded.MCPConnect, 5)
require.Zero(t, seeded.MCPConnectDropped)

// One more preparation still respects the cap and grows the
// dropped count.
turn.RecordMCPConnectSummaries(ctx, database.Chat{ID: chatID}, &debug, []mcpclient.ConnectSummary{{
ConfigID: uuid.New(),
Slug: "server-0",
Outcome: mcpclient.ConnectOutcomeTimeout,
DurationMS: 10000,
}})
turn.RecordOutcome(chatdebug.StatusCompleted)
turn.Finalize(ctx)

var final boundedSummary
require.NoError(t, json.Unmarshal(finalSummary, &final))
require.Len(t, final.MCPConnect, maxMCPConnectSummaryEntries)
require.Equal(t, 26, final.MCPConnectDropped)
// The newest outcomes win: the oldest five preparations were
// dropped, so the retained history starts at preparation 5 and
// ends with the timeout recorded above.
require.Equal(t, int64(5), final.MCPConnect[0].DurationMS)
require.Equal(t, mcpclient.ConnectOutcomeTimeout, final.MCPConnect[maxMCPConnectSummaryEntries-1].Outcome)
}
6 changes: 6 additions & 0 deletions coderd/x/chatd/chatd.go
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,12 @@ const (
DefaultChatHeartbeatInterval = 30 * time.Second
maxChatSteps = 1200

// slowPrepareThreshold is the generation-preparation duration
// above which a warning is logged. Preparation runs before
// every generation step, so sustained slowness (workspace
// dials, MCP connects) taxes the whole turn.
slowPrepareThreshold = 30 * time.Second

// maxConcurrentRecordingUploads caps the number of recording
// stop-and-store operations that can run concurrently. Each
// slot buffers up to MaxRecordingSize + MaxThumbnailSize
Expand Down
Loading
Loading