Thanks to visit codestin.com
Credit goes to github.com

Skip to content

refactor(coderd/x/chatd): extract runGenerationStep from StartGeneration #28968

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
jscottmiller wants to merge 1 commit into
base: main
Choose a base branch
from
+132 −112
Draft

refactor(coderd/x/chatd): extract runGenerationStep from StartGeneration #28968

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
244 changes: 132 additions & 112 deletions coderd/x/chatd/generation.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -441,129 +441,149 @@ func (s *taskStarter) StartGeneration(ctx context.Context, input chatWorkerTaskS
if err != nil {
return xerrors.Errorf("load generation state: %w", err)
}
if s.server.hooks.Enabled() {
result, dispatched, err := s.startGenerationSession(ctx, machine, input, chat, messages)
if err != nil {
if errors.Is(err, errTaskExpectedExit) {
return err
}
return s.finishGenerationError(ctx, machine, input, err, generationAttemptNotRequired)
}
if dispatched {
input.HistoryVersion = result.Chat.HistoryVersion
continue
}
}
prepareInput := generationPrepareInput{
Chat: chat,
Messages: messages,
RecordMCPConnectSummaries: input.DebugTurn.RecordMCPConnectSummaries,
}
prepared, err := retryGenerationPhase(ctx, s, "prepare", func() (generationPrepared, error) {
return s.server.prepareGeneration(ctx, prepareInput)
})
if err != nil {
if errors.Is(err, errTaskExpectedExit) || errors.Is(err, errTaskRetryable) {
return xerrors.Errorf("prepare generation: %w", err)
}
return s.finishGenerationError(ctx, machine, input, err, generationAttemptNotRequired)
}
cleanup := prepared.Cleanup
var decision generationDecision
if input.StopNudges.consume(stopNudgeKey(prepared.Messages)) {
decision = generationDecision{kind: generationActionGenerateAssistant}
} else {
decision, err = retryGenerationPhase(ctx, s, "decide", func() (generationDecision, error) {
return decideGenerationAction(generationDecisionInput{
chat: prepared.Chat,
messages: prepared.Messages,
dynamicToolNames: prepared.DynamicToolNames,
exclusiveToolNames: prepared.ExclusiveToolNames,
stopAfterTools: prepared.StopAfterTools,
maxSteps: prepared.MaxSteps,
compactionEnabled: prepared.Compaction != nil,
compactionNeeded: prepared.Compaction != nil && prepared.Compaction.Required,
compactionThresholdPercent: generationCompactionThreshold(prepared.Compaction),
compactionContextLimit: generationCompactionContextLimit(prepared.Compaction),
})
})
var again bool
input, again, err = s.runGenerationStep(ctx, machine, input, chat, messages)
if again {
continue
}
return err
}
}

// runGenerationStep runs one step of a turn: preparation, the action
// decision, and the action itself. It returns the input to use for
// the next step and whether the caller must reload state and run one.
// The returned input differs from the passed one when a step advances
// the history version.
func (s *taskStarter) runGenerationStep(
ctx context.Context,
machine *chatstate.ChatMachine,
input chatWorkerTaskStartInput,
chat database.Chat,
messages []database.ChatMessage,
) (next chatWorkerTaskStartInput, again bool, err error) {
if s.server.hooks.Enabled() {
result, dispatched, err := s.startGenerationSession(ctx, machine, input, chat, messages)
if err != nil {
cleanup()
if errors.Is(err, errTaskExpectedExit) || errors.Is(err, errTaskRetryable) {
return xerrors.Errorf("decide generation: %w", err)
if errors.Is(err, errTaskExpectedExit) {
return input, false, err
}
if errors.Is(err, errCompactionStillOverLimit) && prepared.Compaction != nil {
metricProvider, metricModel := compactionMetricIdentity(prepared.Compaction)
s.server.metrics.RecordCompaction(
metricProvider,
metricModel,
false,
errCompactionStillOverLimit,
)
}
return s.finishGenerationError(ctx, machine, input, err, generationAttemptNotRequired)
}

var actionErr error
switch decision.kind {
case generationActionEnterRequiresAction:
cleanup()
return s.enterRequiresAction(ctx, machine, input)
case generationActionFinishTurn:
cleanup()
return s.finishGenerationTurn(ctx, machine, input, decision, generationAttemptNotRequired)
case generationActionGenerateAssistant:
actionErr = s.generateAssistant(ctx, machine, input, prepared)
case generationActionExecuteLocalTools:
actionErr = s.executeLocalTools(ctx, machine, input, prepared, decision)
case generationActionCompact:
actionErr = s.generateCompaction(ctx, machine, input, prepared, compactionSourceForDecision(decision))
default:
return s.finishGenerationError(ctx, machine, input, xerrors.Errorf("unknown generation action %q", decision.kind), generationAttemptNotRequired)
return input, false, s.finishGenerationError(ctx, machine, input, err, generationAttemptNotRequired)
}
cleanup()
if actionErr == nil {
return nil
if dispatched {
input.HistoryVersion = result.Chat.HistoryVersion
return input, true, nil
}
// Task cancellation is handled by the runner, not here.
if ctx.Err() != nil && errors.Is(actionErr, context.Canceled) {
return errors.Join(errTaskExpectedExit, xerrors.Errorf("generation action: %w", actionErr), ctx.Err())
}
prepareInput := generationPrepareInput{
Chat: chat,
Messages: messages,
RecordMCPConnectSummaries: input.DebugTurn.RecordMCPConnectSummaries,
}
prepared, err := retryGenerationPhase(ctx, s, "prepare", func() (generationPrepared, error) {
return s.server.prepareGeneration(ctx, prepareInput)
})
if err != nil {
if errors.Is(err, errTaskExpectedExit) || errors.Is(err, errTaskRetryable) {
return input, false, xerrors.Errorf("prepare generation: %w", err)
}
if errors.Is(actionErr, chatloop.ErrInterrupted) {
return errors.Join(errTaskExpectedExit, xerrors.Errorf("generation action: %w", actionErr))
return input, false, s.finishGenerationError(ctx, machine, input, err, generationAttemptNotRequired)
}
cleanup := prepared.Cleanup
var decision generationDecision
if input.StopNudges.consume(stopNudgeKey(prepared.Messages)) {
decision = generationDecision{kind: generationActionGenerateAssistant}
} else {
decision, err = retryGenerationPhase(ctx, s, "decide", func() (generationDecision, error) {
return decideGenerationAction(generationDecisionInput{
chat: prepared.Chat,
messages: prepared.Messages,
dynamicToolNames: prepared.DynamicToolNames,
exclusiveToolNames: prepared.ExclusiveToolNames,
stopAfterTools: prepared.StopAfterTools,
maxSteps: prepared.MaxSteps,
compactionEnabled: prepared.Compaction != nil,
compactionNeeded: prepared.Compaction != nil && prepared.Compaction.Required,
compactionThresholdPercent: generationCompactionThreshold(prepared.Compaction),
compactionContextLimit: generationCompactionContextLimit(prepared.Compaction),
})
})
}
if err != nil {
cleanup()
if errors.Is(err, errTaskExpectedExit) || errors.Is(err, errTaskRetryable) {
return input, false, xerrors.Errorf("decide generation: %w", err)
}
if errors.Is(err, errCompactionStillOverLimit) && prepared.Compaction != nil {
metricProvider, metricModel := compactionMetricIdentity(prepared.Compaction)
s.server.metrics.RecordCompaction(
metricProvider,
metricModel,
false,
errCompactionStillOverLimit,
)
}
if errors.Is(actionErr, errTaskExpectedExit) {
return xerrors.Errorf("generation action: %w", actionErr)
return input, false, s.finishGenerationError(ctx, machine, input, err, generationAttemptNotRequired)
}

var actionErr error
switch decision.kind {
case generationActionEnterRequiresAction:
cleanup()
return input, false, s.enterRequiresAction(ctx, machine, input)
case generationActionFinishTurn:
cleanup()
return input, false, s.finishGenerationTurn(ctx, machine, input, decision, generationAttemptNotRequired)
case generationActionGenerateAssistant:
actionErr = s.generateAssistant(ctx, machine, input, prepared)
case generationActionExecuteLocalTools:
actionErr = s.executeLocalTools(ctx, machine, input, prepared, decision)
case generationActionCompact:
actionErr = s.generateCompaction(ctx, machine, input, prepared, compactionSourceForDecision(decision))
default:
return input, false, s.finishGenerationError(ctx, machine, input, xerrors.Errorf("unknown generation action %q", decision.kind), generationAttemptNotRequired)
}
cleanup()
if actionErr == nil {
return input, false, nil
}
// Task cancellation is handled by the runner, not here.
if ctx.Err() != nil && errors.Is(actionErr, context.Canceled) {
return input, false, errors.Join(errTaskExpectedExit, xerrors.Errorf("generation action: %w", actionErr), ctx.Err())
}
if errors.Is(actionErr, chatloop.ErrInterrupted) {
return input, false, errors.Join(errTaskExpectedExit, xerrors.Errorf("generation action: %w", actionErr))
}
if errors.Is(actionErr, errTaskExpectedExit) {
return input, false, xerrors.Errorf("generation action: %w", actionErr)
}
classified := chaterror.Classify(actionErr)
if classified.Retryable {
action := decision.kind
decision, err := s.recordGenerationRetry(ctx, machine, input, classified)
if err != nil {
return input, false, xerrors.Errorf("record generation retry: %w", err)
}
classified := chaterror.Classify(actionErr)
if classified.Retryable {
action := decision.kind
decision, err := s.recordGenerationRetry(ctx, machine, input, classified)
if err != nil {
return xerrors.Errorf("record generation retry: %w", err)
}
if decision.retry {
s.opts.Logger.Warn(ctx, "chat generation retrying",
slog.F("chat_id", input.ChatID),
slog.F("worker_id", input.WorkerID),
slog.F("action", action),
slog.F("generation_attempt", decision.generationAttempt),
slog.F("delay", decision.delay),
slog.F("error_kind", classified.Kind),
slog.F("provider", classified.Provider),
slog.F("status_code", classified.StatusCode),
slogError(actionErr),
)
if err := s.waitGenerationRetry(ctx, decision.delay); err != nil {
return xerrors.Errorf("wait generation retry: %w", err)
}
continue
if decision.retry {
s.opts.Logger.Warn(ctx, "chat generation retrying",
slog.F("chat_id", input.ChatID),
slog.F("worker_id", input.WorkerID),
slog.F("action", action),
slog.F("generation_attempt", decision.generationAttempt),
slog.F("delay", decision.delay),
slog.F("error_kind", classified.Kind),
slog.F("provider", classified.Provider),
slog.F("status_code", classified.StatusCode),
slogError(actionErr),
)
if err := s.waitGenerationRetry(ctx, decision.delay); err != nil {
return input, false, xerrors.Errorf("wait generation retry: %w", err)
}
return s.finishGenerationError(ctx, machine, input, actionErr, requireGenerationAttempt(decision.generationAttempt))
return input, true, nil
}
return s.finishGenerationError(ctx, machine, input, actionErr, generationAttemptNotRequired)
return input, false, s.finishGenerationError(ctx, machine, input, actionErr, requireGenerationAttempt(decision.generationAttempt))
}
return input, false, s.finishGenerationError(ctx, machine, input, actionErr, generationAttemptNotRequired)
}

func loadGenerationState(
Expand Down
Loading