From fa1fd8207d82563fd36a3aa7d09ecfdb4b3c5169 Mon Sep 17 00:00:00 2001 From: George Katsitadze Date: Thu, 7 May 2026 14:09:34 -0700 Subject: [PATCH 1/3] feat(coderd): add on_success workspace build request handling Add the on_success request shape for workspace builds and validate the supported server-side restart flow. When a stop build includes on_success, persist the requested child start build intent in workspace_build_orchestrations for later processing. Add SDK/API generated docs and tests for validation, persistence, and child request fields. Ref: https://linear.app/codercom/issue/PLAT-143/add-workspace-restart-functionality-to-api Ref: https://github.com/coder/coder/issues/5800 --- coderd/apidoc/docs.go | 40 +++ coderd/apidoc/swagger.json | 36 +++ .../workspacebuildorchestrations.go | 65 +++++ coderd/workspacebuilds.go | 113 +++++++- coderd/workspacebuilds_on_success_test.go | 257 ++++++++++++++++++ codersdk/workspaces.go | 11 + docs/reference/api/builds.md | 11 + docs/reference/api/schemas.md | 65 ++++- site/src/api/typesGenerated.ts | 14 + 9 files changed, 600 insertions(+), 12 deletions(-) create mode 100644 coderd/database/dbtestutil/workspacebuildorchestrations.go create mode 100644 coderd/workspacebuilds_on_success_test.go diff --git a/coderd/apidoc/docs.go b/coderd/apidoc/docs.go index a727d71a5de..c01b8db6d6a 100644 --- a/coderd/apidoc/docs.go +++ b/coderd/apidoc/docs.go @@ -18449,6 +18449,38 @@ const docTemplate = `{ } } }, + "codersdk.CreateWorkspaceBuildOnSuccessRequest": { + "type": "object", + "required": [ + "transition" + ], + "properties": { + "rich_parameter_values": { + "type": "array", + "items": { + "$ref": "#/definitions/codersdk.WorkspaceBuildParameter" + } + }, + "template_version_id": { + "type": "string", + "format": "uuid" + }, + "template_version_preset_id": { + "type": "string", + "format": "uuid" + }, + "transition": { + "enum": [ + "start" + ], + "allOf": [ + { + "$ref": "#/definitions/codersdk.WorkspaceTransition" + } + ] + } + } + }, "codersdk.CreateWorkspaceBuildReason": { "type": "string", "enum": [ @@ -18490,6 +18522,14 @@ const docTemplate = `{ } ] }, + "on_success": { + "description": "OnSuccess queues a follow-up workspace build after this build succeeds.\nIt currently supports restarting a workspace by starting it after a\nsuccessful stop build.", + "allOf": [ + { + "$ref": "#/definitions/codersdk.CreateWorkspaceBuildOnSuccessRequest" + } + ] + }, "orphan": { "description": "Orphan may be set for the Destroy transition.", "type": "boolean" diff --git a/coderd/apidoc/swagger.json b/coderd/apidoc/swagger.json index 20bbcae3227..d331cc56ca0 100644 --- a/coderd/apidoc/swagger.json +++ b/coderd/apidoc/swagger.json @@ -16682,6 +16682,34 @@ } } }, + "codersdk.CreateWorkspaceBuildOnSuccessRequest": { + "type": "object", + "required": ["transition"], + "properties": { + "rich_parameter_values": { + "type": "array", + "items": { + "$ref": "#/definitions/codersdk.WorkspaceBuildParameter" + } + }, + "template_version_id": { + "type": "string", + "format": "uuid" + }, + "template_version_preset_id": { + "type": "string", + "format": "uuid" + }, + "transition": { + "enum": ["start"], + "allOf": [ + { + "$ref": "#/definitions/codersdk.WorkspaceTransition" + } + ] + } + } + }, "codersdk.CreateWorkspaceBuildReason": { "type": "string", "enum": [ @@ -16719,6 +16747,14 @@ } ] }, + "on_success": { + "description": "OnSuccess queues a follow-up workspace build after this build succeeds.\nIt currently supports restarting a workspace by starting it after a\nsuccessful stop build.", + "allOf": [ + { + "$ref": "#/definitions/codersdk.CreateWorkspaceBuildOnSuccessRequest" + } + ] + }, "orphan": { "description": "Orphan may be set for the Destroy transition.", "type": "boolean" diff --git a/coderd/database/dbtestutil/workspacebuildorchestrations.go b/coderd/database/dbtestutil/workspacebuildorchestrations.go new file mode 100644 index 00000000000..18f3b97bc93 --- /dev/null +++ b/coderd/database/dbtestutil/workspacebuildorchestrations.go @@ -0,0 +1,65 @@ +package dbtestutil + +import ( + "context" + "database/sql" + + "github.com/google/uuid" + + "github.com/coder/coder/v2/coderd/database" +) + +const getWorkspaceBuildOrchestrationByParentBuildIDQuery = ` +SELECT + id, + created_at, + updated_at, + parent_build_id, + child_build_id, + child_transition, + child_template_version_id, + child_template_version_preset_id, + child_rich_parameter_values, + child_log_level, + child_reason, + attempt_count, + next_retry_after, + status, + error +FROM + workspace_build_orchestrations +WHERE + parent_build_id = $1 +` + +// GetWorkspaceBuildOrchestrationByParentBuildID reads a workspace +// build orchestration row directly from the database for tests. +func GetWorkspaceBuildOrchestrationByParentBuildID( + ctx context.Context, + sqlDB *sql.DB, + parentBuildID uuid.UUID, +) (database.WorkspaceBuildOrchestration, error) { + var orchestration database.WorkspaceBuildOrchestration + err := sqlDB.QueryRowContext( + ctx, + getWorkspaceBuildOrchestrationByParentBuildIDQuery, + parentBuildID, + ).Scan( + &orchestration.ID, + &orchestration.CreatedAt, + &orchestration.UpdatedAt, + &orchestration.ParentBuildID, + &orchestration.ChildBuildID, + &orchestration.ChildTransition, + &orchestration.ChildTemplateVersionID, + &orchestration.ChildTemplateVersionPresetID, + &orchestration.ChildRichParameterValues, + &orchestration.ChildLogLevel, + &orchestration.ChildReason, + &orchestration.AttemptCount, + &orchestration.NextRetryAfter, + &orchestration.Status, + &orchestration.Error, + ) + return orchestration, err +} diff --git a/coderd/workspacebuilds.go b/coderd/workspacebuilds.go index 8ccb6417b8e..c71f693014d 100644 --- a/coderd/workspacebuilds.go +++ b/coderd/workspacebuilds.go @@ -375,6 +375,30 @@ func (api *API) postWorkspaceBuildsInternal( codersdk.WorkspaceBuild, error, ) { + if err := validateCreateWorkspaceBuildOnSuccess(createBuild); err != nil { + return codersdk.WorkspaceBuild{}, err + } + + var childParameterValuesJSON json.RawMessage + if createBuild.OnSuccess != nil { + childParameterValues := createBuild.OnSuccess.RichParameterValues + if childParameterValues == nil { + childParameterValues = []codersdk.WorkspaceBuildParameter{} + } + + var err error + childParameterValuesJSON, err = json.Marshal(childParameterValues) + if err != nil { + return codersdk.WorkspaceBuild{}, httperror.NewResponseError( + http.StatusInternalServerError, + codersdk.Response{ + Message: "Internal error preparing follow-up workspace build parameters", + Detail: err.Error(), + }, + ) + } + } + transition := database.WorkspaceTransition(createBuild.Transition) builder := wsbuilder.New(workspace, transition, *api.BuildUsageChecker.Load()). Initiator(apiKey.UserID). @@ -481,7 +505,57 @@ func (api *API) postWorkspaceBuildsInternal( }, workspaceBuildBaggage, ) - return err + if err != nil { + return err + } + + if createBuild.OnSuccess != nil { + childBuild := createBuild.OnSuccess + now := dbtime.Now() + _, err = tx.InsertWorkspaceBuildOrchestration(ctx, database.InsertWorkspaceBuildOrchestrationParams{ + ID: uuid.New(), + CreatedAt: now, + UpdatedAt: now, + ParentBuildID: workspaceBuild.ID, + ChildTransition: database.WorkspaceTransition(childBuild.Transition), + ChildTemplateVersionID: uuid.NullUUID{ + UUID: childBuild.TemplateVersionID, + Valid: childBuild.TemplateVersionID != uuid.Nil, + }, + ChildTemplateVersionPresetID: uuid.NullUUID{ + UUID: childBuild.TemplateVersionPresetID, + Valid: childBuild.TemplateVersionPresetID != uuid.Nil, + }, + ChildRichParameterValues: childParameterValuesJSON, + ChildLogLevel: string(createBuild.LogLevel), + ChildReason: database.NullBuildReason{ + BuildReason: database.BuildReason(createBuild.Reason), + Valid: createBuild.Reason != "", + }, + }) + if err != nil { + if dbauthz.IsNotAuthorizedError(err) { + return httperror.NewResponseError(http.StatusForbidden, codersdk.Response{ + Message: "Unauthorized to queue follow-up workspace build.", + Detail: "You do not have permission to queue this follow-up workspace build.", + }) + } + api.Logger.Error(ctx, "failed to queue follow-up workspace build", + slog.F("workspace_id", workspace.ID), + slog.F("workspace_build_id", workspaceBuild.ID), + slog.Error(err), + ) + return httperror.NewResponseError( + http.StatusInternalServerError, + codersdk.Response{ + Message: "Internal error queueing follow-up workspace build", + Detail: err.Error(), + }, + ) + } + } + + return nil }, nil) if err != nil { return codersdk.WorkspaceBuild{}, err @@ -584,6 +658,43 @@ func (api *API) postWorkspaceBuildsInternal( return apiBuild, nil } +// validateCreateWorkspaceBuildOnSuccess enforces the subset of build options +// that currently has well-defined stop-then-start semantics. +func validateCreateWorkspaceBuildOnSuccess(createBuild codersdk.CreateWorkspaceBuildRequest) error { + onSuccess := createBuild.OnSuccess + if onSuccess == nil { + return nil + } + + if createBuild.Transition != codersdk.WorkspaceTransitionStop { + return httperror.NewResponseError(http.StatusBadRequest, codersdk.Response{ + Message: "OnSuccess is only permitted when stopping a workspace.", + }) + } + if onSuccess.Transition != codersdk.WorkspaceTransitionStart { + return httperror.NewResponseError(http.StatusBadRequest, codersdk.Response{ + Message: "OnSuccess transition must be start.", + }) + } + if createBuild.DryRun { + return httperror.NewResponseError(http.StatusBadRequest, codersdk.Response{ + Message: "OnSuccess cannot be set alongside DryRun.", + }) + } + if createBuild.Orphan { + return httperror.NewResponseError(http.StatusBadRequest, codersdk.Response{ + Message: "OnSuccess cannot be set alongside Orphan.", + }) + } + if len(createBuild.ProvisionerState) > 0 { + return httperror.NewResponseError(http.StatusBadRequest, codersdk.Response{ + Message: "OnSuccess cannot be set alongside ProvisionerState.", + }) + } + + return nil +} + func (api *API) notifyWorkspaceUpdated( ctx context.Context, initiatorID uuid.UUID, diff --git a/coderd/workspacebuilds_on_success_test.go b/coderd/workspacebuilds_on_success_test.go new file mode 100644 index 00000000000..921bbc9c95d --- /dev/null +++ b/coderd/workspacebuilds_on_success_test.go @@ -0,0 +1,257 @@ +package coderd_test + +import ( + "encoding/json" + "net/http" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/coder/coder/v2/coderd/coderdtest" + "github.com/coder/coder/v2/coderd/database/dbtestutil" + "github.com/coder/coder/v2/codersdk" + "github.com/coder/coder/v2/provisioner/echo" + "github.com/coder/coder/v2/provisionersdk/proto" + "github.com/coder/coder/v2/testutil" +) + +func TestPostWorkspaceBuildsOnSuccessRestart(t *testing.T) { + t.Parallel() + + const paramName = "foo" + + // GIVEN: a running workspace with an existing rich parameter value. + deploymentValues := coderdtest.DeploymentValues(t) + deploymentValues.EnableTerraformDebugMode = true + db, ps, sqlDB := dbtestutil.NewDBWithSQLDB(t) + client, _, _ := coderdtest.NewWithAPI(t, &coderdtest.Options{ + Database: db, + Pubsub: ps, + IncludeProvisionerDaemon: true, + DeploymentValues: deploymentValues, + }) + first := coderdtest.CreateFirstUser(t, client) + version := coderdtest.CreateTemplateVersion(t, client, first.OrganizationID, + echoResponsesWithRichParameter(paramName, echoResponseOptions{ + blockStopApply: false, + }), + ) + coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID) + template := coderdtest.CreateTemplate(t, client, first.OrganizationID, version.ID) + workspace := coderdtest.CreateWorkspace(t, client, template.ID, func(request *codersdk.CreateWorkspaceRequest) { + request.RichParameterValues = []codersdk.WorkspaceBuildParameter{ + {Name: paramName, Value: "bar"}, + } + }) + initialBuild := coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, workspace.LatestBuild.ID) + require.Equal(t, codersdk.WorkspaceStatusRunning, initialBuild.Status) + + // WHEN: a stop build is created with an on_success start build. + ctx := testutil.Context(t, testutil.WaitLong) + + stopBuild, err := client.CreateWorkspaceBuild(ctx, workspace.ID, codersdk.CreateWorkspaceBuildRequest{ + Transition: codersdk.WorkspaceTransitionStop, + Reason: codersdk.CreateWorkspaceBuildReasonCLI, + LogLevel: codersdk.ProvisionerLogLevelDebug, + OnSuccess: &codersdk.CreateWorkspaceBuildOnSuccessRequest{ + Transition: codersdk.WorkspaceTransitionStart, + TemplateVersionID: template.ActiveVersionID, + RichParameterValues: []codersdk.WorkspaceBuildParameter{ + {Name: paramName, Value: "baz"}, + }, + }, + }) + require.NoError(t, err) + require.Equal(t, codersdk.WorkspaceTransitionStop, stopBuild.Transition) + require.Equal(t, codersdk.BuildReasonCLI, stopBuild.Reason) + + // THEN: the server persists the child start build intent. + orchestration, err := dbtestutil.GetWorkspaceBuildOrchestrationByParentBuildID(ctx, sqlDB, stopBuild.ID) + require.NoError(t, err) + require.Equal(t, codersdk.WorkspaceTransitionStart, codersdk.WorkspaceTransition(orchestration.ChildTransition)) + require.True(t, orchestration.ChildTemplateVersionID.Valid) + require.Equal(t, template.ActiveVersionID, orchestration.ChildTemplateVersionID.UUID) + require.False(t, orchestration.ChildTemplateVersionPresetID.Valid) + require.Equal(t, string(codersdk.ProvisionerLogLevelDebug), orchestration.ChildLogLevel) + require.True(t, orchestration.ChildReason.Valid) + require.Equal(t, codersdk.BuildReasonCLI, codersdk.BuildReason(orchestration.ChildReason.BuildReason)) + + var childRichParameterValues []codersdk.WorkspaceBuildParameter + require.NoError(t, json.Unmarshal(orchestration.ChildRichParameterValues, &childRichParameterValues)) + require.ElementsMatch(t, []codersdk.WorkspaceBuildParameter{ + {Name: paramName, Value: "baz"}, + }, childRichParameterValues) +} + +func TestPostWorkspaceBuildsOnSuccessPinnedChildVersionRequiresTemplateUpdate(t *testing.T) { + t.Parallel() + + // GIVEN: a running workspace owned by a non-template-admin. + client, _ := coderdtest.NewWithDatabase(t, &coderdtest.Options{IncludeProvisionerDaemon: true}) + first := coderdtest.CreateFirstUser(t, client) + userClient, _ := coderdtest.CreateAnotherUser(t, client, first.OrganizationID) + + version := coderdtest.CreateTemplateVersion(t, client, first.OrganizationID, nil) + coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID) + template := coderdtest.CreateTemplate(t, client, first.OrganizationID, version.ID) + workspace := coderdtest.CreateWorkspace(t, userClient, template.ID) + initialBuild := coderdtest.AwaitWorkspaceBuildJobCompleted(t, userClient, workspace.LatestBuild.ID) + require.Equal(t, codersdk.WorkspaceStatusRunning, initialBuild.Status) + + // WHEN: the non-template-admin tries to queue a stop build with a + // pinned on_success child version. + ctx := testutil.Context(t, testutil.WaitLong) + _, err := userClient.CreateWorkspaceBuild(ctx, workspace.ID, codersdk.CreateWorkspaceBuildRequest{ + Transition: codersdk.WorkspaceTransitionStop, + OnSuccess: &codersdk.CreateWorkspaceBuildOnSuccessRequest{ + Transition: codersdk.WorkspaceTransitionStart, + TemplateVersionID: version.ID, + }, + }) + require.Error(t, err) + + // THEN: the API rejects the durable child version pin. + var apiErr *codersdk.Error + require.ErrorAs(t, err, &apiErr) + require.Equal(t, http.StatusForbidden, apiErr.StatusCode()) + + // THEN: no new workspace build is created. + builds, err := userClient.WorkspaceBuilds(ctx, codersdk.WorkspaceBuildsRequest{WorkspaceID: workspace.ID}) + require.NoError(t, err) + require.Len(t, builds, 1) + require.Equal(t, initialBuild.ID, builds[0].ID) +} + +func TestPostWorkspaceBuildsOnSuccessValidation(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + request codersdk.CreateWorkspaceBuildRequest + }{ + { + name: "ParentMustBeStop", + request: codersdk.CreateWorkspaceBuildRequest{ + Transition: codersdk.WorkspaceTransitionStart, + OnSuccess: &codersdk.CreateWorkspaceBuildOnSuccessRequest{ + Transition: codersdk.WorkspaceTransitionStart, + }, + }, + }, + { + name: "ChildMustBeStart", + request: codersdk.CreateWorkspaceBuildRequest{ + Transition: codersdk.WorkspaceTransitionStop, + OnSuccess: &codersdk.CreateWorkspaceBuildOnSuccessRequest{ + Transition: codersdk.WorkspaceTransitionStop, + }, + }, + }, + { + name: "ParentDryRunRejected", + request: codersdk.CreateWorkspaceBuildRequest{ + Transition: codersdk.WorkspaceTransitionStop, + DryRun: true, + OnSuccess: &codersdk.CreateWorkspaceBuildOnSuccessRequest{ + Transition: codersdk.WorkspaceTransitionStart, + }, + }, + }, + { + name: "ParentOrphanRejected", + request: codersdk.CreateWorkspaceBuildRequest{ + Transition: codersdk.WorkspaceTransitionStop, + Orphan: true, + OnSuccess: &codersdk.CreateWorkspaceBuildOnSuccessRequest{ + Transition: codersdk.WorkspaceTransitionStart, + }, + }, + }, + { + name: "ParentProvisionerStateRejected", + request: codersdk.CreateWorkspaceBuildRequest{ + Transition: codersdk.WorkspaceTransitionStop, + ProvisionerState: []byte("state"), + OnSuccess: &codersdk.CreateWorkspaceBuildOnSuccessRequest{ + Transition: codersdk.WorkspaceTransitionStart, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + // GIVEN: a running workspace. + client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerDaemon: true}) + first := coderdtest.CreateFirstUser(t, client) + version := coderdtest.CreateTemplateVersion(t, client, first.OrganizationID, nil) + coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID) + template := coderdtest.CreateTemplate(t, client, first.OrganizationID, version.ID) + workspace := coderdtest.CreateWorkspace(t, client, template.ID) + coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, workspace.LatestBuild.ID) + + // WHEN: an invalid on_success request is posted. + _, err := client.CreateWorkspaceBuild(testutil.Context(t, testutil.WaitLong), workspace.ID, tt.request) + require.Error(t, err) + + // THEN: the API rejects the request before creating a build. + var apiErr *codersdk.Error + require.ErrorAs(t, err, &apiErr) + require.Equal(t, http.StatusBadRequest, apiErr.StatusCode()) + }) + } +} + +type echoResponseOptions struct { + blockStopApply bool + failStopApply bool + validationRegex string +} + +func echoResponsesWithRichParameter(paramName string, options echoResponseOptions) *echo.Responses { + validationError := "" + if options.validationRegex != "" { + validationError = "invalid parameter value" + } + + responses := &echo.Responses{ + Parse: echo.ParseComplete, + ProvisionInit: echo.InitComplete, + ProvisionGraph: []*proto.Response{{ + Type: &proto.Response_Graph{ + Graph: &proto.GraphComplete{ + Parameters: []*proto.RichParameter{{ + Name: paramName, + Type: "string", + DefaultValue: "bar", + Mutable: true, + FormType: proto.ParameterFormType_INPUT, + ValidationRegex: options.validationRegex, + ValidationError: validationError, + }}, + }, + }, + }}, + ProvisionPlan: echo.PlanComplete, + ProvisionApply: echo.ApplyComplete, + } + if options.blockStopApply { + responses.ProvisionApplyMap = map[proto.WorkspaceTransition][]*proto.Response{ + proto.WorkspaceTransition_START: echo.ApplyComplete, + proto.WorkspaceTransition_STOP: {{ + Type: &proto.Response_Log{ + Log: &proto.Log{}, + }, + }}, + } + } + if options.failStopApply { + responses.ProvisionApplyMap = map[proto.WorkspaceTransition][]*proto.Response{ + proto.WorkspaceTransition_START: echo.ApplyComplete, + proto.WorkspaceTransition_STOP: echo.ApplyFailed, + } + } + return responses +} diff --git a/codersdk/workspaces.go b/codersdk/workspaces.go index b520f27e4f8..bc5136215c9 100644 --- a/codersdk/workspaces.go +++ b/codersdk/workspaces.go @@ -136,6 +136,17 @@ type CreateWorkspaceBuildRequest struct { TemplateVersionPresetID uuid.UUID `json:"template_version_preset_id,omitempty" format:"uuid"` // Reason sets the reason for the workspace build. Reason CreateWorkspaceBuildReason `json:"reason,omitempty" validate:"omitempty,oneof=dashboard cli ssh_connection vscode_connection jetbrains_connection task_manual_pause"` + // OnSuccess queues a follow-up workspace build after this build succeeds. + // It currently supports restarting a workspace by starting it after a + // successful stop build. + OnSuccess *CreateWorkspaceBuildOnSuccessRequest `json:"on_success,omitempty"` +} + +type CreateWorkspaceBuildOnSuccessRequest struct { + TemplateVersionID uuid.UUID `json:"template_version_id,omitempty" format:"uuid"` + Transition WorkspaceTransition `json:"transition" validate:"oneof=start,required"` + RichParameterValues []WorkspaceBuildParameter `json:"rich_parameter_values,omitempty"` + TemplateVersionPresetID uuid.UUID `json:"template_version_preset_id,omitempty" format:"uuid"` } type WorkspaceOptions struct { diff --git a/docs/reference/api/builds.md b/docs/reference/api/builds.md index 00db92184dd..5f56524b445 100644 --- a/docs/reference/api/builds.md +++ b/docs/reference/api/builds.md @@ -1767,6 +1767,17 @@ curl -X POST http://coder-server:8080/api/v2/workspaces/{workspace}/builds \ { "dry_run": true, "log_level": "debug", + "on_success": { + "rich_parameter_values": [ + { + "name": "string", + "value": "string" + } + ], + "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", + "template_version_preset_id": "512a53a7-30da-446e-a1fc-713c630baff1", + "transition": "start" + }, "orphan": true, "reason": "dashboard", "rich_parameter_values": [ diff --git a/docs/reference/api/schemas.md b/docs/reference/api/schemas.md index f0102c48c5f..fc589c24cc7 100644 --- a/docs/reference/api/schemas.md +++ b/docs/reference/api/schemas.md @@ -4979,6 +4979,37 @@ This is required on creation to enable a user-flow of validating a template work |-----------|--------|----------|--------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `content` | string | false | | Content must be SKILL.md-format Markdown with YAML frontmatter. The frontmatter must include name, may include description, and must be followed by a non-empty body. | +## codersdk.CreateWorkspaceBuildOnSuccessRequest + +```json +{ + "rich_parameter_values": [ + { + "name": "string", + "value": "string" + } + ], + "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", + "template_version_preset_id": "512a53a7-30da-446e-a1fc-713c630baff1", + "transition": "start" +} +``` + +### Properties + +| Name | Type | Required | Restrictions | Description | +|------------------------------|-------------------------------------------------------------------------------|----------|--------------|-------------| +| `rich_parameter_values` | array of [codersdk.WorkspaceBuildParameter](#codersdkworkspacebuildparameter) | false | | | +| `template_version_id` | string | false | | | +| `template_version_preset_id` | string | false | | | +| `transition` | [codersdk.WorkspaceTransition](#codersdkworkspacetransition) | true | | | + +#### Enumerated Values + +| Property | Value(s) | +|--------------|----------| +| `transition` | `start` | + ## codersdk.CreateWorkspaceBuildReason ```json @@ -4999,6 +5030,17 @@ This is required on creation to enable a user-flow of validating a template work { "dry_run": true, "log_level": "debug", + "on_success": { + "rich_parameter_values": [ + { + "name": "string", + "value": "string" + } + ], + "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", + "template_version_preset_id": "512a53a7-30da-446e-a1fc-713c630baff1", + "transition": "start" + }, "orphan": true, "reason": "dashboard", "rich_parameter_values": [ @@ -5018,17 +5060,18 @@ This is required on creation to enable a user-flow of validating a template work ### Properties -| Name | Type | Required | Restrictions | Description | -|------------------------------|-------------------------------------------------------------------------------|----------|--------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `dry_run` | boolean | false | | | -| `log_level` | [codersdk.ProvisionerLogLevel](#codersdkprovisionerloglevel) | false | | Log level changes the default logging verbosity of a provider ("info" if empty). | -| `orphan` | boolean | false | | Orphan may be set for the Destroy transition. | -| `reason` | [codersdk.CreateWorkspaceBuildReason](#codersdkcreateworkspacebuildreason) | false | | Reason sets the reason for the workspace build. | -| `rich_parameter_values` | array of [codersdk.WorkspaceBuildParameter](#codersdkworkspacebuildparameter) | false | | Rich parameter values are optional. It will write params to the 'workspace' scope. This will overwrite any existing parameters with the same name. This will not delete old params not included in this list. | -| `state` | array of integer | false | | | -| `template_version_id` | string | false | | | -| `template_version_preset_id` | string | false | | Template version preset ID is the ID of the template version preset to use for the build. | -| `transition` | [codersdk.WorkspaceTransition](#codersdkworkspacetransition) | true | | | +| Name | Type | Required | Restrictions | Description | +|------------------------------|------------------------------------------------------------------------------------------------|----------|--------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `dry_run` | boolean | false | | | +| `log_level` | [codersdk.ProvisionerLogLevel](#codersdkprovisionerloglevel) | false | | Log level changes the default logging verbosity of a provider ("info" if empty). | +| `on_success` | [codersdk.CreateWorkspaceBuildOnSuccessRequest](#codersdkcreateworkspacebuildonsuccessrequest) | false | | On success queues a follow-up workspace build after this build succeeds. It currently supports restarting a workspace by starting it after a successful stop build. | +| `orphan` | boolean | false | | Orphan may be set for the Destroy transition. | +| `reason` | [codersdk.CreateWorkspaceBuildReason](#codersdkcreateworkspacebuildreason) | false | | Reason sets the reason for the workspace build. | +| `rich_parameter_values` | array of [codersdk.WorkspaceBuildParameter](#codersdkworkspacebuildparameter) | false | | Rich parameter values are optional. It will write params to the 'workspace' scope. This will overwrite any existing parameters with the same name. This will not delete old params not included in this list. | +| `state` | array of integer | false | | | +| `template_version_id` | string | false | | | +| `template_version_preset_id` | string | false | | Template version preset ID is the ID of the template version preset to use for the build. | +| `transition` | [codersdk.WorkspaceTransition](#codersdkworkspacetransition) | true | | | #### Enumerated Values diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts index a7a9c3ad26d..fd7964fbb2f 100644 --- a/site/src/api/typesGenerated.ts +++ b/site/src/api/typesGenerated.ts @@ -3732,6 +3732,14 @@ export interface CreateUserSkillRequest { readonly content: string; } +// From codersdk/workspaces.go +export interface CreateWorkspaceBuildOnSuccessRequest { + readonly template_version_id?: string; + readonly transition: WorkspaceTransition; + readonly rich_parameter_values?: readonly WorkspaceBuildParameter[]; + readonly template_version_preset_id?: string; +} + // From codersdk/workspaces.go export type CreateWorkspaceBuildReason = | "cli" @@ -3783,6 +3791,12 @@ export interface CreateWorkspaceBuildRequest { * Reason sets the reason for the workspace build. */ readonly reason?: CreateWorkspaceBuildReason; + /** + * OnSuccess queues a follow-up workspace build after this build succeeds. + * It currently supports restarting a workspace by starting it after a + * successful stop build. + */ + readonly on_success?: CreateWorkspaceBuildOnSuccessRequest; } // From codersdk/workspaceproxy.go From 583b3392324a589dfc54c41ab5bd778bd579f5ed Mon Sep 17 00:00:00 2001 From: George Katsitadze Date: Tue, 23 Jun 2026 16:16:10 -0700 Subject: [PATCH 2/3] address pr review comments --- coderd/apidoc/docs.go | 4 + coderd/apidoc/swagger.json | 4 + .../workspacebuildorchestrations.go | 49 ++--------- coderd/workspacebuilds.go | 30 ++++--- coderd/workspacebuilds_on_success_test.go | 83 ++++++++++++++++++- codersdk/workspaces.go | 25 +++++- docs/reference/api/schemas.md | 12 +-- site/src/api/typesGenerated.ts | 27 ++++++ 8 files changed, 170 insertions(+), 64 deletions(-) diff --git a/coderd/apidoc/docs.go b/coderd/apidoc/docs.go index c01b8db6d6a..534f808cb83 100644 --- a/coderd/apidoc/docs.go +++ b/coderd/apidoc/docs.go @@ -18456,20 +18456,24 @@ const docTemplate = `{ ], "properties": { "rich_parameter_values": { + "description": "RichParameterValues are applied to the child build. Parameters\nnot listed here fall back to their values from the previous\nbuild, matching normal build behavior.", "type": "array", "items": { "$ref": "#/definitions/codersdk.WorkspaceBuildParameter" } }, "template_version_id": { + "description": "TemplateVersionID pins the child build to a specific template\nversion. Pinning requires permission to update the template,\nsince the active version may change before the child build\nruns. When empty, the child build uses the template's active\nversion at the time it runs.", "type": "string", "format": "uuid" }, "template_version_preset_id": { + "description": "TemplateVersionPresetID selects a preset for the child build.\nIt requires TemplateVersionID to also be set.", "type": "string", "format": "uuid" }, "transition": { + "description": "Transition must be \"start\". The parent build's transition must\nbe \"stop\".", "enum": [ "start" ], diff --git a/coderd/apidoc/swagger.json b/coderd/apidoc/swagger.json index d331cc56ca0..603029585f6 100644 --- a/coderd/apidoc/swagger.json +++ b/coderd/apidoc/swagger.json @@ -16687,20 +16687,24 @@ "required": ["transition"], "properties": { "rich_parameter_values": { + "description": "RichParameterValues are applied to the child build. Parameters\nnot listed here fall back to their values from the previous\nbuild, matching normal build behavior.", "type": "array", "items": { "$ref": "#/definitions/codersdk.WorkspaceBuildParameter" } }, "template_version_id": { + "description": "TemplateVersionID pins the child build to a specific template\nversion. Pinning requires permission to update the template,\nsince the active version may change before the child build\nruns. When empty, the child build uses the template's active\nversion at the time it runs.", "type": "string", "format": "uuid" }, "template_version_preset_id": { + "description": "TemplateVersionPresetID selects a preset for the child build.\nIt requires TemplateVersionID to also be set.", "type": "string", "format": "uuid" }, "transition": { + "description": "Transition must be \"start\". The parent build's transition must\nbe \"stop\".", "enum": ["start"], "allOf": [ { diff --git a/coderd/database/dbtestutil/workspacebuildorchestrations.go b/coderd/database/dbtestutil/workspacebuildorchestrations.go index 18f3b97bc93..4dce0a92799 100644 --- a/coderd/database/dbtestutil/workspacebuildorchestrations.go +++ b/coderd/database/dbtestutil/workspacebuildorchestrations.go @@ -5,61 +5,28 @@ import ( "database/sql" "github.com/google/uuid" + "github.com/jmoiron/sqlx" "github.com/coder/coder/v2/coderd/database" ) -const getWorkspaceBuildOrchestrationByParentBuildIDQuery = ` -SELECT - id, - created_at, - updated_at, - parent_build_id, - child_build_id, - child_transition, - child_template_version_id, - child_template_version_preset_id, - child_rich_parameter_values, - child_log_level, - child_reason, - attempt_count, - next_retry_after, - status, - error -FROM - workspace_build_orchestrations -WHERE - parent_build_id = $1 -` - // GetWorkspaceBuildOrchestrationByParentBuildID reads a workspace // build orchestration row directly from the database for tests. +// +// It scans into the struct by column name so new columns are picked up +// automatically without updating this helper. func GetWorkspaceBuildOrchestrationByParentBuildID( ctx context.Context, sqlDB *sql.DB, parentBuildID uuid.UUID, ) (database.WorkspaceBuildOrchestration, error) { + db := sqlx.NewDb(sqlDB, "postgres") var orchestration database.WorkspaceBuildOrchestration - err := sqlDB.QueryRowContext( + err := db.GetContext( ctx, - getWorkspaceBuildOrchestrationByParentBuildIDQuery, + &orchestration, + `SELECT * FROM workspace_build_orchestrations WHERE parent_build_id = $1`, parentBuildID, - ).Scan( - &orchestration.ID, - &orchestration.CreatedAt, - &orchestration.UpdatedAt, - &orchestration.ParentBuildID, - &orchestration.ChildBuildID, - &orchestration.ChildTransition, - &orchestration.ChildTemplateVersionID, - &orchestration.ChildTemplateVersionPresetID, - &orchestration.ChildRichParameterValues, - &orchestration.ChildLogLevel, - &orchestration.ChildReason, - &orchestration.AttemptCount, - &orchestration.NextRetryAfter, - &orchestration.Status, - &orchestration.Error, ) return orchestration, err } diff --git a/coderd/workspacebuilds.go b/coderd/workspacebuilds.go index c71f693014d..aecd7a353ca 100644 --- a/coderd/workspacebuilds.go +++ b/coderd/workspacebuilds.go @@ -510,21 +510,22 @@ func (api *API) postWorkspaceBuildsInternal( } if createBuild.OnSuccess != nil { - childBuild := createBuild.OnSuccess - now := dbtime.Now() + onSuccessReq := createBuild.OnSuccess + // Reuse the parent build's timestamps so the orchestration row and + // the parent build agree on created_at. _, err = tx.InsertWorkspaceBuildOrchestration(ctx, database.InsertWorkspaceBuildOrchestrationParams{ ID: uuid.New(), - CreatedAt: now, - UpdatedAt: now, + CreatedAt: workspaceBuild.CreatedAt, + UpdatedAt: workspaceBuild.UpdatedAt, ParentBuildID: workspaceBuild.ID, - ChildTransition: database.WorkspaceTransition(childBuild.Transition), + ChildTransition: database.WorkspaceTransition(onSuccessReq.Transition), ChildTemplateVersionID: uuid.NullUUID{ - UUID: childBuild.TemplateVersionID, - Valid: childBuild.TemplateVersionID != uuid.Nil, + UUID: onSuccessReq.TemplateVersionID, + Valid: onSuccessReq.TemplateVersionID != uuid.Nil, }, ChildTemplateVersionPresetID: uuid.NullUUID{ - UUID: childBuild.TemplateVersionPresetID, - Valid: childBuild.TemplateVersionPresetID != uuid.Nil, + UUID: onSuccessReq.TemplateVersionPresetID, + Valid: onSuccessReq.TemplateVersionPresetID != uuid.Nil, }, ChildRichParameterValues: childParameterValuesJSON, ChildLogLevel: string(createBuild.LogLevel), @@ -535,9 +536,13 @@ func (api *API) postWorkspaceBuildsInternal( }) if err != nil { if dbauthz.IsNotAuthorizedError(err) { + detail := "Queuing the follow-up workspace build requires permission to start the workspace." + if onSuccessReq.TemplateVersionID != uuid.Nil { + detail = "Pinning a template version on the follow-up build requires template update permission. Omit template_version_id to use the active version." + } return httperror.NewResponseError(http.StatusForbidden, codersdk.Response{ Message: "Unauthorized to queue follow-up workspace build.", - Detail: "You do not have permission to queue this follow-up workspace build.", + Detail: detail, }) } api.Logger.Error(ctx, "failed to queue follow-up workspace build", @@ -691,6 +696,11 @@ func validateCreateWorkspaceBuildOnSuccess(createBuild codersdk.CreateWorkspaceB Message: "OnSuccess cannot be set alongside ProvisionerState.", }) } + if onSuccess.TemplateVersionPresetID != uuid.Nil && onSuccess.TemplateVersionID == uuid.Nil { + return httperror.NewResponseError(http.StatusBadRequest, codersdk.Response{ + Message: "OnSuccess TemplateVersionPresetID requires TemplateVersionID.", + }) + } return nil } diff --git a/coderd/workspacebuilds_on_success_test.go b/coderd/workspacebuilds_on_success_test.go index 921bbc9c95d..cbe683c1734 100644 --- a/coderd/workspacebuilds_on_success_test.go +++ b/coderd/workspacebuilds_on_success_test.go @@ -5,6 +5,7 @@ import ( "net/http" "testing" + "github.com/google/uuid" "github.com/stretchr/testify/require" "github.com/coder/coder/v2/coderd/coderdtest" @@ -68,6 +69,7 @@ func TestPostWorkspaceBuildsOnSuccessRestart(t *testing.T) { // THEN: the server persists the child start build intent. orchestration, err := dbtestutil.GetWorkspaceBuildOrchestrationByParentBuildID(ctx, sqlDB, stopBuild.ID) require.NoError(t, err) + require.Equal(t, "pending", orchestration.Status) require.Equal(t, codersdk.WorkspaceTransitionStart, codersdk.WorkspaceTransition(orchestration.ChildTransition)) require.True(t, orchestration.ChildTemplateVersionID.Valid) require.Equal(t, template.ActiveVersionID, orchestration.ChildTemplateVersionID.UUID) @@ -83,6 +85,59 @@ func TestPostWorkspaceBuildsOnSuccessRestart(t *testing.T) { }, childRichParameterValues) } +func TestPostWorkspaceBuildsOnSuccessUnpinnedChildNoParams(t *testing.T) { + t.Parallel() + + // GIVEN: a running workspace owned by a non-template-admin. + db, ps, sqlDB := dbtestutil.NewDBWithSQLDB(t) + client, _, _ := coderdtest.NewWithAPI(t, &coderdtest.Options{ + Database: db, + Pubsub: ps, + IncludeProvisionerDaemon: true, + }) + first := coderdtest.CreateFirstUser(t, client) + userClient, _ := coderdtest.CreateAnotherUser(t, client, first.OrganizationID) + + version := coderdtest.CreateTemplateVersion(t, client, first.OrganizationID, nil) + coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID) + template := coderdtest.CreateTemplate(t, client, first.OrganizationID, version.ID) + workspace := coderdtest.CreateWorkspace(t, userClient, template.ID) + initialBuild := coderdtest.AwaitWorkspaceBuildJobCompleted(t, userClient, workspace.LatestBuild.ID) + require.Equal(t, codersdk.WorkspaceStatusRunning, initialBuild.Status) + + // WHEN: the non-template-admin queues a stop build with an unpinned + // on_success start build that supplies no parameters, reason, or log level. + ctx := testutil.Context(t, testutil.WaitLong) + stopBuild, err := userClient.CreateWorkspaceBuild(ctx, workspace.ID, codersdk.CreateWorkspaceBuildRequest{ + Transition: codersdk.WorkspaceTransitionStop, + OnSuccess: &codersdk.CreateWorkspaceBuildOnSuccessRequest{ + Transition: codersdk.WorkspaceTransitionStart, + }, + }) + // THEN: the request is permitted without template-update privileges, + // because no durable template version pin is requested. + require.NoError(t, err) + require.Equal(t, codersdk.WorkspaceTransitionStop, stopBuild.Transition) + + // THEN: the persisted child build intent leaves the optional fields unset. + orchestration, err := dbtestutil.GetWorkspaceBuildOrchestrationByParentBuildID(ctx, sqlDB, stopBuild.ID) + require.NoError(t, err) + require.Equal(t, "pending", orchestration.Status) + require.Equal(t, workspace.ID, orchestration.WorkspaceID) + require.Equal(t, codersdk.WorkspaceTransitionStart, codersdk.WorkspaceTransition(orchestration.ChildTransition)) + require.False(t, orchestration.ChildTemplateVersionID.Valid) + require.False(t, orchestration.ChildTemplateVersionPresetID.Valid) + require.False(t, orchestration.ChildReason.Valid) + require.Empty(t, orchestration.ChildLogLevel) + + // THEN: nil parameters are coerced to an empty JSON array, not null, to + // satisfy the database CHECK constraint. + require.JSONEq(t, "[]", string(orchestration.ChildRichParameterValues)) + var childRichParameterValues []codersdk.WorkspaceBuildParameter + require.NoError(t, json.Unmarshal(orchestration.ChildRichParameterValues, &childRichParameterValues)) + require.Empty(t, childRichParameterValues) +} + func TestPostWorkspaceBuildsOnSuccessPinnedChildVersionRequiresTemplateUpdate(t *testing.T) { t.Parallel() @@ -110,10 +165,12 @@ func TestPostWorkspaceBuildsOnSuccessPinnedChildVersionRequiresTemplateUpdate(t }) require.Error(t, err) - // THEN: the API rejects the durable child version pin. + // THEN: the API rejects the durable child version pin and explains the + // missing template update permission. var apiErr *codersdk.Error require.ErrorAs(t, err, &apiErr) require.Equal(t, http.StatusForbidden, apiErr.StatusCode()) + require.Contains(t, apiErr.Response.Detail, "template update permission") // THEN: no new workspace build is created. builds, err := userClient.WorkspaceBuilds(ctx, codersdk.WorkspaceBuildsRequest{WorkspaceID: workspace.ID}) @@ -126,8 +183,9 @@ func TestPostWorkspaceBuildsOnSuccessValidation(t *testing.T) { t.Parallel() tests := []struct { - name string - request codersdk.CreateWorkspaceBuildRequest + name string + request codersdk.CreateWorkspaceBuildRequest + wantMessage string }{ { name: "ParentMustBeStop", @@ -137,8 +195,11 @@ func TestPostWorkspaceBuildsOnSuccessValidation(t *testing.T) { Transition: codersdk.WorkspaceTransitionStart, }, }, + wantMessage: "OnSuccess is only permitted when stopping a workspace.", }, { + // The oneof=start struct tag on OnSuccess.Transition rejects this + // during httpapi.Read, before the explicit check is reached. name: "ChildMustBeStart", request: codersdk.CreateWorkspaceBuildRequest{ Transition: codersdk.WorkspaceTransitionStop, @@ -146,6 +207,7 @@ func TestPostWorkspaceBuildsOnSuccessValidation(t *testing.T) { Transition: codersdk.WorkspaceTransitionStop, }, }, + wantMessage: "Validation failed.", }, { name: "ParentDryRunRejected", @@ -156,6 +218,7 @@ func TestPostWorkspaceBuildsOnSuccessValidation(t *testing.T) { Transition: codersdk.WorkspaceTransitionStart, }, }, + wantMessage: "OnSuccess cannot be set alongside DryRun.", }, { name: "ParentOrphanRejected", @@ -166,6 +229,7 @@ func TestPostWorkspaceBuildsOnSuccessValidation(t *testing.T) { Transition: codersdk.WorkspaceTransitionStart, }, }, + wantMessage: "OnSuccess cannot be set alongside Orphan.", }, { name: "ParentProvisionerStateRejected", @@ -176,6 +240,18 @@ func TestPostWorkspaceBuildsOnSuccessValidation(t *testing.T) { Transition: codersdk.WorkspaceTransitionStart, }, }, + wantMessage: "OnSuccess cannot be set alongside ProvisionerState.", + }, + { + name: "ChildPresetWithoutVersionRejected", + request: codersdk.CreateWorkspaceBuildRequest{ + Transition: codersdk.WorkspaceTransitionStop, + OnSuccess: &codersdk.CreateWorkspaceBuildOnSuccessRequest{ + Transition: codersdk.WorkspaceTransitionStart, + TemplateVersionPresetID: uuid.New(), + }, + }, + wantMessage: "OnSuccess TemplateVersionPresetID requires TemplateVersionID.", }, } @@ -200,6 +276,7 @@ func TestPostWorkspaceBuildsOnSuccessValidation(t *testing.T) { var apiErr *codersdk.Error require.ErrorAs(t, err, &apiErr) require.Equal(t, http.StatusBadRequest, apiErr.StatusCode()) + require.Contains(t, apiErr.Message, tt.wantMessage) }) } } diff --git a/codersdk/workspaces.go b/codersdk/workspaces.go index bc5136215c9..7c6f1ea89f2 100644 --- a/codersdk/workspaces.go +++ b/codersdk/workspaces.go @@ -142,11 +142,28 @@ type CreateWorkspaceBuildRequest struct { OnSuccess *CreateWorkspaceBuildOnSuccessRequest `json:"on_success,omitempty"` } +// CreateWorkspaceBuildOnSuccessRequest queues a follow-up build that +// runs after the parent build succeeds. It currently supports +// restarting a workspace: the parent build must be a "stop" and this +// child build a "start". The child build inherits LogLevel and Reason +// from the parent CreateWorkspaceBuildRequest. type CreateWorkspaceBuildOnSuccessRequest struct { - TemplateVersionID uuid.UUID `json:"template_version_id,omitempty" format:"uuid"` - Transition WorkspaceTransition `json:"transition" validate:"oneof=start,required"` - RichParameterValues []WorkspaceBuildParameter `json:"rich_parameter_values,omitempty"` - TemplateVersionPresetID uuid.UUID `json:"template_version_preset_id,omitempty" format:"uuid"` + // TemplateVersionID pins the child build to a specific template + // version. Pinning requires permission to update the template, + // since the active version may change before the child build + // runs. When empty, the child build uses the template's active + // version at the time it runs. + TemplateVersionID uuid.UUID `json:"template_version_id,omitempty" format:"uuid"` + // Transition must be "start". The parent build's transition must + // be "stop". + Transition WorkspaceTransition `json:"transition" validate:"oneof=start,required"` + // RichParameterValues are applied to the child build. Parameters + // not listed here fall back to their values from the previous + // build, matching normal build behavior. + RichParameterValues []WorkspaceBuildParameter `json:"rich_parameter_values,omitempty"` + // TemplateVersionPresetID selects a preset for the child build. + // It requires TemplateVersionID to also be set. + TemplateVersionPresetID uuid.UUID `json:"template_version_preset_id,omitempty" format:"uuid"` } type WorkspaceOptions struct { diff --git a/docs/reference/api/schemas.md b/docs/reference/api/schemas.md index fc589c24cc7..53b1ef18421 100644 --- a/docs/reference/api/schemas.md +++ b/docs/reference/api/schemas.md @@ -4997,12 +4997,12 @@ This is required on creation to enable a user-flow of validating a template work ### Properties -| Name | Type | Required | Restrictions | Description | -|------------------------------|-------------------------------------------------------------------------------|----------|--------------|-------------| -| `rich_parameter_values` | array of [codersdk.WorkspaceBuildParameter](#codersdkworkspacebuildparameter) | false | | | -| `template_version_id` | string | false | | | -| `template_version_preset_id` | string | false | | | -| `transition` | [codersdk.WorkspaceTransition](#codersdkworkspacetransition) | true | | | +| Name | Type | Required | Restrictions | Description | +|------------------------------|-------------------------------------------------------------------------------|----------|--------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `rich_parameter_values` | array of [codersdk.WorkspaceBuildParameter](#codersdkworkspacebuildparameter) | false | | Rich parameter values are applied to the child build. Parameters not listed here fall back to their values from the previous build, matching normal build behavior. | +| `template_version_id` | string | false | | Template version ID pins the child build to a specific template version. Pinning requires permission to update the template, since the active version may change before the child build runs. When empty, the child build uses the template's active version at the time it runs. | +| `template_version_preset_id` | string | false | | Template version preset ID selects a preset for the child build. It requires TemplateVersionID to also be set. | +| `transition` | [codersdk.WorkspaceTransition](#codersdkworkspacetransition) | true | | Transition must be "start". The parent build's transition must be "stop". | #### Enumerated Values diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts index fd7964fbb2f..a290bf8d7f1 100644 --- a/site/src/api/typesGenerated.ts +++ b/site/src/api/typesGenerated.ts @@ -3733,10 +3733,37 @@ export interface CreateUserSkillRequest { } // From codersdk/workspaces.go +/** + * CreateWorkspaceBuildOnSuccessRequest queues a follow-up build that + * runs after the parent build succeeds. It currently supports + * restarting a workspace: the parent build must be a "stop" and this + * child build a "start". The child build inherits LogLevel and Reason + * from the parent CreateWorkspaceBuildRequest. + */ export interface CreateWorkspaceBuildOnSuccessRequest { + /** + * TemplateVersionID pins the child build to a specific template + * version. Pinning requires permission to update the template, + * since the active version may change before the child build + * runs. When empty, the child build uses the template's active + * version at the time it runs. + */ readonly template_version_id?: string; + /** + * Transition must be "start". The parent build's transition must + * be "stop". + */ readonly transition: WorkspaceTransition; + /** + * RichParameterValues are applied to the child build. Parameters + * not listed here fall back to their values from the previous + * build, matching normal build behavior. + */ readonly rich_parameter_values?: readonly WorkspaceBuildParameter[]; + /** + * TemplateVersionPresetID selects a preset for the child build. + * It requires TemplateVersionID to also be set. + */ readonly template_version_preset_id?: string; } From 59461aaea954314e72e470ad39df56a8582874df Mon Sep 17 00:00:00 2001 From: George K Date: Mon, 6 Jul 2026 13:45:10 -0700 Subject: [PATCH 3/3] feat(coderd): process on_success workspace build orchestration (#25759) --- coderd/coderd.go | 18 +- coderd/database/dbauthz/dbauthz.go | 16 +- coderd/database/dbauthz/dbauthz_test.go | 8 + coderd/database/dbmetrics/querymetrics.go | 8 + coderd/database/dbmock/dbmock.go | 15 + coderd/database/dbpurge/dbpurge.go | 16 + coderd/database/dbpurge/dbpurge_test.go | 147 +++++ coderd/database/querier.go | 1 + coderd/database/queries.sql.go | 31 + .../queries/workspacebuildorchestrations.sql | 17 + coderd/pproflabel/pproflabel.go | 3 + .../provisionerdserver/provisionerdserver.go | 16 + coderd/workspacebuilds.go | 4 + coderd/workspacebuilds_on_success_test.go | 506 ++++++++++++++++ coderd/workspaces.go | 15 +- coderd/wsbuildorchestrator/doc.go | 4 + .../wsbuildorchestrator.go | 567 ++++++++++++++++++ .../wsbuildorchestrator_internal_test.go | 427 +++++++++++++ coderd/wspubsub/wspubsub.go | 30 + 19 files changed, 1830 insertions(+), 19 deletions(-) create mode 100644 coderd/wsbuildorchestrator/doc.go create mode 100644 coderd/wsbuildorchestrator/wsbuildorchestrator.go create mode 100644 coderd/wsbuildorchestrator/wsbuildorchestrator_internal_test.go diff --git a/coderd/coderd.go b/coderd/coderd.go index 601669d3210..cba3fc00342 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -96,6 +96,7 @@ import ( "github.com/coder/coder/v2/coderd/workspaceconnwatcher" "github.com/coder/coder/v2/coderd/workspacestats" "github.com/coder/coder/v2/coderd/wsbuilder" + "github.com/coder/coder/v2/coderd/wsbuildorchestrator" "github.com/coder/coder/v2/coderd/x/chatd" "github.com/coder/coder/v2/coderd/x/chatd/chatprovider" "github.com/coder/coder/v2/coderd/x/chatd/mcpclient" @@ -950,6 +951,19 @@ func New(options *Options) *API { api.workspaceAgentConnWatcher = workspaceconnwatcher.New(api.ctx, options.Logger, options.Pubsub, options.Database) + api.workspaceBuildOrchestrator = wsbuildorchestrator.New(wsbuildorchestrator.Options{ + Logger: options.Logger, + Database: options.Database, + Pubsub: options.Pubsub, + FileCache: api.FileCache, + BuildUsageChecker: api.BuildUsageChecker, + DeploymentValues: options.DeploymentValues, + Experiments: api.Experiments, + BuilderMetrics: options.WorkspaceBuilderMetrics, + Clock: api.Clock, + }) + api.workspaceBuildOrchestrator.Start(api.ctx) + apiKeyMiddleware := httpmw.ExtractAPIKeyMW(httpmw.ExtractAPIKeyConfig{ DB: options.Database, ActivateDormantUser: ActivateDormantUser(options.Logger, &api.Auditor, options.Database), @@ -2293,7 +2307,8 @@ type API struct { // profiler is process-global, so concurrent collections would fail. ProfileCollecting atomic.Bool - workspaceAgentConnWatcher *workspaceconnwatcher.Watcher + workspaceAgentConnWatcher *workspaceconnwatcher.Watcher + workspaceBuildOrchestrator *wsbuildorchestrator.Orchestrator } // Close waits for all WebSocket connections to drain before returning. @@ -2358,6 +2373,7 @@ func (api *API) Close() error { _ = api.AppEncryptionKeyCache.Close() _ = api.UpdatesProvider.Close() api.workspaceAgentConnWatcher.Close() + api.workspaceBuildOrchestrator.Close() if current := api.PrebuildsReconciler.Load(); current != nil { ctx, giveUp := context.WithTimeoutCause(context.Background(), time.Second*30, xerrors.New("gave up waiting for reconciler to stop before shutdown")) diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index 46c5e41addc..bd5ae625852 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -671,10 +671,11 @@ var ( Identifier: rbac.RoleIdentifier{Name: "dbpurge"}, DisplayName: "DB Purge Daemon", Site: rbac.Permissions(map[string][]policy.Action{ - rbac.ResourceSystem.Type: {policy.ActionDelete}, - rbac.ResourceNotificationMessage.Type: {policy.ActionDelete}, - rbac.ResourceApiKey.Type: {policy.ActionDelete}, - rbac.ResourceAibridgeInterception.Type: {policy.ActionDelete}, + rbac.ResourceSystem.Type: {policy.ActionDelete}, + rbac.ResourceNotificationMessage.Type: {policy.ActionDelete}, + rbac.ResourceApiKey.Type: {policy.ActionDelete}, + rbac.ResourceAibridgeInterception.Type: {policy.ActionDelete}, + rbac.ResourceWorkspaceBuildOrchestration.Type: {policy.ActionDelete}, // Chat auto-archive sets archived=true on inactive chats. rbac.ResourceChat.Type: {policy.ActionRead, policy.ActionUpdate}, // Purge old boundary logs past the retention period. @@ -2389,6 +2390,13 @@ func (q *querier) DeleteOldWorkspaceAgentStats(ctx context.Context) error { return q.db.DeleteOldWorkspaceAgentStats(ctx) } +func (q *querier) DeleteOldWorkspaceBuildOrchestrations(ctx context.Context, arg database.DeleteOldWorkspaceBuildOrchestrationsParams) (int64, error) { + if err := q.authorizeContext(ctx, policy.ActionDelete, rbac.ResourceWorkspaceBuildOrchestration.AnyOrganization()); err != nil { + return 0, err + } + return q.db.DeleteOldWorkspaceBuildOrchestrations(ctx, arg) +} + func (q *querier) DeleteOrganizationMember(ctx context.Context, arg database.DeleteOrganizationMemberParams) error { return deleteQ[database.OrganizationMember](q.log, q.auth, func(ctx context.Context, arg database.DeleteOrganizationMemberParams) (database.OrganizationMember, error) { member, err := database.ExpectOne(q.OrganizationMembers(ctx, database.OrganizationMembersParams{ diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index 8f76c9bc2fd..a71ab78532f 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -4156,6 +4156,14 @@ func (s *MethodTestSuite) TestWorkspace() { Asserts(rbac.ResourceWorkspaceBuildOrchestration.AnyOrganization(), policy.ActionUpdate). Returns(orchestration) })) + s.Run("DeleteOldWorkspaceBuildOrchestrations", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) { + arg := database.DeleteOldWorkspaceBuildOrchestrationsParams{ + BeforeTime: dbtime.Now(), + LimitCount: 100, + } + dbm.EXPECT().DeleteOldWorkspaceBuildOrchestrations(gomock.Any(), arg).Return(int64(0), nil).AnyTimes() + check.Args(arg).Asserts(rbac.ResourceWorkspaceBuildOrchestration.AnyOrganization(), policy.ActionDelete) + })) s.Run("Start/InsertWorkspaceBuildParameters", s.Mocked(func(dbm *dbmock.MockStore, faker *gofakeit.Faker, check *expects) { w := testutil.Fake(s.T(), faker, database.Workspace{}) b := testutil.Fake(s.T(), faker, database.WorkspaceBuild{ diff --git a/coderd/database/dbmetrics/querymetrics.go b/coderd/database/dbmetrics/querymetrics.go index 3b294898f9b..89f1d82c2b6 100644 --- a/coderd/database/dbmetrics/querymetrics.go +++ b/coderd/database/dbmetrics/querymetrics.go @@ -818,6 +818,14 @@ func (m queryMetricsStore) DeleteOldWorkspaceAgentStats(ctx context.Context) err return r0 } +func (m queryMetricsStore) DeleteOldWorkspaceBuildOrchestrations(ctx context.Context, arg database.DeleteOldWorkspaceBuildOrchestrationsParams) (int64, error) { + start := time.Now() + r0, r1 := m.s.DeleteOldWorkspaceBuildOrchestrations(ctx, arg) + m.queryLatencies.WithLabelValues("DeleteOldWorkspaceBuildOrchestrations").Observe(time.Since(start).Seconds()) + m.queryCounts.WithLabelValues(httpmw.ExtractHTTPRoute(ctx), httpmw.ExtractHTTPMethod(ctx), "DeleteOldWorkspaceBuildOrchestrations").Inc() + return r0, r1 +} + func (m queryMetricsStore) DeleteOrganizationMember(ctx context.Context, arg database.DeleteOrganizationMemberParams) error { start := time.Now() r0 := m.s.DeleteOrganizationMember(ctx, arg) diff --git a/coderd/database/dbmock/dbmock.go b/coderd/database/dbmock/dbmock.go index 6ca5ce63839..47afc753d37 100644 --- a/coderd/database/dbmock/dbmock.go +++ b/coderd/database/dbmock/dbmock.go @@ -1381,6 +1381,21 @@ func (mr *MockStoreMockRecorder) DeleteOldWorkspaceAgentStats(ctx any) *gomock.C return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldWorkspaceAgentStats", reflect.TypeOf((*MockStore)(nil).DeleteOldWorkspaceAgentStats), ctx) } +// DeleteOldWorkspaceBuildOrchestrations mocks base method. +func (m *MockStore) DeleteOldWorkspaceBuildOrchestrations(ctx context.Context, arg database.DeleteOldWorkspaceBuildOrchestrationsParams) (int64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteOldWorkspaceBuildOrchestrations", ctx, arg) + ret0, _ := ret[0].(int64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteOldWorkspaceBuildOrchestrations indicates an expected call of DeleteOldWorkspaceBuildOrchestrations. +func (mr *MockStoreMockRecorder) DeleteOldWorkspaceBuildOrchestrations(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldWorkspaceBuildOrchestrations", reflect.TypeOf((*MockStore)(nil).DeleteOldWorkspaceBuildOrchestrations), ctx, arg) +} + // DeleteOrganizationMember mocks base method. func (m *MockStore) DeleteOrganizationMember(ctx context.Context, arg database.DeleteOrganizationMemberParams) error { m.ctrl.T.Helper() diff --git a/coderd/database/dbpurge/dbpurge.go b/coderd/database/dbpurge/dbpurge.go index b50bfe3ae8b..51e034417e0 100644 --- a/coderd/database/dbpurge/dbpurge.go +++ b/coderd/database/dbpurge/dbpurge.go @@ -39,6 +39,11 @@ const ( // long enough to cover the maximum interval of a heartbeat event (currently // 1 hour) plus some buffer. maxTelemetryHeartbeatAge = 24 * time.Hour + // Operational handoff state; terminal rows are kept for debugging, then + // purged. + workspaceBuildOrchestrationTerminalRetention = 24 * time.Hour + // Batch size for workspace build orchestration deletion. + workspaceBuildOrchestrationsBatchSize = 10000 // Chat and chat file batch sizes stay smaller than audit/connection // log batches because chat_files rows carry bytea blobs. chatsBatchSize = 1000 @@ -275,6 +280,15 @@ func (i *instance) purgeTick(ctx context.Context, db database.Store, start time. } } + deleteOldWorkspaceBuildOrchestrationsBefore := start.Add(-workspaceBuildOrchestrationTerminalRetention) + purgedWorkspaceBuildOrchestrations, err := tx.DeleteOldWorkspaceBuildOrchestrations(ctx, database.DeleteOldWorkspaceBuildOrchestrationsParams{ + BeforeTime: deleteOldWorkspaceBuildOrchestrationsBefore, + LimitCount: workspaceBuildOrchestrationsBatchSize, + }) + if err != nil { + return xerrors.Errorf("failed to delete old workspace build orchestrations: %w", err) + } + var purgedChats, purgedChatFiles, purgedChatDebugRuns int64 if purgeChats { purgedChats, purgedChatFiles, err = i.purgeChatsInTx(ctx, tx, start, chatRetentionDays) @@ -304,6 +318,7 @@ func (i *instance) purgeTick(ctx context.Context, db database.Store, start time. slog.F("audit_logs", purgedAuditLogs), slog.F("boundary_logs", purgedBoundaryLogs), slog.F("boundary_sessions", purgedBoundarySessions), + slog.F("workspace_build_orchestrations", purgedWorkspaceBuildOrchestrations), slog.F("chats", purgedChats), slog.F("chat_files", purgedChatFiles), slog.F("chat_debug_runs", purgedChatDebugRuns), @@ -318,6 +333,7 @@ func (i *instance) purgeTick(ctx context.Context, db database.Store, start time. i.recordsPurged.WithLabelValues("audit_logs").Add(float64(purgedAuditLogs)) i.recordsPurged.WithLabelValues("boundary_logs").Add(float64(purgedBoundaryLogs)) i.recordsPurged.WithLabelValues("boundary_sessions").Add(float64(purgedBoundarySessions)) + i.recordsPurged.WithLabelValues("workspace_build_orchestrations").Add(float64(purgedWorkspaceBuildOrchestrations)) i.recordsPurged.WithLabelValues("chats").Add(float64(purgedChats)) i.recordsPurged.WithLabelValues("chat_debug_runs").Add(float64(purgedChatDebugRuns)) i.recordsPurged.WithLabelValues("chat_files").Add(float64(purgedChatFiles)) diff --git a/coderd/database/dbpurge/dbpurge_test.go b/coderd/database/dbpurge/dbpurge_test.go index 18d7866cac5..6efe954360a 100644 --- a/coderd/database/dbpurge/dbpurge_test.go +++ b/coderd/database/dbpurge/dbpurge_test.go @@ -129,6 +129,11 @@ func TestMetrics(t *testing.T) { }) require.GreaterOrEqual(t, auditLogs, 0) + workspaceBuildOrchestrations := promhelp.CounterValue(t, reg, "coderd_dbpurge_records_purged_total", prometheus.Labels{ + "record_type": "workspace_build_orchestrations", + }) + require.GreaterOrEqual(t, workspaceBuildOrchestrations, 0) + chats := promhelp.CounterValue(t, reg, "coderd_dbpurge_records_purged_total", prometheus.Labels{ "record_type": "chats", }) @@ -247,6 +252,7 @@ func TestMetrics(t *testing.T) { mDB.EXPECT().DeleteOldNotificationMessages(gomock.Any()).Return(nil).AnyTimes() mDB.EXPECT().ExpirePrebuildsAPIKeys(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() mDB.EXPECT().DeleteOldTelemetryLocks(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + mDB.EXPECT().DeleteOldWorkspaceBuildOrchestrations(gomock.Any(), gomock.Any()).Return(int64(0), nil).AnyTimes() mDB.EXPECT().DeleteOldAuditLogConnectionEvents(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() mDB.EXPECT().DeleteOldChatDebugRuns(gomock.Any(), gomock.AssignableToTypeOf(database.DeleteOldChatDebugRunsParams{})).Return(int64(0), nil).MinTimes(1) mDB.EXPECT().InTx(gomock.Any(), database.DefaultTXOptions().WithID("db_purge")). @@ -297,6 +303,7 @@ func TestMetrics(t *testing.T) { mDB.EXPECT().DeleteOldNotificationMessages(gomock.Any()).Return(nil).AnyTimes() mDB.EXPECT().ExpirePrebuildsAPIKeys(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() mDB.EXPECT().DeleteOldTelemetryLocks(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + mDB.EXPECT().DeleteOldWorkspaceBuildOrchestrations(gomock.Any(), gomock.Any()).Return(int64(0), nil).AnyTimes() mDB.EXPECT().DeleteOldAuditLogConnectionEvents(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() mDB.EXPECT().DeleteOldChats(gomock.Any(), gomock.AssignableToTypeOf(database.DeleteOldChatsParams{})).Return(int64(0), nil).MinTimes(1) mDB.EXPECT().DeleteOldChatFiles(gomock.Any(), gomock.AssignableToTypeOf(database.DeleteOldChatFilesParams{})).Return(int64(0), nil).MinTimes(1) @@ -1117,6 +1124,146 @@ func TestDeleteOldTelemetryHeartbeats(t *testing.T) { }, testutil.WaitShort, testutil.IntervalFast, "it should delete old telemetry heartbeats") } +func TestDeleteOldWorkspaceBuildOrchestrations(t *testing.T) { + t.Parallel() + + ctx := testutil.Context(t, testutil.WaitShort) + db, _, rawDB := dbtestutil.NewDBWithSQLDB(t) + + org := dbgen.Organization(t, db, database.Organization{}) + user := dbgen.User(t, db, database.User{}) + versionJob := dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{ + OrganizationID: org.ID, + Type: database.ProvisionerJobTypeTemplateVersionImport, + }) + version := dbgen.TemplateVersion(t, db, database.TemplateVersion{ + OrganizationID: org.ID, + JobID: versionJob.ID, + CreatedBy: user.ID, + }) + template := dbgen.Template(t, db, database.Template{ + OrganizationID: org.ID, + ActiveVersionID: version.ID, + CreatedBy: user.ID, + }) + workspace := dbgen.Workspace(t, db, database.WorkspaceTable{ + OwnerID: user.ID, + OrganizationID: org.ID, + TemplateID: template.ID, + }) + + now := dbtime.Now() + cutoff := now.Add(-24 * time.Hour) + buildTime := cutoff.Add(-time.Hour) + oldCompletedTime := cutoff.Add(-3 * time.Minute) + oldFailedTime := cutoff.Add(-2 * time.Minute) + oldCanceledTime := cutoff.Add(-time.Minute) + oldPendingTime := cutoff.Add(-time.Minute) + recentTime := cutoff.Add(time.Minute) + + createBuild := func(buildNumber int32, createdAt time.Time) database.WorkspaceBuild { + return mustCreateWorkspaceBuild(t, db, org, version, workspace.ID, createdAt, buildNumber) + } + insertOrchestration := func(parentBuild database.WorkspaceBuild, updatedAt time.Time) database.WorkspaceBuildOrchestration { + orchestration, err := db.InsertWorkspaceBuildOrchestration(ctx, database.InsertWorkspaceBuildOrchestrationParams{ + ID: uuid.New(), + CreatedAt: updatedAt, + UpdatedAt: updatedAt, + ParentBuildID: parentBuild.ID, + ChildTransition: database.WorkspaceTransitionStart, + ChildRichParameterValues: json.RawMessage("[]"), + }) + require.NoError(t, err) + return orchestration + } + + // Given: old terminal orchestration rows (completed, failed, + // canceled), an old pending row, and a recent terminal row. + oldCompletedParent := createBuild(1, buildTime) + oldCompletedChild := createBuild(2, buildTime) + oldCompleted := insertOrchestration(oldCompletedParent, oldCompletedTime) + _, err := db.UpdateWorkspaceBuildOrchestrationCompletedByID(ctx, database.UpdateWorkspaceBuildOrchestrationCompletedByIDParams{ + ID: oldCompleted.ID, + ChildBuildID: uuid.NullUUID{UUID: oldCompletedChild.ID, Valid: true}, + UpdatedAt: oldCompletedTime, + }) + require.NoError(t, err) + + oldFailedParent := createBuild(3, buildTime) + oldFailed := insertOrchestration(oldFailedParent, oldFailedTime) + _, err = db.UpdateWorkspaceBuildOrchestrationFailedByID(ctx, database.UpdateWorkspaceBuildOrchestrationFailedByIDParams{ + ID: oldFailed.ID, + Error: sql.NullString{String: "failed", Valid: true}, + UpdatedAt: oldFailedTime, + }) + require.NoError(t, err) + + oldCanceledParent := createBuild(4, buildTime) + oldCanceled := insertOrchestration(oldCanceledParent, oldCanceledTime) + _, err = db.UpdateWorkspaceBuildOrchestrationCanceledByID(ctx, database.UpdateWorkspaceBuildOrchestrationCanceledByIDParams{ + ID: oldCanceled.ID, + UpdatedAt: oldCanceledTime, + }) + require.NoError(t, err) + + oldPendingParent := createBuild(5, buildTime) + oldPending := insertOrchestration(oldPendingParent, oldPendingTime) + + recentCompletedParent := createBuild(6, buildTime) + recentCompletedChild := createBuild(7, buildTime) + recentCompleted := insertOrchestration(recentCompletedParent, recentTime) + _, err = db.UpdateWorkspaceBuildOrchestrationCompletedByID(ctx, database.UpdateWorkspaceBuildOrchestrationCompletedByIDParams{ + ID: recentCompleted.ID, + ChildBuildID: uuid.NullUUID{UUID: recentCompletedChild.ID, Valid: true}, + UpdatedAt: recentTime, + }) + require.NoError(t, err) + + // When: old workspace build orchestrations are deleted with LimitCount 1 + deleted, err := db.DeleteOldWorkspaceBuildOrchestrations(ctx, database.DeleteOldWorkspaceBuildOrchestrationsParams{ + BeforeTime: cutoff, + LimitCount: 1, + }) + require.NoError(t, err) + require.EqualValues(t, 1, deleted) + + // Then: only the oldest terminal row is deleted. + assertOrchestrationDeleted(ctx, t, rawDB, oldCompletedParent.ID) + assertOrchestrationExists(ctx, t, rawDB, oldFailedParent.ID, oldFailed.ID) + assertOrchestrationExists(ctx, t, rawDB, oldCanceledParent.ID, oldCanceled.ID) + assertOrchestrationExists(ctx, t, rawDB, oldPendingParent.ID, oldPending.ID) + assertOrchestrationExists(ctx, t, rawDB, recentCompletedParent.ID, recentCompleted.ID) + + // When: old workspace build orchestrations are deleted again. + deleted, err = db.DeleteOldWorkspaceBuildOrchestrations(ctx, database.DeleteOldWorkspaceBuildOrchestrationsParams{ + BeforeTime: cutoff, + LimitCount: 10, + }) + require.NoError(t, err) + require.EqualValues(t, 2, deleted) + + // Then: the remaining old terminal rows are deleted. + assertOrchestrationDeleted(ctx, t, rawDB, oldFailedParent.ID) + assertOrchestrationDeleted(ctx, t, rawDB, oldCanceledParent.ID) + assertOrchestrationExists(ctx, t, rawDB, oldPendingParent.ID, oldPending.ID) + assertOrchestrationExists(ctx, t, rawDB, recentCompletedParent.ID, recentCompleted.ID) +} + +func assertOrchestrationDeleted(ctx context.Context, t *testing.T, rawDB *sql.DB, parentBuildID uuid.UUID) { + t.Helper() + + _, err := dbtestutil.GetWorkspaceBuildOrchestrationByParentBuildID(ctx, rawDB, parentBuildID) + require.ErrorIs(t, err, sql.ErrNoRows) +} + +func assertOrchestrationExists(ctx context.Context, t *testing.T, rawDB *sql.DB, parentBuildID uuid.UUID, orchestrationID uuid.UUID) { + t.Helper() + + orchestration, err := dbtestutil.GetWorkspaceBuildOrchestrationByParentBuildID(ctx, rawDB, parentBuildID) + require.NoError(t, err) + require.Equal(t, orchestrationID, orchestration.ID) +} + func TestDeleteOldConnectionLogs(t *testing.T) { t.Parallel() diff --git a/coderd/database/querier.go b/coderd/database/querier.go index d5259b3820c..3f48a4697a9 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -222,6 +222,7 @@ type sqlcQuerier interface { // Logs can take up a lot of space, so it's important we clean up frequently. DeleteOldWorkspaceAgentLogs(ctx context.Context, threshold time.Time) (int64, error) DeleteOldWorkspaceAgentStats(ctx context.Context) error + DeleteOldWorkspaceBuildOrchestrations(ctx context.Context, arg DeleteOldWorkspaceBuildOrchestrationsParams) (int64, error) DeleteOrganizationMember(ctx context.Context, arg DeleteOrganizationMemberParams) error DeleteProvisionerKey(ctx context.Context, id uuid.UUID) error DeleteReplicasUpdatedBefore(ctx context.Context, updatedAt time.Time) error diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index b83febb163e..bbf8ed756c0 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -35299,6 +35299,37 @@ func (q *sqlQuerier) InsertWorkspaceAppStats(ctx context.Context, arg InsertWork return err } +const deleteOldWorkspaceBuildOrchestrations = `-- name: DeleteOldWorkspaceBuildOrchestrations :execrows +WITH deletable AS ( + SELECT + id + FROM + workspace_build_orchestrations + WHERE + status IN ('completed', 'failed', 'canceled') + AND updated_at < $1::timestamptz + ORDER BY + updated_at ASC + LIMIT $2::int +) +DELETE FROM workspace_build_orchestrations +USING deletable +WHERE workspace_build_orchestrations.id = deletable.id +` + +type DeleteOldWorkspaceBuildOrchestrationsParams struct { + BeforeTime time.Time `db:"before_time" json:"before_time"` + LimitCount int32 `db:"limit_count" json:"limit_count"` +} + +func (q *sqlQuerier) DeleteOldWorkspaceBuildOrchestrations(ctx context.Context, arg DeleteOldWorkspaceBuildOrchestrationsParams) (int64, error) { + result, err := q.db.ExecContext(ctx, deleteOldWorkspaceBuildOrchestrations, arg.BeforeTime, arg.LimitCount) + if err != nil { + return 0, err + } + return result.RowsAffected() +} + const getNextPendingWorkspaceBuildOrchestrationForUpdate = `-- name: GetNextPendingWorkspaceBuildOrchestrationForUpdate :one SELECT wbo.id, wbo.created_at, wbo.updated_at, wbo.workspace_id, wbo.parent_build_id, wbo.child_build_id, wbo.child_transition, wbo.child_template_version_id, wbo.child_template_version_preset_id, wbo.child_rich_parameter_values, wbo.child_log_level, wbo.child_reason, wbo.attempt_count, wbo.next_retry_after, wbo.status, wbo.error diff --git a/coderd/database/queries/workspacebuildorchestrations.sql b/coderd/database/queries/workspacebuildorchestrations.sql index f256d89a7ad..fe8279a20e5 100644 --- a/coderd/database/queries/workspacebuildorchestrations.sql +++ b/coderd/database/queries/workspacebuildorchestrations.sql @@ -114,3 +114,20 @@ WHERE id = @id AND status = 'pending' RETURNING *; + +-- name: DeleteOldWorkspaceBuildOrchestrations :execrows +WITH deletable AS ( + SELECT + id + FROM + workspace_build_orchestrations + WHERE + status IN ('completed', 'failed', 'canceled') + AND updated_at < @before_time::timestamptz + ORDER BY + updated_at ASC + LIMIT @limit_count::int +) +DELETE FROM workspace_build_orchestrations +USING deletable +WHERE workspace_build_orchestrations.id = deletable.id; diff --git a/coderd/pproflabel/pproflabel.go b/coderd/pproflabel/pproflabel.go index f686c1c4288..5204a0681df 100644 --- a/coderd/pproflabel/pproflabel.go +++ b/coderd/pproflabel/pproflabel.go @@ -35,6 +35,9 @@ const ( // ServiceTallymanPublisher publishes usage events to coder/tallyman. ServiceTallymanPublisher = "tallyman-publisher" ServiceUsageEventCron = "usage-event-cron" + // ServiceWorkspaceBuildOrchestrator fulfills workspace build + // orchestrations once their parent build reaches a terminal state. + ServiceWorkspaceBuildOrchestrator = "workspace-build-orchestrator" RequestTypeTag = "coder_request_type" ) diff --git a/coderd/provisionerdserver/provisionerdserver.go b/coderd/provisionerdserver/provisionerdserver.go index d233cb41dd9..cb65747ed76 100644 --- a/coderd/provisionerdserver/provisionerdserver.go +++ b/coderd/provisionerdserver/provisionerdserver.go @@ -1285,6 +1285,13 @@ func (s *server) FailJob(ctx context.Context, failJob *proto.FailedJob) (*proto. s.notifyWorkspaceBuildFailed(ctx, workspace, build) + // Wake the orchestrator before the workspace event publish + // below, which returns on error, so a failed UI event cannot + // skip the wake. + if err := wspubsub.PublishWorkspaceBuildOrchestrationWake(ctx, s.Pubsub); err != nil { + s.Logger.Warn(ctx, "failed to publish workspace build orchestration wake", slog.Error(err)) + } + msg, err := json.Marshal(wspubsub.WorkspaceEvent{ Kind: wspubsub.WorkspaceEventKindStateChange, WorkspaceID: workspace.ID, @@ -2532,6 +2539,15 @@ func (s *server) completeWorkspaceBuildJob(ctx context.Context, job database.Pro } } + // Wake the orchestrator before the workspace event publish below, + // which returns on error, so a failed UI event cannot skip the + // wake. + if err := wspubsub.PublishWorkspaceBuildOrchestrationWake(ctx, s.Pubsub); err != nil { + s.Logger.Warn(ctx, "failed to publish workspace build orchestration wake", + slog.Error(err), + ) + } + msg, err := json.Marshal(wspubsub.WorkspaceEvent{ Kind: wspubsub.WorkspaceEventKindStateChange, WorkspaceID: workspace.ID, diff --git a/coderd/workspacebuilds.go b/coderd/workspacebuilds.go index aecd7a353ca..9678defee55 100644 --- a/coderd/workspacebuilds.go +++ b/coderd/workspacebuilds.go @@ -886,6 +886,10 @@ func (api *API) patchCancelWorkspaceBuild(rw http.ResponseWriter, r *http.Reques Kind: wspubsub.WorkspaceEventKindStateChange, WorkspaceID: workspace.ID, }) + err = wspubsub.PublishWorkspaceBuildOrchestrationWake(ctx, api.Pubsub) + if err != nil { + api.Logger.Warn(ctx, "failed to publish workspace build orchestration wake", slog.Error(err)) + } // Publish workspace build update to the all builds channel if the experiment is enabled. if api.Experiments.Enabled(codersdk.ExperimentWorkspaceBuildUpdates) { diff --git a/coderd/workspacebuilds_on_success_test.go b/coderd/workspacebuilds_on_success_test.go index cbe683c1734..a49ef56147e 100644 --- a/coderd/workspacebuilds_on_success_test.go +++ b/coderd/workspacebuilds_on_success_test.go @@ -3,12 +3,15 @@ package coderd_test import ( "encoding/json" "net/http" + "strconv" "testing" "github.com/google/uuid" "github.com/stretchr/testify/require" "github.com/coder/coder/v2/coderd/coderdtest" + "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/database/dbgen" "github.com/coder/coder/v2/coderd/database/dbtestutil" "github.com/coder/coder/v2/codersdk" "github.com/coder/coder/v2/provisioner/echo" @@ -49,6 +52,8 @@ func TestPostWorkspaceBuildsOnSuccessRestart(t *testing.T) { // WHEN: a stop build is created with an on_success start build. ctx := testutil.Context(t, testutil.WaitLong) + user, err := client.User(ctx, codersdk.Me) + require.NoError(t, err) stopBuild, err := client.CreateWorkspaceBuild(ctx, workspace.ID, codersdk.CreateWorkspaceBuildRequest{ Transition: codersdk.WorkspaceTransitionStop, @@ -83,6 +88,178 @@ func TestPostWorkspaceBuildsOnSuccessRestart(t *testing.T) { require.ElementsMatch(t, []codersdk.WorkspaceBuildParameter{ {Name: paramName, Value: "baz"}, }, childRichParameterValues) + + // THEN: the returned parent stop build completes successfully. + stopBuild = coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, stopBuild.ID) + require.Equal(t, codersdk.ProvisionerJobSucceeded, stopBuild.Job.Status) + require.Equal(t, codersdk.WorkspaceStatusStopped, stopBuild.Status) + + // THEN: the server creates and completes the child start build. + var childBuild codersdk.WorkspaceBuild + require.Eventually(t, func() bool { + childBuild, err = client.WorkspaceBuildByUsernameAndWorkspaceNameAndBuildNumber( + ctx, + user.Username, + workspace.Name, + strconv.FormatInt(int64(stopBuild.BuildNumber+1), 10), + ) + return err == nil && + childBuild.Transition == codersdk.WorkspaceTransitionStart + }, testutil.WaitMedium, testutil.IntervalFast) + + childBuild = coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, childBuild.ID) + require.Equal(t, codersdk.ProvisionerJobSucceeded, childBuild.Job.Status) + require.Equal(t, codersdk.WorkspaceStatusRunning, childBuild.Status) + require.Equal(t, codersdk.BuildReasonCLI, childBuild.Reason) + require.Equal(t, template.ActiveVersionID, childBuild.TemplateVersionID) + + // THEN: the child build uses the on_success parameter values. + params, err := client.WorkspaceBuildParameters(ctx, childBuild.ID) + require.NoError(t, err) + require.ElementsMatch(t, []codersdk.WorkspaceBuildParameter{ + {Name: paramName, Value: "baz"}, + }, params) +} + +func TestPostWorkspaceBuildsOnSuccessTemplateVersionPreset(t *testing.T) { + t.Parallel() + + // GIVEN: a running workspace and a preset on its active template + // version. + db, ps, sqlDB := dbtestutil.NewDBWithSQLDB(t) + client, _, _ := coderdtest.NewWithAPI(t, &coderdtest.Options{ + Database: db, + Pubsub: ps, + IncludeProvisionerDaemon: true, + }) + first := coderdtest.CreateFirstUser(t, client) + version := coderdtest.CreateTemplateVersion(t, client, first.OrganizationID, nil) + coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID) + template := coderdtest.CreateTemplate(t, client, first.OrganizationID, version.ID) + preset := dbgen.Preset(t, db, database.InsertPresetParams{ + Name: "on-success-preset", + TemplateVersionID: version.ID, + }) + workspace := coderdtest.CreateWorkspace(t, client, template.ID) + initialBuild := coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, workspace.LatestBuild.ID) + require.Equal(t, codersdk.WorkspaceStatusRunning, initialBuild.Status) + + // WHEN: a stop build is created with an on_success start build + // that requests the preset. + ctx := testutil.Context(t, testutil.WaitLong) + user, err := client.User(ctx, codersdk.Me) + require.NoError(t, err) + + stopBuild, err := client.CreateWorkspaceBuild(ctx, workspace.ID, codersdk.CreateWorkspaceBuildRequest{ + Transition: codersdk.WorkspaceTransitionStop, + OnSuccess: &codersdk.CreateWorkspaceBuildOnSuccessRequest{ + Transition: codersdk.WorkspaceTransitionStart, + TemplateVersionID: template.ActiveVersionID, + TemplateVersionPresetID: preset.ID, + }, + }) + require.NoError(t, err) + + // THEN: the server persists the child preset intent. + orchestration, err := dbtestutil.GetWorkspaceBuildOrchestrationByParentBuildID(ctx, sqlDB, stopBuild.ID) + require.NoError(t, err) + require.True(t, orchestration.ChildTemplateVersionPresetID.Valid) + require.Equal(t, preset.ID, orchestration.ChildTemplateVersionPresetID.UUID) + + stopBuild = coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, stopBuild.ID) + require.Equal(t, codersdk.ProvisionerJobSucceeded, stopBuild.Job.Status) + require.Equal(t, codersdk.WorkspaceStatusStopped, stopBuild.Status) + + // THEN: the child start build uses the preset. + var childBuild codersdk.WorkspaceBuild + require.Eventually(t, func() bool { + childBuild, err = client.WorkspaceBuildByUsernameAndWorkspaceNameAndBuildNumber( + ctx, + user.Username, + workspace.Name, + strconv.FormatInt(int64(stopBuild.BuildNumber+1), 10), + ) + return err == nil && + childBuild.Transition == codersdk.WorkspaceTransitionStart + }, testutil.WaitShort, testutil.IntervalFast) + + childBuild = coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, childBuild.ID) + require.Equal(t, codersdk.ProvisionerJobSucceeded, childBuild.Job.Status) + require.Equal(t, codersdk.WorkspaceStatusRunning, childBuild.Status) + require.NotNil(t, childBuild.TemplateVersionPresetID) + require.Equal(t, preset.ID, *childBuild.TemplateVersionPresetID) +} + +func TestPostWorkspaceBuildsOnSuccessUnpinnedChildUsesActiveTemplateVersion(t *testing.T) { + t.Parallel() + + // GIVEN: a running workspace and a second completed template + // version that is not active yet. + db, ps, sqlDB := dbtestutil.NewDBWithSQLDB(t) + client, provisionerCloser, api := coderdtest.NewWithAPI(t, &coderdtest.Options{ + Database: db, + Pubsub: ps, + IncludeProvisionerDaemon: true, + }) + first := coderdtest.CreateFirstUser(t, client) + userClient, user := coderdtest.CreateAnotherUser(t, client, first.OrganizationID) + + version := coderdtest.CreateTemplateVersion(t, client, first.OrganizationID, nil) + coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID) + template := coderdtest.CreateTemplate(t, client, first.OrganizationID, version.ID) + workspace := coderdtest.CreateWorkspace(t, userClient, template.ID) + initialBuild := coderdtest.AwaitWorkspaceBuildJobCompleted(t, userClient, workspace.LatestBuild.ID) + require.Equal(t, codersdk.WorkspaceStatusRunning, initialBuild.Status) + + newVersion := coderdtest.UpdateTemplateVersion(t, client, first.OrganizationID, nil, template.ID) + coderdtest.AwaitTemplateVersionJobCompleted(t, client, newVersion.ID) + + // WHEN: a non-template-admin queues an unpinned on_success child + // build. + + // Stop the provisioner so the parent build cannot complete before + // the test updates the active template version. + require.NoError(t, provisionerCloser.Close()) + ctx := testutil.Context(t, testutil.WaitLong) + stopBuild, err := userClient.CreateWorkspaceBuild(ctx, workspace.ID, codersdk.CreateWorkspaceBuildRequest{ + Transition: codersdk.WorkspaceTransitionStop, + OnSuccess: &codersdk.CreateWorkspaceBuildOnSuccessRequest{ + Transition: codersdk.WorkspaceTransitionStart, + }, + }) + require.NoError(t, err) + + // THEN: the child build remains unpinned in the orchestration row. + orchestration, err := dbtestutil.GetWorkspaceBuildOrchestrationByParentBuildID(ctx, sqlDB, stopBuild.ID) + require.NoError(t, err) + require.False(t, orchestration.ChildTemplateVersionID.Valid, "child build should remain unpinned") + + // WHEN: the active version changes before the parent succeeds. + coderdtest.UpdateActiveTemplateVersion(t, client, template.ID, newVersion.ID) + coderdtest.NewProvisionerDaemon(t, api) + + stopBuild = coderdtest.AwaitWorkspaceBuildJobCompleted(t, userClient, stopBuild.ID) + require.Equal(t, codersdk.ProvisionerJobSucceeded, stopBuild.Job.Status) + require.Equal(t, codersdk.WorkspaceStatusStopped, stopBuild.Status) + + // THEN: the child build uses the active version when the + // orchestrator creates it. + var childBuild codersdk.WorkspaceBuild + require.Eventually(t, func() bool { + childBuild, err = userClient.WorkspaceBuildByUsernameAndWorkspaceNameAndBuildNumber( + ctx, + user.Username, + workspace.Name, + strconv.FormatInt(int64(stopBuild.BuildNumber+1), 10), + ) + return err == nil && + childBuild.Transition == codersdk.WorkspaceTransitionStart + }, testutil.WaitMedium, testutil.IntervalFast) + + childBuild = coderdtest.AwaitWorkspaceBuildJobCompleted(t, userClient, childBuild.ID) + require.Equal(t, codersdk.ProvisionerJobSucceeded, childBuild.Job.Status) + require.Equal(t, codersdk.WorkspaceStatusRunning, childBuild.Status) + require.Equal(t, newVersion.ID, childBuild.TemplateVersionID) } func TestPostWorkspaceBuildsOnSuccessUnpinnedChildNoParams(t *testing.T) { @@ -281,6 +458,335 @@ func TestPostWorkspaceBuildsOnSuccessValidation(t *testing.T) { } } +// Canceling an already-running job resolves the orchestration as +// "failed", not "canceled". This hits the same orchestrator branch as +// TestPostWorkspaceBuildsOnSuccessParentFailed below; despite that +// overlap, the test pins this non-obvious end-to-end behavior. +func TestPostWorkspaceBuildsOnSuccessParentCanceledMidFlight(t *testing.T) { + t.Parallel() + + // GIVEN: a running workspace whose stop apply will block. + db, ps, sqlDB := dbtestutil.NewDBWithSQLDB(t) + client, _, _ := coderdtest.NewWithAPI(t, &coderdtest.Options{ + Database: db, + Pubsub: ps, + IncludeProvisionerDaemon: true, + }) + first := coderdtest.CreateFirstUser(t, client) + version := coderdtest.CreateTemplateVersion(t, client, first.OrganizationID, + echoResponsesWithRichParameter("foo", echoResponseOptions{ + blockStopApply: true, + }), + ) + coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID) + template := coderdtest.CreateTemplate(t, client, first.OrganizationID, version.ID) + workspace := coderdtest.CreateWorkspace(t, client, template.ID) + initialBuild := coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, workspace.LatestBuild.ID) + require.Equal(t, codersdk.WorkspaceStatusRunning, initialBuild.Status) + + // WHEN: a stop build is created with an on_success start build. + ctx := testutil.Context(t, testutil.WaitLong) + user, err := client.User(ctx, codersdk.Me) + require.NoError(t, err) + + stopBuild, err := client.CreateWorkspaceBuild(ctx, workspace.ID, codersdk.CreateWorkspaceBuildRequest{ + Transition: codersdk.WorkspaceTransitionStop, + OnSuccess: &codersdk.CreateWorkspaceBuildOnSuccessRequest{ + Transition: codersdk.WorkspaceTransitionStart, + TemplateVersionID: template.ActiveVersionID, + }, + }) + require.NoError(t, err) + require.Equal(t, codersdk.WorkspaceTransitionStop, stopBuild.Transition) + require.Equal(t, codersdk.BuildReasonInitiator, stopBuild.Reason) + + // WHEN: the parent stop build starts running and is canceled. + require.Eventually(t, func() bool { + var err error + stopBuild, err = client.WorkspaceBuild(ctx, stopBuild.ID) + return err == nil && + stopBuild.Job.Status == codersdk.ProvisionerJobRunning + }, testutil.WaitShort, testutil.IntervalFast) + + require.NoError(t, client.CancelWorkspaceBuild(ctx, stopBuild.ID, codersdk.CancelWorkspaceBuildParams{})) + require.Eventually(t, func() bool { + var err error + stopBuild, err = client.WorkspaceBuild(ctx, stopBuild.ID) + if err != nil { + return false + } + return stopBuild.Job.Status == codersdk.ProvisionerJobFailed && + stopBuild.Job.Error == "canceled" + }, testutil.WaitShort, testutil.IntervalFast) + + // THEN: the server resolves the orchestration without creating the + // child start build. + require.Eventually(t, func() bool { + orchestration, err := dbtestutil.GetWorkspaceBuildOrchestrationByParentBuildID(ctx, sqlDB, stopBuild.ID) + return err == nil && + orchestration.Status == "failed" && + !orchestration.ChildBuildID.Valid && + orchestration.Error.Valid && + orchestration.Error.String == "parent workspace build failed: canceled" + }, testutil.WaitShort, testutil.IntervalFast) + + _, err = client.WorkspaceBuildByUsernameAndWorkspaceNameAndBuildNumber( + ctx, + user.Username, + workspace.Name, + strconv.FormatInt(int64(stopBuild.BuildNumber+1), 10), + ) + var apiErr *codersdk.Error + require.ErrorAs(t, err, &apiErr) + require.Equal(t, http.StatusNotFound, apiErr.StatusCode()) +} + +func TestPostWorkspaceBuildsOnSuccessParentFailed(t *testing.T) { + t.Parallel() + + // GIVEN: a running workspace whose stop apply will fail. + db, ps, sqlDB := dbtestutil.NewDBWithSQLDB(t) + client, _, _ := coderdtest.NewWithAPI(t, &coderdtest.Options{ + Database: db, + Pubsub: ps, + IncludeProvisionerDaemon: true, + }) + first := coderdtest.CreateFirstUser(t, client) + version := coderdtest.CreateTemplateVersion(t, client, first.OrganizationID, + echoResponsesWithRichParameter("foo", echoResponseOptions{ + failStopApply: true, + }), + ) + coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID) + template := coderdtest.CreateTemplate(t, client, first.OrganizationID, version.ID) + workspace := coderdtest.CreateWorkspace(t, client, template.ID) + initialBuild := coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, workspace.LatestBuild.ID) + require.Equal(t, codersdk.WorkspaceStatusRunning, initialBuild.Status) + + // WHEN: a stop build is created with an on_success start build. + ctx := testutil.Context(t, testutil.WaitLong) + user, err := client.User(ctx, codersdk.Me) + require.NoError(t, err) + + stopBuild, err := client.CreateWorkspaceBuild(ctx, workspace.ID, codersdk.CreateWorkspaceBuildRequest{ + Transition: codersdk.WorkspaceTransitionStop, + OnSuccess: &codersdk.CreateWorkspaceBuildOnSuccessRequest{ + Transition: codersdk.WorkspaceTransitionStart, + TemplateVersionID: template.ActiveVersionID, + }, + }) + require.NoError(t, err) + + // WHEN: the parent stop build fails. + stopBuild = coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, stopBuild.ID) + require.Equal(t, codersdk.ProvisionerJobFailed, stopBuild.Job.Status) + + // THEN: the server resolves the orchestration without creating the + // child start build. + require.Eventually(t, func() bool { + orchestration, err := dbtestutil.GetWorkspaceBuildOrchestrationByParentBuildID(ctx, sqlDB, stopBuild.ID) + return err == nil && + orchestration.Status == "failed" && + !orchestration.ChildBuildID.Valid && + orchestration.Error.Valid && + orchestration.Error.String == "parent workspace build failed: failed!" + }, testutil.WaitShort, testutil.IntervalFast) + + _, err = client.WorkspaceBuildByUsernameAndWorkspaceNameAndBuildNumber( + ctx, + user.Username, + workspace.Name, + strconv.FormatInt(int64(stopBuild.BuildNumber+1), 10), + ) + var apiErr *codersdk.Error + require.ErrorAs(t, err, &apiErr) + require.Equal(t, http.StatusNotFound, apiErr.StatusCode()) +} + +func TestPostWorkspaceBuildsOnSuccessNonRetryableChildBuildFailure(t *testing.T) { + t.Parallel() + + // GIVEN: a running workspace with a rich parameter value that + // satisfies the template regex validation. + db, ps, sqlDB := dbtestutil.NewDBWithSQLDB(t) + client, _, _ := coderdtest.NewWithAPI(t, &coderdtest.Options{ + Database: db, + Pubsub: ps, + IncludeProvisionerDaemon: true, + }) + first := coderdtest.CreateFirstUser(t, client) + version := coderdtest.CreateTemplateVersion(t, client, first.OrganizationID, + echoResponsesWithRichParameter("foo", echoResponseOptions{ + validationRegex: "^good$", + }), + ) + coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID) + template := coderdtest.CreateTemplate(t, client, first.OrganizationID, version.ID) + workspace := coderdtest.CreateWorkspace(t, client, template.ID, func(request *codersdk.CreateWorkspaceRequest) { + request.RichParameterValues = []codersdk.WorkspaceBuildParameter{ + {Name: "foo", Value: "good"}, + } + }) + initialBuild := coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, workspace.LatestBuild.ID) + require.Equal(t, codersdk.WorkspaceStatusRunning, initialBuild.Status) + + // WHEN: a stop build is created with an on_success child build + // that has an invalid rich parameter value. + ctx := testutil.Context(t, testutil.WaitLong) + user, err := client.User(ctx, codersdk.Me) + require.NoError(t, err) + + stopBuild, err := client.CreateWorkspaceBuild(ctx, workspace.ID, codersdk.CreateWorkspaceBuildRequest{ + Transition: codersdk.WorkspaceTransitionStop, + OnSuccess: &codersdk.CreateWorkspaceBuildOnSuccessRequest{ + Transition: codersdk.WorkspaceTransitionStart, + TemplateVersionID: template.ActiveVersionID, + // The invalid value triggers a non-retryable child build + // creation error when the orchestrator processes the row. + RichParameterValues: []codersdk.WorkspaceBuildParameter{ + {Name: "foo", Value: "bad"}, + }, + }, + }) + require.NoError(t, err) + + stopBuild = coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, stopBuild.ID) + require.Equal(t, codersdk.ProvisionerJobSucceeded, stopBuild.Job.Status) + require.Equal(t, codersdk.WorkspaceStatusStopped, stopBuild.Status) + + // THEN: the server marks the orchestration as failed without + // retrying or creating the child start build. + var orchestration database.WorkspaceBuildOrchestration + require.Eventually(t, func() bool { + orchestration, err = dbtestutil.GetWorkspaceBuildOrchestrationByParentBuildID(ctx, sqlDB, stopBuild.ID) + return err == nil && + orchestration.Status == "failed" && + !orchestration.ChildBuildID.Valid && + orchestration.AttemptCount == 0 && + !orchestration.NextRetryAfter.Valid && + orchestration.Error.Valid + }, testutil.WaitShort, testutil.IntervalFast) + require.Contains(t, orchestration.Error.String, "Unable to validate parameters") + + _, err = client.WorkspaceBuildByUsernameAndWorkspaceNameAndBuildNumber( + ctx, + user.Username, + workspace.Name, + strconv.FormatInt(int64(stopBuild.BuildNumber+1), 10), + ) + var apiErr *codersdk.Error + require.ErrorAs(t, err, &apiErr) + require.Equal(t, http.StatusNotFound, apiErr.StatusCode()) +} + +func TestPostWorkspaceBuildsOnSuccessRetryableChildBuildFailureDoesNotBlockLaterRestart(t *testing.T) { + t.Parallel() + + // GIVEN: two provisioners, one holding a template import job + // open to make the child build fail retryably while the other + // processes workspace builds. + db, ps, sqlDB := dbtestutil.NewDBWithSQLDB(t) + client, _, api := coderdtest.NewWithAPI(t, &coderdtest.Options{ + Database: db, + Pubsub: ps, + IncludeProvisionerDaemon: true, + }) + coderdtest.NewProvisionerDaemon(t, api) + + first := coderdtest.CreateFirstUser(t, client) + version := coderdtest.CreateTemplateVersion(t, client, first.OrganizationID, nil) + coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID) + template := coderdtest.CreateTemplate(t, client, first.OrganizationID, version.ID) + + workspace := coderdtest.CreateWorkspace(t, client, template.ID) + startBuild := coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, workspace.LatestBuild.ID) + require.Equal(t, codersdk.WorkspaceStatusRunning, startBuild.Status) + + blockedVersion := coderdtest.UpdateTemplateVersion(t, client, first.OrganizationID, + // Without a PlanComplete response, the echo provisioner will + // keep the template version import job running. + &echo.Responses{ + Parse: echo.ParseComplete, + ProvisionPlan: []*proto.Response{{ + Type: &proto.Response_Log{ + Log: &proto.Log{}, + }, + }}, + }, template.ID, + ) + coderdtest.AwaitTemplateVersionJobRunning(t, client, blockedVersion.ID) + + // WHEN: a stop build is created with an on_success child build + // request that references a template version whose import job is + // still running, causing child build creation to be retried later. + ctx := testutil.Context(t, testutil.WaitLong) + badStopBuild, err := client.CreateWorkspaceBuild(ctx, workspace.ID, codersdk.CreateWorkspaceBuildRequest{ + Transition: codersdk.WorkspaceTransitionStop, + OnSuccess: &codersdk.CreateWorkspaceBuildOnSuccessRequest{ + Transition: codersdk.WorkspaceTransitionStart, + TemplateVersionID: blockedVersion.ID, + }, + }) + require.NoError(t, err) + + badStopBuild = coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, badStopBuild.ID) + require.Equal(t, codersdk.ProvisionerJobSucceeded, badStopBuild.Job.Status) + require.Equal(t, codersdk.WorkspaceStatusStopped, badStopBuild.Status) + + // THEN: the orchestrator records a delayed retry after child + // build creation fails because the requested template version + // is still importing. + var badOrchestration database.WorkspaceBuildOrchestration + require.Eventually(t, func() bool { + badOrchestration, err = dbtestutil.GetWorkspaceBuildOrchestrationByParentBuildID(ctx, sqlDB, badStopBuild.ID) + return err == nil && + badOrchestration.Status == "pending" && + badOrchestration.AttemptCount == 1 && + badOrchestration.NextRetryAfter.Valid + }, testutil.WaitShort, testutil.IntervalFast) + require.False(t, badOrchestration.ChildBuildID.Valid) + require.True(t, badOrchestration.Error.Valid) + require.Contains(t, badOrchestration.Error.String, "template version is running") + + // WHEN: a later restart uses a valid child build request. + goodWorkspace := coderdtest.CreateWorkspace(t, client, template.ID) + goodStartBuild := coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, goodWorkspace.LatestBuild.ID) + require.Equal(t, codersdk.WorkspaceStatusRunning, goodStartBuild.Status) + + user, err := client.User(ctx, codersdk.Me) + require.NoError(t, err) + + goodStopBuild, err := client.CreateWorkspaceBuild(ctx, goodWorkspace.ID, codersdk.CreateWorkspaceBuildRequest{ + Transition: codersdk.WorkspaceTransitionStop, + OnSuccess: &codersdk.CreateWorkspaceBuildOnSuccessRequest{ + Transition: codersdk.WorkspaceTransitionStart, + TemplateVersionID: template.ActiveVersionID, + }, + }) + require.NoError(t, err) + + goodStopBuild = coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, goodStopBuild.ID) + require.Equal(t, codersdk.ProvisionerJobSucceeded, goodStopBuild.Job.Status) + require.Equal(t, codersdk.WorkspaceStatusStopped, goodStopBuild.Status) + + // THEN: the delayed retry row does not block the later orchestration. + var goodChildBuild codersdk.WorkspaceBuild + require.Eventually(t, func() bool { + goodChildBuild, err = client.WorkspaceBuildByUsernameAndWorkspaceNameAndBuildNumber( + ctx, + user.Username, + goodWorkspace.Name, + strconv.FormatInt(int64(goodStopBuild.BuildNumber+1), 10), + ) + return err == nil && + goodChildBuild.Transition == codersdk.WorkspaceTransitionStart + }, testutil.WaitMedium, testutil.IntervalFast) + + goodChildBuild = coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, goodChildBuild.ID) + require.Equal(t, codersdk.ProvisionerJobSucceeded, goodChildBuild.Job.Status) + require.Equal(t, codersdk.WorkspaceStatusRunning, goodChildBuild.Status) +} + type echoResponseOptions struct { blockStopApply bool failStopApply bool diff --git a/coderd/workspaces.go b/coderd/workspaces.go index 8d2a3887303..d53989d2baf 100644 --- a/coderd/workspaces.go +++ b/coderd/workspaces.go @@ -2943,20 +2943,7 @@ func validWorkspaceSchedule(s *string) (sql.NullString, error) { } func (api *API) publishWorkspaceUpdate(ctx context.Context, ownerID uuid.UUID, event wspubsub.WorkspaceEvent) { - err := event.Validate() - if err != nil { - api.Logger.Warn(ctx, "invalid workspace update event", - slog.F("workspace_id", event.WorkspaceID), - slog.F("event_kind", event.Kind), slog.Error(err)) - return - } - msg, err := json.Marshal(event) - if err != nil { - api.Logger.Warn(ctx, "failed to marshal workspace update", - slog.F("workspace_id", event.WorkspaceID), slog.Error(err)) - return - } - err = api.Pubsub.Publish(wspubsub.WorkspaceEventChannel(ownerID), msg) + err := wspubsub.PublishWorkspaceEvent(ctx, api.Pubsub, ownerID, event) if err != nil { api.Logger.Warn(ctx, "failed to publish workspace update", slog.F("workspace_id", event.WorkspaceID), slog.Error(err)) diff --git a/coderd/wsbuildorchestrator/doc.go b/coderd/wsbuildorchestrator/doc.go new file mode 100644 index 00000000000..5c0c79e63ef --- /dev/null +++ b/coderd/wsbuildorchestrator/doc.go @@ -0,0 +1,4 @@ +// Package wsbuildorchestrator runs the background worker that +// fulfills workspace build orchestrations once their parent build +// reaches a terminal state. +package wsbuildorchestrator diff --git a/coderd/wsbuildorchestrator/wsbuildorchestrator.go b/coderd/wsbuildorchestrator/wsbuildorchestrator.go new file mode 100644 index 00000000000..58a6fa1234f --- /dev/null +++ b/coderd/wsbuildorchestrator/wsbuildorchestrator.go @@ -0,0 +1,567 @@ +package wsbuildorchestrator + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "net/http" + "sync" + "sync/atomic" + "time" + + "github.com/cenkalti/backoff/v4" + "github.com/google/uuid" + "golang.org/x/xerrors" + + "cdr.dev/slog/v3" + "github.com/coder/coder/v2/coderd/audit" + "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/database/dbauthz" + "github.com/coder/coder/v2/coderd/database/dbtime" + "github.com/coder/coder/v2/coderd/database/provisionerjobs" + "github.com/coder/coder/v2/coderd/database/pubsub" + "github.com/coder/coder/v2/coderd/files" + "github.com/coder/coder/v2/coderd/pproflabel" + "github.com/coder/coder/v2/coderd/wsbuilder" + "github.com/coder/coder/v2/coderd/wspubsub" + "github.com/coder/coder/v2/codersdk" + "github.com/coder/quartz" +) + +const ( + subscribeMaxBackoff = 10 * time.Second + // Pubsub should wake the worker promptly, while occasional + // polling prevents missed wakes from leaving rows pending + // indefinitely. + backupPollInterval = 30 * time.Second + maxAttempts = 3 + retryDelay = 30 * time.Second +) + +// Orchestrator fulfills workspace build orchestrations after their +// parent builds reach a terminal state. +type Orchestrator struct { + logger slog.Logger + db database.Store + pubsub pubsub.Pubsub + fileCache *files.Cache + buildUsageChecker *atomic.Pointer[wsbuilder.UsageChecker] + deploymentValues *codersdk.DeploymentValues + experiments codersdk.Experiments + builderMetrics *wsbuilder.Metrics + clock quartz.Clock + + wakeCh chan struct{} + + // startOnce ensures the background goroutines are launched at most + // once, even if Start is called more than once. + startOnce sync.Once + // cancel cancels the context on all running jobs. If the ctx + // passed into `Start` is canceled, the jobs will also stop. + cancel context.CancelFunc + // wg ensures all job goroutines have exited before Close returns. + wg sync.WaitGroup +} + +type Options struct { + Logger slog.Logger + Database database.Store + Pubsub pubsub.Pubsub + FileCache *files.Cache + BuildUsageChecker *atomic.Pointer[wsbuilder.UsageChecker] + DeploymentValues *codersdk.DeploymentValues + Experiments codersdk.Experiments + BuilderMetrics *wsbuilder.Metrics + Clock quartz.Clock +} + +// New constructs an Orchestrator. Call Start to begin processing. +func New(opts Options) *Orchestrator { + clock := opts.Clock + if clock == nil { + clock = quartz.NewReal() + } + return &Orchestrator{ + logger: opts.Logger.Named("workspace_build_orchestrator"), + db: opts.Database, + pubsub: opts.Pubsub, + fileCache: opts.FileCache, + buildUsageChecker: opts.BuildUsageChecker, + deploymentValues: opts.DeploymentValues, + experiments: opts.Experiments, + builderMetrics: opts.BuilderMetrics, + clock: clock, + // Keep one pending wake signal while the worker is between + // runs. One is enough because each run drains all ready + // orchestration rows. + wakeCh: make(chan struct{}, 1), + } +} + +// Start launches the orchestrator's background goroutines. It is safe +// to call more than once; only the first call has any effect. Call +// Close to stop the goroutines and wait for their exit. +func (o *Orchestrator) Start(ctx context.Context) { + o.startOnce.Do(func() { + ctx, o.cancel = context.WithCancel(ctx) + o.wg.Add(2) + pproflabel.Go(ctx, pproflabel.Service(pproflabel.ServiceWorkspaceBuildOrchestrator, "goroutine", "subscribe"), func(ctx context.Context) { + defer o.wg.Done() + o.subscribe(ctx) + }) + pproflabel.Go(ctx, pproflabel.Service(pproflabel.ServiceWorkspaceBuildOrchestrator, "goroutine", "run"), func(ctx context.Context) { + defer o.wg.Done() + o.run(ctx) + }) + }) +} + +// Close stops the orchestrator and waits for its goroutines to exit. +func (o *Orchestrator) Close() { + if o.cancel != nil { + o.cancel() + } + o.wg.Wait() +} + +func (o *Orchestrator) subscribe(ctx context.Context) { + eb := backoff.NewExponentialBackOff() + eb.MaxElapsedTime = 0 + eb.MaxInterval = subscribeMaxBackoff + bkoff := backoff.WithContext(eb, ctx) + + var cancelSubscribe func() + err := backoff.Retry(func() error { + cancelFn, err := o.pubsub.SubscribeWithErr( + wspubsub.WorkspaceBuildOrchestrationWakeChannel, + o.listen, + ) + if err != nil { + o.logger.Warn(ctx, "failed to subscribe to wake channel", slog.Error(err)) + return err + } + cancelSubscribe = cancelFn + return nil + }, bkoff) + if err != nil { + if ctx.Err() == nil { + o.logger.Error(ctx, "code bug: retry failed before context canceled", slog.Error(err)) + } + return + } + defer cancelSubscribe() + o.logger.Debug(ctx, "subscribed to wake channel") + + // Reconcile rows that may have become ready while the worker was + // not subscribed. + o.wake() + + <-ctx.Done() +} + +func (o *Orchestrator) listen(ctx context.Context, _ []byte, err error) { + if xerrors.Is(err, pubsub.ErrDroppedMessages) { + o.logger.Warn(ctx, "pubsub may have dropped wake signals") + o.wake() + return + } + if err != nil { + o.logger.Warn(ctx, "unhandled pubsub error", slog.Error(err)) + return + } + o.wake() +} + +func (o *Orchestrator) wake() { + select { + case o.wakeCh <- struct{}{}: + default: + } +} + +func (o *Orchestrator) run(ctx context.Context) { + ticker := o.clock.NewTicker(backupPollInterval) + defer ticker.Stop() + + for { + // wakeCh can win the select below even when ctx is canceled, + // so re-check here. Once canceled, do not begin another + // processing round. + if ctx.Err() != nil { + return + } + + err := o.processAll(ctx) + if err != nil && ctx.Err() == nil { + o.logger.Error(ctx, "failed to process orchestrations", slog.Error(err)) + } + + select { + case <-o.wakeCh: + case <-ticker.C: + case <-ctx.Done(): + return + } + } +} + +// processAll processes all pending orchestration rows whose parent +// builds have reached a terminal state. +func (o *Orchestrator) processAll(ctx context.Context) error { + for { + found, err := o.processNext(ctx) + if err != nil { + return err + } + if !found { + return nil + } + } +} + +func (o *Orchestrator) processNext(ctx context.Context) (bool, error) { + //nolint:gocritic // Inserting the orchestration row required + // authorization for the parent and child transitions. The worker + // uses system authority to fulfill that durable intent after the + // parent build completes. + sysCtx := dbauthz.AsSystemRestricted(ctx) + + var ( + found bool + workspace database.Workspace + childJob *database.ProvisionerJob + orchestrationID uuid.UUID + childBuildErr error + ) + + err := o.db.InTx(func(tx database.Store) error { + orchestration, err := tx.GetNextPendingWorkspaceBuildOrchestrationForUpdate(sysCtx) + if xerrors.Is(err, sql.ErrNoRows) { + return nil + } + if err != nil { + return xerrors.Errorf("get next pending workspace build orchestration: %w", err) + } + + found = true + orchestrationID = orchestration.ID + + // markFailed resolves the locked orchestration as failed with + // a message, so a row that cannot make progress does not keep + // blocking later ones. + markFailed := func(msg string) error { + _, err := tx.UpdateWorkspaceBuildOrchestrationFailedByID(sysCtx, database.UpdateWorkspaceBuildOrchestrationFailedByIDParams{ + Error: sql.NullString{String: msg, Valid: true}, + UpdatedAt: dbtime.Now(), + ID: orchestration.ID, + }) + if err != nil { + return xerrors.Errorf("mark workspace build orchestration as failed: %w", err) + } + return nil + } + + // parentBuild and parentJob are guaranteed to exist by + // foreign keys on the locked orchestration row, so an error + // here is unexpected and likely transient. Return it to + // retry, rather than resolving the orchestration as failed. + parentBuild, err := tx.GetWorkspaceBuildByID(sysCtx, orchestration.ParentBuildID) + if err != nil { + return xerrors.Errorf("get parent workspace build: %w", err) + } + + parentJob, err := tx.GetProvisionerJobByID(sysCtx, parentBuild.JobID) + if err != nil { + return xerrors.Errorf("get parent provisioner job: %w", err) + } + + // Resolve terminal parent outcomes that do not create a child + // build. Successful parents continue below. + switch parentJob.JobStatus { + case database.ProvisionerJobStatusSucceeded: + case database.ProvisionerJobStatusCanceled: + _, err = tx.UpdateWorkspaceBuildOrchestrationCanceledByID(sysCtx, database.UpdateWorkspaceBuildOrchestrationCanceledByIDParams{ + ID: orchestration.ID, + UpdatedAt: dbtime.Now(), + }) + if err != nil { + return xerrors.Errorf("mark workspace build orchestration as canceled: %w", err) + } + return nil + case database.ProvisionerJobStatusFailed: + parentFailure := "parent workspace build failed" + if parentJob.Error.Valid && parentJob.Error.String != "" { + parentFailure = fmt.Sprintf("parent workspace build failed: %s", parentJob.Error.String) + } + return markFailed(parentFailure) + default: + // This should be unreachable because the row-locking query + // only selects terminal parent jobs. Mark the row as failed + // because retrying would block later orchestrations. + return markFailed(fmt.Sprintf("unexpected parent job status %q", parentJob.JobStatus)) + } + + childBuildRequest, err := childBuildRequestFromOrchestration(orchestration) + if err != nil { + // Mark the row failed to avoid retrying work that cannot + // make progress. + return markFailed(err.Error()) + } + + workspace, err = tx.GetWorkspaceByID(sysCtx, parentBuild.WorkspaceID) + if err != nil { + return xerrors.Errorf("get workspace: %w", err) + } + + // GetWorkspaceByID returns soft-deleted rows. + if workspace.Deleted { + return markFailed("workspace was deleted") + } + + // A dormant workspace must be woken before it can start. + // Starting it while still dormant would leave it running but + // still subject to deleting_at, which could auto-delete it. + if workspace.DormantAt.Valid { + return markFailed("workspace is dormant") + } + + childBuild, provisionerJob, err := o.createBuild(sysCtx, tx, workspace, parentBuild.InitiatorID, childBuildRequest) + if err != nil { + // Carry the builder error out of the transaction; the + // fail-vs-retry decision runs after the rollback. + childBuildErr = err + return xerrors.Errorf("create child workspace build: %w", err) + } + childJob = provisionerJob + + _, err = tx.UpdateWorkspaceBuildOrchestrationCompletedByID(sysCtx, database.UpdateWorkspaceBuildOrchestrationCompletedByIDParams{ + ChildBuildID: uuid.NullUUID{ + UUID: childBuild.ID, + Valid: true, + }, + UpdatedAt: dbtime.Now(), + ID: orchestration.ID, + }) + if err != nil { + return xerrors.Errorf("complete workspace build orchestration: %w", err) + } + + return nil + }, nil) + if err != nil { + if !found { + // A persistent error here blocks the whole queue, but + // that is systemic, not a poison row. Surface for retry. + return false, err + } + + if ctx.Err() != nil { + // On shutdown, don't resolve or log it as unexpected + // error below. + return false, err + } + + // A row was locked but processing failed. Resolve so it does + // not stay pending and block newer orchestrations. + errMsg := err.Error() + failNow := false + if childBuildErr != nil { + // The child build error carries an HTTP status we can + // classify into retryable vs permanent. + errMsg = childBuildErrorMessage(childBuildErr) + failNow = childBuildErrorShouldFailOrchestration(childBuildErr) + } else { + o.logger.Error(ctx, "unexpected error processing orchestration", + slog.F("workspace_build_orchestration_id", orchestrationID), + slog.Error(err)) + } + + var markErr error + if failNow { + // Mark the orchestration failed so one bad row does not + // block later orchestrations. + _, markErr = o.db.UpdateWorkspaceBuildOrchestrationFailedByID(sysCtx, database.UpdateWorkspaceBuildOrchestrationFailedByIDParams{ + Error: sql.NullString{ + String: errMsg, + Valid: true, + }, + UpdatedAt: dbtime.Now(), + ID: orchestrationID, + }) + } else { + // Back off and retry, eventually failing after maxAttempts + // so a persistently failing row stops blocking the queue. + now := dbtime.Now() + _, markErr = o.db.UpdateWorkspaceBuildOrchestrationRetryByID(sysCtx, database.UpdateWorkspaceBuildOrchestrationRetryByIDParams{ + Error: sql.NullString{ + String: errMsg, + Valid: true, + }, + NextRetryAfter: now.Add(retryDelay), + UpdatedAt: now, + ID: orchestrationID, + MaxAttemptCount: maxAttempts, + }) + } + + if markErr != nil { + if xerrors.Is(markErr, sql.ErrNoRows) { + // This update runs after the transaction has ended, so + // another worker may have resolved the orchestration + // first. Treat that race as success because the row no + // longer needs processing. + return found, nil + } + // Preserve the original error because the orchestration row + // could not be updated with it. + return false, errors.Join( + err, + xerrors.Errorf("resolve workspace build orchestration: %w", markErr), + ) + } + + return found, nil + } + + // These post-commit notifications are best-effort. The child + // build and provisioner job are already persisted, so missing + // pubsub does not corrupt state. It can delay workers or + // subscribers until another wake or refresh. + if childJob != nil { + if err := provisionerjobs.PostJob(o.pubsub, *childJob); err != nil { + o.logger.Error(ctx, "failed to post child provisioner job to pubsub", + slog.F("workspace_build_orchestration_id", orchestrationID), + slog.F("workspace_id", workspace.ID), + slog.Error(err), + ) + } + + err := wspubsub.PublishWorkspaceEvent(ctx, o.pubsub, workspace.OwnerID, wspubsub.WorkspaceEvent{ + Kind: wspubsub.WorkspaceEventKindStateChange, + WorkspaceID: workspace.ID, + }) + if err != nil { + o.logger.Warn(ctx, "failed to publish workspace update", + slog.F("workspace_build_orchestration_id", orchestrationID), + slog.F("workspace_id", workspace.ID), slog.Error(err)) + } + } + + return found, nil +} + +func childBuildRequestFromOrchestration(orchestration database.WorkspaceBuildOrchestration) (codersdk.CreateWorkspaceBuildRequest, error) { + var childParameterValues []codersdk.WorkspaceBuildParameter + if len(orchestration.ChildRichParameterValues) > 0 { + err := json.Unmarshal(orchestration.ChildRichParameterValues, &childParameterValues) + if err != nil { + return codersdk.CreateWorkspaceBuildRequest{}, xerrors.Errorf("unmarshal child rich parameter values: %w", err) + } + } + if childParameterValues == nil { + childParameterValues = []codersdk.WorkspaceBuildParameter{} + } + + request := codersdk.CreateWorkspaceBuildRequest{ + Transition: codersdk.WorkspaceTransition(orchestration.ChildTransition), + RichParameterValues: childParameterValues, + LogLevel: codersdk.ProvisionerLogLevel(orchestration.ChildLogLevel), + } + + if orchestration.ChildTemplateVersionID.Valid { + request.TemplateVersionID = orchestration.ChildTemplateVersionID.UUID + } + if orchestration.ChildTemplateVersionPresetID.Valid { + request.TemplateVersionPresetID = orchestration.ChildTemplateVersionPresetID.UUID + } + if orchestration.ChildReason.Valid { + request.Reason = codersdk.CreateWorkspaceBuildReason(orchestration.ChildReason.BuildReason) + } + + return request, nil +} + +func (o *Orchestrator) createBuild( + ctx context.Context, + tx database.Store, + workspace database.Workspace, + initiatorID uuid.UUID, + request codersdk.CreateWorkspaceBuildRequest, +) (*database.WorkspaceBuild, *database.ProvisionerJob, error) { + transition := database.WorkspaceTransition(request.Transition) + builder := wsbuilder.New(workspace, transition, *o.buildUsageChecker.Load()). + Initiator(initiatorID). + RichParameterValues(request.RichParameterValues). + LogLevel(string(request.LogLevel)). + DeploymentValues(o.deploymentValues). + Experiments(o.experiments). + TemplateVersionPresetID(request.TemplateVersionPresetID). + BuildMetrics(o.builderMetrics) + + if request.TemplateVersionID != uuid.Nil { + builder = builder.VersionID(request.TemplateVersionID) + } else if transition == database.WorkspaceTransitionStart { + builder = builder.ActiveVersion() + } + if request.Reason != "" { + builder = builder.Reason(database.BuildReason(request.Reason)) + } + + workspaceBuild, provisionerJob, _, err := builder.Build(ctx, tx, o.fileCache, + // nil authorization function skips the builder's RBAC and + // config checks. The parent and child transitions were + // authorized when the orchestration row was inserted, and the + // child reuses the parent build's already-validated log + // level. + nil, + // The child build is created by a background worker, so there + // is no request IP to attach. Its initiator is still set from + // the parent build. + audit.WorkspaceBuildBaggage{}, + ) + if err != nil { + return nil, nil, err + } + + return workspaceBuild, provisionerJob, nil +} + +// childBuildErrorShouldFailOrchestration reports whether a child build +// error should be persisted as a failed orchestration instead of retried. +func childBuildErrorShouldFailOrchestration(err error) bool { + buildErr, ok := errors.AsType[wsbuilder.BuildError](err) + if !ok { + return false + } + + switch buildErr.Status { + case http.StatusBadRequest, http.StatusForbidden, http.StatusNotFound: + // These statuses indicate invalid stored build input or a + // permission/resource state that retrying the same request + // will not fix. + return true + default: + return false + } +} + +// childBuildErrorMessage returns the error text to be stored on the +// orchestration row. Build errors can expose cleaner response +// messages than Error(), which may contain only the wrapped cause. +func childBuildErrorMessage(err error) string { + buildErr, ok := errors.AsType[wsbuilder.BuildError](err) + if !ok { + return err.Error() + } + + _, response := buildErr.Response() + if response.Detail != "" && response.Detail != response.Message { + return fmt.Sprintf("%s: %s", response.Message, response.Detail) + } + if response.Message != "" { + return response.Message + } + return buildErr.Error() +} diff --git a/coderd/wsbuildorchestrator/wsbuildorchestrator_internal_test.go b/coderd/wsbuildorchestrator/wsbuildorchestrator_internal_test.go new file mode 100644 index 00000000000..d8fe1445bf3 --- /dev/null +++ b/coderd/wsbuildorchestrator/wsbuildorchestrator_internal_test.go @@ -0,0 +1,427 @@ +package wsbuildorchestrator + +import ( + "context" + "database/sql" + "encoding/json" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + "go.uber.org/goleak" + "golang.org/x/xerrors" + + "cdr.dev/slog/v3/sloggers/slogtest" + "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/database/dbgen" + "github.com/coder/coder/v2/coderd/database/dbtestutil" + "github.com/coder/coder/v2/coderd/database/dbtime" + "github.com/coder/coder/v2/coderd/database/pubsub" + "github.com/coder/coder/v2/coderd/wspubsub" + "github.com/coder/coder/v2/testutil" + "github.com/coder/quartz" +) + +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m, testutil.GoleakOptions...) +} + +func newTestOrchestrator(t *testing.T, db database.Store, ps pubsub.Pubsub) *Orchestrator { + t.Helper() + + return New(Options{ + Logger: testutil.Logger(t), + Database: db, + Pubsub: ps, + }) +} + +func TestWorkspaceBuildOrchestratorSubscribeQueuesWakeOnPubsub(t *testing.T) { + t.Parallel() + + ctx := testutil.Context(t, testutil.WaitShort) + ps := pubsub.NewInMemory() + o := newTestOrchestrator(t, nil, ps) + + go o.subscribe(ctx) + + // subscribe sends an initial wake after registration. Drain it so + // the publish below tests pubsub delivery without racing setup. + testutil.RequireReceive(ctx, t, o.wakeCh) + + err := wspubsub.PublishWorkspaceBuildOrchestrationWake(ctx, ps) + require.NoError(t, err) + + testutil.RequireReceive(ctx, t, o.wakeCh) +} + +func TestWorkspaceBuildOrchestratorRunProcessesOnWakeAndPoll(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + // trigger causes the run loop to process another pass, either + // via a wake signal or by advancing past the backup poll. + trigger func(ctx context.Context, o *Orchestrator, mClock *quartz.Mock) + }{ + { + name: "Wake", + trigger: func(_ context.Context, o *Orchestrator, _ *quartz.Mock) { + o.wake() + }, + }, + { + name: "BackupPoll", + trigger: func(ctx context.Context, _ *Orchestrator, mClock *quartz.Mock) { + mClock.Advance(backupPollInterval).MustWait(ctx) + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + ctx := testutil.Context(t, testutil.WaitShort) + mClock := quartz.NewMock(t) + store := &runStore{ + calls: make(chan struct{}), + } + o := New(Options{ + Logger: testutil.Logger(t), + Database: store, + Clock: mClock, + }) + + go o.run(ctx) + + // Drain the initial pass. run() creates the backup poll + // ticker and then processes (sends) once before + // waiting. So, receiving here also ensures the ticker + // exists before we advance the clock. + testutil.RequireReceive(ctx, t, store.calls) + + tc.trigger(ctx, o, mClock) + // Now this pass can only come from the trigger. + testutil.RequireReceive(ctx, t, store.calls) + }) + } +} + +type runStore struct { + database.Store + calls chan struct{} +} + +func (s *runStore) InTx(fn func(database.Store) error, _ *database.TxOptions) error { + return fn(s) +} + +func (s *runStore) GetNextPendingWorkspaceBuildOrchestrationForUpdate(ctx context.Context) ( + database.WorkspaceBuildOrchestration, error, +) { + select { + case s.calls <- struct{}{}: + case <-ctx.Done(): + return database.WorkspaceBuildOrchestration{}, ctx.Err() + } + return database.WorkspaceBuildOrchestration{}, sql.ErrNoRows +} + +// Note: it overwrites parentJob's OrganizationID and Type. +func seedPendingOrchestration( + ctx context.Context, + t *testing.T, + db database.Store, + workspaceDeleted bool, + parentJob database.ProvisionerJob, +) (database.ProvisionerJob, database.WorkspaceBuild) { + t.Helper() + + org := dbgen.Organization(t, db, database.Organization{}) + user := dbgen.User(t, db, database.User{}) + versionJob := dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{ + OrganizationID: org.ID, + Type: database.ProvisionerJobTypeTemplateVersionImport, + }) + version := dbgen.TemplateVersion(t, db, database.TemplateVersion{ + OrganizationID: org.ID, + JobID: versionJob.ID, + CreatedBy: user.ID, + }) + template := dbgen.Template(t, db, database.Template{ + OrganizationID: org.ID, + ActiveVersionID: version.ID, + CreatedBy: user.ID, + }) + workspace := dbgen.Workspace(t, db, database.WorkspaceTable{ + OwnerID: user.ID, + OrganizationID: org.ID, + TemplateID: template.ID, + Deleted: workspaceDeleted, + }) + + parentJob.OrganizationID = org.ID + parentJob.Type = database.ProvisionerJobTypeWorkspaceBuild + job := dbgen.ProvisionerJob(t, db, nil, parentJob) + parentBuild := dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{ + WorkspaceID: workspace.ID, + TemplateVersionID: version.ID, + JobID: job.ID, + Transition: database.WorkspaceTransitionStop, + Reason: database.BuildReasonInitiator, + }) + + now := dbtime.Now() + _, err := db.InsertWorkspaceBuildOrchestration(ctx, database.InsertWorkspaceBuildOrchestrationParams{ + ID: uuid.New(), + CreatedAt: now, + UpdatedAt: now, + ParentBuildID: parentBuild.ID, + ChildTransition: database.WorkspaceTransitionStart, + ChildRichParameterValues: json.RawMessage("[]"), + }) + require.NoError(t, err) + + return job, parentBuild +} + +// succeededJob returns a provisioner job in the succeeded state. +func succeededJob() database.ProvisionerJob { + now := dbtime.Now() + return database.ProvisionerJob{ + StartedAt: sql.NullTime{Time: now, Valid: true}, + CompletedAt: sql.NullTime{Time: now, Valid: true}, + } +} + +// A succeeded parent whose workspace can no longer be started +// (deleted or dormant) must fail the orchestration without creating a +// child build. +func TestWorkspaceBuildOrchestratorFailsForUnstartableWorkspace(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + // seed builds a pending orchestration whose workspace cannot + // start, and returns its parent build. + seed func(ctx context.Context, t *testing.T, db database.Store) database.WorkspaceBuild + wantError string + }{ + { + name: "Deleted", + seed: func(ctx context.Context, t *testing.T, db database.Store) database.WorkspaceBuild { + job, build := seedPendingOrchestration(ctx, t, db, true, succeededJob()) + require.Equal(t, database.ProvisionerJobStatusSucceeded, job.JobStatus) + return build + }, + wantError: "workspace was deleted", + }, + { + name: "Dormant", + seed: func(ctx context.Context, t *testing.T, db database.Store) database.WorkspaceBuild { + job, build := seedPendingOrchestration(ctx, t, db, false, succeededJob()) + require.Equal(t, database.ProvisionerJobStatusSucceeded, job.JobStatus) + _, err := db.UpdateWorkspaceDormantDeletingAt(ctx, database.UpdateWorkspaceDormantDeletingAtParams{ + ID: build.WorkspaceID, + DormantAt: sql.NullTime{Time: dbtime.Now(), Valid: true}, + }) + require.NoError(t, err) + return build + }, + wantError: "workspace is dormant", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + // GIVEN: a pending orchestration whose workspace cannot start. + ctx := testutil.Context(t, testutil.WaitShort) + db, _, rawDB := dbtestutil.NewDBWithSQLDB(t) + parentBuild := tc.seed(ctx, t, db) + + o := newTestOrchestrator(t, db, nil) + + // WHEN: the orchestrator processes the row. + found, err := o.processNext(ctx) + require.NoError(t, err) + require.True(t, found) + + // THEN: the orchestration resolves as failed without creating + // a child build. + orchestration, err := dbtestutil.GetWorkspaceBuildOrchestrationByParentBuildID(ctx, rawDB, parentBuild.ID) + require.NoError(t, err) + require.Equal(t, "failed", orchestration.Status) + require.False(t, orchestration.ChildBuildID.Valid) + require.True(t, orchestration.Error.Valid) + require.Equal(t, tc.wantError, orchestration.Error.String) + }) + } +} + +// If a pending parent build is canceled before any provisioner +// acquires it, the orchestration resolves as canceled, with no error +// and without creating a child build. +func TestWorkspaceBuildOrchestratorCancelsForCanceledParent(t *testing.T) { + t.Parallel() + + // GIVEN: a workspace whose parent stop build was canceled + // (without a provisioner acquiring it) and a pending + // orchestration to start it. + ctx := testutil.Context(t, testutil.WaitShort) + db, _, rawDB := dbtestutil.NewDBWithSQLDB(t) + + now := dbtime.Now() + parentJob, parentBuild := seedPendingOrchestration(ctx, t, db, false, database.ProvisionerJob{ + CanceledAt: sql.NullTime{Time: now, Valid: true}, + CompletedAt: sql.NullTime{Time: now, Valid: true}, + }) + require.Equal(t, database.ProvisionerJobStatusCanceled, parentJob.JobStatus) + + o := newTestOrchestrator(t, db, nil) + + // WHEN: the orchestrator processes the row. + found, err := o.processNext(ctx) + require.NoError(t, err) + require.True(t, found) + + // THEN: the orchestration resolves as canceled, with no error and + // without creating a child build. + orchestration, err := dbtestutil.GetWorkspaceBuildOrchestrationByParentBuildID(ctx, rawDB, parentBuild.ID) + require.NoError(t, err) + require.Equal(t, "canceled", orchestration.Status) + require.False(t, orchestration.ChildBuildID.Valid) + require.False(t, orchestration.Error.Valid) +} + +// emptyStore reports no pending orchestrations so the run loop stays +// idle. +type emptyStore struct { + database.Store +} + +func (s emptyStore) InTx(fn func(database.Store) error, _ *database.TxOptions) error { + return fn(s) +} + +func (emptyStore) GetNextPendingWorkspaceBuildOrchestrationForUpdate(context.Context) ( + database.WorkspaceBuildOrchestration, error, +) { + return database.WorkspaceBuildOrchestration{}, sql.ErrNoRows +} + +func TestWorkspaceBuildOrchestratorCloseStopsGoroutines(t *testing.T) { + t.Parallel() + + o := New(Options{ + Logger: testutil.Logger(t), + Database: emptyStore{}, + Pubsub: pubsub.NewInMemory(), + }) + o.Start(context.Background()) + + // Close blocks on wg.Wait, so it returns only once both + // background goroutines have exited. A timeout here means a + // goroutine never exited after cancellation, leaving Close + // blocked. + closed := make(chan struct{}) + go func() { + o.Close() + close(closed) + }() + + ctx := testutil.Context(t, testutil.WaitShort) + select { + case <-ctx.Done(): + t.Fatal("Close did not stop background goroutines") + case <-closed: + } +} + +// lookupErrorStore returns a pending orchestration, then fails the +// parent provisioner job lookup. This exercises the non-child-build +// error path in processNext and captures the resulting retry update. +type lookupErrorStore struct { + database.Store + orchestrationID uuid.UUID + jobErr error + + retryCalled bool + retryParams database.UpdateWorkspaceBuildOrchestrationRetryByIDParams +} + +func (s *lookupErrorStore) InTx(fn func(database.Store) error, _ *database.TxOptions) error { + return fn(s) +} + +func (s *lookupErrorStore) GetNextPendingWorkspaceBuildOrchestrationForUpdate(context.Context) ( + database.WorkspaceBuildOrchestration, error, +) { + return database.WorkspaceBuildOrchestration{ + ID: s.orchestrationID, + ParentBuildID: uuid.New(), + }, nil +} + +func (*lookupErrorStore) GetWorkspaceBuildByID(_ context.Context, id uuid.UUID) ( + database.WorkspaceBuild, error, +) { + return database.WorkspaceBuild{ID: id, JobID: uuid.New()}, nil +} + +func (s *lookupErrorStore) GetProvisionerJobByID(context.Context, uuid.UUID) ( + database.ProvisionerJob, error, +) { + return database.ProvisionerJob{}, s.jobErr +} + +func (s *lookupErrorStore) UpdateWorkspaceBuildOrchestrationRetryByID( + _ context.Context, + arg database.UpdateWorkspaceBuildOrchestrationRetryByIDParams, +) (database.WorkspaceBuildOrchestration, error) { + s.retryCalled = true + s.retryParams = arg + return database.WorkspaceBuildOrchestration{}, nil +} + +// TestWorkspaceBuildOrchestratorRetriesUnexpectedError verifies that +// an unexpected error while processing a row makes processNext +// request a bounded retry rather than surfacing the error, which +// would leave the row pending and block newer orchestrations. +func TestWorkspaceBuildOrchestratorRetriesUnexpectedError(t *testing.T) { + t.Parallel() + + // GIVEN: a store that returns a pending orchestration, then fails + // the parent provisioner job lookup with an unexpected + // (non-child-build) error. + store := &lookupErrorStore{ + orchestrationID: uuid.New(), + jobErr: xerrors.New("boom"), + } + o := New(Options{ + // The unexpected-error path logs at error level by design, so + // tolerate it here instead of failing via slogtest. + Logger: slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}), + Database: store, + Pubsub: pubsub.NewInMemory(), + }) + + ctx := testutil.Context(t, testutil.WaitShort) + + // WHEN: the orchestrator processes the row. + found, err := o.processNext(ctx) + + // THEN: processNext requests a bounded retry (next_retry_after + // and maxAttempts passed) and returns without error, instead of + // surfacing the error, which would leave the row pending. + require.NoError(t, err) + require.True(t, found) + require.True(t, store.retryCalled) + require.Equal(t, store.orchestrationID, store.retryParams.ID) + require.Equal(t, int32(maxAttempts), store.retryParams.MaxAttemptCount) + require.False(t, store.retryParams.NextRetryAfter.IsZero()) + require.True(t, store.retryParams.Error.Valid) + require.Contains(t, store.retryParams.Error.String, "boom") +} diff --git a/coderd/wspubsub/wspubsub.go b/coderd/wspubsub/wspubsub.go index c648022e1da..c2bdda859aa 100644 --- a/coderd/wspubsub/wspubsub.go +++ b/coderd/wspubsub/wspubsub.go @@ -17,6 +17,10 @@ import ( // creating N separate subscriptions. const AllWorkspaceEventChannel = "workspace_updates:all" +// WorkspaceBuildOrchestrationWakeChannel wakes the internal worker that +// processes pending workspace build orchestration rows. +const WorkspaceBuildOrchestrationWakeChannel = "workspace_build_orchestrations:wake" + // HandleWorkspaceBuildUpdate wraps a callback to parse WorkspaceBuildUpdate // messages from the pubsub. func HandleWorkspaceBuildUpdate(cb func(ctx context.Context, payload codersdk.WorkspaceBuildUpdate, err error)) func(ctx context.Context, message []byte, err error) { @@ -48,12 +52,38 @@ func PublishWorkspaceBuildUpdate(_ context.Context, ps pubsub.Pubsub, update cod return nil } +// PublishWorkspaceBuildOrchestrationWake wakes coderd instances that can +// process pending workspace build orchestration rows. Call this after any +// workspace build reaches a terminal state: succeeded, failed, or canceled. +func PublishWorkspaceBuildOrchestrationWake(_ context.Context, ps pubsub.Pubsub) error { + if err := ps.Publish(WorkspaceBuildOrchestrationWakeChannel, []byte("{}")); err != nil { + return xerrors.Errorf("publish workspace build orchestration wake: %w", err) + } + return nil +} + // WorkspaceEventChannel can be used to subscribe to events for // workspaces owned by the provided user ID. func WorkspaceEventChannel(ownerID uuid.UUID) string { return fmt.Sprintf("workspace_owner:%s", ownerID) } +// PublishWorkspaceEvent validates and publishes a workspace event to +// the owner's event channel. +func PublishWorkspaceEvent(_ context.Context, ps pubsub.Pubsub, ownerID uuid.UUID, event WorkspaceEvent) error { + if err := event.Validate(); err != nil { + return xerrors.Errorf("validate workspace event: %w", err) + } + msg, err := json.Marshal(event) + if err != nil { + return xerrors.Errorf("marshal workspace event: %w", err) + } + if err := ps.Publish(WorkspaceEventChannel(ownerID), msg); err != nil { + return xerrors.Errorf("publish workspace event: %w", err) + } + return nil +} + func HandleWorkspaceEvent(cb func(ctx context.Context, payload WorkspaceEvent, err error)) func(ctx context.Context, message []byte, err error) { return func(ctx context.Context, message []byte, err error) { if err != nil {