diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index 46c5e41addce9..bd5ae6258529f 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -671,10 +671,11 @@ var ( Identifier: rbac.RoleIdentifier{Name: "dbpurge"}, DisplayName: "DB Purge Daemon", Site: rbac.Permissions(map[string][]policy.Action{ - rbac.ResourceSystem.Type: {policy.ActionDelete}, - rbac.ResourceNotificationMessage.Type: {policy.ActionDelete}, - rbac.ResourceApiKey.Type: {policy.ActionDelete}, - rbac.ResourceAibridgeInterception.Type: {policy.ActionDelete}, + rbac.ResourceSystem.Type: {policy.ActionDelete}, + rbac.ResourceNotificationMessage.Type: {policy.ActionDelete}, + rbac.ResourceApiKey.Type: {policy.ActionDelete}, + rbac.ResourceAibridgeInterception.Type: {policy.ActionDelete}, + rbac.ResourceWorkspaceBuildOrchestration.Type: {policy.ActionDelete}, // Chat auto-archive sets archived=true on inactive chats. rbac.ResourceChat.Type: {policy.ActionRead, policy.ActionUpdate}, // Purge old boundary logs past the retention period. @@ -2389,6 +2390,13 @@ func (q *querier) DeleteOldWorkspaceAgentStats(ctx context.Context) error { return q.db.DeleteOldWorkspaceAgentStats(ctx) } +func (q *querier) DeleteOldWorkspaceBuildOrchestrations(ctx context.Context, arg database.DeleteOldWorkspaceBuildOrchestrationsParams) (int64, error) { + if err := q.authorizeContext(ctx, policy.ActionDelete, rbac.ResourceWorkspaceBuildOrchestration.AnyOrganization()); err != nil { + return 0, err + } + return q.db.DeleteOldWorkspaceBuildOrchestrations(ctx, arg) +} + func (q *querier) DeleteOrganizationMember(ctx context.Context, arg database.DeleteOrganizationMemberParams) error { return deleteQ[database.OrganizationMember](q.log, q.auth, func(ctx context.Context, arg database.DeleteOrganizationMemberParams) (database.OrganizationMember, error) { member, err := database.ExpectOne(q.OrganizationMembers(ctx, database.OrganizationMembersParams{ diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index 8f76c9bc2fd31..a71ab78532ff2 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -4156,6 +4156,14 @@ func (s *MethodTestSuite) TestWorkspace() { Asserts(rbac.ResourceWorkspaceBuildOrchestration.AnyOrganization(), policy.ActionUpdate). Returns(orchestration) })) + s.Run("DeleteOldWorkspaceBuildOrchestrations", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) { + arg := database.DeleteOldWorkspaceBuildOrchestrationsParams{ + BeforeTime: dbtime.Now(), + LimitCount: 100, + } + dbm.EXPECT().DeleteOldWorkspaceBuildOrchestrations(gomock.Any(), arg).Return(int64(0), nil).AnyTimes() + check.Args(arg).Asserts(rbac.ResourceWorkspaceBuildOrchestration.AnyOrganization(), policy.ActionDelete) + })) s.Run("Start/InsertWorkspaceBuildParameters", s.Mocked(func(dbm *dbmock.MockStore, faker *gofakeit.Faker, check *expects) { w := testutil.Fake(s.T(), faker, database.Workspace{}) b := testutil.Fake(s.T(), faker, database.WorkspaceBuild{ diff --git a/coderd/database/dbmetrics/querymetrics.go b/coderd/database/dbmetrics/querymetrics.go index 3b294898f9bc9..89f1d82c2b6e9 100644 --- a/coderd/database/dbmetrics/querymetrics.go +++ b/coderd/database/dbmetrics/querymetrics.go @@ -818,6 +818,14 @@ func (m queryMetricsStore) DeleteOldWorkspaceAgentStats(ctx context.Context) err return r0 } +func (m queryMetricsStore) DeleteOldWorkspaceBuildOrchestrations(ctx context.Context, arg database.DeleteOldWorkspaceBuildOrchestrationsParams) (int64, error) { + start := time.Now() + r0, r1 := m.s.DeleteOldWorkspaceBuildOrchestrations(ctx, arg) + m.queryLatencies.WithLabelValues("DeleteOldWorkspaceBuildOrchestrations").Observe(time.Since(start).Seconds()) + m.queryCounts.WithLabelValues(httpmw.ExtractHTTPRoute(ctx), httpmw.ExtractHTTPMethod(ctx), "DeleteOldWorkspaceBuildOrchestrations").Inc() + return r0, r1 +} + func (m queryMetricsStore) DeleteOrganizationMember(ctx context.Context, arg database.DeleteOrganizationMemberParams) error { start := time.Now() r0 := m.s.DeleteOrganizationMember(ctx, arg) diff --git a/coderd/database/dbmock/dbmock.go b/coderd/database/dbmock/dbmock.go index 6ca5ce6383916..47afc753d37de 100644 --- a/coderd/database/dbmock/dbmock.go +++ b/coderd/database/dbmock/dbmock.go @@ -1381,6 +1381,21 @@ func (mr *MockStoreMockRecorder) DeleteOldWorkspaceAgentStats(ctx any) *gomock.C return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldWorkspaceAgentStats", reflect.TypeOf((*MockStore)(nil).DeleteOldWorkspaceAgentStats), ctx) } +// DeleteOldWorkspaceBuildOrchestrations mocks base method. +func (m *MockStore) DeleteOldWorkspaceBuildOrchestrations(ctx context.Context, arg database.DeleteOldWorkspaceBuildOrchestrationsParams) (int64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteOldWorkspaceBuildOrchestrations", ctx, arg) + ret0, _ := ret[0].(int64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteOldWorkspaceBuildOrchestrations indicates an expected call of DeleteOldWorkspaceBuildOrchestrations. +func (mr *MockStoreMockRecorder) DeleteOldWorkspaceBuildOrchestrations(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldWorkspaceBuildOrchestrations", reflect.TypeOf((*MockStore)(nil).DeleteOldWorkspaceBuildOrchestrations), ctx, arg) +} + // DeleteOrganizationMember mocks base method. func (m *MockStore) DeleteOrganizationMember(ctx context.Context, arg database.DeleteOrganizationMemberParams) error { m.ctrl.T.Helper() diff --git a/coderd/database/dbpurge/dbpurge.go b/coderd/database/dbpurge/dbpurge.go index b50bfe3ae8b6c..51e034417e079 100644 --- a/coderd/database/dbpurge/dbpurge.go +++ b/coderd/database/dbpurge/dbpurge.go @@ -39,6 +39,11 @@ const ( // long enough to cover the maximum interval of a heartbeat event (currently // 1 hour) plus some buffer. maxTelemetryHeartbeatAge = 24 * time.Hour + // Operational handoff state; terminal rows are kept for debugging, then + // purged. + workspaceBuildOrchestrationTerminalRetention = 24 * time.Hour + // Batch size for workspace build orchestration deletion. + workspaceBuildOrchestrationsBatchSize = 10000 // Chat and chat file batch sizes stay smaller than audit/connection // log batches because chat_files rows carry bytea blobs. chatsBatchSize = 1000 @@ -275,6 +280,15 @@ func (i *instance) purgeTick(ctx context.Context, db database.Store, start time. } } + deleteOldWorkspaceBuildOrchestrationsBefore := start.Add(-workspaceBuildOrchestrationTerminalRetention) + purgedWorkspaceBuildOrchestrations, err := tx.DeleteOldWorkspaceBuildOrchestrations(ctx, database.DeleteOldWorkspaceBuildOrchestrationsParams{ + BeforeTime: deleteOldWorkspaceBuildOrchestrationsBefore, + LimitCount: workspaceBuildOrchestrationsBatchSize, + }) + if err != nil { + return xerrors.Errorf("failed to delete old workspace build orchestrations: %w", err) + } + var purgedChats, purgedChatFiles, purgedChatDebugRuns int64 if purgeChats { purgedChats, purgedChatFiles, err = i.purgeChatsInTx(ctx, tx, start, chatRetentionDays) @@ -304,6 +318,7 @@ func (i *instance) purgeTick(ctx context.Context, db database.Store, start time. slog.F("audit_logs", purgedAuditLogs), slog.F("boundary_logs", purgedBoundaryLogs), slog.F("boundary_sessions", purgedBoundarySessions), + slog.F("workspace_build_orchestrations", purgedWorkspaceBuildOrchestrations), slog.F("chats", purgedChats), slog.F("chat_files", purgedChatFiles), slog.F("chat_debug_runs", purgedChatDebugRuns), @@ -318,6 +333,7 @@ func (i *instance) purgeTick(ctx context.Context, db database.Store, start time. i.recordsPurged.WithLabelValues("audit_logs").Add(float64(purgedAuditLogs)) i.recordsPurged.WithLabelValues("boundary_logs").Add(float64(purgedBoundaryLogs)) i.recordsPurged.WithLabelValues("boundary_sessions").Add(float64(purgedBoundarySessions)) + i.recordsPurged.WithLabelValues("workspace_build_orchestrations").Add(float64(purgedWorkspaceBuildOrchestrations)) i.recordsPurged.WithLabelValues("chats").Add(float64(purgedChats)) i.recordsPurged.WithLabelValues("chat_debug_runs").Add(float64(purgedChatDebugRuns)) i.recordsPurged.WithLabelValues("chat_files").Add(float64(purgedChatFiles)) diff --git a/coderd/database/dbpurge/dbpurge_test.go b/coderd/database/dbpurge/dbpurge_test.go index 18d7866cac50b..6efe954360a6b 100644 --- a/coderd/database/dbpurge/dbpurge_test.go +++ b/coderd/database/dbpurge/dbpurge_test.go @@ -129,6 +129,11 @@ func TestMetrics(t *testing.T) { }) require.GreaterOrEqual(t, auditLogs, 0) + workspaceBuildOrchestrations := promhelp.CounterValue(t, reg, "coderd_dbpurge_records_purged_total", prometheus.Labels{ + "record_type": "workspace_build_orchestrations", + }) + require.GreaterOrEqual(t, workspaceBuildOrchestrations, 0) + chats := promhelp.CounterValue(t, reg, "coderd_dbpurge_records_purged_total", prometheus.Labels{ "record_type": "chats", }) @@ -247,6 +252,7 @@ func TestMetrics(t *testing.T) { mDB.EXPECT().DeleteOldNotificationMessages(gomock.Any()).Return(nil).AnyTimes() mDB.EXPECT().ExpirePrebuildsAPIKeys(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() mDB.EXPECT().DeleteOldTelemetryLocks(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + mDB.EXPECT().DeleteOldWorkspaceBuildOrchestrations(gomock.Any(), gomock.Any()).Return(int64(0), nil).AnyTimes() mDB.EXPECT().DeleteOldAuditLogConnectionEvents(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() mDB.EXPECT().DeleteOldChatDebugRuns(gomock.Any(), gomock.AssignableToTypeOf(database.DeleteOldChatDebugRunsParams{})).Return(int64(0), nil).MinTimes(1) mDB.EXPECT().InTx(gomock.Any(), database.DefaultTXOptions().WithID("db_purge")). @@ -297,6 +303,7 @@ func TestMetrics(t *testing.T) { mDB.EXPECT().DeleteOldNotificationMessages(gomock.Any()).Return(nil).AnyTimes() mDB.EXPECT().ExpirePrebuildsAPIKeys(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() mDB.EXPECT().DeleteOldTelemetryLocks(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + mDB.EXPECT().DeleteOldWorkspaceBuildOrchestrations(gomock.Any(), gomock.Any()).Return(int64(0), nil).AnyTimes() mDB.EXPECT().DeleteOldAuditLogConnectionEvents(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() mDB.EXPECT().DeleteOldChats(gomock.Any(), gomock.AssignableToTypeOf(database.DeleteOldChatsParams{})).Return(int64(0), nil).MinTimes(1) mDB.EXPECT().DeleteOldChatFiles(gomock.Any(), gomock.AssignableToTypeOf(database.DeleteOldChatFilesParams{})).Return(int64(0), nil).MinTimes(1) @@ -1117,6 +1124,146 @@ func TestDeleteOldTelemetryHeartbeats(t *testing.T) { }, testutil.WaitShort, testutil.IntervalFast, "it should delete old telemetry heartbeats") } +func TestDeleteOldWorkspaceBuildOrchestrations(t *testing.T) { + t.Parallel() + + ctx := testutil.Context(t, testutil.WaitShort) + db, _, rawDB := dbtestutil.NewDBWithSQLDB(t) + + org := dbgen.Organization(t, db, database.Organization{}) + user := dbgen.User(t, db, database.User{}) + versionJob := dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{ + OrganizationID: org.ID, + Type: database.ProvisionerJobTypeTemplateVersionImport, + }) + version := dbgen.TemplateVersion(t, db, database.TemplateVersion{ + OrganizationID: org.ID, + JobID: versionJob.ID, + CreatedBy: user.ID, + }) + template := dbgen.Template(t, db, database.Template{ + OrganizationID: org.ID, + ActiveVersionID: version.ID, + CreatedBy: user.ID, + }) + workspace := dbgen.Workspace(t, db, database.WorkspaceTable{ + OwnerID: user.ID, + OrganizationID: org.ID, + TemplateID: template.ID, + }) + + now := dbtime.Now() + cutoff := now.Add(-24 * time.Hour) + buildTime := cutoff.Add(-time.Hour) + oldCompletedTime := cutoff.Add(-3 * time.Minute) + oldFailedTime := cutoff.Add(-2 * time.Minute) + oldCanceledTime := cutoff.Add(-time.Minute) + oldPendingTime := cutoff.Add(-time.Minute) + recentTime := cutoff.Add(time.Minute) + + createBuild := func(buildNumber int32, createdAt time.Time) database.WorkspaceBuild { + return mustCreateWorkspaceBuild(t, db, org, version, workspace.ID, createdAt, buildNumber) + } + insertOrchestration := func(parentBuild database.WorkspaceBuild, updatedAt time.Time) database.WorkspaceBuildOrchestration { + orchestration, err := db.InsertWorkspaceBuildOrchestration(ctx, database.InsertWorkspaceBuildOrchestrationParams{ + ID: uuid.New(), + CreatedAt: updatedAt, + UpdatedAt: updatedAt, + ParentBuildID: parentBuild.ID, + ChildTransition: database.WorkspaceTransitionStart, + ChildRichParameterValues: json.RawMessage("[]"), + }) + require.NoError(t, err) + return orchestration + } + + // Given: old terminal orchestration rows (completed, failed, + // canceled), an old pending row, and a recent terminal row. + oldCompletedParent := createBuild(1, buildTime) + oldCompletedChild := createBuild(2, buildTime) + oldCompleted := insertOrchestration(oldCompletedParent, oldCompletedTime) + _, err := db.UpdateWorkspaceBuildOrchestrationCompletedByID(ctx, database.UpdateWorkspaceBuildOrchestrationCompletedByIDParams{ + ID: oldCompleted.ID, + ChildBuildID: uuid.NullUUID{UUID: oldCompletedChild.ID, Valid: true}, + UpdatedAt: oldCompletedTime, + }) + require.NoError(t, err) + + oldFailedParent := createBuild(3, buildTime) + oldFailed := insertOrchestration(oldFailedParent, oldFailedTime) + _, err = db.UpdateWorkspaceBuildOrchestrationFailedByID(ctx, database.UpdateWorkspaceBuildOrchestrationFailedByIDParams{ + ID: oldFailed.ID, + Error: sql.NullString{String: "failed", Valid: true}, + UpdatedAt: oldFailedTime, + }) + require.NoError(t, err) + + oldCanceledParent := createBuild(4, buildTime) + oldCanceled := insertOrchestration(oldCanceledParent, oldCanceledTime) + _, err = db.UpdateWorkspaceBuildOrchestrationCanceledByID(ctx, database.UpdateWorkspaceBuildOrchestrationCanceledByIDParams{ + ID: oldCanceled.ID, + UpdatedAt: oldCanceledTime, + }) + require.NoError(t, err) + + oldPendingParent := createBuild(5, buildTime) + oldPending := insertOrchestration(oldPendingParent, oldPendingTime) + + recentCompletedParent := createBuild(6, buildTime) + recentCompletedChild := createBuild(7, buildTime) + recentCompleted := insertOrchestration(recentCompletedParent, recentTime) + _, err = db.UpdateWorkspaceBuildOrchestrationCompletedByID(ctx, database.UpdateWorkspaceBuildOrchestrationCompletedByIDParams{ + ID: recentCompleted.ID, + ChildBuildID: uuid.NullUUID{UUID: recentCompletedChild.ID, Valid: true}, + UpdatedAt: recentTime, + }) + require.NoError(t, err) + + // When: old workspace build orchestrations are deleted with LimitCount 1 + deleted, err := db.DeleteOldWorkspaceBuildOrchestrations(ctx, database.DeleteOldWorkspaceBuildOrchestrationsParams{ + BeforeTime: cutoff, + LimitCount: 1, + }) + require.NoError(t, err) + require.EqualValues(t, 1, deleted) + + // Then: only the oldest terminal row is deleted. + assertOrchestrationDeleted(ctx, t, rawDB, oldCompletedParent.ID) + assertOrchestrationExists(ctx, t, rawDB, oldFailedParent.ID, oldFailed.ID) + assertOrchestrationExists(ctx, t, rawDB, oldCanceledParent.ID, oldCanceled.ID) + assertOrchestrationExists(ctx, t, rawDB, oldPendingParent.ID, oldPending.ID) + assertOrchestrationExists(ctx, t, rawDB, recentCompletedParent.ID, recentCompleted.ID) + + // When: old workspace build orchestrations are deleted again. + deleted, err = db.DeleteOldWorkspaceBuildOrchestrations(ctx, database.DeleteOldWorkspaceBuildOrchestrationsParams{ + BeforeTime: cutoff, + LimitCount: 10, + }) + require.NoError(t, err) + require.EqualValues(t, 2, deleted) + + // Then: the remaining old terminal rows are deleted. + assertOrchestrationDeleted(ctx, t, rawDB, oldFailedParent.ID) + assertOrchestrationDeleted(ctx, t, rawDB, oldCanceledParent.ID) + assertOrchestrationExists(ctx, t, rawDB, oldPendingParent.ID, oldPending.ID) + assertOrchestrationExists(ctx, t, rawDB, recentCompletedParent.ID, recentCompleted.ID) +} + +func assertOrchestrationDeleted(ctx context.Context, t *testing.T, rawDB *sql.DB, parentBuildID uuid.UUID) { + t.Helper() + + _, err := dbtestutil.GetWorkspaceBuildOrchestrationByParentBuildID(ctx, rawDB, parentBuildID) + require.ErrorIs(t, err, sql.ErrNoRows) +} + +func assertOrchestrationExists(ctx context.Context, t *testing.T, rawDB *sql.DB, parentBuildID uuid.UUID, orchestrationID uuid.UUID) { + t.Helper() + + orchestration, err := dbtestutil.GetWorkspaceBuildOrchestrationByParentBuildID(ctx, rawDB, parentBuildID) + require.NoError(t, err) + require.Equal(t, orchestrationID, orchestration.ID) +} + func TestDeleteOldConnectionLogs(t *testing.T) { t.Parallel() diff --git a/coderd/database/querier.go b/coderd/database/querier.go index d5259b3820c52..3f48a4697a97d 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -222,6 +222,7 @@ type sqlcQuerier interface { // Logs can take up a lot of space, so it's important we clean up frequently. DeleteOldWorkspaceAgentLogs(ctx context.Context, threshold time.Time) (int64, error) DeleteOldWorkspaceAgentStats(ctx context.Context) error + DeleteOldWorkspaceBuildOrchestrations(ctx context.Context, arg DeleteOldWorkspaceBuildOrchestrationsParams) (int64, error) DeleteOrganizationMember(ctx context.Context, arg DeleteOrganizationMemberParams) error DeleteProvisionerKey(ctx context.Context, id uuid.UUID) error DeleteReplicasUpdatedBefore(ctx context.Context, updatedAt time.Time) error diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index b83febb163ee7..bbf8ed756c0e2 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -35299,6 +35299,37 @@ func (q *sqlQuerier) InsertWorkspaceAppStats(ctx context.Context, arg InsertWork return err } +const deleteOldWorkspaceBuildOrchestrations = `-- name: DeleteOldWorkspaceBuildOrchestrations :execrows +WITH deletable AS ( + SELECT + id + FROM + workspace_build_orchestrations + WHERE + status IN ('completed', 'failed', 'canceled') + AND updated_at < $1::timestamptz + ORDER BY + updated_at ASC + LIMIT $2::int +) +DELETE FROM workspace_build_orchestrations +USING deletable +WHERE workspace_build_orchestrations.id = deletable.id +` + +type DeleteOldWorkspaceBuildOrchestrationsParams struct { + BeforeTime time.Time `db:"before_time" json:"before_time"` + LimitCount int32 `db:"limit_count" json:"limit_count"` +} + +func (q *sqlQuerier) DeleteOldWorkspaceBuildOrchestrations(ctx context.Context, arg DeleteOldWorkspaceBuildOrchestrationsParams) (int64, error) { + result, err := q.db.ExecContext(ctx, deleteOldWorkspaceBuildOrchestrations, arg.BeforeTime, arg.LimitCount) + if err != nil { + return 0, err + } + return result.RowsAffected() +} + const getNextPendingWorkspaceBuildOrchestrationForUpdate = `-- name: GetNextPendingWorkspaceBuildOrchestrationForUpdate :one SELECT wbo.id, wbo.created_at, wbo.updated_at, wbo.workspace_id, wbo.parent_build_id, wbo.child_build_id, wbo.child_transition, wbo.child_template_version_id, wbo.child_template_version_preset_id, wbo.child_rich_parameter_values, wbo.child_log_level, wbo.child_reason, wbo.attempt_count, wbo.next_retry_after, wbo.status, wbo.error diff --git a/coderd/database/queries/workspacebuildorchestrations.sql b/coderd/database/queries/workspacebuildorchestrations.sql index f256d89a7adcf..fe8279a20e5db 100644 --- a/coderd/database/queries/workspacebuildorchestrations.sql +++ b/coderd/database/queries/workspacebuildorchestrations.sql @@ -114,3 +114,20 @@ WHERE id = @id AND status = 'pending' RETURNING *; + +-- name: DeleteOldWorkspaceBuildOrchestrations :execrows +WITH deletable AS ( + SELECT + id + FROM + workspace_build_orchestrations + WHERE + status IN ('completed', 'failed', 'canceled') + AND updated_at < @before_time::timestamptz + ORDER BY + updated_at ASC + LIMIT @limit_count::int +) +DELETE FROM workspace_build_orchestrations +USING deletable +WHERE workspace_build_orchestrations.id = deletable.id;