From 86668c889e3016de27c401ab2e969722e79cc856 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 1 Sep 2026 13:09:05 +0000 Subject: [PATCH 1/4] fix: close the insert-vs-soft-delete race on user child tables A child-table insert racing a user soft-delete could commit after delete_deleted_user_resources ran, resurrecting rows (including live session tokens in api_keys) for a deleted account. Migration 000591 delegates the four existing per-table guard functions to one shared check_user_not_deleted() via CREATE OR REPLACE (no DROP TRIGGER, no ACCESS EXCLUSIVE on hot tables), locks the users row with FOR NO KEY UPDATE on INSERT and owner reassignment, adds guards to user_ai_provider_keys, organization_members, user_ai_budget_overrides, and group_members, and covers UPDATE ... SET user_id everywhere while keeping same-owner updates unlocked. There is no backfill in the migration: orphaned rows of already-deleted users are removed by the idempotent dbpurge reaper (PurgeSoftDeletedUserResources) at startup and on the purge cadence. Transactions that write a guarded child row and later insert one for the same user take the users lock first via AcquireUserSoftDeleteGuardLock (OAuth2 token grants, oauthLogin, regenerateSessionToken) so their lock order matches the cleanup trigger and cannot deadlock; TestOAuth2ProviderTokenExchangeLockOrder pins both the lock and its position via pg_locks. --- coderd/database/dbauthz/dbauthz.go | 21 + coderd/database/dbauthz/dbauthz_test.go | 9 + coderd/database/dbmetrics/querymetrics.go | 16 + coderd/database/dbmock/dbmock.go | 29 + coderd/database/dbpurge/dbpurge.go | 6 + coderd/database/dbpurge/dbpurge_test.go | 96 +++ coderd/database/dbtestutil/softdelete.go | 40 + coderd/database/dump.sql | 140 +++- ...0591_lock_user_soft_delete_guards.down.sql | 74 ++ ...000591_lock_user_soft_delete_guards.up.sql | 223 ++++++ coderd/database/migrations/migrate_test.go | 168 +++++ coderd/database/querier.go | 32 + coderd/database/queries.sql.go | 69 ++ coderd/database/queries/users.sql | 55 ++ .../database/user_soft_delete_guards_test.go | 690 ++++++++++++++++++ coderd/database/usersoftdeleteguards.go | 25 + coderd/exp_chats.go | 8 + coderd/exp_chats_test.go | 22 + coderd/members.go | 9 + coderd/members_test.go | 21 + coderd/oauth2_test.go | 103 +++ coderd/oauth2provider/tokens.go | 23 + .../provisionerdserver/provisionerdserver.go | 12 +- coderd/userauth.go | 12 + coderd/userskills.go | 10 +- enterprise/cli/server_dbcrypt_test.go | 22 +- enterprise/dbcrypt/cliutil_test.go | 8 +- 27 files changed, 1901 insertions(+), 42 deletions(-) create mode 100644 coderd/database/dbtestutil/softdelete.go create mode 100644 coderd/database/migrations/000591_lock_user_soft_delete_guards.down.sql create mode 100644 coderd/database/migrations/000591_lock_user_soft_delete_guards.up.sql create mode 100644 coderd/database/user_soft_delete_guards_test.go create mode 100644 coderd/database/usersoftdeleteguards.go diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index e27731ab0fe..fa18f764e2a 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -1815,6 +1815,20 @@ func (q *querier) AcquireStaleChatDiffStatuses(ctx context.Context, limitVal int return q.db.AcquireStaleChatDiffStatuses(ctx, limitVal) } +func (q *querier) AcquireUserSoftDeleteGuardLock(ctx context.Context, userID uuid.UUID) (uuid.UUID, error) { + // The lock is a deadlock-avoidance primitive, not a user capability: it + // orders a transaction's locks (users first, then child rows), granting + // nothing the guard triggers do not already do for free on every insert. + // Authorizing it on the target user would be role-dependent (owners + // pass, members fail) while protecting nothing, so it authorizes as a + // system primitive and call sites on user-scoped paths wrap only this + // call in dbauthz.AsSystemRestricted. + if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceSystem); err != nil { + return uuid.Nil, err + } + return q.db.AcquireUserSoftDeleteGuardLock(ctx, userID) +} + func (q *querier) ActivityBumpWorkspace(ctx context.Context, arg database.ActivityBumpWorkspaceParams) error { fetch := func(ctx context.Context, arg database.ActivityBumpWorkspaceParams) (database.Workspace, error) { return q.db.GetWorkspaceByID(ctx, arg.WorkspaceID) @@ -7223,6 +7237,13 @@ func (q *querier) PopNextQueuedMessage(ctx context.Context, chatID uuid.UUID) (d return q.db.PopNextQueuedMessage(ctx, chatID) } +func (q *querier) PurgeSoftDeletedUserResources(ctx context.Context) error { + if err := q.authorizeContext(ctx, policy.ActionDelete, rbac.ResourceSystem); err != nil { + return err + } + return q.db.PurgeSoftDeletedUserResources(ctx) +} + func (q *querier) ReduceWorkspaceAgentShareLevelToAuthenticatedByTemplate(ctx context.Context, templateID uuid.UUID) error { template, err := q.db.GetTemplateByID(ctx, templateID) if err != nil { diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index 5dd49f43068..cf4e0d8c03f 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -3093,6 +3093,15 @@ func (s *MethodTestSuite) TestUser() { dbm.EXPECT().GetUserByID(gomock.Any(), u.ID).Return(u, nil).AnyTimes() check.Args(u.ID).Asserts(u, policy.ActionRead).Returns(u) })) + s.Run("AcquireUserSoftDeleteGuardLock", s.Mocked(func(dbm *dbmock.MockStore, faker *gofakeit.Faker, check *expects) { + u := testutil.Fake(s.T(), faker, database.User{}) + dbm.EXPECT().AcquireUserSoftDeleteGuardLock(gomock.Any(), u.ID).Return(u.ID, nil).AnyTimes() + check.Args(u.ID).Asserts(rbac.ResourceSystem, policy.ActionUpdate).Returns(u.ID) + })) + s.Run("PurgeSoftDeletedUserResources", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) { + dbm.EXPECT().PurgeSoftDeletedUserResources(gomock.Any()).Return(nil).AnyTimes() + check.Args().Asserts(rbac.ResourceSystem, policy.ActionDelete) + })) s.Run("GetUsersByIDs", s.Mocked(func(dbm *dbmock.MockStore, faker *gofakeit.Faker, check *expects) { a := testutil.Fake(s.T(), faker, database.User{CreatedAt: dbtime.Now().Add(-time.Hour)}) b := testutil.Fake(s.T(), faker, database.User{CreatedAt: dbtime.Now()}) diff --git a/coderd/database/dbmetrics/querymetrics.go b/coderd/database/dbmetrics/querymetrics.go index 365899afbd2..bc74f267120 100644 --- a/coderd/database/dbmetrics/querymetrics.go +++ b/coderd/database/dbmetrics/querymetrics.go @@ -144,6 +144,14 @@ func (m queryMetricsStore) AcquireStaleChatDiffStatuses(ctx context.Context, lim return r0, r1 } +func (m queryMetricsStore) AcquireUserSoftDeleteGuardLock(ctx context.Context, userID uuid.UUID) (uuid.UUID, error) { + start := time.Now() + r0, r1 := m.s.AcquireUserSoftDeleteGuardLock(ctx, userID) + m.queryLatencies.WithLabelValues("AcquireUserSoftDeleteGuardLock").Observe(time.Since(start).Seconds()) + m.queryCounts.WithLabelValues(httpmw.ExtractHTTPRoute(ctx), httpmw.ExtractHTTPMethod(ctx), "AcquireUserSoftDeleteGuardLock").Inc() + return r0, r1 +} + func (m queryMetricsStore) ActivityBumpWorkspace(ctx context.Context, arg database.ActivityBumpWorkspaceParams) error { start := time.Now() r0 := m.s.ActivityBumpWorkspace(ctx, arg) @@ -5064,6 +5072,14 @@ func (m queryMetricsStore) PopNextQueuedMessage(ctx context.Context, chatID uuid return r0, r1 } +func (m queryMetricsStore) PurgeSoftDeletedUserResources(ctx context.Context) error { + start := time.Now() + r0 := m.s.PurgeSoftDeletedUserResources(ctx) + m.queryLatencies.WithLabelValues("PurgeSoftDeletedUserResources").Observe(time.Since(start).Seconds()) + m.queryCounts.WithLabelValues(httpmw.ExtractHTTPRoute(ctx), httpmw.ExtractHTTPMethod(ctx), "PurgeSoftDeletedUserResources").Inc() + return r0 +} + func (m queryMetricsStore) ReduceWorkspaceAgentShareLevelToAuthenticatedByTemplate(ctx context.Context, templateID uuid.UUID) error { start := time.Now() r0 := m.s.ReduceWorkspaceAgentShareLevelToAuthenticatedByTemplate(ctx, templateID) diff --git a/coderd/database/dbmock/dbmock.go b/coderd/database/dbmock/dbmock.go index f52286cf511..36afb8d054e 100644 --- a/coderd/database/dbmock/dbmock.go +++ b/coderd/database/dbmock/dbmock.go @@ -118,6 +118,21 @@ func (mr *MockStoreMockRecorder) AcquireStaleChatDiffStatuses(ctx, limitVal any) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AcquireStaleChatDiffStatuses", reflect.TypeOf((*MockStore)(nil).AcquireStaleChatDiffStatuses), ctx, limitVal) } +// AcquireUserSoftDeleteGuardLock mocks base method. +func (m *MockStore) AcquireUserSoftDeleteGuardLock(ctx context.Context, userID uuid.UUID) (uuid.UUID, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AcquireUserSoftDeleteGuardLock", ctx, userID) + ret0, _ := ret[0].(uuid.UUID) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AcquireUserSoftDeleteGuardLock indicates an expected call of AcquireUserSoftDeleteGuardLock. +func (mr *MockStoreMockRecorder) AcquireUserSoftDeleteGuardLock(ctx, userID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AcquireUserSoftDeleteGuardLock", reflect.TypeOf((*MockStore)(nil).AcquireUserSoftDeleteGuardLock), ctx, userID) +} + // ActivityBumpWorkspace mocks base method. func (m *MockStore) ActivityBumpWorkspace(ctx context.Context, arg database.ActivityBumpWorkspaceParams) error { m.ctrl.T.Helper() @@ -9624,6 +9639,20 @@ func (mr *MockStoreMockRecorder) PopNextQueuedMessage(ctx, chatID any) *gomock.C return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PopNextQueuedMessage", reflect.TypeOf((*MockStore)(nil).PopNextQueuedMessage), ctx, chatID) } +// PurgeSoftDeletedUserResources mocks base method. +func (m *MockStore) PurgeSoftDeletedUserResources(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PurgeSoftDeletedUserResources", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// PurgeSoftDeletedUserResources indicates an expected call of PurgeSoftDeletedUserResources. +func (mr *MockStoreMockRecorder) PurgeSoftDeletedUserResources(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PurgeSoftDeletedUserResources", reflect.TypeOf((*MockStore)(nil).PurgeSoftDeletedUserResources), ctx) +} + // ReduceWorkspaceAgentShareLevelToAuthenticatedByTemplate mocks base method. func (m *MockStore) ReduceWorkspaceAgentShareLevelToAuthenticatedByTemplate(ctx context.Context, templateID uuid.UUID) error { m.ctrl.T.Helper() diff --git a/coderd/database/dbpurge/dbpurge.go b/coderd/database/dbpurge/dbpurge.go index 6f809e68f41..b184087d2fc 100644 --- a/coderd/database/dbpurge/dbpurge.go +++ b/coderd/database/dbpurge/dbpurge.go @@ -238,6 +238,12 @@ func (i *instance) purgeTick(ctx context.Context, db database.Store, start time. if err := tx.ExpirePrebuildsAPIKeys(ctx, dbtime.Time(start)); err != nil { return xerrors.Errorf("failed to expire prebuilds user api keys: %w", err) } + // Remove child rows orphaned by a user soft-delete that predates the + // guard triggers and cleanup coverage (migration 000591). The guards + // prevent new orphans, so after the first pass this is a no-op. + if err := tx.PurgeSoftDeletedUserResources(ctx); err != nil { + return xerrors.Errorf("failed to purge soft-deleted user resources: %w", err) + } var expiredAPIKeys int64 apiKeysRetention := i.vals.Retention.APIKeys.Value() diff --git a/coderd/database/dbpurge/dbpurge_test.go b/coderd/database/dbpurge/dbpurge_test.go index 26a0b003a0c..5fe56c79f11 100644 --- a/coderd/database/dbpurge/dbpurge_test.go +++ b/coderd/database/dbpurge/dbpurge_test.go @@ -3551,3 +3551,99 @@ func TestDeleteIdentifiedModuleCacheFiles(t *testing.T) { assertFileExists(late.ID, "archive inserted after the one-off pass") assertCacheRef(lateTV, late.ID, true, "archive inserted after the one-off pass") } + +// TestPurgeSoftDeletedUserResources verifies the reaper removes child rows +// orphaned by a user soft-delete that predates the guard triggers and +// cleanup coverage (migration 000591 deliberately has no backfill), while a +// live user's rows survive. +// +//nolint:paralleltest // It uses LockIDDBPurge. +func TestPurgeSoftDeletedUserResources(t *testing.T) { + db, _, sqlDB := dbtestutil.NewDBWithSQLDB(t) + logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}) + ctx := testutil.Context(t, testutil.WaitShort) + + org := dbgen.Organization(t, db, database.Organization{}) + provider := dbgen.AIProvider(t, db, database.AIProvider{}) + group := dbgen.Group(t, db, database.Group{OrganizationID: org.ID}) + + liveUser := dbgen.User(t, db, database.User{}) + doomedUser := dbgen.User(t, db, database.User{}) + + seed := func(userID uuid.UUID) { + _, err := sqlDB.ExecContext(ctx, ` + INSERT INTO api_keys (id, hashed_secret, user_id, last_used, expires_at, created_at, updated_at, login_type, scopes, allow_list) + VALUES ($1, 'reap-hash'::bytea, $2, now(), now() + interval '1 hour', now(), now(), 'password', '{}'::api_key_scope[], ARRAY['*']) + `, uuid.NewString(), userID) + require.NoError(t, err) + _, err = sqlDB.ExecContext(ctx, + `INSERT INTO user_links (user_id, login_type, linked_id) VALUES ($1, 'github', $2)`, + userID, "reap-link-"+userID.String()) + require.NoError(t, err) + _, err = sqlDB.ExecContext(ctx, ` + INSERT INTO user_secrets (id, user_id, name, description, value, env_name) + VALUES ($1, $2, 'reap-secret', '', 'value', 'REAP_SECRET') + `, uuid.New(), userID) + require.NoError(t, err) + _, err = sqlDB.ExecContext(ctx, ` + INSERT INTO user_skills (id, user_id, name, description, content) + VALUES ($1, $2, 'reap-skill', '', 'content') + `, uuid.New(), userID) + require.NoError(t, err) + _, err = sqlDB.ExecContext(ctx, ` + INSERT INTO user_ai_provider_keys (id, user_id, ai_provider_id, api_key) + VALUES ($1, $2, $3, 'reap-key') + `, uuid.New(), userID, provider.ID) + require.NoError(t, err) + _, err = sqlDB.ExecContext(ctx, ` + INSERT INTO organization_members (user_id, organization_id, created_at, updated_at) + VALUES ($1, $2, now(), now()) + `, userID, org.ID) + require.NoError(t, err) + _, err = sqlDB.ExecContext(ctx, + `INSERT INTO group_members (user_id, group_id) VALUES ($1, $2)`, + userID, group.ID) + require.NoError(t, err) + _, err = sqlDB.ExecContext(ctx, ` + INSERT INTO user_ai_budget_overrides (user_id, group_id, spend_limit_micros) + VALUES ($1, $2, 1000000) + `, userID, group.ID) + require.NoError(t, err) + } + seed(liveUser.ID) + seed(doomedUser.ID) + + // Reconstruct pre-guard orphans: soft-delete with the cleanup trigger + // suppressed, so every child row survives. + dbtestutil.SoftDeleteUserKeepingRows(ctx, t, sqlDB, doomedUser.ID) + + guardedTables := []string{"api_keys", "user_links", "user_secrets", "user_skills", "user_ai_provider_keys", "organization_members", "group_members", "user_ai_budget_overrides"} + countRows := func(table string, userID uuid.UUID) int { + var count int + //nolint:gosec // The table name comes from the fixed list above. + err := sqlDB.QueryRowContext(ctx, + `SELECT count(*) FROM `+table+` WHERE user_id = $1`, userID).Scan(&count) + require.NoError(t, err) + return count + } + for _, table := range guardedTables { + require.Equal(t, 1, countRows(table, doomedUser.ID), "pre-purge: %s orphan must exist", table) + } + + // The initial tick runs the purge immediately. + closer := dbpurge.New(ctx, logger, db, &codersdk.DeploymentValues{}, prometheus.NewRegistry(), dbpurge.WithClock(quartz.NewReal())) + defer closer.Close() + + require.Eventually(t, func() bool { + for _, table := range guardedTables { + if countRows(table, doomedUser.ID) != 0 { + return false + } + } + return true + }, testutil.WaitShort, testutil.IntervalFast, "all orphaned child rows of the soft-deleted user must be reaped") + + for _, table := range guardedTables { + require.Equal(t, 1, countRows(table, liveUser.ID), "post-purge: %s row for the live user must survive", table) + } +} diff --git a/coderd/database/dbtestutil/softdelete.go b/coderd/database/dbtestutil/softdelete.go new file mode 100644 index 00000000000..dd8ac73d8c6 --- /dev/null +++ b/coderd/database/dbtestutil/softdelete.go @@ -0,0 +1,40 @@ +package dbtestutil + +import ( + "context" + "database/sql" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" +) + +// SoftDeleteUserKeepingRows marks the user deleted while suppressing the +// delete_deleted_user_resources cleanup trigger, so the user's child rows +// (api_keys, user_links, and the other guarded tables) survive. This +// reconstructs the orphaned-row state that could exist before migration +// 000591 closed the insert-vs-soft-delete race (the insert guards now also +// reject new rows for deleted users, so the state can only be constructed +// this way). Tests use it to prove such legacy rows stay inert. +func SoftDeleteUserKeepingRows(ctx context.Context, t testing.TB, sqlDB *sql.DB, userID uuid.UUID) { + t.Helper() + // One transaction: transactional DDL keeps the disabled trigger + // invisible to concurrent sessions (which may share this database under + // CODER_PG_CONNECTION_URL) and rolls the disable back on failure. + tx, err := sqlDB.BeginTx(ctx, nil) + require.NoError(t, err) + committed := false + defer func() { + if !committed { + _ = tx.Rollback() + } + }() + _, err = tx.ExecContext(ctx, `ALTER TABLE users DISABLE TRIGGER trigger_update_users`) + require.NoError(t, err) + _, err = tx.ExecContext(ctx, `UPDATE users SET deleted = true WHERE id = $1`, userID) + require.NoError(t, err) + _, err = tx.ExecContext(ctx, `ALTER TABLE users ENABLE TRIGGER trigger_update_users`) + require.NoError(t, err) + require.NoError(t, tx.Commit()) + committed = true +} diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql index df596fa8005..5d912ce1941 100644 --- a/coderd/database/dump.sql +++ b/coderd/database/dump.sql @@ -885,6 +885,68 @@ $$; COMMENT ON FUNCTION chat_message_search_text(content jsonb) IS 'Extracts searchable content from chat_messages. Returns NULL for scalar JSON strings (content_version=0). Immutable as it is used in indexes.'; +CREATE FUNCTION check_user_not_deleted(target_user_id uuid, take_lock boolean, tg_operation text, display_name text, deleted_constraint text) RETURNS void + LANGUAGE plpgsql + AS $$ +DECLARE + user_deleted boolean; +BEGIN + -- Serialize child-table inserts against a concurrent user soft-delete: + -- an unlocked insert can read deleted = false, lose the race to the + -- soft-delete UPDATE and its cleanup, then commit a resurrected row. + -- FOR NO KEY UPDATE conflicts with the soft-delete UPDATE but not with + -- the FOR KEY SHARE locks that foreign-key validation takes on the + -- users row for other child tables. + -- + -- The lock is taken only when the row starts belonging to + -- target_user_id (INSERT, or UPDATE that reassigns the owner). On + -- same-owner UPDATE statements the unlocked read is kept: taking the users lock + -- there deadlocks against delete_deleted_user_resources (a multi-row + -- UPDATE or ON CONFLICT path can hold one child tuple and wait on + -- users while the cleanup holds users and waits on a child tuple), and + -- would serialize routine child updates on the hot users row. An + -- existing row is cleaned up by the soft-delete either way. + -- + -- The locking path imposes an ordering contract on writers: a + -- transaction that writes a guarded child row (INSERT, UPDATE, or + -- DELETE) and later inserts a guarded row for the same user must call + -- AcquireUserSoftDeleteGuardLock first, so its lock order (users, then + -- child rows) matches delete_deleted_user_resources. The same contract + -- covers the cap triggers' advisory locks (migration 000590): without + -- the users lock first, an update-then-insert writer can cycle with a + -- concurrent insert that holds the users lock and waits on the + -- advisory lock. coderd/database/user_soft_delete_guards_test.go + -- replays each known such path as a deterministic deadlock regression, + -- and the OAuth2 token exchange is driven through its real entry point. + -- + -- Isolation: the locking read is correct at READ COMMITTED, which every + -- production writer of the guarded tables uses. Under REPEATABLE READ + -- or SERIALIZABLE the lock wait fails with a serialization error + -- (40001) whenever any transaction committed an update to the users + -- row after the snapshot (users is written on ordinary browsing to + -- bump last_seen_at), and coderd does not retry 40001; do not wrap + -- guarded inserts in database.ReadModifyUpdate or other + -- stronger-isolation transactions. + IF take_lock THEN + SELECT deleted INTO user_deleted + FROM users + WHERE id = target_user_id + FOR NO KEY UPDATE; + ELSE + SELECT deleted INTO user_deleted + FROM users + WHERE id = target_user_id; + END IF; + IF (user_deleted) THEN + RAISE EXCEPTION 'Cannot % % for deleted user', + CASE WHEN tg_operation = 'INSERT' THEN 'create' ELSE 'modify' END, + display_name + USING ERRCODE = 'check_violation', + CONSTRAINT = deleted_constraint; + END IF; +END; +$$; + CREATE FUNCTION check_workspace_agent_name_unique() RETURNS trigger LANGUAGE plpgsql AS $$ @@ -1146,6 +1208,24 @@ BEGIN END; $$; +CREATE FUNCTION fail_if_user_deleted() RETURNS trigger + LANGUAGE plpgsql + AS $$ +DECLARE + reassigned boolean := false; +BEGIN + IF (TG_OP = 'UPDATE') THEN + reassigned := NEW.user_id IS DISTINCT FROM OLD.user_id; + -- Same-owner updates on the tables using this function are + -- filtered out by the triggers' UPDATE OF user_id column list; + -- reassigned still guards the no-op UPDATE ... SET user_id = user_id. + END IF; + PERFORM check_user_not_deleted( + NEW.user_id, TG_OP = 'INSERT' OR reassigned, TG_OP, TG_ARGV[0], TG_ARGV[1]); + RETURN NEW; +END; +$$; + CREATE FUNCTION inhibit_enqueue_if_disabled() RETURNS trigger LANGUAGE plpgsql AS $$ @@ -1177,14 +1257,14 @@ $$; CREATE FUNCTION insert_apikey_fail_if_user_deleted() RETURNS trigger LANGUAGE plpgsql AS $$ - DECLARE + reassigned boolean := false; BEGIN - IF (NEW.user_id IS NOT NULL) THEN - IF (SELECT deleted FROM users WHERE id = NEW.user_id LIMIT 1) THEN - RAISE EXCEPTION 'Cannot create API key for deleted user'; - END IF; + IF (TG_OP = 'UPDATE') THEN + reassigned := NEW.user_id IS DISTINCT FROM OLD.user_id; END IF; + PERFORM check_user_not_deleted( + NEW.user_id, TG_OP = 'INSERT' OR reassigned, TG_OP, 'API key', 'api_key_user_deleted'); RETURN NEW; END; $$; @@ -1236,14 +1316,14 @@ $$; CREATE FUNCTION insert_user_links_fail_if_user_deleted() RETURNS trigger LANGUAGE plpgsql AS $$ - DECLARE + reassigned boolean := false; BEGIN - IF (NEW.user_id IS NOT NULL) THEN - IF (SELECT deleted FROM users WHERE id = NEW.user_id LIMIT 1) THEN - RAISE EXCEPTION 'Cannot create user_link for deleted user'; - END IF; + IF (TG_OP = 'UPDATE') THEN + reassigned := NEW.user_id IS DISTINCT FROM OLD.user_id; END IF; + PERFORM check_user_not_deleted( + NEW.user_id, TG_OP = 'INSERT' OR reassigned, TG_OP, 'user_link', 'user_link_user_deleted'); RETURN NEW; END; $$; @@ -1251,14 +1331,14 @@ $$; CREATE FUNCTION insert_user_secret_fail_if_user_deleted() RETURNS trigger LANGUAGE plpgsql AS $$ - DECLARE + reassigned boolean := false; BEGIN - IF (NEW.user_id IS NOT NULL) THEN - IF (SELECT deleted FROM users WHERE id = NEW.user_id LIMIT 1) THEN - RAISE EXCEPTION 'Cannot create user_secret for deleted user'; - END IF; + IF (TG_OP = 'UPDATE') THEN + reassigned := NEW.user_id IS DISTINCT FROM OLD.user_id; END IF; + PERFORM check_user_not_deleted( + NEW.user_id, TG_OP = 'INSERT' OR reassigned, TG_OP, 'user_secret', 'user_secret_user_deleted'); RETURN NEW; END; $$; @@ -1266,19 +1346,15 @@ $$; CREATE FUNCTION insert_user_skill_fail_if_user_deleted() RETURNS trigger LANGUAGE plpgsql AS $$ - +DECLARE + reassigned boolean := false; BEGIN - PERFORM 1 - FROM users - WHERE id = NEW.user_id - AND deleted = true - LIMIT 1; - IF FOUND THEN - RAISE EXCEPTION 'Cannot create user_skill for deleted user' - USING ERRCODE = 'check_violation', - CONSTRAINT = 'user_skill_user_deleted'; - END IF; - RETURN NEW; + IF (TG_OP = 'UPDATE') THEN + reassigned := NEW.user_id IS DISTINCT FROM OLD.user_id; + END IF; + PERFORM check_user_not_deleted( + NEW.user_id, TG_OP = 'INSERT' OR reassigned, TG_OP, 'user_skill', 'user_skill_user_deleted'); + RETURN NEW; END; $$; @@ -5225,8 +5301,16 @@ CREATE TRIGGER trigger_enforce_user_ai_budget_override_membership BEFORE INSERT CREATE TRIGGER trigger_insert_apikeys BEFORE INSERT ON api_keys FOR EACH ROW EXECUTE FUNCTION insert_apikey_fail_if_user_deleted(); +CREATE TRIGGER trigger_insert_group_members BEFORE INSERT OR UPDATE OF user_id ON group_members FOR EACH ROW EXECUTE FUNCTION fail_if_user_deleted('group_member', 'group_member_user_deleted'); + +CREATE TRIGGER trigger_insert_organization_members BEFORE INSERT OR UPDATE OF user_id ON organization_members FOR EACH ROW EXECUTE FUNCTION fail_if_user_deleted('organization_member', 'organization_member_user_deleted'); + CREATE TRIGGER trigger_insert_organization_system_roles AFTER INSERT ON organizations FOR EACH ROW EXECUTE FUNCTION insert_organization_system_roles(); +CREATE TRIGGER trigger_insert_user_ai_budget_overrides BEFORE INSERT OR UPDATE OF user_id ON user_ai_budget_overrides FOR EACH ROW EXECUTE FUNCTION fail_if_user_deleted('user_ai_budget_override', 'user_ai_budget_override_user_deleted'); + +CREATE TRIGGER trigger_insert_user_ai_provider_keys BEFORE INSERT OR UPDATE OF user_id ON user_ai_provider_keys FOR EACH ROW EXECUTE FUNCTION fail_if_user_deleted('user_ai_provider_key', 'user_ai_provider_key_user_deleted'); + CREATE TRIGGER trigger_nullify_next_start_at_on_workspace_autostart_modificati AFTER UPDATE ON workspaces FOR EACH ROW EXECUTE FUNCTION nullify_next_start_at_on_workspace_autostart_modification(); CREATE TRIGGER trigger_set_chat_message_revision_on_insert BEFORE INSERT ON chat_messages FOR EACH ROW EXECUTE FUNCTION set_chat_message_revision_before(); @@ -5235,6 +5319,8 @@ CREATE TRIGGER trigger_set_chat_message_revision_on_update BEFORE UPDATE ON chat CREATE TRIGGER trigger_sync_chat_retry_state BEFORE UPDATE OF retry_state, retry_state_version, generation_attempt ON chats FOR EACH ROW EXECUTE FUNCTION sync_chat_retry_state(); +CREATE TRIGGER trigger_update_apikeys_owner BEFORE UPDATE OF user_id ON api_keys FOR EACH ROW WHEN ((new.user_id IS DISTINCT FROM old.user_id)) EXECUTE FUNCTION insert_apikey_fail_if_user_deleted(); + CREATE TRIGGER trigger_update_chat_history_after_message_insert AFTER INSERT ON chat_messages REFERENCING NEW TABLE AS chat_message_history_new_rows FOR EACH STATEMENT EXECUTE FUNCTION update_chat_history_after_message_insert(); CREATE TRIGGER trigger_update_chat_history_after_message_update AFTER UPDATE ON chat_messages REFERENCING OLD TABLE AS chat_message_history_old_rows NEW TABLE AS chat_message_history_new_rows FOR EACH STATEMENT EXECUTE FUNCTION update_chat_history_after_message_update(); diff --git a/coderd/database/migrations/000591_lock_user_soft_delete_guards.down.sql b/coderd/database/migrations/000591_lock_user_soft_delete_guards.down.sql new file mode 100644 index 00000000000..d5b2cf0a019 --- /dev/null +++ b/coderd/database/migrations/000591_lock_user_soft_delete_guards.down.sql @@ -0,0 +1,74 @@ +DROP TRIGGER IF EXISTS trigger_insert_group_members ON group_members; +DROP TRIGGER IF EXISTS trigger_insert_user_ai_budget_overrides ON user_ai_budget_overrides; +DROP TRIGGER IF EXISTS trigger_insert_organization_members ON organization_members; +DROP TRIGGER IF EXISTS trigger_insert_user_ai_provider_keys ON user_ai_provider_keys; +DROP TRIGGER IF EXISTS trigger_update_apikeys_owner ON api_keys; +DROP FUNCTION IF EXISTS fail_if_user_deleted(); + +-- Restore the standalone per-table guard bodies this migration replaced +-- (as reproduced from the pre-migration dump.sql). +CREATE OR REPLACE FUNCTION insert_apikey_fail_if_user_deleted() RETURNS trigger + LANGUAGE plpgsql + AS $$ + +DECLARE +BEGIN + IF (NEW.user_id IS NOT NULL) THEN + IF (SELECT deleted FROM users WHERE id = NEW.user_id LIMIT 1) THEN + RAISE EXCEPTION 'Cannot create API key for deleted user'; + END IF; + END IF; + RETURN NEW; +END; +$$; + +CREATE OR REPLACE FUNCTION insert_user_links_fail_if_user_deleted() RETURNS trigger + LANGUAGE plpgsql + AS $$ + +DECLARE +BEGIN + IF (NEW.user_id IS NOT NULL) THEN + IF (SELECT deleted FROM users WHERE id = NEW.user_id LIMIT 1) THEN + RAISE EXCEPTION 'Cannot create user_link for deleted user'; + END IF; + END IF; + RETURN NEW; +END; +$$; + +CREATE OR REPLACE FUNCTION insert_user_secret_fail_if_user_deleted() RETURNS trigger + LANGUAGE plpgsql + AS $$ + +DECLARE +BEGIN + IF (NEW.user_id IS NOT NULL) THEN + IF (SELECT deleted FROM users WHERE id = NEW.user_id LIMIT 1) THEN + RAISE EXCEPTION 'Cannot create user_secret for deleted user'; + END IF; + END IF; + RETURN NEW; +END; +$$; + +CREATE OR REPLACE FUNCTION insert_user_skill_fail_if_user_deleted() RETURNS trigger + LANGUAGE plpgsql + AS $$ + +BEGIN + PERFORM 1 + FROM users + WHERE id = NEW.user_id + AND deleted = true + LIMIT 1; + IF FOUND THEN + RAISE EXCEPTION 'Cannot create user_skill for deleted user' + USING ERRCODE = 'check_violation', + CONSTRAINT = 'user_skill_user_deleted'; + END IF; + RETURN NEW; +END; +$$; + +DROP FUNCTION IF EXISTS check_user_not_deleted(uuid, boolean, text, text, text); diff --git a/coderd/database/migrations/000591_lock_user_soft_delete_guards.up.sql b/coderd/database/migrations/000591_lock_user_soft_delete_guards.up.sql new file mode 100644 index 00000000000..f3bd8631edc --- /dev/null +++ b/coderd/database/migrations/000591_lock_user_soft_delete_guards.up.sql @@ -0,0 +1,223 @@ +-- Close the insert-vs-soft-delete race for every table that +-- delete_deleted_user_resources deletes directly: an in-flight insert could +-- observe deleted = false, a concurrent soft-delete UPDATE (and its cleanup) +-- could commit, and the insert could then commit afterwards, resurrecting a +-- row for a soft-deleted user. For api_keys that resurrects a live session +-- token on an account the operator believes they deleted. +-- +-- The guard and lock-ordering rationale lives inside check_user_not_deleted() +-- below so it survives into dump.sql. This migration: +-- +-- 1. Replaces the bodies of the four per-table guard functions (api_keys, +-- user_links, user_secrets, user_skills) with delegation to one shared +-- check_user_not_deleted() function, so the lock and the operation gate +-- exist in exactly one place. CREATE OR REPLACE FUNCTION swaps the +-- bodies without touching the triggers: no DROP TRIGGER, no +-- ACCESS EXCLUSIVE lock on the hot tables during the upgrade. +-- 2. Adds guard triggers to the directly cleaned tables that had none +-- (user_ai_provider_keys, organization_members) plus +-- user_ai_budget_overrides and group_members, whose rows feed readers +-- that do not filter users.deleted (GetOverBudgetUsersPerGroup reads +-- override rows into a budget metric; GetAuthorizationUserRoles reads +-- group_members directly into rbac.Subject.Groups). CREATE TRIGGER +-- takes SHARE ROW EXCLUSIVE only (blocks writes briefly, never reads). +-- 3. Adds owner-reassignment coverage: every guarded table also checks +-- UPDATE ... SET user_id, so a live child row cannot be re-parented +-- onto a soft-deleted user. The users-row lock is taken exactly when +-- the row starts belonging to the target user (INSERT or owner +-- reassignment); same-owner updates keep the unlocked read. +-- +-- There are deliberately no backfill DELETE statements here: unbounded deletes inside +-- the single-transaction migration would hold locks fleet-wide during the +-- upgrade. Orphaned child rows of already-soft-deleted users are removed by +-- the idempotent dbpurge reaper (PurgeSoftDeletedUserResources), which runs +-- at startup and periodically; the triggers installed here guarantee no new +-- orphans are created once this migration commits. + +-- The shared soft-delete guard. take_lock selects the locking read (INSERT +-- and owner-reassignment paths); tg_operation feeds the error message; +-- display_name and deleted_constraint parameterize the raised error, with +-- the constraint name being the stable identifier API handlers match on. +CREATE FUNCTION check_user_not_deleted( + target_user_id uuid, + take_lock boolean, + tg_operation text, + display_name text, + deleted_constraint text +) RETURNS void + LANGUAGE plpgsql +AS $$ +DECLARE + user_deleted boolean; +BEGIN + -- Serialize child-table inserts against a concurrent user soft-delete: + -- an unlocked insert can read deleted = false, lose the race to the + -- soft-delete UPDATE and its cleanup, then commit a resurrected row. + -- FOR NO KEY UPDATE conflicts with the soft-delete UPDATE but not with + -- the FOR KEY SHARE locks that foreign-key validation takes on the + -- users row for other child tables. + -- + -- The lock is taken only when the row starts belonging to + -- target_user_id (INSERT, or UPDATE that reassigns the owner). On + -- same-owner UPDATE statements the unlocked read is kept: taking the users lock + -- there deadlocks against delete_deleted_user_resources (a multi-row + -- UPDATE or ON CONFLICT path can hold one child tuple and wait on + -- users while the cleanup holds users and waits on a child tuple), and + -- would serialize routine child updates on the hot users row. An + -- existing row is cleaned up by the soft-delete either way. + -- + -- The locking path imposes an ordering contract on writers: a + -- transaction that writes a guarded child row (INSERT, UPDATE, or + -- DELETE) and later inserts a guarded row for the same user must call + -- AcquireUserSoftDeleteGuardLock first, so its lock order (users, then + -- child rows) matches delete_deleted_user_resources. The same contract + -- covers the cap triggers' advisory locks (migration 000590): without + -- the users lock first, an update-then-insert writer can cycle with a + -- concurrent insert that holds the users lock and waits on the + -- advisory lock. coderd/database/user_soft_delete_guards_test.go + -- replays each known such path as a deterministic deadlock regression, + -- and the OAuth2 token exchange is driven through its real entry point. + -- + -- Isolation: the locking read is correct at READ COMMITTED, which every + -- production writer of the guarded tables uses. Under REPEATABLE READ + -- or SERIALIZABLE the lock wait fails with a serialization error + -- (40001) whenever any transaction committed an update to the users + -- row after the snapshot (users is written on ordinary browsing to + -- bump last_seen_at), and coderd does not retry 40001; do not wrap + -- guarded inserts in database.ReadModifyUpdate or other + -- stronger-isolation transactions. + IF take_lock THEN + SELECT deleted INTO user_deleted + FROM users + WHERE id = target_user_id + FOR NO KEY UPDATE; + ELSE + SELECT deleted INTO user_deleted + FROM users + WHERE id = target_user_id; + END IF; + IF (user_deleted) THEN + RAISE EXCEPTION 'Cannot % % for deleted user', + CASE WHEN tg_operation = 'INSERT' THEN 'create' ELSE 'modify' END, + display_name + USING ERRCODE = 'check_violation', + CONSTRAINT = deleted_constraint; + END IF; +END; +$$; + +-- The generic trigger form for tables whose triggers this migration +-- creates. TG_ARGV[0] is the display name for the error message, TG_ARGV[1] +-- the stable constraint name callers match on. +CREATE FUNCTION fail_if_user_deleted() RETURNS trigger + LANGUAGE plpgsql +AS $$ +DECLARE + reassigned boolean := false; +BEGIN + IF (TG_OP = 'UPDATE') THEN + reassigned := NEW.user_id IS DISTINCT FROM OLD.user_id; + -- Same-owner updates on the tables using this function are + -- filtered out by the triggers' UPDATE OF user_id column list; + -- reassigned still guards the no-op UPDATE ... SET user_id = user_id. + END IF; + PERFORM check_user_not_deleted( + NEW.user_id, TG_OP = 'INSERT' OR reassigned, TG_OP, TG_ARGV[0], TG_ARGV[1]); + RETURN NEW; +END; +$$; + +-- Delegate the four pre-existing per-table guard functions to the shared +-- check. Their triggers are untouched; the INSERT-only tables additionally +-- get an owner-reassignment trigger below. +CREATE OR REPLACE FUNCTION insert_apikey_fail_if_user_deleted() RETURNS trigger + LANGUAGE plpgsql +AS $$ +DECLARE + reassigned boolean := false; +BEGIN + IF (TG_OP = 'UPDATE') THEN + reassigned := NEW.user_id IS DISTINCT FROM OLD.user_id; + END IF; + PERFORM check_user_not_deleted( + NEW.user_id, TG_OP = 'INSERT' OR reassigned, TG_OP, 'API key', 'api_key_user_deleted'); + RETURN NEW; +END; +$$; + +CREATE OR REPLACE FUNCTION insert_user_links_fail_if_user_deleted() RETURNS trigger + LANGUAGE plpgsql +AS $$ +DECLARE + reassigned boolean := false; +BEGIN + IF (TG_OP = 'UPDATE') THEN + reassigned := NEW.user_id IS DISTINCT FROM OLD.user_id; + END IF; + PERFORM check_user_not_deleted( + NEW.user_id, TG_OP = 'INSERT' OR reassigned, TG_OP, 'user_link', 'user_link_user_deleted'); + RETURN NEW; +END; +$$; + +CREATE OR REPLACE FUNCTION insert_user_secret_fail_if_user_deleted() RETURNS trigger + LANGUAGE plpgsql +AS $$ +DECLARE + reassigned boolean := false; +BEGIN + IF (TG_OP = 'UPDATE') THEN + reassigned := NEW.user_id IS DISTINCT FROM OLD.user_id; + END IF; + PERFORM check_user_not_deleted( + NEW.user_id, TG_OP = 'INSERT' OR reassigned, TG_OP, 'user_secret', 'user_secret_user_deleted'); + RETURN NEW; +END; +$$; + +CREATE OR REPLACE FUNCTION insert_user_skill_fail_if_user_deleted() RETURNS trigger + LANGUAGE plpgsql +AS $$ +DECLARE + reassigned boolean := false; +BEGIN + IF (TG_OP = 'UPDATE') THEN + reassigned := NEW.user_id IS DISTINCT FROM OLD.user_id; + END IF; + PERFORM check_user_not_deleted( + NEW.user_id, TG_OP = 'INSERT' OR reassigned, TG_OP, 'user_skill', 'user_skill_user_deleted'); + RETURN NEW; +END; +$$; + +-- Owner-reassignment coverage for api_keys, whose existing trigger is +-- INSERT-only. The UPDATE OF user_id column list plus the WHEN clause keep +-- every ordinary api_keys update (last_used bumps on each authenticated +-- request) out of plpgsql entirely. +CREATE TRIGGER trigger_update_apikeys_owner + BEFORE UPDATE OF user_id ON api_keys + FOR EACH ROW + WHEN (NEW.user_id IS DISTINCT FROM OLD.user_id) +EXECUTE FUNCTION insert_apikey_fail_if_user_deleted(); + +-- Guards for the directly cleaned tables that had none, plus the two whose +-- rows feed deleted-user-blind readers (see the header). +CREATE TRIGGER trigger_insert_user_ai_provider_keys + BEFORE INSERT OR UPDATE OF user_id ON user_ai_provider_keys + FOR EACH ROW +EXECUTE FUNCTION fail_if_user_deleted('user_ai_provider_key', 'user_ai_provider_key_user_deleted'); + +CREATE TRIGGER trigger_insert_organization_members + BEFORE INSERT OR UPDATE OF user_id ON organization_members + FOR EACH ROW +EXECUTE FUNCTION fail_if_user_deleted('organization_member', 'organization_member_user_deleted'); + +CREATE TRIGGER trigger_insert_user_ai_budget_overrides + BEFORE INSERT OR UPDATE OF user_id ON user_ai_budget_overrides + FOR EACH ROW +EXECUTE FUNCTION fail_if_user_deleted('user_ai_budget_override', 'user_ai_budget_override_user_deleted'); + +CREATE TRIGGER trigger_insert_group_members + BEFORE INSERT OR UPDATE OF user_id ON group_members + FOR EACH ROW +EXECUTE FUNCTION fail_if_user_deleted('group_member', 'group_member_user_deleted'); diff --git a/coderd/database/migrations/migrate_test.go b/coderd/database/migrations/migrate_test.go index 9844d8ec4bd..3a354c1f477 100644 --- a/coderd/database/migrations/migrate_test.go +++ b/coderd/database/migrations/migrate_test.go @@ -3742,3 +3742,171 @@ func TestMigration000583ChatModelOverrideOrgScope(t *testing.T) { _, err = db.ExecContext(ctx, string(upSQL)) require.NoError(t, err) } + +func TestMigration000591LockUserSoftDeleteGuards(t *testing.T) { + t.Parallel() + if testing.Short() { + t.SkipNow() + } + + // The stepping constant is the version stepped up to before applying + // the tested migration (000591). + const migrationVersion = 590 + + sqlDB := testSQLDB(t) + + // Step up to migrationVersion. + next, err := migrations.Stepper(sqlDB) + require.NoError(t, err) + for { + version, more, err := next() + require.NoError(t, err) + if !more { + t.Fatalf("migration %d not found", migrationVersion) + } + if version == migrationVersion { + break + } + } + + ctx := testutil.Context(t, testutil.WaitLong) + now := time.Now().UTC().Truncate(time.Microsecond) + + liveUser := uuid.New() + doomedUser := uuid.New() + for i, id := range []uuid.UUID{liveUser, doomedUser} { + _, err = sqlDB.ExecContext(ctx, + `INSERT INTO users (id, username, email, hashed_password, created_at, updated_at, status, rbac_roles, login_type) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`, + id, fmt.Sprintf("guards-user-%d", i), fmt.Sprintf("guards-%d@test.com", i), []byte{}, now, now, "active", pq.StringArray{}, "password", + ) + require.NoError(t, err) + } + + orgID := uuid.New() + _, err = sqlDB.ExecContext(ctx, + `INSERT INTO organizations (id, name, display_name, description, icon, created_at, updated_at, is_default, default_org_member_roles) + VALUES ($1, 'guards-org', 'Guards Org', '', '', $2, $2, false, '{}')`, + orgID, now, + ) + require.NoError(t, err) + providerID := uuid.New() + _, err = sqlDB.ExecContext(ctx, + `INSERT INTO ai_providers (id, type, name, base_url) VALUES ($1, 'openai', 'guards-provider', 'https://example.com')`, + providerID, + ) + require.NoError(t, err) + groupID := uuid.New() + _, err = sqlDB.ExecContext(ctx, + `INSERT INTO groups (id, name, organization_id) VALUES ($1, 'guards-group', $2)`, + groupID, orgID, + ) + require.NoError(t, err) + + // One row per guarded table for both users. + for _, id := range []uuid.UUID{liveUser, doomedUser} { + _, err = sqlDB.ExecContext(ctx, + `INSERT INTO api_keys (id, hashed_secret, user_id, last_used, expires_at, created_at, updated_at, login_type, scopes, allow_list) + VALUES ($1, 'hash'::bytea, $2, $3, $3, $3, $3, 'password', '{}'::api_key_scope[], ARRAY['*'])`, + "key-"+id.String()[:13], id, now, + ) + require.NoError(t, err) + _, err = sqlDB.ExecContext(ctx, + `INSERT INTO user_links (user_id, login_type, linked_id) VALUES ($1, 'github', $2)`, + id, "link-"+id.String(), + ) + require.NoError(t, err) + _, err = sqlDB.ExecContext(ctx, + `INSERT INTO user_secrets (id, user_id, name, description, value, env_name) + VALUES ($1, $2, 'seed-secret', '', 'value', 'SEED_SECRET')`, + uuid.New(), id, + ) + require.NoError(t, err) + _, err = sqlDB.ExecContext(ctx, + `INSERT INTO user_skills (id, user_id, name, description, content) + VALUES ($1, $2, 'seed-skill', '', 'content')`, + uuid.New(), id, + ) + require.NoError(t, err) + _, err = sqlDB.ExecContext(ctx, + `INSERT INTO user_ai_provider_keys (id, user_id, ai_provider_id, api_key) + VALUES ($1, $2, $3, 'seed-key')`, + uuid.New(), id, providerID, + ) + require.NoError(t, err) + _, err = sqlDB.ExecContext(ctx, + `INSERT INTO organization_members (user_id, organization_id, created_at, updated_at) + VALUES ($1, $2, $3, $3)`, + id, orgID, now, + ) + require.NoError(t, err) + _, err = sqlDB.ExecContext(ctx, + `INSERT INTO group_members (user_id, group_id) VALUES ($1, $2)`, + id, groupID, + ) + require.NoError(t, err) + _, err = sqlDB.ExecContext(ctx, + `INSERT INTO user_ai_budget_overrides (user_id, group_id, spend_limit_micros) + VALUES ($1, $2, 1000000)`, + id, groupID, + ) + require.NoError(t, err) + } + + // Reproduce the race outcome the guards close: a user soft-deleted + // while their child rows survive. Suppressing the cleanup trigger + // simulates the insert committing after the soft-delete. + dbtestutil.SoftDeleteUserKeepingRows(ctx, t, sqlDB, doomedUser) + + countRows := func(table string, userID uuid.UUID) int { + var count int + //nolint:gosec // The table name comes from the fixed list below. + err := sqlDB.QueryRowContext(ctx, + `SELECT count(*) FROM `+table+` WHERE user_id = $1`, userID).Scan(&count) + require.NoError(t, err) + return count + } + guardedTables := []string{"api_keys", "user_links", "user_secrets", "user_skills", "user_ai_provider_keys", "organization_members", "group_members", "user_ai_budget_overrides"} + for _, table := range guardedTables { + require.Equal(t, 1, countRows(table, doomedUser), "pre-migration: %s row for the doomed user must exist", table) + } + + // Apply migration 000591. + version, more, err := next() + require.NoError(t, err) + require.True(t, more) + require.EqualValues(t, migrationVersion+1, version) + + // The migration deliberately has no backfill DELETEs (they would hold + // locks fleet-wide inside the single-transaction migration): existing + // orphans survive and are removed by the dbpurge reaper + // (PurgeSoftDeletedUserResources) instead. + for _, table := range guardedTables { + require.Equal(t, 1, countRows(table, doomedUser), "post-migration: %s orphan must survive (reaper owns cleanup)", table) + require.Equal(t, 1, countRows(table, liveUser), "post-migration: %s row for the live user must survive", table) + } + + // The guards are live: inserting for the doomed user fails with the + // stable constraint name, on both a pre-existing table (delegated + // function body) and a newly guarded one. + _, err = sqlDB.ExecContext(ctx, + `INSERT INTO api_keys (id, hashed_secret, user_id, last_used, expires_at, created_at, updated_at, login_type, scopes, allow_list) + VALUES ($1, 'hash'::bytea, $2, $3, $3, $3, $3, 'password', '{}'::api_key_scope[], ARRAY['*'])`, + "key2-"+doomedUser.String()[:12], doomedUser, now, + ) + require.Error(t, err) + require.True(t, database.IsCheckViolation(err, database.CheckAPIKeyUserDeleted), "expected api_key_user_deleted, got: %v", err) + _, err = sqlDB.ExecContext(ctx, + `INSERT INTO group_members (user_id, group_id) VALUES ($1, $2)`, + doomedUser, groupID, + ) + require.Error(t, err) + require.True(t, database.IsCheckViolation(err, database.CheckGroupMemberUserDeleted), "expected group_member_user_deleted, got: %v", err) + + // The owner-reassignment leg is live: re-parenting the live user's + // api_keys row onto the doomed user fails. + _, err = sqlDB.ExecContext(ctx, + `UPDATE api_keys SET user_id = $1 WHERE user_id = $2`, doomedUser, liveUser) + require.Error(t, err) + require.True(t, database.IsCheckViolation(err, database.CheckAPIKeyUserDeleted), "expected api_key_user_deleted on reassignment, got: %v", err) +} diff --git a/coderd/database/querier.go b/coderd/database/querier.go index 88e86e97304..a1808bfe433 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -40,6 +40,25 @@ type sqlcQuerier interface { // https://www.postgresql.org/docs/9.5/sql-select.html#SQL-FOR-UPDATE-SHARE AcquireProvisionerJob(ctx context.Context, arg AcquireProvisionerJobParams) (ProvisionerJob, error) AcquireStaleChatDiffStatuses(ctx context.Context, limitVal int32) ([]AcquireStaleChatDiffStatusesRow, error) + // Acquires the users-row lock that the soft-delete guard triggers take on + // child-table inserts and owner reassignments (see check_user_not_deleted in + // migration 000591). Any transaction that writes a guarded child row + // (INSERT, UPDATE, or DELETE) and later inserts a guarded row for the same + // user (for example the OAuth2 token exchange, which replaces api_keys rows) + // must call this first so its lock order (users first, then child rows) + // matches delete_deleted_user_resources and cannot deadlock with a + // concurrent user soft-delete. + // Must run inside the transaction that performs the child writes: outside + // one, the lock is released at the implicit statement commit and protects + // nothing. + // The query is dbauthz-authorized as a system primitive: callers must run + // it under a system-authorized context (dbauthz.AsSystemRestricted at the + // call site when the surrounding request runs as an ordinary user). + // Returns sql.ErrNoRows when the user does not exist. It does NOT detect a + // wrong id that belongs to some other real user: that row is locked and the + // transaction proceeds with the guard defeated, so callers must derive the + // id from the same variable the child writes use. + AcquireUserSoftDeleteGuardLock(ctx context.Context, userID uuid.UUID) (uuid.UUID, error) // Bumps the workspace deadline by the template's configured "activity_bump" // duration (default 1h). If the workspace bump will cross an autostart // threshold, then the bump is autostart + TTL. This is the deadline behavior if @@ -1367,6 +1386,19 @@ type sqlcQuerier interface { // sequence, so this is acceptable. PinChatByID(ctx context.Context, id uuid.UUID) error PopNextQueuedMessage(ctx context.Context, chatID uuid.UUID) (ChatQueuedMessage, error) + // Deletes child rows belonging to already-soft-deleted users. The guard + // triggers (migration 000591) prevent new rows from being created for + // soft-deleted users, and delete_deleted_user_resources cleans rows at + // soft-delete time; this reaper removes what predates both (legacy orphans + // from before cleanup coverage, and race products from before the guards). + // It deletes in the same table order as delete_deleted_user_resources to + // minimize deadlock exposure against a concurrent soft-delete; a lost + // deadlock surfaces as a failed purge cycle and is retried on the next one. + // group_members and user_ai_budget_overrides rows are normally wiped + // transitively (BEFORE DELETE triggers on organization_members); the direct + // deletes catch rows orphaned after the user's organization_members rows + // were already cleaned up. + PurgeSoftDeletedUserResources(ctx context.Context) error ReduceWorkspaceAgentShareLevelToAuthenticatedByTemplate(ctx context.Context, templateID uuid.UUID) error RegisterWorkspaceProxy(ctx context.Context, arg RegisterWorkspaceProxyParams) (WorkspaceProxy, error) ReindexStaleChatMessagesSearchTsv(ctx context.Context, batchSize int32) (int64, error) diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index ccbd4a5af2a..c04209d6140 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -31296,6 +31296,37 @@ func (q *sqlQuerier) UpdateUserSkillByUserIDAndName(ctx context.Context, arg Upd return i, err } +const acquireUserSoftDeleteGuardLock = `-- name: AcquireUserSoftDeleteGuardLock :one +SELECT id FROM users +WHERE id = $1 +FOR NO KEY UPDATE +` + +// Acquires the users-row lock that the soft-delete guard triggers take on +// child-table inserts and owner reassignments (see check_user_not_deleted in +// migration 000591). Any transaction that writes a guarded child row +// (INSERT, UPDATE, or DELETE) and later inserts a guarded row for the same +// user (for example the OAuth2 token exchange, which replaces api_keys rows) +// must call this first so its lock order (users first, then child rows) +// matches delete_deleted_user_resources and cannot deadlock with a +// concurrent user soft-delete. +// Must run inside the transaction that performs the child writes: outside +// one, the lock is released at the implicit statement commit and protects +// nothing. +// The query is dbauthz-authorized as a system primitive: callers must run +// it under a system-authorized context (dbauthz.AsSystemRestricted at the +// call site when the surrounding request runs as an ordinary user). +// Returns sql.ErrNoRows when the user does not exist. It does NOT detect a +// wrong id that belongs to some other real user: that row is locked and the +// transaction proceeds with the guard defeated, so callers must derive the +// id from the same variable the child writes use. +func (q *sqlQuerier) AcquireUserSoftDeleteGuardLock(ctx context.Context, userID uuid.UUID) (uuid.UUID, error) { + row := q.db.QueryRowContext(ctx, acquireUserSoftDeleteGuardLock, userID) + var id uuid.UUID + err := row.Scan(&id) + return id, err +} + const allUserIDs = `-- name: AllUserIDs :many SELECT DISTINCT id FROM USERS WHERE CASE WHEN $1::bool THEN TRUE ELSE is_system = false END @@ -32243,6 +32274,44 @@ func (q *sqlQuerier) ListUserChatCompactionThresholds(ctx context.Context, userI return items, nil } +const purgeSoftDeletedUserResources = `-- name: PurgeSoftDeletedUserResources :exec +WITH doomed_users AS ( + SELECT id FROM users WHERE deleted +), delete_api_keys AS ( + DELETE FROM api_keys WHERE user_id IN (SELECT id FROM doomed_users) +), delete_user_links AS ( + DELETE FROM user_links WHERE user_id IN (SELECT id FROM doomed_users) +), delete_user_secrets AS ( + DELETE FROM user_secrets WHERE user_id IN (SELECT id FROM doomed_users) +), delete_user_ai_provider_keys AS ( + DELETE FROM user_ai_provider_keys WHERE user_id IN (SELECT id FROM doomed_users) +), delete_organization_members AS ( + DELETE FROM organization_members WHERE user_id IN (SELECT id FROM doomed_users) +), delete_user_skills AS ( + DELETE FROM user_skills WHERE user_id IN (SELECT id FROM doomed_users) +), delete_group_members AS ( + DELETE FROM group_members WHERE user_id IN (SELECT id FROM doomed_users) +) +DELETE FROM user_ai_budget_overrides WHERE user_id IN (SELECT id FROM doomed_users) +` + +// Deletes child rows belonging to already-soft-deleted users. The guard +// triggers (migration 000591) prevent new rows from being created for +// soft-deleted users, and delete_deleted_user_resources cleans rows at +// soft-delete time; this reaper removes what predates both (legacy orphans +// from before cleanup coverage, and race products from before the guards). +// It deletes in the same table order as delete_deleted_user_resources to +// minimize deadlock exposure against a concurrent soft-delete; a lost +// deadlock surfaces as a failed purge cycle and is retried on the next one. +// group_members and user_ai_budget_overrides rows are normally wiped +// transitively (BEFORE DELETE triggers on organization_members); the direct +// deletes catch rows orphaned after the user's organization_members rows +// were already cleaned up. +func (q *sqlQuerier) PurgeSoftDeletedUserResources(ctx context.Context) error { + _, err := q.db.ExecContext(ctx, purgeSoftDeletedUserResources) + return err +} + const updateInactiveUsersToDormant = `-- name: UpdateInactiveUsersToDormant :many UPDATE users diff --git a/coderd/database/queries/users.sql b/coderd/database/queries/users.sql index fe6bbd291e6..2fb64a8ce3f 100644 --- a/coderd/database/queries/users.sql +++ b/coderd/database/queries/users.sql @@ -742,3 +742,58 @@ WHERE SELECT * FROM users WHERE id = @id::uuid; + +-- Acquires the users-row lock that the soft-delete guard triggers take on +-- child-table inserts and owner reassignments (see check_user_not_deleted in +-- migration 000591). Any transaction that writes a guarded child row +-- (INSERT, UPDATE, or DELETE) and later inserts a guarded row for the same +-- user (for example the OAuth2 token exchange, which replaces api_keys rows) +-- must call this first so its lock order (users first, then child rows) +-- matches delete_deleted_user_resources and cannot deadlock with a +-- concurrent user soft-delete. +-- Must run inside the transaction that performs the child writes: outside +-- one, the lock is released at the implicit statement commit and protects +-- nothing. +-- The query is dbauthz-authorized as a system primitive: callers must run +-- it under a system-authorized context (dbauthz.AsSystemRestricted at the +-- call site when the surrounding request runs as an ordinary user). +-- Returns sql.ErrNoRows when the user does not exist. It does NOT detect a +-- wrong id that belongs to some other real user: that row is locked and the +-- transaction proceeds with the guard defeated, so callers must derive the +-- id from the same variable the child writes use. +-- name: AcquireUserSoftDeleteGuardLock :one +SELECT id FROM users +WHERE id = @user_id +FOR NO KEY UPDATE; + +-- Deletes child rows belonging to already-soft-deleted users. The guard +-- triggers (migration 000591) prevent new rows from being created for +-- soft-deleted users, and delete_deleted_user_resources cleans rows at +-- soft-delete time; this reaper removes what predates both (legacy orphans +-- from before cleanup coverage, and race products from before the guards). +-- It deletes in the same table order as delete_deleted_user_resources to +-- minimize deadlock exposure against a concurrent soft-delete; a lost +-- deadlock surfaces as a failed purge cycle and is retried on the next one. +-- group_members and user_ai_budget_overrides rows are normally wiped +-- transitively (BEFORE DELETE triggers on organization_members); the direct +-- deletes catch rows orphaned after the user's organization_members rows +-- were already cleaned up. +-- name: PurgeSoftDeletedUserResources :exec +WITH doomed_users AS ( + SELECT id FROM users WHERE deleted +), delete_api_keys AS ( + DELETE FROM api_keys WHERE user_id IN (SELECT id FROM doomed_users) +), delete_user_links AS ( + DELETE FROM user_links WHERE user_id IN (SELECT id FROM doomed_users) +), delete_user_secrets AS ( + DELETE FROM user_secrets WHERE user_id IN (SELECT id FROM doomed_users) +), delete_user_ai_provider_keys AS ( + DELETE FROM user_ai_provider_keys WHERE user_id IN (SELECT id FROM doomed_users) +), delete_organization_members AS ( + DELETE FROM organization_members WHERE user_id IN (SELECT id FROM doomed_users) +), delete_user_skills AS ( + DELETE FROM user_skills WHERE user_id IN (SELECT id FROM doomed_users) +), delete_group_members AS ( + DELETE FROM group_members WHERE user_id IN (SELECT id FROM doomed_users) +) +DELETE FROM user_ai_budget_overrides WHERE user_id IN (SELECT id FROM doomed_users); diff --git a/coderd/database/user_soft_delete_guards_test.go b/coderd/database/user_soft_delete_guards_test.go new file mode 100644 index 00000000000..f07f8785183 --- /dev/null +++ b/coderd/database/user_soft_delete_guards_test.go @@ -0,0 +1,690 @@ +package database_test + +import ( + "context" + "database/sql" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + + "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/database/dbgen" + "github.com/coder/coder/v2/coderd/database/dbtestutil" + "github.com/coder/coder/v2/testutil" +) + +// TestSoftDeleteGuardWinsConcurrentInsert verifies that all eight soft-delete +// guard triggers serialize against a concurrent user soft-delete via the +// parent-row lock added in migration 000591: the insert blocks on the locked +// users row and, once the soft-delete commits, fails with the guard's +// constraint instead of resurrecting a row for the deleted user. Each +// subtest also pins its database.Check* constant against the live trigger by +// matching the raised constraint name. +func TestSoftDeleteGuardWinsConcurrentInsert(t *testing.T) { + t.Parallel() + if testing.Short() { + t.SkipNow() + } + + db, _, sqlDB := dbtestutil.NewDBWithSQLDB(t) + + // Shared parents for the FK-bearing tables. + org := dbgen.Organization(t, db, database.Organization{}) + provider := dbgen.AIProvider(t, db, database.AIProvider{}) + group := dbgen.Group(t, db, database.Group{OrganizationID: org.ID}) + + testCases := []struct { + name string + table string + constraint database.CheckConstraint + // seed prepares rows the insert depends on (memberships etc.). + seed func(ctx context.Context, t *testing.T, userID uuid.UUID) + insert func(userID uuid.UUID) stmt + }{ + { + name: "APIKey", + table: "api_keys", + constraint: database.CheckAPIKeyUserDeleted, + insert: func(userID uuid.UUID) stmt { + return stmt{` + INSERT INTO api_keys (id, hashed_secret, user_id, last_used, expires_at, created_at, updated_at, login_type, scopes, allow_list) + VALUES ($1, 'race-hash'::bytea, $2, now(), now() + interval '1 hour', now(), now(), 'password', '{}'::api_key_scope[], ARRAY['*']) + `, []any{uuid.NewString(), userID}} + }, + }, + { + name: "UserLink", + table: "user_links", + constraint: database.CheckUserLinkUserDeleted, + insert: func(userID uuid.UUID) stmt { + return stmt{` + INSERT INTO user_links (user_id, login_type, linked_id) + VALUES ($1, 'github', 'race-link') + `, []any{userID}} + }, + }, + { + name: "UserSecret", + table: "user_secrets", + constraint: database.CheckUserSecretUserDeleted, + insert: func(userID uuid.UUID) stmt { + return stmt{` + INSERT INTO user_secrets (id, user_id, name, description, value, env_name) + VALUES ($1, $2, 'race-secret', '', 'value', 'RACE_SECRET') + `, []any{uuid.New(), userID}} + }, + }, + { + name: "UserSkill", + table: "user_skills", + constraint: database.CheckUserSkillUserDeleted, + insert: func(userID uuid.UUID) stmt { + return stmt{` + INSERT INTO user_skills (id, user_id, name, description, content) + VALUES ($1, $2, 'race-skill', '', 'content') + `, []any{uuid.New(), userID}} + }, + }, + { + name: "UserAIProviderKey", + table: "user_ai_provider_keys", + constraint: database.CheckUserAIProviderKeyUserDeleted, + insert: func(userID uuid.UUID) stmt { + return stmt{` + INSERT INTO user_ai_provider_keys (id, user_id, ai_provider_id, api_key) + VALUES ($1, $2, $3, 'race-key') + `, []any{uuid.New(), userID, provider.ID}} + }, + }, + { + name: "OrganizationMember", + table: "organization_members", + constraint: database.CheckOrganizationMemberUserDeleted, + insert: func(userID uuid.UUID) stmt { + return stmt{` + INSERT INTO organization_members (user_id, organization_id, created_at, updated_at) + VALUES ($1, $2, now(), now()) + `, []any{userID, org.ID}} + }, + }, + { + name: "GroupMember", + table: "group_members", + constraint: database.CheckGroupMemberUserDeleted, + insert: func(userID uuid.UUID) stmt { + return stmt{` + INSERT INTO group_members (user_id, group_id) VALUES ($1, $2) + `, []any{userID, group.ID}} + }, + }, + { + name: "UserAIBudgetOverride", + table: "user_ai_budget_overrides", + constraint: database.CheckUserAIBudgetOverrideUserDeleted, + // The membership trigger on this table fires before the guard + // (name order) and rejects non-members outright, so the racing + // user must be an org and group member for the insert to reach + // the guard's users lock. + seed: func(ctx context.Context, t *testing.T, userID uuid.UUID) { + _, err := sqlDB.ExecContext(ctx, ` + INSERT INTO organization_members (user_id, organization_id, created_at, updated_at) + VALUES ($1, $2, now(), now()) + `, userID, org.ID) + require.NoError(t, err) + _, err = sqlDB.ExecContext(ctx, + `INSERT INTO group_members (user_id, group_id) VALUES ($1, $2)`, + userID, group.ID) + require.NoError(t, err) + }, + insert: func(userID uuid.UUID) stmt { + return stmt{` + INSERT INTO user_ai_budget_overrides (user_id, group_id, spend_limit_micros) + VALUES ($1, $2, 0) + `, []any{userID, group.ID}} + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + ctx := testutil.Context(t, testutil.WaitLong) + user := dbgen.User(t, db, database.User{}) + if tc.seed != nil { + tc.seed(ctx, t, user.ID) + } + + // Hold the same lock the guard trigger takes so the insert + // blocks, then soft-delete before releasing it. + err := runLockRace(ctx, t, sqlDB, + []stmt{{`SELECT id FROM users WHERE id = $1 FOR NO KEY UPDATE`, []any{user.ID}}}, + tc.insert(user.ID), + []stmt{{`UPDATE users SET deleted = true WHERE id = $1`, []any{user.ID}}}, + ) + require.Error(t, err) + require.True(t, database.IsCheckViolation(err, tc.constraint), "expected constraint %q, got: %v", tc.constraint, err) + + var remaining int + //nolint:gosec // The table name comes from the test case definition. + err = sqlDB.QueryRowContext(ctx, + `SELECT count(*) FROM `+tc.table+` WHERE user_id = $1`, user.ID, + ).Scan(&remaining) + require.NoError(t, err) + require.Zero(t, remaining, "no rows may survive for the soft-deleted user") + }) + } +} + +// TestSoftDeleteGuardBlocksOwnerReassignment pins the UPDATE ... SET user_id +// leg added by migration 000591: re-parenting a live child row onto a +// soft-deleted user is rejected, both when the target is already deleted and +// when the soft-delete races the reassignment (the reassignment takes the +// same users-row lock as an insert). api_keys covers the dedicated +// owner-reassignment trigger on a previously INSERT-only table; +// user_ai_provider_keys covers the shared fail_if_user_deleted form. +func TestSoftDeleteGuardBlocksOwnerReassignment(t *testing.T) { + t.Parallel() + if testing.Short() { + t.SkipNow() + } + + db, _, sqlDB := dbtestutil.NewDBWithSQLDB(t) + provider := dbgen.AIProvider(t, db, database.AIProvider{}) + + insertAPIKey := func(ctx context.Context, t *testing.T, userID uuid.UUID) string { + id := uuid.NewString() + _, err := sqlDB.ExecContext(ctx, ` + INSERT INTO api_keys (id, hashed_secret, user_id, last_used, expires_at, created_at, updated_at, login_type, scopes, allow_list) + VALUES ($1, 'reassign-hash'::bytea, $2, now(), now() + interval '1 hour', now(), now(), 'password', '{}'::api_key_scope[], ARRAY['*']) + `, id, userID) + require.NoError(t, err) + return id + } + + t.Run("APIKeyOntoDeletedUser", func(t *testing.T) { + t.Parallel() + ctx := testutil.Context(t, testutil.WaitLong) + owner := dbgen.User(t, db, database.User{}) + doomed := dbgen.User(t, db, database.User{}) + keyID := insertAPIKey(ctx, t, owner.ID) + + _, err := sqlDB.ExecContext(ctx, + `UPDATE users SET deleted = true WHERE id = $1`, doomed.ID) + require.NoError(t, err) + + _, err = sqlDB.ExecContext(ctx, + `UPDATE api_keys SET user_id = $1 WHERE id = $2`, doomed.ID, keyID) + require.Error(t, err, "re-parenting a live api_keys row onto a deleted user must fail") + require.True(t, database.IsCheckViolation(err, database.CheckAPIKeyUserDeleted), + "expected constraint %q, got: %v", database.CheckAPIKeyUserDeleted, err) + + // Reassignment onto a live user succeeds. + liveTarget := dbgen.User(t, db, database.User{}) + _, err = sqlDB.ExecContext(ctx, + `UPDATE api_keys SET user_id = $1 WHERE id = $2`, liveTarget.ID, keyID) + require.NoError(t, err, "re-parenting onto a live user must succeed") + }) + + t.Run("AIProviderKeyOntoDeletedUser", func(t *testing.T) { + t.Parallel() + ctx := testutil.Context(t, testutil.WaitLong) + owner := dbgen.User(t, db, database.User{}) + doomed := dbgen.User(t, db, database.User{}) + rowID := uuid.New() + _, err := sqlDB.ExecContext(ctx, ` + INSERT INTO user_ai_provider_keys (id, user_id, ai_provider_id, api_key) + VALUES ($1, $2, $3, 'reassign-key') + `, rowID, owner.ID, provider.ID) + require.NoError(t, err) + + _, err = sqlDB.ExecContext(ctx, + `UPDATE users SET deleted = true WHERE id = $1`, doomed.ID) + require.NoError(t, err) + + _, err = sqlDB.ExecContext(ctx, + `UPDATE user_ai_provider_keys SET user_id = $1 WHERE id = $2`, doomed.ID, rowID) + require.Error(t, err, "re-parenting a provider key row onto a deleted user must fail") + require.True(t, database.IsCheckViolation(err, database.CheckUserAIProviderKeyUserDeleted), + "expected constraint %q, got: %v", database.CheckUserAIProviderKeyUserDeleted, err) + }) + + // The reassignment leg takes the same users-row lock as the INSERT leg, + // so it loses a race against a concurrent soft-delete instead of + // re-parenting onto a user whose soft-delete commits first. + t.Run("APIKeyLosesSoftDeleteRace", func(t *testing.T) { + t.Parallel() + ctx := testutil.Context(t, testutil.WaitLong) + owner := dbgen.User(t, db, database.User{}) + target := dbgen.User(t, db, database.User{}) + keyID := insertAPIKey(ctx, t, owner.ID) + + err := runLockRace(ctx, t, sqlDB, + []stmt{{`SELECT id FROM users WHERE id = $1 FOR NO KEY UPDATE`, []any{target.ID}}}, + stmt{`UPDATE api_keys SET user_id = $1 WHERE id = $2`, []any{target.ID, keyID}}, + []stmt{{`UPDATE users SET deleted = true WHERE id = $1`, []any{target.ID}}}, + ) + require.Error(t, err) + require.True(t, database.IsCheckViolation(err, database.CheckAPIKeyUserDeleted), + "expected constraint %q, got: %v", database.CheckAPIKeyUserDeleted, err) + }) +} + +// TestSoftDeleteGuardUpdatePathTakesNoUserLock pins the lock gates: no +// soft-delete guard takes a users-row lock on a same-owner UPDATE, so routine +// child-row updates proceed while the users row is locked. Without the gates +// this would block here and could deadlock in production against +// delete_deleted_user_resources. api_keys covers the hottest path (the +// last_used bump on every authenticated request), which the trigger's +// UPDATE OF user_id column list keeps out of plpgsql entirely. +func TestSoftDeleteGuardUpdatePathTakesNoUserLock(t *testing.T) { + t.Parallel() + if testing.Short() { + t.SkipNow() + } + + db, _, sqlDB := dbtestutil.NewDBWithSQLDB(t) + ctx := testutil.Context(t, testutil.WaitLong) + user := dbgen.User(t, db, database.User{}) + + keyID := uuid.NewString() + _, err := sqlDB.ExecContext(ctx, ` + INSERT INTO api_keys (id, hashed_secret, user_id, last_used, expires_at, created_at, updated_at, login_type, scopes, allow_list) + VALUES ($1, 'gate-hash'::bytea, $2, now(), now() + interval '1 hour', now(), now(), 'password', '{}'::api_key_scope[], ARRAY['*']) + `, keyID, user.ID) + require.NoError(t, err) + _, err = sqlDB.ExecContext(ctx, ` + INSERT INTO user_skills (id, user_id, name, description, content) + VALUES ($1, $2, 'gate-skill', '', 'content') + `, uuid.New(), user.ID) + require.NoError(t, err) + _, err = sqlDB.ExecContext(ctx, ` + INSERT INTO user_links (user_id, login_type, linked_id) + VALUES ($1, 'github', 'gate-link') + `, user.ID) + require.NoError(t, err) + _, err = sqlDB.ExecContext(ctx, ` + INSERT INTO user_secrets (id, user_id, name, description, value, env_name) + VALUES ($1, $2, 'gate-secret', '', 'value', 'GATE_SECRET') + `, uuid.New(), user.ID) + require.NoError(t, err) + + lockTx, err := sqlDB.BeginTx(ctx, nil) + require.NoError(t, err) + defer func() { _ = lockTx.Rollback() }() + var lockedUserID uuid.UUID + err = lockTx.QueryRowContext(ctx, + `SELECT id FROM users WHERE id = $1 FOR NO KEY UPDATE`, user.ID, + ).Scan(&lockedUserID) + require.NoError(t, err) + require.Equal(t, user.ID, lockedUserID) + + // All updates must complete while the users row is locked; blocking + // here would mean a trigger locked the parent on the same-owner UPDATE + // path, so the lock_timeout turns a missing gate into a failure. + updateConn := lockTimeoutConn(ctx, t, sqlDB, "5s") + + _, err = updateConn.ExecContext(ctx, + `UPDATE api_keys SET last_used = now() WHERE id = $1`, keyID) + require.NoError(t, err) + _, err = updateConn.ExecContext(ctx, + `UPDATE user_skills SET description = 'edited' WHERE user_id = $1`, user.ID) + require.NoError(t, err) + _, err = updateConn.ExecContext(ctx, + `UPDATE user_links SET linked_id = 'edited' WHERE user_id = $1`, user.ID) + require.NoError(t, err) + _, err = updateConn.ExecContext(ctx, + `UPDATE user_secrets SET description = 'edited' WHERE user_id = $1`, user.ID) + require.NoError(t, err) +} + +// TestSoftDeleteGuardTriggerOrder pins the trigger firing order the advisory +// cap locks depend on: BEFORE ROW triggers fire in name order, and the +// soft-delete guards (users lock) must fire before the zz_-prefixed cap +// triggers (advisory lock). A transaction that held the advisory lock while +// waiting on the users lock could cycle with an UPDATE-path advisory waiter +// and the soft-delete cleanup. Both the name order and the BEFORE ROW timing +// are asserted: an AFTER trigger sorts identically but fires after the cap +// trigger, silently inverting the real order. +func TestSoftDeleteGuardTriggerOrder(t *testing.T) { + t.Parallel() + if testing.Short() { + t.SkipNow() + } + + _, _, sqlDB := dbtestutil.NewDBWithSQLDB(t) + ctx := testutil.Context(t, testutil.WaitShort) + + for _, tc := range []struct { + table string + guard string + capName string + }{ + {"user_secrets", "trigger_upsert_user_secrets", "trigger_zz_user_secrets_per_user_limits"}, + {"user_skills", "trigger_upsert_user_skills", "trigger_zz_user_skills_per_user_limit"}, + } { + // Fetch the two triggers from the live catalog in firing (name) + // order, so the assertion derives from the database rather than + // comparing the test's own literals. tgtype bit 0 is ROW, bit 1 + // is BEFORE (an unset bit 1 with bit 6 unset means AFTER). + rows, err := sqlDB.QueryContext(ctx, ` + SELECT tgname, (tgtype::int & 1) = 1 AS is_row, (tgtype::int & 2) = 2 AS is_before + FROM pg_trigger + WHERE tgrelid = $1::regclass AND NOT tgisinternal AND tgname IN ($2, $3) + ORDER BY tgname + `, tc.table, tc.guard, tc.capName) + require.NoError(t, err) + var firingOrder []string + for rows.Next() { + var name string + var isRow, isBefore bool + require.NoError(t, rows.Scan(&name, &isRow, &isBefore)) + require.True(t, isRow, "%s: %s must be a FOR EACH ROW trigger", tc.table, name) + require.True(t, isBefore, "%s: %s must be a BEFORE trigger for name order to equal firing order", tc.table, name) + firingOrder = append(firingOrder, name) + } + require.NoError(t, rows.Err()) + require.NoError(t, rows.Close()) + require.Equal(t, []string{tc.guard, tc.capName}, firingOrder, + "%s: both triggers must exist and the guard must sort (and therefore fire) before the cap trigger", tc.table) + } +} + +// lockTimeoutConn returns a dedicated connection whose lock_timeout bounds +// every lock wait, so a statement that unexpectedly blocks fails the test +// instead of waiting for the shared context deadline to release the lock +// (which would let the statement succeed and mask the regression). The +// timeout is RESET before the connection returns to the pool: database/sql +// does not reset session state, and cleanups run LIFO, so the RESET runs +// before Close. +func lockTimeoutConn(ctx context.Context, t *testing.T, sqlDB *sql.DB, timeout string) *sql.Conn { + t.Helper() + conn, err := sqlDB.Conn(ctx) + require.NoError(t, err) + t.Cleanup(func() { _ = conn.Close() }) + _, err = conn.ExecContext(ctx, `SET lock_timeout = '`+timeout+`'`) + require.NoError(t, err) + t.Cleanup(func() { _, _ = conn.ExecContext(context.Background(), `RESET lock_timeout`) }) + return conn +} + +// runGuardedWriteRace deterministically replays a delete-or-update-then-insert +// transaction against a concurrent user soft-delete: +// +// 1. An outside transaction locks the child row the app transaction will +// touch, so the app transaction blocks mid-flight while already holding +// the users lock it took first (mirroring AcquireUserSoftDeleteGuardLock). +// 2. The soft-delete starts and queues behind the app transaction's users +// lock instead of interleaving into a lock-order inversion. +// 3. The outside lock is released; the app transaction finishes cleanly and +// the soft-delete then runs its cleanup. +// +// Remove the users-lock SELECT at the top of the app transaction below (the +// statement mirroring AcquireUserSoftDeleteGuardLock) and the replay +// deadlocks: the soft-delete's cleanup waits on the child row while the app +// transaction waits on the users row (SQLSTATE 40P01). +func runGuardedWriteRace(ctx context.Context, t *testing.T, sqlDB *sql.DB, userID uuid.UUID, childLock stmt, appStmts []stmt) { + t.Helper() + + outsideTx, err := sqlDB.BeginTx(ctx, nil) + require.NoError(t, err) + released := false + t.Cleanup(func() { + if !released { + _ = outsideTx.Rollback() + } + }) + _, err = outsideTx.ExecContext(ctx, childLock.sql, childLock.args...) + require.NoError(t, err) + + appConn, err := sqlDB.Conn(ctx) + require.NoError(t, err) + t.Cleanup(func() { _ = appConn.Close() }) + var appPID int + require.NoError(t, appConn.QueryRowContext(ctx, `SELECT pg_backend_pid()`).Scan(&appPID)) + appResult := make(chan error, 1) + go func() { + appResult <- func() error { + tx, err := appConn.BeginTx(ctx, nil) + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + var lockedID uuid.UUID + err = tx.QueryRowContext(ctx, + `SELECT id FROM users WHERE id = $1 FOR NO KEY UPDATE`, userID, + ).Scan(&lockedID) + if err != nil { + return err + } + for _, s := range appStmts { + if _, err := tx.ExecContext(ctx, s.sql, s.args...); err != nil { + return err + } + } + return tx.Commit() + }() + }() + waitForBackendBlocked(ctx, t, sqlDB, appPID) + + deleteConn, err := sqlDB.Conn(ctx) + require.NoError(t, err) + t.Cleanup(func() { _ = deleteConn.Close() }) + var deletePID int + require.NoError(t, deleteConn.QueryRowContext(ctx, `SELECT pg_backend_pid()`).Scan(&deletePID)) + deleteResult := make(chan error, 1) + go func() { + _, err := deleteConn.ExecContext(ctx, `UPDATE users SET deleted = true WHERE id = $1`, userID) + deleteResult <- err + }() + waitForBackendBlocked(ctx, t, sqlDB, deletePID) + + require.NoError(t, outsideTx.Rollback()) + released = true + + select { + case err := <-appResult: + require.NoError(t, err, "the app transaction must finish without deadlocking") + case <-ctx.Done(): + t.Fatalf("app transaction did not finish: %v", ctx.Err()) + } + select { + case err := <-deleteResult: + require.NoError(t, err, "the soft-delete must finish without deadlocking") + case <-ctx.Done(): + t.Fatalf("soft-delete did not finish: %v", ctx.Err()) + } +} + +// TestSoftDeleteGuardLockOrderPaths is the per-path deadlock-regression suite +// for the ordering contract documented on AcquireUserSoftDeleteGuardLock: +// each subtest mirrors the exact statement order of one Go transaction that +// locks a guarded child row and later inserts one, and fails with a deadlock +// if the users lock is not taken first. +func TestSoftDeleteGuardLockOrderPaths(t *testing.T) { + t.Parallel() + if testing.Short() { + t.SkipNow() + } + + db, _, sqlDB := dbtestutil.NewDBWithSQLDB(t) + org := dbgen.Organization(t, db, database.Organization{}) + + insertAPIKey := func(userID uuid.UUID) stmt { + return stmt{` + INSERT INTO api_keys (id, hashed_secret, user_id, last_used, expires_at, created_at, updated_at, login_type, scopes, allow_list) + VALUES ($1, 'lock-order-hash'::bytea, $2, now(), now() + interval '1 hour', now(), now(), 'password', '{}'::api_key_scope[], ARRAY['*']) + `, []any{uuid.NewString(), userID}} + } + + // Mirrors coderd/oauth2provider/tokens.go: both token grants delete the + // previous api_keys row and insert its replacement. + t.Run("OAuth2TokenReplacement", func(t *testing.T) { + t.Parallel() + ctx := testutil.Context(t, testutil.WaitLong) + user := dbgen.User(t, db, database.User{}) + prevKey, _ := dbgen.APIKey(t, db, database.APIKey{UserID: user.ID}) + + runGuardedWriteRace(ctx, t, sqlDB, user.ID, + stmt{`SELECT 1 FROM api_keys WHERE id = $1 FOR UPDATE`, []any{prevKey.ID}}, + []stmt{ + {`DELETE FROM api_keys WHERE id = $1`, []any{prevKey.ID}}, + insertAPIKey(user.ID), + }, + ) + }) + + // Mirrors coderd/provisionerdserver regenerateSessionToken: delete the + // workspace session token by name, insert the replacement. + t.Run("RegenerateSessionToken", func(t *testing.T) { + t.Parallel() + ctx := testutil.Context(t, testutil.WaitLong) + user := dbgen.User(t, db, database.User{}) + sessionKey, _ := dbgen.APIKey(t, db, database.APIKey{UserID: user.ID, TokenName: "session-token-lock-order"}) + + runGuardedWriteRace(ctx, t, sqlDB, user.ID, + stmt{`SELECT 1 FROM api_keys WHERE id = $1 FOR UPDATE`, []any{sessionKey.ID}}, + []stmt{ + {`DELETE FROM api_keys WHERE user_id = $1 AND token_name = $2`, []any{user.ID, "session-token-lock-order"}}, + insertAPIKey(user.ID), + }, + ) + }) + + // Mirrors coderd/userauth.go oauthLogin: update the user_links row for + // the fresh OAuth tokens, then insert organization_members via org sync. + t.Run("OAuthLoginOrgSync", func(t *testing.T) { + t.Parallel() + ctx := testutil.Context(t, testutil.WaitLong) + user := dbgen.User(t, db, database.User{}) + _, err := sqlDB.ExecContext(ctx, ` + INSERT INTO user_links (user_id, login_type, linked_id) + VALUES ($1, 'oidc', 'lock-order-link') + `, user.ID) + require.NoError(t, err) + + runGuardedWriteRace(ctx, t, sqlDB, user.ID, + stmt{`SELECT 1 FROM user_links WHERE user_id = $1 AND login_type = 'oidc' FOR UPDATE`, []any{user.ID}}, + []stmt{ + {`UPDATE user_links SET oauth_access_token = 'refreshed' WHERE user_id = $1 AND login_type = 'oidc'`, []any{user.ID}}, + {`INSERT INTO organization_members (user_id, organization_id, created_at, updated_at) VALUES ($1, $2, now(), now())`, []any{user.ID, org.ID}}, + }, + ) + }) + + // The advisory-lock leg of the ordering contract: an update-then-insert + // user_secrets writer holds the per-user advisory lock (from the + // UPDATE-path cap trigger, migration 000590) with no users lock, so a + // concurrent insert that holds the users lock and waits on the advisory + // lock would cycle with it. Taking the users lock first (as the + // contract requires) serializes the two: the concurrent insert queues + // behind the users lock and both finish. No Go path does + // update-then-insert on user_secrets today; this pins the contract the + // cap comments state. + t.Run("SecretsUpdateThenInsert", func(t *testing.T) { + t.Parallel() + ctx := testutil.Context(t, testutil.WaitLong) + user := dbgen.User(t, db, database.User{}) + secret := uuid.New() + _, err := sqlDB.ExecContext(ctx, ` + INSERT INTO user_secrets (id, user_id, name, description, value, env_name, file_path) + VALUES ($1, $2, 'update-then-insert', '', 'value', '', '/tmp/update-then-insert') + `, secret, user.ID) + require.NoError(t, err) + + // The contract-following writer: users lock, then UPDATE (advisory + // lock), then INSERT. The concurrent insert blocks on the users + // lock instead of interleaving into the advisory cycle, and + // succeeds once the writer commits. + err = runLockRace(ctx, t, sqlDB, + []stmt{ + {`SELECT id FROM users WHERE id = $1 FOR NO KEY UPDATE`, []any{user.ID}}, + {`UPDATE user_secrets SET value = 'edited' WHERE id = $1`, []any{secret}}, + }, + stmt{` + INSERT INTO user_secrets (id, user_id, name, description, value, env_name, file_path) + VALUES ($1, $2, 'concurrent-insert', '', 'value', '', '/tmp/concurrent-insert') + `, []any{uuid.New(), user.ID}}, + []stmt{ + {` + INSERT INTO user_secrets (id, user_id, name, description, value, env_name, file_path) + VALUES ($1, $2, 'writer-insert', '', 'value', '', '/tmp/writer-insert') + `, []any{uuid.New(), user.ID}}, + }, + ) + require.NoError(t, err, "with the users lock taken first, neither side may deadlock") + }) +} + +// TestSoftDeleteGuardRejectsUpdatesForDeletedUser pins the guard's UPDATE +// branch (the unlocked deleted-check) on the three upsert tables: a child +// row that survived cleanup must not keep being updated, or an orphaned +// user_links row could keep having its OAuth tokens refreshed. +func TestSoftDeleteGuardRejectsUpdatesForDeletedUser(t *testing.T) { + t.Parallel() + if testing.Short() { + t.SkipNow() + } + + db, _, sqlDB := dbtestutil.NewDBWithSQLDB(t) + + testCases := []struct { + name string + constraint database.CheckConstraint + seed func(ctx context.Context, t *testing.T, userID uuid.UUID) + update stmt + }{ + { + name: "UserLink", + constraint: database.CheckUserLinkUserDeleted, + seed: func(ctx context.Context, t *testing.T, userID uuid.UUID) { + _, err := sqlDB.ExecContext(ctx, ` + INSERT INTO user_links (user_id, login_type, linked_id) + VALUES ($1, 'oidc', 'update-branch-link') + `, userID) + require.NoError(t, err) + }, + update: stmt{`UPDATE user_links SET oauth_access_token = 'refreshed' WHERE user_id = $1`, nil}, + }, + { + name: "UserSecret", + constraint: database.CheckUserSecretUserDeleted, + seed: func(ctx context.Context, t *testing.T, userID uuid.UUID) { + _, err := sqlDB.ExecContext(ctx, ` + INSERT INTO user_secrets (id, user_id, name, description, value, env_name, file_path) + VALUES ($1, $2, 'update-branch-secret', '', 'value', '', '/tmp/update-branch-secret') + `, uuid.New(), userID) + require.NoError(t, err) + }, + update: stmt{`UPDATE user_secrets SET value = 'edited' WHERE user_id = $1`, nil}, + }, + { + name: "UserSkill", + constraint: database.CheckUserSkillUserDeleted, + seed: func(ctx context.Context, t *testing.T, userID uuid.UUID) { + _, err := sqlDB.ExecContext(ctx, ` + INSERT INTO user_skills (id, user_id, name, description, content) + VALUES ($1, $2, 'update-branch-skill', '', 'content') + `, uuid.New(), userID) + require.NoError(t, err) + }, + update: stmt{`UPDATE user_skills SET description = 'edited' WHERE user_id = $1`, nil}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + ctx := testutil.Context(t, testutil.WaitLong) + user := dbgen.User(t, db, database.User{}) + tc.seed(ctx, t, user.ID) + + dbtestutil.SoftDeleteUserKeepingRows(ctx, t, sqlDB, user.ID) + + _, err := sqlDB.ExecContext(ctx, tc.update.sql, user.ID) + require.Error(t, err, "updating a surviving child row of a deleted user must fail") + require.True(t, database.IsCheckViolation(err, tc.constraint), + "expected constraint %q, got: %v", tc.constraint, err) + }) + } +} diff --git a/coderd/database/usersoftdeleteguards.go b/coderd/database/usersoftdeleteguards.go new file mode 100644 index 00000000000..2414eeb79c8 --- /dev/null +++ b/coderd/database/usersoftdeleteguards.go @@ -0,0 +1,25 @@ +package database + +// Constraint names raised by the user soft-delete guard trigger functions +// installed by migration 000591 (check_user_not_deleted and the per-table +// functions delegating to it). These are raised with USING CONSTRAINT from +// plpgsql, not declared as table CHECK constraints, so dbgen does not emit +// them in check_constraint.go; they are declared once here so handlers and +// tests share one set of literals. TestSoftDeleteGuardWinsConcurrentInsert +// pins each name against the live trigger by matching the constraint a real +// failing insert raises. +const ( + // Raised by the guard triggers when inserting a child row for a + // soft-deleted user, reassigning a child row onto one, or (for the + // upsert tables user_links, user_secrets, and user_skills) updating a + // surviving child row of one. + //nolint:gosec // A trigger constraint name, not a credential. + CheckAPIKeyUserDeleted CheckConstraint = "api_key_user_deleted" + CheckUserLinkUserDeleted CheckConstraint = "user_link_user_deleted" + CheckUserSecretUserDeleted CheckConstraint = "user_secret_user_deleted" + CheckUserSkillUserDeleted CheckConstraint = "user_skill_user_deleted" + CheckUserAIProviderKeyUserDeleted CheckConstraint = "user_ai_provider_key_user_deleted" + CheckOrganizationMemberUserDeleted CheckConstraint = "organization_member_user_deleted" + CheckUserAIBudgetOverrideUserDeleted CheckConstraint = "user_ai_budget_override_user_deleted" + CheckGroupMemberUserDeleted CheckConstraint = "group_member_user_deleted" +) diff --git a/coderd/exp_chats.go b/coderd/exp_chats.go index 9f21f1c0adb..0ba6d6d382c 100644 --- a/coderd/exp_chats.go +++ b/coderd/exp_chats.go @@ -6847,6 +6847,14 @@ func (api *API) upsertUserAIProviderKey(rw http.ResponseWriter, r *http.Request) CreatedAt: now, UpdatedAt: now, }) + // 409: the request is well-formed and fails on the target's state, and + // the sibling deleted-user guard in userskills.go already uses Conflict. + if database.IsCheckViolation(err, database.CheckUserAIProviderKeyUserDeleted) { + httpapi.Write(ctx, rw, http.StatusConflict, codersdk.Response{ + Message: "Cannot store an AI provider key for a deleted user.", + }) + return + } if err != nil { api.Logger.Error(ctx, "failed to update user AI provider key", slog.Error(err), slog.F("user_id", targetUser.ID), slog.F("ai_provider_id", providerID)) httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{Message: "Failed to update user AI provider key."}) diff --git a/coderd/exp_chats_test.go b/coderd/exp_chats_test.go index 4644dab67f5..ec67c880ded 100644 --- a/coderd/exp_chats_test.go +++ b/coderd/exp_chats_test.go @@ -3399,6 +3399,28 @@ func TestUserAIProviderKeys(t *testing.T) { return nil } + t.Run("DeletedUser", func(t *testing.T) { + t.Parallel() + + ctx := testutil.Context(t, testutil.WaitLong) + adminClient := newChatClient(t) + firstUser := coderdtest.CreateFirstUser(t, adminClient.Client) + _, member := coderdtest.CreateAnotherUser(t, adminClient.Client, firstUser.OrganizationID) + provider := createOpenAIProvider(t, adminClient, "test-deleted-user-"+uuid.NewString(), true, "test-provider-api-key") + + err := adminClient.DeleteUser(ctx, member.ID) + require.NoError(t, err) + + // The {user} param resolves deleted users, so the request reaches + // the insert; the guard trigger rejects it and the handler maps the + // violation to a 409 instead of a raw pq error. + _, err = adminClient.UpsertUserAIProviderKey(ctx, member.ID.String(), provider.ID, codersdk.CreateUserAIProviderKeyRequest{APIKey: "orphan-key"}) + var apiErr *codersdk.Error + require.ErrorAs(t, err, &apiErr) + require.Equal(t, http.StatusConflict, apiErr.StatusCode()) + require.Equal(t, "Cannot store an AI provider key for a deleted user.", apiErr.Message) + }) + t.Run("SelfServiceLifecycle", func(t *testing.T) { t.Parallel() diff --git a/coderd/members.go b/coderd/members.go index 21728c065d5..ac1945ad3df 100644 --- a/coderd/members.go +++ b/coderd/members.go @@ -70,6 +70,15 @@ func (api *API) postOrganizationMember(rw http.ResponseWriter, r *http.Request) }) return } + // 409: the request is well-formed and fails on the target's state, and + // the sibling deleted-user guard in userskills.go already uses Conflict. + if database.IsCheckViolation(err, database.CheckOrganizationMemberUserDeleted) { + httpapi.Write(ctx, rw, http.StatusConflict, codersdk.Response{ + Message: "Cannot add a deleted user to an organization", + Detail: fmt.Sprintf("%s has been deleted.", user.Username), + }) + return + } if err != nil { httpapi.InternalServerError(rw, err) return diff --git a/coderd/members_test.go b/coderd/members_test.go index c2bf219c1eb..a9f0de3b9c6 100644 --- a/coderd/members_test.go +++ b/coderd/members_test.go @@ -3,6 +3,7 @@ package coderd_test import ( "context" "database/sql" + "net/http" "testing" "github.com/google/uuid" @@ -48,6 +49,26 @@ func TestAddMember(t *testing.T) { require.NoError(t, err) require.Equal(t, member.UserID, first.UserID) }) + + t.Run("DeletedUser", func(t *testing.T) { + t.Parallel() + ctx := testutil.Context(t, testutil.WaitMedium) + // The soft-delete removes the user's memberships; re-adding by ID + // reaches the insert (the {user} param resolves deleted users) and + // the guard trigger rejects it, which the handler maps to a 409. + _, deletedUser := coderdtest.CreateAnotherUser(t, owner, first.OrganizationID) + // nolint:gocritic // deleting a user requires owner permission. + err := owner.DeleteUser(ctx, deletedUser.ID) + require.NoError(t, err) + + // nolint:gocritic // adding an organization member requires owner permission. + _, err = owner.PostOrganizationMember(ctx, first.OrganizationID, deletedUser.ID.String()) + var apiErr *codersdk.Error + require.ErrorAs(t, err, &apiErr) + require.Equal(t, http.StatusConflict, apiErr.StatusCode()) + require.Equal(t, "Cannot add a deleted user to an organization", apiErr.Message) + require.Contains(t, apiErr.Detail, "has been deleted") + }) } func TestDeleteMember(t *testing.T) { diff --git a/coderd/oauth2_test.go b/coderd/oauth2_test.go index 57ddb4790d4..d47e98e3861 100644 --- a/coderd/oauth2_test.go +++ b/coderd/oauth2_test.go @@ -572,6 +572,109 @@ func TestOAuth2ProviderTokenExchange(t *testing.T) { } } +// TestOAuth2ProviderTokenExchangeLockOrder drives the token exchange through +// its real HTTP entry point as an ordinary member while the test holds the +// member's users-row lock, pinning two contracts the SQL-replay suite in +// coderd/database/user_soft_delete_guards_test.go cannot: +// 1. dbauthz authorizes AcquireUserSoftDeleteGuardLock as a system +// primitive, so a member's own token exchange passes authorization. +// 2. The real exchange transaction takes the users lock first: a backend +// blocks inside AcquireUserSoftDeleteGuardLock (identified by the sqlc +// query name in pg_stat_activity) before any api_keys or oauth2 code +// row is written. Remove the Go lock call, or weaken the query below +// FOR NO KEY UPDATE (which no longer conflicts with the held lock), +// and no backend ever blocks there: the Eventually times out red. +// "Before" is asserted directly: while the exchange is blocked, its +// backend must hold zero RowExclusiveLocks on api_keys or +// oauth2_provider_app_codes, so moving the lock call after the child +// writes (the deadlock-reintroducing inversion) also fails red. +func TestOAuth2ProviderTokenExchangeLockOrder(t *testing.T) { + t.Parallel() + + db, pubsub, sqlDB := dbtestutil.NewDBWithSQLDB(t) + ownerClient := coderdtest.New(t, &coderdtest.Options{ + Database: db, + Pubsub: pubsub, + }) + owner := coderdtest.CreateFirstUser(t, ownerClient) + ctx := testutil.Context(t, testutil.WaitLong) + apps := generateApps(ctx, t, ownerClient, "token-lock-order") + + //nolint:gocritic // OAauth2 app management requires owner permission. + secret, err := ownerClient.PostOAuth2ProviderAppSecret(ctx, apps.Default.ID) + require.NoError(t, err) + + userClient, user := coderdtest.CreateAnotherUser(t, ownerClient, owner.OrganizationID) + cfg := &oauth2.Config{ + ClientID: apps.Default.ID.String(), + ClientSecret: secret.ClientSecretFull, + Endpoint: oauth2.Endpoint{ + AuthURL: apps.Default.Endpoints.Authorization, + DeviceAuthURL: apps.Default.Endpoints.DeviceAuth, + TokenURL: apps.Default.Endpoints.Token, + AuthStyle: oauth2.AuthStyleInParams, + }, + RedirectURL: apps.Default.CallbackURL, + Scopes: []string{}, + } + code, verifier, err := authorizationFlow(ctx, userClient, cfg) + require.NoError(t, err) + + // Hold the same users-row lock a concurrent soft-delete would take. + lockTx, err := sqlDB.BeginTx(ctx, nil) + require.NoError(t, err) + defer func() { _ = lockTx.Rollback() }() + var lockedID uuid.UUID + require.NoError(t, lockTx.QueryRowContext(ctx, + `SELECT id FROM users WHERE id = $1 FOR NO KEY UPDATE`, user.ID, + ).Scan(&lockedID)) + + exchangeDone := make(chan error, 1) + var token *oauth2.Token + go func() { + tok, err := cfg.Exchange(ctx, code, oauth2.SetAuthURLParam("code_verifier", verifier)) + token = tok + exchangeDone <- err + }() + + // The exchange must block inside AcquireUserSoftDeleteGuardLock before + // writing any child row; sqlc embeds the query name in the SQL text. + var blockedPID int + testutil.Eventually(ctx, t, func(ctx context.Context) bool { + err := sqlDB.QueryRowContext(ctx, ` + SELECT pid FROM pg_stat_activity + WHERE datname = current_database() + AND wait_event_type = 'Lock' + AND query LIKE '%AcquireUserSoftDeleteGuardLock%' + `).Scan(&blockedPID) + return err == nil && blockedPID != 0 + }, testutil.IntervalFast, "the exchange must block on the users lock inside AcquireUserSoftDeleteGuardLock") + + // The lock is taken FIRST: while blocked on the users row, the exchange + // transaction must not yet hold a write lock on either child table it + // later writes. Reordering the AcquireUserSoftDeleteGuardLock call after + // DeleteOAuth2ProviderAppCodeByID / DeleteAPIKeyByID (the CRF-4 lock + // inversion) leaves a RowExclusiveLock here and fails this assertion. + var childLocks int + require.NoError(t, sqlDB.QueryRowContext(ctx, ` + SELECT count(*) FROM pg_locks + WHERE pid = $1 + AND mode = 'RowExclusiveLock' + AND relation IN ('api_keys'::regclass, 'oauth2_provider_app_codes'::regclass) + `, blockedPID).Scan(&childLocks)) + require.Zero(t, childLocks, "the exchange must take the users lock before writing any api_keys or oauth2_provider_app_codes row") + + require.NoError(t, lockTx.Rollback()) + + select { + case err := <-exchangeDone: + require.NoError(t, err, "a member's token exchange must succeed once the users lock is released") + require.NotEmpty(t, token.AccessToken) + case <-ctx.Done(): + t.Fatalf("exchange did not finish: %v", ctx.Err()) + } +} + // TestOAuth2ProviderTokenExchangeCodeBelongsToDifferentApp covers // authorizationCodeGrant's code-ownership check in isolation. The token // endpoint validates redirect_uri against the app resolved from client_id diff --git a/coderd/oauth2provider/tokens.go b/coderd/oauth2provider/tokens.go index 640f1f45659..61a4d17880f 100644 --- a/coderd/oauth2provider/tokens.go +++ b/coderd/oauth2provider/tokens.go @@ -497,6 +497,17 @@ func authorizationCodeGrant(ctx context.Context, db database.Store, logger slog. err = db.InTx(func(tx database.Store) error { ctx := dbauthz.As(ctx, actor) + + // Lock order: users before api_keys (deleted and inserted below for + // dbCode.UserID). See AcquireUserSoftDeleteGuardLock. + // The lock is a system primitive; the surrounding transaction stays + // under the end user's own actor. + //nolint:gocritic // see above + _, err = tx.AcquireUserSoftDeleteGuardLock(dbauthz.AsSystemRestricted(ctx), dbCode.UserID) + if err != nil { + return xerrors.Errorf("acquire user soft-delete guard lock: %w", err) + } + err = tx.DeleteOAuth2ProviderAppCodeByID(ctx, dbCode.ID) if err != nil { return xerrors.Errorf("delete oauth2 app code: %w", err) @@ -642,6 +653,18 @@ func refreshTokenGrant(ctx context.Context, db database.Store, app database.OAut err = db.InTx(func(tx database.Store) error { ctx := dbauthz.As(ctx, actor) + + // Lock order: users before api_keys. See AcquireUserSoftDeleteGuardLock. + // prevKey.UserID matches the rows deleted and inserted below; every + // other statement in this block derives from prevKey. + // The lock is a system primitive; the surrounding transaction stays + // under the end user's own actor. + //nolint:gocritic // see above + _, err = tx.AcquireUserSoftDeleteGuardLock(dbauthz.AsSystemRestricted(ctx), prevKey.UserID) + if err != nil { + return xerrors.Errorf("acquire user soft-delete guard lock: %w", err) + } + err = tx.DeleteAPIKeyByID(ctx, prevKey.ID) // This cascades to the token. if err != nil { return xerrors.Errorf("delete oauth2 app token: %w", err) diff --git a/coderd/provisionerdserver/provisionerdserver.go b/coderd/provisionerdserver/provisionerdserver.go index 381ab67c773..0e5e2cd2327 100644 --- a/coderd/provisionerdserver/provisionerdserver.go +++ b/coderd/provisionerdserver/provisionerdserver.go @@ -3317,7 +3317,17 @@ func (s *server) regenerateSessionToken(ctx context.Context, user database.User, } err = s.Database.InTx(func(tx database.Store) error { - err := deleteSessionToken(ctx, tx, workspace) + // Lock order: users before api_keys. See AcquireUserSoftDeleteGuardLock. + // user.ID is the same variable the inserted key derives from (the + // single caller passes the workspace owner), so the lock and the + // insert cannot diverge. + //nolint:gocritic // Session token rotation runs as the provisioner daemon, not the workspace owner. + _, err := tx.AcquireUserSoftDeleteGuardLock(dbauthz.AsSystemRestricted(ctx), user.ID) + if err != nil { + return xerrors.Errorf("acquire user soft-delete guard lock: %w", err) + } + + err = deleteSessionToken(ctx, tx, workspace) if err != nil { return xerrors.Errorf("delete session token: %w", err) } diff --git a/coderd/userauth.go b/coderd/userauth.go index fb80c2230b5..da2687df472 100644 --- a/coderd/userauth.go +++ b/coderd/userauth.go @@ -1759,6 +1759,18 @@ func (api *API) oauthLogin(r *http.Request, params *oauthLoginParams) ([]*http.C user = params.User link = params.Link + if user.ID != uuid.Nil { + // Lock order: users before user_links and organization_members. + // See AcquireUserSoftDeleteGuardLock. New signups skip this: + // their users row is created inside this transaction and is + // invisible to a concurrent soft-delete. + // nolint:gocritic // The user is not authenticated yet; the lock runs as the system. + _, err = tx.AcquireUserSoftDeleteGuardLock(dbauthz.AsSystemRestricted(ctx), user.ID) + if err != nil { + return xerrors.Errorf("acquire user soft-delete guard lock: %w", err) + } + } + // If you do a convert to OIDC and your email does not match, we need to // catch this and not make a new account. if isMergeStateString(params.State.StateString) { diff --git a/coderd/userskills.go b/coderd/userskills.go index ba0f35dd52b..9deb00ad474 100644 --- a/coderd/userskills.go +++ b/coderd/userskills.go @@ -27,12 +27,6 @@ const ( // maxPersonalSkillRequestBytes allows worst-case JSON string escaping for // otherwise valid raw skill content. maxPersonalSkillRequestBytes = skills.MaxPersonalSkillSizeBytes*personalSkillJSONEscapeExpansion + personalSkillRequestEnvelopeBytes - - // Raised by the insert_user_skill_fail_if_user_deleted trigger with - // USING CONSTRAINT. Not a table CHECK constraint, so dbgen does not - // emit it in check_constraint.go. The cap constraint lives in - // database.CheckUserSkillsPerUserLimit. - userSkillUserDeletedConstraint database.CheckConstraint = "user_skill_user_deleted" ) // @Summary Create a user skill @@ -84,7 +78,7 @@ func (api *API) postUserSkill(rw http.ResponseWriter, r *http.Request) { httpapi.Forbidden(rw) return } - if database.IsCheckViolation(err, userSkillUserDeletedConstraint) { + if database.IsCheckViolation(err, database.CheckUserSkillUserDeleted) { writeCannotCreateUserSkillForDeletedUser(ctx, rw) return } @@ -246,7 +240,7 @@ func (api *API) patchUserSkill(rw http.ResponseWriter, r *http.Request) { httpapi.Forbidden(rw) return } - if database.IsCheckViolation(err, userSkillUserDeletedConstraint) { + if database.IsCheckViolation(err, database.CheckUserSkillUserDeleted) { writeCannotModifyUserSkillForDeletedUser(ctx, rw) return } diff --git a/enterprise/cli/server_dbcrypt_test.go b/enterprise/cli/server_dbcrypt_test.go index 0915f2d3157..55015f4733f 100644 --- a/enterprise/cli/server_dbcrypt_test.go +++ b/enterprise/cli/server_dbcrypt_test.go @@ -43,7 +43,7 @@ func TestServerDBCrypt(t *testing.T) { // Populate the database with some unencrypted data. t.Log("Generating unencrypted data") - users := genData(t, db) + users := genData(t, db, sqlDB) // Setup an initial cipher A keyA := testutil.MustRandString(t, 32) @@ -56,7 +56,7 @@ func TestServerDBCrypt(t *testing.T) { // Populate the database with some encrypted data using cipher A. t.Log("Generating data encrypted with cipher A") - newUsers := genData(t, cryptdb) + newUsers := genData(t, cryptdb, sqlDB) // Validate that newly created users were encrypted with cipher A for _, usr := range newUsers { @@ -205,7 +205,7 @@ func TestServerDBCrypt(t *testing.T) { } } -func genData(t *testing.T, db database.Store) []database.User { +func genData(t *testing.T, db database.Store, sqlDB *sql.DB) []database.User { t.Helper() var users []database.User // Make some users @@ -213,12 +213,17 @@ func genData(t *testing.T, db database.Store) []database.User { for _, loginType := range database.AllLoginTypeValues() { for _, deleted := range []bool{false, true} { randName := testutil.MustRandString(t, 32) + // Users in the deleted lane are created live, seeded, and + // soft-deleted below with the cleanup trigger suppressed: + // the guard triggers (migration 000591) reject inserting + // child rows for already-deleted users, and the point of + // the deleted lane is encrypting the orphaned rows that + // predate them. usr := dbgen.User(t, db, database.User{ Username: randName, Email: randName + "@notcoder.com", LoginType: loginType, Status: status, - Deleted: deleted, }) _ = dbgen.ExternalAuthLink(t, db, database.ExternalAuthLink{ UserID: usr.ID, @@ -241,6 +246,9 @@ func genData(t *testing.T, db database.Store) []database.User { PrivateKey: "private-" + usr.ID.String(), PublicKey: "public-" + usr.ID.String(), }) + // Seeded for every user; the deleted lane keeps this row as + // an orphan via the trigger-suppressed soft-delete below, + // preserving deleted-user encryption coverage. now := time.Now() _, err := db.UpsertUserAIProviderKey(context.Background(), database.UpsertUserAIProviderKeyParams{ ID: uuid.New(), @@ -272,6 +280,12 @@ func genData(t *testing.T, db database.Store) []database.User { FilePath: "", }) } + if deleted { + // Reconstructs orphaned child rows that predate the + // migration 000591 guards; rotation must handle them. + dbtestutil.SoftDeleteUserKeepingRows(context.Background(), t, sqlDB, usr.ID) + usr.Deleted = true + } users = append(users, usr) } } diff --git a/enterprise/dbcrypt/cliutil_test.go b/enterprise/dbcrypt/cliutil_test.go index fef03753191..c8eb112d230 100644 --- a/enterprise/dbcrypt/cliutil_test.go +++ b/enterprise/dbcrypt/cliutil_test.go @@ -632,11 +632,13 @@ func TestRotateUserAIProviderKeys(t *testing.T) { // alongside a live user's row purely to confirm Rotate doesn't need // one. liveUser := dbgen.User(t, f.rawDB, database.User{}) - deletedUser := dbgen.User(t, f.rawDB, database.User{Deleted: true}) + deletedUser := dbgen.User(t, f.rawDB, database.User{}) provider := dbgen.AIProvider(t, f.rawDB, database.AIProvider{}) liveKey := upsertUserAIProviderKey(f.ctx, t, f.cryptDBA, liveUser.ID, provider.ID, "user-key-live") deletedKey := upsertUserAIProviderKey(f.ctx, t, f.cryptDBA, deletedUser.ID, provider.ID, "user-key-deleted-user") + // Rotation and decryption must still handle legacy orphaned rows. + dbtestutil.SoftDeleteUserKeepingRows(f.ctx, t, f.sqlDB, deletedUser.ID) f.rotate(t) @@ -1223,11 +1225,13 @@ func TestDecryptUserAIProviderKeys(t *testing.T) { f := newDecryptFixture(t) liveUser := dbgen.User(t, f.rawDB, database.User{}) - deletedUser := dbgen.User(t, f.rawDB, database.User{Deleted: true}) + deletedUser := dbgen.User(t, f.rawDB, database.User{}) provider := dbgen.AIProvider(t, f.rawDB, database.AIProvider{}) liveKey := upsertUserAIProviderKey(f.ctx, t, f.cryptDBA, liveUser.ID, provider.ID, "user-key-live") deletedKey := upsertUserAIProviderKey(f.ctx, t, f.cryptDBA, deletedUser.ID, provider.ID, "user-key-deleted-user") + // Rotation and decryption must still handle legacy orphaned rows. + dbtestutil.SoftDeleteUserKeepingRows(f.ctx, t, f.sqlDB, deletedUser.ID) f.decrypt(t) From d3befa7fc6d5e4a0b5e60ab26613f370bb527eb6 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 1 Sep 2026 13:57:10 +0000 Subject: [PATCH 2/4] fix(coderd/database/dbpurge): expect the reaper call in purge metrics mocks The strict-mock TestMetrics chat-retention subtests enumerate every store call purgeTick makes; the new PurgeSoftDeletedUserResources call made the mock abort before the chat purges ran. --- coderd/database/dbpurge/dbpurge_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/coderd/database/dbpurge/dbpurge_test.go b/coderd/database/dbpurge/dbpurge_test.go index 5fe56c79f11..cbf85eb7216 100644 --- a/coderd/database/dbpurge/dbpurge_test.go +++ b/coderd/database/dbpurge/dbpurge_test.go @@ -253,6 +253,7 @@ func TestMetrics(t *testing.T) { mDB.EXPECT().DeleteOldProvisionerDaemons(gomock.Any()).Return(nil).AnyTimes() mDB.EXPECT().DeleteOldNotificationMessages(gomock.Any()).Return(nil).AnyTimes() mDB.EXPECT().ExpirePrebuildsAPIKeys(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + mDB.EXPECT().PurgeSoftDeletedUserResources(gomock.Any()).Return(nil).AnyTimes() mDB.EXPECT().DeleteOldTelemetryLocks(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() mDB.EXPECT().DeleteOldWorkspaceBuildOrchestrations(gomock.Any(), gomock.Any()).Return(int64(0), nil).AnyTimes() mDB.EXPECT().DeleteOldAuditLogConnectionEvents(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() @@ -307,6 +308,7 @@ func TestMetrics(t *testing.T) { mDB.EXPECT().DeleteOldProvisionerDaemons(gomock.Any()).Return(nil).AnyTimes() mDB.EXPECT().DeleteOldNotificationMessages(gomock.Any()).Return(nil).AnyTimes() mDB.EXPECT().ExpirePrebuildsAPIKeys(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + mDB.EXPECT().PurgeSoftDeletedUserResources(gomock.Any()).Return(nil).AnyTimes() mDB.EXPECT().DeleteOldTelemetryLocks(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() mDB.EXPECT().DeleteOldWorkspaceBuildOrchestrations(gomock.Any(), gomock.Any()).Return(int64(0), nil).AnyTimes() mDB.EXPECT().DeleteOldAuditLogConnectionEvents(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() From 60c6b21a20dccb762f1e008ae7f1a3b2d8f96833 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 1 Sep 2026 16:44:57 +0000 Subject: [PATCH 3/4] fix: map soft-delete guard violations to 409 in key and budget handlers Review round 5 body-only findings: POST /users/{user}/keys and /keys/tokens surfaced the api_keys guard as a raw 500 (CRF-60), and upsertUserAIBudgetOverride did the same for the budget-override guard in exactly the deletion race it exists to catch (CRF-59). Map both check violations to 409 like members.go and cover them with handler tests; the budget test holds an uncommitted soft-delete so the guard, not the alphabetically-earlier membership trigger, rejects the insert. Also thread the beforeCommit hook through the lock-race harness here: the base cap branch dropped it as unexercised, and this branch's guard tests are the consumers that flip users.deleted while the racing insert is parked on the users-row lock. Reword the stale dbcrypt test comment about deleted users' user_links/user_secrets. --- coderd/apikey.go | 20 ++++++++ coderd/apikey_test.go | 46 ++++++++++++++++++ coderd/database/lockrace_test.go | 17 +++++-- coderd/database/user_caps_test.go | 3 ++ enterprise/cli/server_dbcrypt_test.go | 4 +- enterprise/coderd/aibridge.go | 10 ++++ enterprise/coderd/aibridge_test.go | 67 +++++++++++++++++++++++++++ 7 files changed, 161 insertions(+), 6 deletions(-) diff --git a/coderd/apikey.go b/coderd/apikey.go index 79f437d55f5..df78a0f46fc 100644 --- a/coderd/apikey.go +++ b/coderd/apikey.go @@ -176,6 +176,16 @@ func (api *API) postToken(rw http.ResponseWriter, r *http.Request) { }) return } + // The soft-delete guard rejects API keys for a user deleted after + // the middleware fetched them. 409: the request is well-formed and + // fails on the target's state, matching members.go. + if database.IsCheckViolation(err, database.CheckAPIKeyUserDeleted) { + httpapi.Write(ctx, rw, http.StatusConflict, codersdk.Response{ + Message: "Cannot create a token for a deleted user.", + Detail: fmt.Sprintf("%s has been deleted.", user.Username), + }) + return + } httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ Message: "Failed to create API key.", Detail: err.Error(), @@ -226,6 +236,16 @@ func (api *API) postAPIKey(rw http.ResponseWriter, r *http.Request) { RemoteAddr: r.RemoteAddr, }) if err != nil { + // The soft-delete guard rejects API keys for a user deleted after + // the middleware fetched them. 409: the request is well-formed and + // fails on the target's state, matching members.go. + if database.IsCheckViolation(err, database.CheckAPIKeyUserDeleted) { + httpapi.Write(ctx, rw, http.StatusConflict, codersdk.Response{ + Message: "Cannot create an API key for a deleted user.", + Detail: fmt.Sprintf("%s has been deleted.", user.Username), + }) + return + } httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ Message: "Failed to create API key.", Detail: err.Error(), diff --git a/coderd/apikey_test.go b/coderd/apikey_test.go index 8ba07c5a2d3..3d4dd2e3d5a 100644 --- a/coderd/apikey_test.go +++ b/coderd/apikey_test.go @@ -70,6 +70,52 @@ func TestTokenCRUD(t *testing.T) { require.Equal(t, database.AuditActionDelete, auditor.AuditLogs()[numLogs-1].Action) } +// TestAPIKeysDeletedUser verifies both key-creation handlers map the +// api_keys soft-delete guard to a 409: the {user} parameter resolves +// deleted users by ID, so a stale ID reaches the insert and the guard +// trigger (migration 000591) rejects it instead of surfacing a 500. +func TestAPIKeysDeletedUser(t *testing.T) { + t.Parallel() + + client := coderdtest.New(t, nil) + owner := coderdtest.CreateFirstUser(t, client) + + deleteUser := func(ctx context.Context, t *testing.T) codersdk.User { + t.Helper() + _, user := coderdtest.CreateAnotherUser(t, client, owner.OrganizationID) + // nolint:gocritic // deleting a user requires owner permission. + err := client.DeleteUser(ctx, user.ID) + require.NoError(t, err) + return user + } + + t.Run("Token", func(t *testing.T) { + t.Parallel() + ctx := testutil.Context(t, testutil.WaitMedium) + deletedUser := deleteUser(ctx, t) + + _, err := client.CreateToken(ctx, deletedUser.ID.String(), codersdk.CreateTokenRequest{}) + var apiErr *codersdk.Error + require.ErrorAs(t, err, &apiErr) + require.Equal(t, http.StatusConflict, apiErr.StatusCode()) + require.Equal(t, "Cannot create a token for a deleted user.", apiErr.Message) + require.Contains(t, apiErr.Detail, "has been deleted") + }) + + t.Run("SessionKey", func(t *testing.T) { + t.Parallel() + ctx := testutil.Context(t, testutil.WaitMedium) + deletedUser := deleteUser(ctx, t) + + _, err := client.CreateAPIKey(ctx, deletedUser.ID.String()) + var apiErr *codersdk.Error + require.ErrorAs(t, err, &apiErr) + require.Equal(t, http.StatusConflict, apiErr.StatusCode()) + require.Equal(t, "Cannot create an API key for a deleted user.", apiErr.Message) + require.Contains(t, apiErr.Detail, "has been deleted") + }) +} + func TestTokensFilterExpired(t *testing.T) { t.Parallel() diff --git a/coderd/database/lockrace_test.go b/coderd/database/lockrace_test.go index f54e831ef2c..b30a45afbe2 100644 --- a/coderd/database/lockrace_test.go +++ b/coderd/database/lockrace_test.go @@ -40,11 +40,14 @@ func waitForBackendBlocked(ctx context.Context, t *testing.T, sqlDB *sql.DB, pid // runLockRace executes the blocking statements in one transaction, launches // racing in its own transaction on a dedicated connection, deterministically -// waits for it to block on a lock held by the blocking transaction, commits -// the blocking transaction, commits the racing transaction when its -// statement succeeded, and returns the racing side's error. Both -// transactions run at the default isolation level. -func runLockRace(ctx context.Context, t *testing.T, sqlDB *sql.DB, blocking []stmt, racing stmt) error { +// waits for it to block on a lock held by the blocking transaction, executes +// beforeCommit inside the blocking transaction, commits it, commits the +// racing transaction when its statement succeeded, and returns the racing +// side's error. Both transactions run at the default isolation level. +// beforeCommit lets the soft-delete guard tests flip users.deleted while +// the racing insert is provably parked on the users-row lock; callers +// without such a step pass nil. +func runLockRace(ctx context.Context, t *testing.T, sqlDB *sql.DB, blocking []stmt, racing stmt, beforeCommit []stmt) error { t.Helper() blockTx, err := sqlDB.BeginTx(ctx, nil) @@ -82,6 +85,10 @@ func runLockRace(ctx context.Context, t *testing.T, sqlDB *sql.DB, blocking []st waitForBackendBlocked(ctx, t, sqlDB, racePID) + for _, s := range beforeCommit { + _, err := blockTx.ExecContext(ctx, s.sql, s.args...) + require.NoError(t, err) + } require.NoError(t, blockTx.Commit()) committed = true diff --git a/coderd/database/user_caps_test.go b/coderd/database/user_caps_test.go index 882cffef496..7d3940c2cb7 100644 --- a/coderd/database/user_caps_test.go +++ b/coderd/database/user_caps_test.go @@ -82,6 +82,7 @@ func TestUserSecretsCapConcurrentUpdates(t *testing.T) { err := runLockRace(ctx, t, sqlDB, []stmt{{`UPDATE user_secrets SET value = $1 WHERE id = $2`, []any{bigValue, secretA}}}, stmt{`UPDATE user_secrets SET value = $1 WHERE id = $2`, []any{bigValue, secretB}}, + nil, ) require.Error(t, err, "the second update must not bypass the byte cap") require.True(t, database.IsCheckViolation(err, database.CheckUserSecretsPerUserTotalBytesLimit), @@ -178,6 +179,7 @@ func TestUserSkillsCapConcurrentInserts(t *testing.T) { err = runLockRace(ctx, t, sqlDB, []stmt{insert("winner-skill")}, insert("loser-skill"), + nil, ) require.Error(t, err, "the racing insert must recount and fail the cap") require.True(t, database.IsCheckViolation(err, database.CheckUserSkillsPerUserLimit), @@ -229,6 +231,7 @@ func TestUserSkillsCapConcurrentReassignment(t *testing.T) { VALUES ($1, $2, 'winner-skill', '', 'content') `, []any{uuid.New(), target.ID}}}, stmt{`UPDATE user_skills SET user_id = $1 WHERE id = $2`, []any{target.ID, movingSkill}}, + nil, ) require.Error(t, err, "the racing reassignment must recount and fail the cap") require.True(t, database.IsCheckViolation(err, database.CheckUserSkillsPerUserLimit), diff --git a/enterprise/cli/server_dbcrypt_test.go b/enterprise/cli/server_dbcrypt_test.go index 55015f4733f..18588a2fae1 100644 --- a/enterprise/cli/server_dbcrypt_test.go +++ b/enterprise/cli/server_dbcrypt_test.go @@ -260,7 +260,9 @@ func genData(t *testing.T, db database.Store, sqlDB *sql.DB) []database.User { }) require.NoError(t, err) - // Deleted users cannot have user_links or user_secrets. + // The soft-delete guards reject inserting user_links or + // user_secrets for a deleted user, so seed them only for + // live users. if !deleted { // Fun fact: our schema allows _all_ login types to have // a user_link. Even though I'm not sure how it could occur diff --git a/enterprise/coderd/aibridge.go b/enterprise/coderd/aibridge.go index 760f3ee176d..7cf7cedde59 100644 --- a/enterprise/coderd/aibridge.go +++ b/enterprise/coderd/aibridge.go @@ -876,6 +876,16 @@ func (api *API) upsertUserAIBudgetOverride(rw http.ResponseWriter, r *http.Reque }) return } + // The soft-delete guard rejects overrides for a user deleted after the + // middleware fetched them. 409: the request is well-formed and fails on + // the target's state, matching coderd/members.go. + if database.IsCheckViolation(err, database.CheckUserAIBudgetOverrideUserDeleted) { + httpapi.Write(ctx, rw, http.StatusConflict, codersdk.Response{ + Message: "Cannot set an AI budget override for a deleted user.", + Detail: fmt.Sprintf("%s has been deleted.", user.Username), + }) + return + } if httpapi.Is404Error(err) { httpapi.ResourceNotFound(rw) return diff --git a/enterprise/coderd/aibridge_test.go b/enterprise/coderd/aibridge_test.go index 38fcb9ad6af..9d6427b356f 100644 --- a/enterprise/coderd/aibridge_test.go +++ b/enterprise/coderd/aibridge_test.go @@ -2861,6 +2861,73 @@ func TestUserAIBudgetOverride(t *testing.T) { require.EqualValues(t, 1_000_000_000, currentOverride.SpendLimitMicros) }) + t.Run("Upsert/DeletedUserRace", func(t *testing.T) { + t.Parallel() + + // The membership trigger fires before the soft-delete guard + // (alphabetical trigger order) and a committed soft-delete removes + // the memberships it checks, so the guard's check violation + // surfaces only in the true race: a soft-delete that commits while + // the upsert's insert is parked on the locked users row. Hold that + // deletion uncommitted, let the upsert block on it, commit, and + // require the handler to map the guard violation to a 409. + db, ps, sqlDB := dbtestutil.NewDBWithSQLDB(t) + adminClient, targetUser, group := setupAICostControlTest(t, aiCostControlTestOptions{ + GroupName: "deleted-user-race-group", + Database: db, + Pubsub: ps, + }) + ctx := testutil.Context(t, testutil.WaitLong) + + deleteTx, err := sqlDB.BeginTx(ctx, nil) + require.NoError(t, err) + committed := false + defer func() { + if !committed { + _ = deleteTx.Rollback() + } + }() + _, err = deleteTx.ExecContext(ctx, + `UPDATE users SET deleted = true WHERE id = $1`, targetUser.ID) + require.NoError(t, err) + + upsertDone := make(chan error, 1) + go func() { + _, err := adminClient.UpsertUserAIBudgetOverride(ctx, targetUser.ID, codersdk.UpsertUserAIBudgetOverrideRequest{ + GroupID: group.ID, + SpendLimitMicros: 500_000_000, + }) + upsertDone <- err + }() + + // Wait until the upsert's insert is provably parked on the users + // row held by the deletion; sqlc embeds the query name in the SQL. + testutil.Eventually(ctx, t, func(ctx context.Context) bool { + var blockedPID int + err := sqlDB.QueryRowContext(ctx, ` + SELECT pid FROM pg_stat_activity + WHERE datname = current_database() + AND wait_event_type = 'Lock' + AND query LIKE '%UpsertUserAIBudgetOverride%' + `).Scan(&blockedPID) + return err == nil && blockedPID != 0 + }, testutil.IntervalFast, "the upsert must block on the users row inside the soft-delete guard") + + require.NoError(t, deleteTx.Commit()) + committed = true + + select { + case err := <-upsertDone: + var apiErr *codersdk.Error + require.ErrorAs(t, err, &apiErr) + require.Equal(t, http.StatusConflict, apiErr.StatusCode()) + require.Equal(t, "Cannot set an AI budget override for a deleted user.", apiErr.Message) + require.Contains(t, apiErr.Detail, "has been deleted") + case <-ctx.Done(): + t.Fatalf("upsert did not finish: %v", ctx.Err()) + } + }) + t.Run("Upsert/ReassignsGroup", func(t *testing.T) { t.Parallel() From dd747c862a130f2e3d8bc71988d5f8ed3220a867 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 7 Sep 2026 21:22:30 +0000 Subject: [PATCH 4/4] chore: renumber soft-delete guard migration to 000592 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Trunk took 000590 for workspace_agent_session_counts (#28126), which shifted the cap advisory-lock migration to 000591 and this one to 000592. The migration test's stepping constant, its name, and the comments citing either migration number follow; dump.sql and the sqlc output regenerate identically apart from those comments. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `xhigh`_ --- coderd/apikey_test.go | 2 +- coderd/database/dbpurge/dbpurge.go | 2 +- coderd/database/dbpurge/dbpurge_test.go | 2 +- coderd/database/dbtestutil/softdelete.go | 2 +- coderd/database/dump.sql | 2 +- ...n.sql => 000592_lock_user_soft_delete_guards.down.sql} | 0 ....up.sql => 000592_lock_user_soft_delete_guards.up.sql} | 2 +- coderd/database/migrations/migrate_test.go | 8 ++++---- coderd/database/querier.go | 4 ++-- coderd/database/queries.sql.go | 4 ++-- coderd/database/queries/users.sql | 4 ++-- coderd/database/user_soft_delete_guards_test.go | 6 +++--- coderd/database/usersoftdeleteguards.go | 2 +- enterprise/cli/server_dbcrypt_test.go | 4 ++-- 14 files changed, 22 insertions(+), 22 deletions(-) rename coderd/database/migrations/{000591_lock_user_soft_delete_guards.down.sql => 000592_lock_user_soft_delete_guards.down.sql} (100%) rename coderd/database/migrations/{000591_lock_user_soft_delete_guards.up.sql => 000592_lock_user_soft_delete_guards.up.sql} (99%) diff --git a/coderd/apikey_test.go b/coderd/apikey_test.go index 3d4dd2e3d5a..4b2eb67cec5 100644 --- a/coderd/apikey_test.go +++ b/coderd/apikey_test.go @@ -73,7 +73,7 @@ func TestTokenCRUD(t *testing.T) { // TestAPIKeysDeletedUser verifies both key-creation handlers map the // api_keys soft-delete guard to a 409: the {user} parameter resolves // deleted users by ID, so a stale ID reaches the insert and the guard -// trigger (migration 000591) rejects it instead of surfacing a 500. +// trigger (migration 000592) rejects it instead of surfacing a 500. func TestAPIKeysDeletedUser(t *testing.T) { t.Parallel() diff --git a/coderd/database/dbpurge/dbpurge.go b/coderd/database/dbpurge/dbpurge.go index b184087d2fc..6e8067cc16b 100644 --- a/coderd/database/dbpurge/dbpurge.go +++ b/coderd/database/dbpurge/dbpurge.go @@ -239,7 +239,7 @@ func (i *instance) purgeTick(ctx context.Context, db database.Store, start time. return xerrors.Errorf("failed to expire prebuilds user api keys: %w", err) } // Remove child rows orphaned by a user soft-delete that predates the - // guard triggers and cleanup coverage (migration 000591). The guards + // guard triggers and cleanup coverage (migration 000592). The guards // prevent new orphans, so after the first pass this is a no-op. if err := tx.PurgeSoftDeletedUserResources(ctx); err != nil { return xerrors.Errorf("failed to purge soft-deleted user resources: %w", err) diff --git a/coderd/database/dbpurge/dbpurge_test.go b/coderd/database/dbpurge/dbpurge_test.go index cbf85eb7216..d38d770e70a 100644 --- a/coderd/database/dbpurge/dbpurge_test.go +++ b/coderd/database/dbpurge/dbpurge_test.go @@ -3556,7 +3556,7 @@ func TestDeleteIdentifiedModuleCacheFiles(t *testing.T) { // TestPurgeSoftDeletedUserResources verifies the reaper removes child rows // orphaned by a user soft-delete that predates the guard triggers and -// cleanup coverage (migration 000591 deliberately has no backfill), while a +// cleanup coverage (migration 000592 deliberately has no backfill), while a // live user's rows survive. // //nolint:paralleltest // It uses LockIDDBPurge. diff --git a/coderd/database/dbtestutil/softdelete.go b/coderd/database/dbtestutil/softdelete.go index dd8ac73d8c6..df5024085e4 100644 --- a/coderd/database/dbtestutil/softdelete.go +++ b/coderd/database/dbtestutil/softdelete.go @@ -13,7 +13,7 @@ import ( // delete_deleted_user_resources cleanup trigger, so the user's child rows // (api_keys, user_links, and the other guarded tables) survive. This // reconstructs the orphaned-row state that could exist before migration -// 000591 closed the insert-vs-soft-delete race (the insert guards now also +// 000592 closed the insert-vs-soft-delete race (the insert guards now also // reject new rows for deleted users, so the state can only be constructed // this way). Tests use it to prove such legacy rows stay inert. func SoftDeleteUserKeepingRows(ctx context.Context, t testing.TB, sqlDB *sql.DB, userID uuid.UUID) { diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql index 5d912ce1941..8a4ab628f60 100644 --- a/coderd/database/dump.sql +++ b/coderd/database/dump.sql @@ -912,7 +912,7 @@ BEGIN -- DELETE) and later inserts a guarded row for the same user must call -- AcquireUserSoftDeleteGuardLock first, so its lock order (users, then -- child rows) matches delete_deleted_user_resources. The same contract - -- covers the cap triggers' advisory locks (migration 000590): without + -- covers the cap triggers' advisory locks (migration 000591): without -- the users lock first, an update-then-insert writer can cycle with a -- concurrent insert that holds the users lock and waits on the -- advisory lock. coderd/database/user_soft_delete_guards_test.go diff --git a/coderd/database/migrations/000591_lock_user_soft_delete_guards.down.sql b/coderd/database/migrations/000592_lock_user_soft_delete_guards.down.sql similarity index 100% rename from coderd/database/migrations/000591_lock_user_soft_delete_guards.down.sql rename to coderd/database/migrations/000592_lock_user_soft_delete_guards.down.sql diff --git a/coderd/database/migrations/000591_lock_user_soft_delete_guards.up.sql b/coderd/database/migrations/000592_lock_user_soft_delete_guards.up.sql similarity index 99% rename from coderd/database/migrations/000591_lock_user_soft_delete_guards.up.sql rename to coderd/database/migrations/000592_lock_user_soft_delete_guards.up.sql index f3bd8631edc..05814273e60 100644 --- a/coderd/database/migrations/000591_lock_user_soft_delete_guards.up.sql +++ b/coderd/database/migrations/000592_lock_user_soft_delete_guards.up.sql @@ -71,7 +71,7 @@ BEGIN -- DELETE) and later inserts a guarded row for the same user must call -- AcquireUserSoftDeleteGuardLock first, so its lock order (users, then -- child rows) matches delete_deleted_user_resources. The same contract - -- covers the cap triggers' advisory locks (migration 000590): without + -- covers the cap triggers' advisory locks (migration 000591): without -- the users lock first, an update-then-insert writer can cycle with a -- concurrent insert that holds the users lock and waits on the -- advisory lock. coderd/database/user_soft_delete_guards_test.go diff --git a/coderd/database/migrations/migrate_test.go b/coderd/database/migrations/migrate_test.go index 3a354c1f477..3d015e52eda 100644 --- a/coderd/database/migrations/migrate_test.go +++ b/coderd/database/migrations/migrate_test.go @@ -3743,15 +3743,15 @@ func TestMigration000583ChatModelOverrideOrgScope(t *testing.T) { require.NoError(t, err) } -func TestMigration000591LockUserSoftDeleteGuards(t *testing.T) { +func TestMigration000592LockUserSoftDeleteGuards(t *testing.T) { t.Parallel() if testing.Short() { t.SkipNow() } // The stepping constant is the version stepped up to before applying - // the tested migration (000591). - const migrationVersion = 590 + // the tested migration (000592). + const migrationVersion = 591 sqlDB := testSQLDB(t) @@ -3871,7 +3871,7 @@ func TestMigration000591LockUserSoftDeleteGuards(t *testing.T) { require.Equal(t, 1, countRows(table, doomedUser), "pre-migration: %s row for the doomed user must exist", table) } - // Apply migration 000591. + // Apply migration 000592. version, more, err := next() require.NoError(t, err) require.True(t, more) diff --git a/coderd/database/querier.go b/coderd/database/querier.go index a1808bfe433..a60b4d8b9b0 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -42,7 +42,7 @@ type sqlcQuerier interface { AcquireStaleChatDiffStatuses(ctx context.Context, limitVal int32) ([]AcquireStaleChatDiffStatusesRow, error) // Acquires the users-row lock that the soft-delete guard triggers take on // child-table inserts and owner reassignments (see check_user_not_deleted in - // migration 000591). Any transaction that writes a guarded child row + // migration 000592). Any transaction that writes a guarded child row // (INSERT, UPDATE, or DELETE) and later inserts a guarded row for the same // user (for example the OAuth2 token exchange, which replaces api_keys rows) // must call this first so its lock order (users first, then child rows) @@ -1387,7 +1387,7 @@ type sqlcQuerier interface { PinChatByID(ctx context.Context, id uuid.UUID) error PopNextQueuedMessage(ctx context.Context, chatID uuid.UUID) (ChatQueuedMessage, error) // Deletes child rows belonging to already-soft-deleted users. The guard - // triggers (migration 000591) prevent new rows from being created for + // triggers (migration 000592) prevent new rows from being created for // soft-deleted users, and delete_deleted_user_resources cleans rows at // soft-delete time; this reaper removes what predates both (legacy orphans // from before cleanup coverage, and race products from before the guards). diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index c04209d6140..dc46525226e 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -31304,7 +31304,7 @@ FOR NO KEY UPDATE // Acquires the users-row lock that the soft-delete guard triggers take on // child-table inserts and owner reassignments (see check_user_not_deleted in -// migration 000591). Any transaction that writes a guarded child row +// migration 000592). Any transaction that writes a guarded child row // (INSERT, UPDATE, or DELETE) and later inserts a guarded row for the same // user (for example the OAuth2 token exchange, which replaces api_keys rows) // must call this first so its lock order (users first, then child rows) @@ -32296,7 +32296,7 @@ DELETE FROM user_ai_budget_overrides WHERE user_id IN (SELECT id FROM doomed_use ` // Deletes child rows belonging to already-soft-deleted users. The guard -// triggers (migration 000591) prevent new rows from being created for +// triggers (migration 000592) prevent new rows from being created for // soft-deleted users, and delete_deleted_user_resources cleans rows at // soft-delete time; this reaper removes what predates both (legacy orphans // from before cleanup coverage, and race products from before the guards). diff --git a/coderd/database/queries/users.sql b/coderd/database/queries/users.sql index 2fb64a8ce3f..006d68582cd 100644 --- a/coderd/database/queries/users.sql +++ b/coderd/database/queries/users.sql @@ -745,7 +745,7 @@ WHERE id = @id::uuid; -- Acquires the users-row lock that the soft-delete guard triggers take on -- child-table inserts and owner reassignments (see check_user_not_deleted in --- migration 000591). Any transaction that writes a guarded child row +-- migration 000592). Any transaction that writes a guarded child row -- (INSERT, UPDATE, or DELETE) and later inserts a guarded row for the same -- user (for example the OAuth2 token exchange, which replaces api_keys rows) -- must call this first so its lock order (users first, then child rows) @@ -767,7 +767,7 @@ WHERE id = @user_id FOR NO KEY UPDATE; -- Deletes child rows belonging to already-soft-deleted users. The guard --- triggers (migration 000591) prevent new rows from being created for +-- triggers (migration 000592) prevent new rows from being created for -- soft-deleted users, and delete_deleted_user_resources cleans rows at -- soft-delete time; this reaper removes what predates both (legacy orphans -- from before cleanup coverage, and race products from before the guards). diff --git a/coderd/database/user_soft_delete_guards_test.go b/coderd/database/user_soft_delete_guards_test.go index f07f8785183..bacd488b205 100644 --- a/coderd/database/user_soft_delete_guards_test.go +++ b/coderd/database/user_soft_delete_guards_test.go @@ -16,7 +16,7 @@ import ( // TestSoftDeleteGuardWinsConcurrentInsert verifies that all eight soft-delete // guard triggers serialize against a concurrent user soft-delete via the -// parent-row lock added in migration 000591: the insert blocks on the locked +// parent-row lock added in migration 000592: the insert blocks on the locked // users row and, once the soft-delete commits, fails with the guard's // constraint instead of resurrecting a row for the deleted user. Each // subtest also pins its database.Check* constant against the live trigger by @@ -177,7 +177,7 @@ func TestSoftDeleteGuardWinsConcurrentInsert(t *testing.T) { } // TestSoftDeleteGuardBlocksOwnerReassignment pins the UPDATE ... SET user_id -// leg added by migration 000591: re-parenting a live child row onto a +// leg added by migration 000592: re-parenting a live child row onto a // soft-deleted user is rejected, both when the target is already deleted and // when the soft-delete races the reassignment (the reassignment takes the // same users-row lock as an insert). api_keys covers the dedicated @@ -574,7 +574,7 @@ func TestSoftDeleteGuardLockOrderPaths(t *testing.T) { // The advisory-lock leg of the ordering contract: an update-then-insert // user_secrets writer holds the per-user advisory lock (from the - // UPDATE-path cap trigger, migration 000590) with no users lock, so a + // UPDATE-path cap trigger, migration 000591) with no users lock, so a // concurrent insert that holds the users lock and waits on the advisory // lock would cycle with it. Taking the users lock first (as the // contract requires) serializes the two: the concurrent insert queues diff --git a/coderd/database/usersoftdeleteguards.go b/coderd/database/usersoftdeleteguards.go index 2414eeb79c8..729ac666f51 100644 --- a/coderd/database/usersoftdeleteguards.go +++ b/coderd/database/usersoftdeleteguards.go @@ -1,7 +1,7 @@ package database // Constraint names raised by the user soft-delete guard trigger functions -// installed by migration 000591 (check_user_not_deleted and the per-table +// installed by migration 000592 (check_user_not_deleted and the per-table // functions delegating to it). These are raised with USING CONSTRAINT from // plpgsql, not declared as table CHECK constraints, so dbgen does not emit // them in check_constraint.go; they are declared once here so handlers and diff --git a/enterprise/cli/server_dbcrypt_test.go b/enterprise/cli/server_dbcrypt_test.go index 18588a2fae1..1fae5bc0100 100644 --- a/enterprise/cli/server_dbcrypt_test.go +++ b/enterprise/cli/server_dbcrypt_test.go @@ -215,7 +215,7 @@ func genData(t *testing.T, db database.Store, sqlDB *sql.DB) []database.User { randName := testutil.MustRandString(t, 32) // Users in the deleted lane are created live, seeded, and // soft-deleted below with the cleanup trigger suppressed: - // the guard triggers (migration 000591) reject inserting + // the guard triggers (migration 000592) reject inserting // child rows for already-deleted users, and the point of // the deleted lane is encrypting the orphaned rows that // predate them. @@ -284,7 +284,7 @@ func genData(t *testing.T, db database.Store, sqlDB *sql.DB) []database.User { } if deleted { // Reconstructs orphaned child rows that predate the - // migration 000591 guards; rotation must handle them. + // migration 000592 guards; rotation must handle them. dbtestutil.SoftDeleteUserKeepingRows(context.Background(), t, sqlDB, usr.ID) usr.Deleted = true }