diff --git a/coderd/apikey.go b/coderd/apikey.go index 79f437d55f5..df78a0f46fc 100644 --- a/coderd/apikey.go +++ b/coderd/apikey.go @@ -176,6 +176,16 @@ func (api *API) postToken(rw http.ResponseWriter, r *http.Request) { }) return } + // The soft-delete guard rejects API keys for a user deleted after + // the middleware fetched them. 409: the request is well-formed and + // fails on the target's state, matching members.go. + if database.IsCheckViolation(err, database.CheckAPIKeyUserDeleted) { + httpapi.Write(ctx, rw, http.StatusConflict, codersdk.Response{ + Message: "Cannot create a token for a deleted user.", + Detail: fmt.Sprintf("%s has been deleted.", user.Username), + }) + return + } httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ Message: "Failed to create API key.", Detail: err.Error(), @@ -226,6 +236,16 @@ func (api *API) postAPIKey(rw http.ResponseWriter, r *http.Request) { RemoteAddr: r.RemoteAddr, }) if err != nil { + // The soft-delete guard rejects API keys for a user deleted after + // the middleware fetched them. 409: the request is well-formed and + // fails on the target's state, matching members.go. + if database.IsCheckViolation(err, database.CheckAPIKeyUserDeleted) { + httpapi.Write(ctx, rw, http.StatusConflict, codersdk.Response{ + Message: "Cannot create an API key for a deleted user.", + Detail: fmt.Sprintf("%s has been deleted.", user.Username), + }) + return + } httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ Message: "Failed to create API key.", Detail: err.Error(), diff --git a/coderd/apikey_test.go b/coderd/apikey_test.go index 8ba07c5a2d3..4b2eb67cec5 100644 --- a/coderd/apikey_test.go +++ b/coderd/apikey_test.go @@ -70,6 +70,52 @@ func TestTokenCRUD(t *testing.T) { require.Equal(t, database.AuditActionDelete, auditor.AuditLogs()[numLogs-1].Action) } +// TestAPIKeysDeletedUser verifies both key-creation handlers map the +// api_keys soft-delete guard to a 409: the {user} parameter resolves +// deleted users by ID, so a stale ID reaches the insert and the guard +// trigger (migration 000592) rejects it instead of surfacing a 500. +func TestAPIKeysDeletedUser(t *testing.T) { + t.Parallel() + + client := coderdtest.New(t, nil) + owner := coderdtest.CreateFirstUser(t, client) + + deleteUser := func(ctx context.Context, t *testing.T) codersdk.User { + t.Helper() + _, user := coderdtest.CreateAnotherUser(t, client, owner.OrganizationID) + // nolint:gocritic // deleting a user requires owner permission. + err := client.DeleteUser(ctx, user.ID) + require.NoError(t, err) + return user + } + + t.Run("Token", func(t *testing.T) { + t.Parallel() + ctx := testutil.Context(t, testutil.WaitMedium) + deletedUser := deleteUser(ctx, t) + + _, err := client.CreateToken(ctx, deletedUser.ID.String(), codersdk.CreateTokenRequest{}) + var apiErr *codersdk.Error + require.ErrorAs(t, err, &apiErr) + require.Equal(t, http.StatusConflict, apiErr.StatusCode()) + require.Equal(t, "Cannot create a token for a deleted user.", apiErr.Message) + require.Contains(t, apiErr.Detail, "has been deleted") + }) + + t.Run("SessionKey", func(t *testing.T) { + t.Parallel() + ctx := testutil.Context(t, testutil.WaitMedium) + deletedUser := deleteUser(ctx, t) + + _, err := client.CreateAPIKey(ctx, deletedUser.ID.String()) + var apiErr *codersdk.Error + require.ErrorAs(t, err, &apiErr) + require.Equal(t, http.StatusConflict, apiErr.StatusCode()) + require.Equal(t, "Cannot create an API key for a deleted user.", apiErr.Message) + require.Contains(t, apiErr.Detail, "has been deleted") + }) +} + func TestTokensFilterExpired(t *testing.T) { t.Parallel() diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index e27731ab0fe..fa18f764e2a 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -1815,6 +1815,20 @@ func (q *querier) AcquireStaleChatDiffStatuses(ctx context.Context, limitVal int return q.db.AcquireStaleChatDiffStatuses(ctx, limitVal) } +func (q *querier) AcquireUserSoftDeleteGuardLock(ctx context.Context, userID uuid.UUID) (uuid.UUID, error) { + // The lock is a deadlock-avoidance primitive, not a user capability: it + // orders a transaction's locks (users first, then child rows), granting + // nothing the guard triggers do not already do for free on every insert. + // Authorizing it on the target user would be role-dependent (owners + // pass, members fail) while protecting nothing, so it authorizes as a + // system primitive and call sites on user-scoped paths wrap only this + // call in dbauthz.AsSystemRestricted. + if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceSystem); err != nil { + return uuid.Nil, err + } + return q.db.AcquireUserSoftDeleteGuardLock(ctx, userID) +} + func (q *querier) ActivityBumpWorkspace(ctx context.Context, arg database.ActivityBumpWorkspaceParams) error { fetch := func(ctx context.Context, arg database.ActivityBumpWorkspaceParams) (database.Workspace, error) { return q.db.GetWorkspaceByID(ctx, arg.WorkspaceID) @@ -7223,6 +7237,13 @@ func (q *querier) PopNextQueuedMessage(ctx context.Context, chatID uuid.UUID) (d return q.db.PopNextQueuedMessage(ctx, chatID) } +func (q *querier) PurgeSoftDeletedUserResources(ctx context.Context) error { + if err := q.authorizeContext(ctx, policy.ActionDelete, rbac.ResourceSystem); err != nil { + return err + } + return q.db.PurgeSoftDeletedUserResources(ctx) +} + func (q *querier) ReduceWorkspaceAgentShareLevelToAuthenticatedByTemplate(ctx context.Context, templateID uuid.UUID) error { template, err := q.db.GetTemplateByID(ctx, templateID) if err != nil { diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index 5dd49f43068..cf4e0d8c03f 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -3093,6 +3093,15 @@ func (s *MethodTestSuite) TestUser() { dbm.EXPECT().GetUserByID(gomock.Any(), u.ID).Return(u, nil).AnyTimes() check.Args(u.ID).Asserts(u, policy.ActionRead).Returns(u) })) + s.Run("AcquireUserSoftDeleteGuardLock", s.Mocked(func(dbm *dbmock.MockStore, faker *gofakeit.Faker, check *expects) { + u := testutil.Fake(s.T(), faker, database.User{}) + dbm.EXPECT().AcquireUserSoftDeleteGuardLock(gomock.Any(), u.ID).Return(u.ID, nil).AnyTimes() + check.Args(u.ID).Asserts(rbac.ResourceSystem, policy.ActionUpdate).Returns(u.ID) + })) + s.Run("PurgeSoftDeletedUserResources", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) { + dbm.EXPECT().PurgeSoftDeletedUserResources(gomock.Any()).Return(nil).AnyTimes() + check.Args().Asserts(rbac.ResourceSystem, policy.ActionDelete) + })) s.Run("GetUsersByIDs", s.Mocked(func(dbm *dbmock.MockStore, faker *gofakeit.Faker, check *expects) { a := testutil.Fake(s.T(), faker, database.User{CreatedAt: dbtime.Now().Add(-time.Hour)}) b := testutil.Fake(s.T(), faker, database.User{CreatedAt: dbtime.Now()}) diff --git a/coderd/database/dbmetrics/querymetrics.go b/coderd/database/dbmetrics/querymetrics.go index 365899afbd2..bc74f267120 100644 --- a/coderd/database/dbmetrics/querymetrics.go +++ b/coderd/database/dbmetrics/querymetrics.go @@ -144,6 +144,14 @@ func (m queryMetricsStore) AcquireStaleChatDiffStatuses(ctx context.Context, lim return r0, r1 } +func (m queryMetricsStore) AcquireUserSoftDeleteGuardLock(ctx context.Context, userID uuid.UUID) (uuid.UUID, error) { + start := time.Now() + r0, r1 := m.s.AcquireUserSoftDeleteGuardLock(ctx, userID) + m.queryLatencies.WithLabelValues("AcquireUserSoftDeleteGuardLock").Observe(time.Since(start).Seconds()) + m.queryCounts.WithLabelValues(httpmw.ExtractHTTPRoute(ctx), httpmw.ExtractHTTPMethod(ctx), "AcquireUserSoftDeleteGuardLock").Inc() + return r0, r1 +} + func (m queryMetricsStore) ActivityBumpWorkspace(ctx context.Context, arg database.ActivityBumpWorkspaceParams) error { start := time.Now() r0 := m.s.ActivityBumpWorkspace(ctx, arg) @@ -5064,6 +5072,14 @@ func (m queryMetricsStore) PopNextQueuedMessage(ctx context.Context, chatID uuid return r0, r1 } +func (m queryMetricsStore) PurgeSoftDeletedUserResources(ctx context.Context) error { + start := time.Now() + r0 := m.s.PurgeSoftDeletedUserResources(ctx) + m.queryLatencies.WithLabelValues("PurgeSoftDeletedUserResources").Observe(time.Since(start).Seconds()) + m.queryCounts.WithLabelValues(httpmw.ExtractHTTPRoute(ctx), httpmw.ExtractHTTPMethod(ctx), "PurgeSoftDeletedUserResources").Inc() + return r0 +} + func (m queryMetricsStore) ReduceWorkspaceAgentShareLevelToAuthenticatedByTemplate(ctx context.Context, templateID uuid.UUID) error { start := time.Now() r0 := m.s.ReduceWorkspaceAgentShareLevelToAuthenticatedByTemplate(ctx, templateID) diff --git a/coderd/database/dbmock/dbmock.go b/coderd/database/dbmock/dbmock.go index f52286cf511..36afb8d054e 100644 --- a/coderd/database/dbmock/dbmock.go +++ b/coderd/database/dbmock/dbmock.go @@ -118,6 +118,21 @@ func (mr *MockStoreMockRecorder) AcquireStaleChatDiffStatuses(ctx, limitVal any) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AcquireStaleChatDiffStatuses", reflect.TypeOf((*MockStore)(nil).AcquireStaleChatDiffStatuses), ctx, limitVal) } +// AcquireUserSoftDeleteGuardLock mocks base method. +func (m *MockStore) AcquireUserSoftDeleteGuardLock(ctx context.Context, userID uuid.UUID) (uuid.UUID, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AcquireUserSoftDeleteGuardLock", ctx, userID) + ret0, _ := ret[0].(uuid.UUID) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AcquireUserSoftDeleteGuardLock indicates an expected call of AcquireUserSoftDeleteGuardLock. +func (mr *MockStoreMockRecorder) AcquireUserSoftDeleteGuardLock(ctx, userID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AcquireUserSoftDeleteGuardLock", reflect.TypeOf((*MockStore)(nil).AcquireUserSoftDeleteGuardLock), ctx, userID) +} + // ActivityBumpWorkspace mocks base method. func (m *MockStore) ActivityBumpWorkspace(ctx context.Context, arg database.ActivityBumpWorkspaceParams) error { m.ctrl.T.Helper() @@ -9624,6 +9639,20 @@ func (mr *MockStoreMockRecorder) PopNextQueuedMessage(ctx, chatID any) *gomock.C return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PopNextQueuedMessage", reflect.TypeOf((*MockStore)(nil).PopNextQueuedMessage), ctx, chatID) } +// PurgeSoftDeletedUserResources mocks base method. +func (m *MockStore) PurgeSoftDeletedUserResources(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PurgeSoftDeletedUserResources", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// PurgeSoftDeletedUserResources indicates an expected call of PurgeSoftDeletedUserResources. +func (mr *MockStoreMockRecorder) PurgeSoftDeletedUserResources(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PurgeSoftDeletedUserResources", reflect.TypeOf((*MockStore)(nil).PurgeSoftDeletedUserResources), ctx) +} + // ReduceWorkspaceAgentShareLevelToAuthenticatedByTemplate mocks base method. func (m *MockStore) ReduceWorkspaceAgentShareLevelToAuthenticatedByTemplate(ctx context.Context, templateID uuid.UUID) error { m.ctrl.T.Helper() diff --git a/coderd/database/dbpurge/dbpurge.go b/coderd/database/dbpurge/dbpurge.go index 6f809e68f41..6e8067cc16b 100644 --- a/coderd/database/dbpurge/dbpurge.go +++ b/coderd/database/dbpurge/dbpurge.go @@ -238,6 +238,12 @@ func (i *instance) purgeTick(ctx context.Context, db database.Store, start time. if err := tx.ExpirePrebuildsAPIKeys(ctx, dbtime.Time(start)); err != nil { return xerrors.Errorf("failed to expire prebuilds user api keys: %w", err) } + // Remove child rows orphaned by a user soft-delete that predates the + // guard triggers and cleanup coverage (migration 000592). The guards + // prevent new orphans, so after the first pass this is a no-op. + if err := tx.PurgeSoftDeletedUserResources(ctx); err != nil { + return xerrors.Errorf("failed to purge soft-deleted user resources: %w", err) + } var expiredAPIKeys int64 apiKeysRetention := i.vals.Retention.APIKeys.Value() diff --git a/coderd/database/dbpurge/dbpurge_test.go b/coderd/database/dbpurge/dbpurge_test.go index 26a0b003a0c..d38d770e70a 100644 --- a/coderd/database/dbpurge/dbpurge_test.go +++ b/coderd/database/dbpurge/dbpurge_test.go @@ -253,6 +253,7 @@ func TestMetrics(t *testing.T) { mDB.EXPECT().DeleteOldProvisionerDaemons(gomock.Any()).Return(nil).AnyTimes() mDB.EXPECT().DeleteOldNotificationMessages(gomock.Any()).Return(nil).AnyTimes() mDB.EXPECT().ExpirePrebuildsAPIKeys(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + mDB.EXPECT().PurgeSoftDeletedUserResources(gomock.Any()).Return(nil).AnyTimes() mDB.EXPECT().DeleteOldTelemetryLocks(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() mDB.EXPECT().DeleteOldWorkspaceBuildOrchestrations(gomock.Any(), gomock.Any()).Return(int64(0), nil).AnyTimes() mDB.EXPECT().DeleteOldAuditLogConnectionEvents(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() @@ -307,6 +308,7 @@ func TestMetrics(t *testing.T) { mDB.EXPECT().DeleteOldProvisionerDaemons(gomock.Any()).Return(nil).AnyTimes() mDB.EXPECT().DeleteOldNotificationMessages(gomock.Any()).Return(nil).AnyTimes() mDB.EXPECT().ExpirePrebuildsAPIKeys(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + mDB.EXPECT().PurgeSoftDeletedUserResources(gomock.Any()).Return(nil).AnyTimes() mDB.EXPECT().DeleteOldTelemetryLocks(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() mDB.EXPECT().DeleteOldWorkspaceBuildOrchestrations(gomock.Any(), gomock.Any()).Return(int64(0), nil).AnyTimes() mDB.EXPECT().DeleteOldAuditLogConnectionEvents(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() @@ -3551,3 +3553,99 @@ func TestDeleteIdentifiedModuleCacheFiles(t *testing.T) { assertFileExists(late.ID, "archive inserted after the one-off pass") assertCacheRef(lateTV, late.ID, true, "archive inserted after the one-off pass") } + +// TestPurgeSoftDeletedUserResources verifies the reaper removes child rows +// orphaned by a user soft-delete that predates the guard triggers and +// cleanup coverage (migration 000592 deliberately has no backfill), while a +// live user's rows survive. +// +//nolint:paralleltest // It uses LockIDDBPurge. +func TestPurgeSoftDeletedUserResources(t *testing.T) { + db, _, sqlDB := dbtestutil.NewDBWithSQLDB(t) + logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}) + ctx := testutil.Context(t, testutil.WaitShort) + + org := dbgen.Organization(t, db, database.Organization{}) + provider := dbgen.AIProvider(t, db, database.AIProvider{}) + group := dbgen.Group(t, db, database.Group{OrganizationID: org.ID}) + + liveUser := dbgen.User(t, db, database.User{}) + doomedUser := dbgen.User(t, db, database.User{}) + + seed := func(userID uuid.UUID) { + _, err := sqlDB.ExecContext(ctx, ` + INSERT INTO api_keys (id, hashed_secret, user_id, last_used, expires_at, created_at, updated_at, login_type, scopes, allow_list) + VALUES ($1, 'reap-hash'::bytea, $2, now(), now() + interval '1 hour', now(), now(), 'password', '{}'::api_key_scope[], ARRAY['*']) + `, uuid.NewString(), userID) + require.NoError(t, err) + _, err = sqlDB.ExecContext(ctx, + `INSERT INTO user_links (user_id, login_type, linked_id) VALUES ($1, 'github', $2)`, + userID, "reap-link-"+userID.String()) + require.NoError(t, err) + _, err = sqlDB.ExecContext(ctx, ` + INSERT INTO user_secrets (id, user_id, name, description, value, env_name) + VALUES ($1, $2, 'reap-secret', '', 'value', 'REAP_SECRET') + `, uuid.New(), userID) + require.NoError(t, err) + _, err = sqlDB.ExecContext(ctx, ` + INSERT INTO user_skills (id, user_id, name, description, content) + VALUES ($1, $2, 'reap-skill', '', 'content') + `, uuid.New(), userID) + require.NoError(t, err) + _, err = sqlDB.ExecContext(ctx, ` + INSERT INTO user_ai_provider_keys (id, user_id, ai_provider_id, api_key) + VALUES ($1, $2, $3, 'reap-key') + `, uuid.New(), userID, provider.ID) + require.NoError(t, err) + _, err = sqlDB.ExecContext(ctx, ` + INSERT INTO organization_members (user_id, organization_id, created_at, updated_at) + VALUES ($1, $2, now(), now()) + `, userID, org.ID) + require.NoError(t, err) + _, err = sqlDB.ExecContext(ctx, + `INSERT INTO group_members (user_id, group_id) VALUES ($1, $2)`, + userID, group.ID) + require.NoError(t, err) + _, err = sqlDB.ExecContext(ctx, ` + INSERT INTO user_ai_budget_overrides (user_id, group_id, spend_limit_micros) + VALUES ($1, $2, 1000000) + `, userID, group.ID) + require.NoError(t, err) + } + seed(liveUser.ID) + seed(doomedUser.ID) + + // Reconstruct pre-guard orphans: soft-delete with the cleanup trigger + // suppressed, so every child row survives. + dbtestutil.SoftDeleteUserKeepingRows(ctx, t, sqlDB, doomedUser.ID) + + guardedTables := []string{"api_keys", "user_links", "user_secrets", "user_skills", "user_ai_provider_keys", "organization_members", "group_members", "user_ai_budget_overrides"} + countRows := func(table string, userID uuid.UUID) int { + var count int + //nolint:gosec // The table name comes from the fixed list above. + err := sqlDB.QueryRowContext(ctx, + `SELECT count(*) FROM `+table+` WHERE user_id = $1`, userID).Scan(&count) + require.NoError(t, err) + return count + } + for _, table := range guardedTables { + require.Equal(t, 1, countRows(table, doomedUser.ID), "pre-purge: %s orphan must exist", table) + } + + // The initial tick runs the purge immediately. + closer := dbpurge.New(ctx, logger, db, &codersdk.DeploymentValues{}, prometheus.NewRegistry(), dbpurge.WithClock(quartz.NewReal())) + defer closer.Close() + + require.Eventually(t, func() bool { + for _, table := range guardedTables { + if countRows(table, doomedUser.ID) != 0 { + return false + } + } + return true + }, testutil.WaitShort, testutil.IntervalFast, "all orphaned child rows of the soft-deleted user must be reaped") + + for _, table := range guardedTables { + require.Equal(t, 1, countRows(table, liveUser.ID), "post-purge: %s row for the live user must survive", table) + } +} diff --git a/coderd/database/dbtestutil/softdelete.go b/coderd/database/dbtestutil/softdelete.go new file mode 100644 index 00000000000..df5024085e4 --- /dev/null +++ b/coderd/database/dbtestutil/softdelete.go @@ -0,0 +1,40 @@ +package dbtestutil + +import ( + "context" + "database/sql" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" +) + +// SoftDeleteUserKeepingRows marks the user deleted while suppressing the +// delete_deleted_user_resources cleanup trigger, so the user's child rows +// (api_keys, user_links, and the other guarded tables) survive. This +// reconstructs the orphaned-row state that could exist before migration +// 000592 closed the insert-vs-soft-delete race (the insert guards now also +// reject new rows for deleted users, so the state can only be constructed +// this way). Tests use it to prove such legacy rows stay inert. +func SoftDeleteUserKeepingRows(ctx context.Context, t testing.TB, sqlDB *sql.DB, userID uuid.UUID) { + t.Helper() + // One transaction: transactional DDL keeps the disabled trigger + // invisible to concurrent sessions (which may share this database under + // CODER_PG_CONNECTION_URL) and rolls the disable back on failure. + tx, err := sqlDB.BeginTx(ctx, nil) + require.NoError(t, err) + committed := false + defer func() { + if !committed { + _ = tx.Rollback() + } + }() + _, err = tx.ExecContext(ctx, `ALTER TABLE users DISABLE TRIGGER trigger_update_users`) + require.NoError(t, err) + _, err = tx.ExecContext(ctx, `UPDATE users SET deleted = true WHERE id = $1`, userID) + require.NoError(t, err) + _, err = tx.ExecContext(ctx, `ALTER TABLE users ENABLE TRIGGER trigger_update_users`) + require.NoError(t, err) + require.NoError(t, tx.Commit()) + committed = true +} diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql index df596fa8005..8a4ab628f60 100644 --- a/coderd/database/dump.sql +++ b/coderd/database/dump.sql @@ -885,6 +885,68 @@ $$; COMMENT ON FUNCTION chat_message_search_text(content jsonb) IS 'Extracts searchable content from chat_messages. Returns NULL for scalar JSON strings (content_version=0). Immutable as it is used in indexes.'; +CREATE FUNCTION check_user_not_deleted(target_user_id uuid, take_lock boolean, tg_operation text, display_name text, deleted_constraint text) RETURNS void + LANGUAGE plpgsql + AS $$ +DECLARE + user_deleted boolean; +BEGIN + -- Serialize child-table inserts against a concurrent user soft-delete: + -- an unlocked insert can read deleted = false, lose the race to the + -- soft-delete UPDATE and its cleanup, then commit a resurrected row. + -- FOR NO KEY UPDATE conflicts with the soft-delete UPDATE but not with + -- the FOR KEY SHARE locks that foreign-key validation takes on the + -- users row for other child tables. + -- + -- The lock is taken only when the row starts belonging to + -- target_user_id (INSERT, or UPDATE that reassigns the owner). On + -- same-owner UPDATE statements the unlocked read is kept: taking the users lock + -- there deadlocks against delete_deleted_user_resources (a multi-row + -- UPDATE or ON CONFLICT path can hold one child tuple and wait on + -- users while the cleanup holds users and waits on a child tuple), and + -- would serialize routine child updates on the hot users row. An + -- existing row is cleaned up by the soft-delete either way. + -- + -- The locking path imposes an ordering contract on writers: a + -- transaction that writes a guarded child row (INSERT, UPDATE, or + -- DELETE) and later inserts a guarded row for the same user must call + -- AcquireUserSoftDeleteGuardLock first, so its lock order (users, then + -- child rows) matches delete_deleted_user_resources. The same contract + -- covers the cap triggers' advisory locks (migration 000591): without + -- the users lock first, an update-then-insert writer can cycle with a + -- concurrent insert that holds the users lock and waits on the + -- advisory lock. coderd/database/user_soft_delete_guards_test.go + -- replays each known such path as a deterministic deadlock regression, + -- and the OAuth2 token exchange is driven through its real entry point. + -- + -- Isolation: the locking read is correct at READ COMMITTED, which every + -- production writer of the guarded tables uses. Under REPEATABLE READ + -- or SERIALIZABLE the lock wait fails with a serialization error + -- (40001) whenever any transaction committed an update to the users + -- row after the snapshot (users is written on ordinary browsing to + -- bump last_seen_at), and coderd does not retry 40001; do not wrap + -- guarded inserts in database.ReadModifyUpdate or other + -- stronger-isolation transactions. + IF take_lock THEN + SELECT deleted INTO user_deleted + FROM users + WHERE id = target_user_id + FOR NO KEY UPDATE; + ELSE + SELECT deleted INTO user_deleted + FROM users + WHERE id = target_user_id; + END IF; + IF (user_deleted) THEN + RAISE EXCEPTION 'Cannot % % for deleted user', + CASE WHEN tg_operation = 'INSERT' THEN 'create' ELSE 'modify' END, + display_name + USING ERRCODE = 'check_violation', + CONSTRAINT = deleted_constraint; + END IF; +END; +$$; + CREATE FUNCTION check_workspace_agent_name_unique() RETURNS trigger LANGUAGE plpgsql AS $$ @@ -1146,6 +1208,24 @@ BEGIN END; $$; +CREATE FUNCTION fail_if_user_deleted() RETURNS trigger + LANGUAGE plpgsql + AS $$ +DECLARE + reassigned boolean := false; +BEGIN + IF (TG_OP = 'UPDATE') THEN + reassigned := NEW.user_id IS DISTINCT FROM OLD.user_id; + -- Same-owner updates on the tables using this function are + -- filtered out by the triggers' UPDATE OF user_id column list; + -- reassigned still guards the no-op UPDATE ... SET user_id = user_id. + END IF; + PERFORM check_user_not_deleted( + NEW.user_id, TG_OP = 'INSERT' OR reassigned, TG_OP, TG_ARGV[0], TG_ARGV[1]); + RETURN NEW; +END; +$$; + CREATE FUNCTION inhibit_enqueue_if_disabled() RETURNS trigger LANGUAGE plpgsql AS $$ @@ -1177,14 +1257,14 @@ $$; CREATE FUNCTION insert_apikey_fail_if_user_deleted() RETURNS trigger LANGUAGE plpgsql AS $$ - DECLARE + reassigned boolean := false; BEGIN - IF (NEW.user_id IS NOT NULL) THEN - IF (SELECT deleted FROM users WHERE id = NEW.user_id LIMIT 1) THEN - RAISE EXCEPTION 'Cannot create API key for deleted user'; - END IF; + IF (TG_OP = 'UPDATE') THEN + reassigned := NEW.user_id IS DISTINCT FROM OLD.user_id; END IF; + PERFORM check_user_not_deleted( + NEW.user_id, TG_OP = 'INSERT' OR reassigned, TG_OP, 'API key', 'api_key_user_deleted'); RETURN NEW; END; $$; @@ -1236,14 +1316,14 @@ $$; CREATE FUNCTION insert_user_links_fail_if_user_deleted() RETURNS trigger LANGUAGE plpgsql AS $$ - DECLARE + reassigned boolean := false; BEGIN - IF (NEW.user_id IS NOT NULL) THEN - IF (SELECT deleted FROM users WHERE id = NEW.user_id LIMIT 1) THEN - RAISE EXCEPTION 'Cannot create user_link for deleted user'; - END IF; + IF (TG_OP = 'UPDATE') THEN + reassigned := NEW.user_id IS DISTINCT FROM OLD.user_id; END IF; + PERFORM check_user_not_deleted( + NEW.user_id, TG_OP = 'INSERT' OR reassigned, TG_OP, 'user_link', 'user_link_user_deleted'); RETURN NEW; END; $$; @@ -1251,14 +1331,14 @@ $$; CREATE FUNCTION insert_user_secret_fail_if_user_deleted() RETURNS trigger LANGUAGE plpgsql AS $$ - DECLARE + reassigned boolean := false; BEGIN - IF (NEW.user_id IS NOT NULL) THEN - IF (SELECT deleted FROM users WHERE id = NEW.user_id LIMIT 1) THEN - RAISE EXCEPTION 'Cannot create user_secret for deleted user'; - END IF; + IF (TG_OP = 'UPDATE') THEN + reassigned := NEW.user_id IS DISTINCT FROM OLD.user_id; END IF; + PERFORM check_user_not_deleted( + NEW.user_id, TG_OP = 'INSERT' OR reassigned, TG_OP, 'user_secret', 'user_secret_user_deleted'); RETURN NEW; END; $$; @@ -1266,19 +1346,15 @@ $$; CREATE FUNCTION insert_user_skill_fail_if_user_deleted() RETURNS trigger LANGUAGE plpgsql AS $$ - +DECLARE + reassigned boolean := false; BEGIN - PERFORM 1 - FROM users - WHERE id = NEW.user_id - AND deleted = true - LIMIT 1; - IF FOUND THEN - RAISE EXCEPTION 'Cannot create user_skill for deleted user' - USING ERRCODE = 'check_violation', - CONSTRAINT = 'user_skill_user_deleted'; - END IF; - RETURN NEW; + IF (TG_OP = 'UPDATE') THEN + reassigned := NEW.user_id IS DISTINCT FROM OLD.user_id; + END IF; + PERFORM check_user_not_deleted( + NEW.user_id, TG_OP = 'INSERT' OR reassigned, TG_OP, 'user_skill', 'user_skill_user_deleted'); + RETURN NEW; END; $$; @@ -5225,8 +5301,16 @@ CREATE TRIGGER trigger_enforce_user_ai_budget_override_membership BEFORE INSERT CREATE TRIGGER trigger_insert_apikeys BEFORE INSERT ON api_keys FOR EACH ROW EXECUTE FUNCTION insert_apikey_fail_if_user_deleted(); +CREATE TRIGGER trigger_insert_group_members BEFORE INSERT OR UPDATE OF user_id ON group_members FOR EACH ROW EXECUTE FUNCTION fail_if_user_deleted('group_member', 'group_member_user_deleted'); + +CREATE TRIGGER trigger_insert_organization_members BEFORE INSERT OR UPDATE OF user_id ON organization_members FOR EACH ROW EXECUTE FUNCTION fail_if_user_deleted('organization_member', 'organization_member_user_deleted'); + CREATE TRIGGER trigger_insert_organization_system_roles AFTER INSERT ON organizations FOR EACH ROW EXECUTE FUNCTION insert_organization_system_roles(); +CREATE TRIGGER trigger_insert_user_ai_budget_overrides BEFORE INSERT OR UPDATE OF user_id ON user_ai_budget_overrides FOR EACH ROW EXECUTE FUNCTION fail_if_user_deleted('user_ai_budget_override', 'user_ai_budget_override_user_deleted'); + +CREATE TRIGGER trigger_insert_user_ai_provider_keys BEFORE INSERT OR UPDATE OF user_id ON user_ai_provider_keys FOR EACH ROW EXECUTE FUNCTION fail_if_user_deleted('user_ai_provider_key', 'user_ai_provider_key_user_deleted'); + CREATE TRIGGER trigger_nullify_next_start_at_on_workspace_autostart_modificati AFTER UPDATE ON workspaces FOR EACH ROW EXECUTE FUNCTION nullify_next_start_at_on_workspace_autostart_modification(); CREATE TRIGGER trigger_set_chat_message_revision_on_insert BEFORE INSERT ON chat_messages FOR EACH ROW EXECUTE FUNCTION set_chat_message_revision_before(); @@ -5235,6 +5319,8 @@ CREATE TRIGGER trigger_set_chat_message_revision_on_update BEFORE UPDATE ON chat CREATE TRIGGER trigger_sync_chat_retry_state BEFORE UPDATE OF retry_state, retry_state_version, generation_attempt ON chats FOR EACH ROW EXECUTE FUNCTION sync_chat_retry_state(); +CREATE TRIGGER trigger_update_apikeys_owner BEFORE UPDATE OF user_id ON api_keys FOR EACH ROW WHEN ((new.user_id IS DISTINCT FROM old.user_id)) EXECUTE FUNCTION insert_apikey_fail_if_user_deleted(); + CREATE TRIGGER trigger_update_chat_history_after_message_insert AFTER INSERT ON chat_messages REFERENCING NEW TABLE AS chat_message_history_new_rows FOR EACH STATEMENT EXECUTE FUNCTION update_chat_history_after_message_insert(); CREATE TRIGGER trigger_update_chat_history_after_message_update AFTER UPDATE ON chat_messages REFERENCING OLD TABLE AS chat_message_history_old_rows NEW TABLE AS chat_message_history_new_rows FOR EACH STATEMENT EXECUTE FUNCTION update_chat_history_after_message_update(); diff --git a/coderd/database/lockrace_test.go b/coderd/database/lockrace_test.go index f54e831ef2c..b30a45afbe2 100644 --- a/coderd/database/lockrace_test.go +++ b/coderd/database/lockrace_test.go @@ -40,11 +40,14 @@ func waitForBackendBlocked(ctx context.Context, t *testing.T, sqlDB *sql.DB, pid // runLockRace executes the blocking statements in one transaction, launches // racing in its own transaction on a dedicated connection, deterministically -// waits for it to block on a lock held by the blocking transaction, commits -// the blocking transaction, commits the racing transaction when its -// statement succeeded, and returns the racing side's error. Both -// transactions run at the default isolation level. -func runLockRace(ctx context.Context, t *testing.T, sqlDB *sql.DB, blocking []stmt, racing stmt) error { +// waits for it to block on a lock held by the blocking transaction, executes +// beforeCommit inside the blocking transaction, commits it, commits the +// racing transaction when its statement succeeded, and returns the racing +// side's error. Both transactions run at the default isolation level. +// beforeCommit lets the soft-delete guard tests flip users.deleted while +// the racing insert is provably parked on the users-row lock; callers +// without such a step pass nil. +func runLockRace(ctx context.Context, t *testing.T, sqlDB *sql.DB, blocking []stmt, racing stmt, beforeCommit []stmt) error { t.Helper() blockTx, err := sqlDB.BeginTx(ctx, nil) @@ -82,6 +85,10 @@ func runLockRace(ctx context.Context, t *testing.T, sqlDB *sql.DB, blocking []st waitForBackendBlocked(ctx, t, sqlDB, racePID) + for _, s := range beforeCommit { + _, err := blockTx.ExecContext(ctx, s.sql, s.args...) + require.NoError(t, err) + } require.NoError(t, blockTx.Commit()) committed = true diff --git a/coderd/database/migrations/000592_lock_user_soft_delete_guards.down.sql b/coderd/database/migrations/000592_lock_user_soft_delete_guards.down.sql new file mode 100644 index 00000000000..d5b2cf0a019 --- /dev/null +++ b/coderd/database/migrations/000592_lock_user_soft_delete_guards.down.sql @@ -0,0 +1,74 @@ +DROP TRIGGER IF EXISTS trigger_insert_group_members ON group_members; +DROP TRIGGER IF EXISTS trigger_insert_user_ai_budget_overrides ON user_ai_budget_overrides; +DROP TRIGGER IF EXISTS trigger_insert_organization_members ON organization_members; +DROP TRIGGER IF EXISTS trigger_insert_user_ai_provider_keys ON user_ai_provider_keys; +DROP TRIGGER IF EXISTS trigger_update_apikeys_owner ON api_keys; +DROP FUNCTION IF EXISTS fail_if_user_deleted(); + +-- Restore the standalone per-table guard bodies this migration replaced +-- (as reproduced from the pre-migration dump.sql). +CREATE OR REPLACE FUNCTION insert_apikey_fail_if_user_deleted() RETURNS trigger + LANGUAGE plpgsql + AS $$ + +DECLARE +BEGIN + IF (NEW.user_id IS NOT NULL) THEN + IF (SELECT deleted FROM users WHERE id = NEW.user_id LIMIT 1) THEN + RAISE EXCEPTION 'Cannot create API key for deleted user'; + END IF; + END IF; + RETURN NEW; +END; +$$; + +CREATE OR REPLACE FUNCTION insert_user_links_fail_if_user_deleted() RETURNS trigger + LANGUAGE plpgsql + AS $$ + +DECLARE +BEGIN + IF (NEW.user_id IS NOT NULL) THEN + IF (SELECT deleted FROM users WHERE id = NEW.user_id LIMIT 1) THEN + RAISE EXCEPTION 'Cannot create user_link for deleted user'; + END IF; + END IF; + RETURN NEW; +END; +$$; + +CREATE OR REPLACE FUNCTION insert_user_secret_fail_if_user_deleted() RETURNS trigger + LANGUAGE plpgsql + AS $$ + +DECLARE +BEGIN + IF (NEW.user_id IS NOT NULL) THEN + IF (SELECT deleted FROM users WHERE id = NEW.user_id LIMIT 1) THEN + RAISE EXCEPTION 'Cannot create user_secret for deleted user'; + END IF; + END IF; + RETURN NEW; +END; +$$; + +CREATE OR REPLACE FUNCTION insert_user_skill_fail_if_user_deleted() RETURNS trigger + LANGUAGE plpgsql + AS $$ + +BEGIN + PERFORM 1 + FROM users + WHERE id = NEW.user_id + AND deleted = true + LIMIT 1; + IF FOUND THEN + RAISE EXCEPTION 'Cannot create user_skill for deleted user' + USING ERRCODE = 'check_violation', + CONSTRAINT = 'user_skill_user_deleted'; + END IF; + RETURN NEW; +END; +$$; + +DROP FUNCTION IF EXISTS check_user_not_deleted(uuid, boolean, text, text, text); diff --git a/coderd/database/migrations/000592_lock_user_soft_delete_guards.up.sql b/coderd/database/migrations/000592_lock_user_soft_delete_guards.up.sql new file mode 100644 index 00000000000..05814273e60 --- /dev/null +++ b/coderd/database/migrations/000592_lock_user_soft_delete_guards.up.sql @@ -0,0 +1,223 @@ +-- Close the insert-vs-soft-delete race for every table that +-- delete_deleted_user_resources deletes directly: an in-flight insert could +-- observe deleted = false, a concurrent soft-delete UPDATE (and its cleanup) +-- could commit, and the insert could then commit afterwards, resurrecting a +-- row for a soft-deleted user. For api_keys that resurrects a live session +-- token on an account the operator believes they deleted. +-- +-- The guard and lock-ordering rationale lives inside check_user_not_deleted() +-- below so it survives into dump.sql. This migration: +-- +-- 1. Replaces the bodies of the four per-table guard functions (api_keys, +-- user_links, user_secrets, user_skills) with delegation to one shared +-- check_user_not_deleted() function, so the lock and the operation gate +-- exist in exactly one place. CREATE OR REPLACE FUNCTION swaps the +-- bodies without touching the triggers: no DROP TRIGGER, no +-- ACCESS EXCLUSIVE lock on the hot tables during the upgrade. +-- 2. Adds guard triggers to the directly cleaned tables that had none +-- (user_ai_provider_keys, organization_members) plus +-- user_ai_budget_overrides and group_members, whose rows feed readers +-- that do not filter users.deleted (GetOverBudgetUsersPerGroup reads +-- override rows into a budget metric; GetAuthorizationUserRoles reads +-- group_members directly into rbac.Subject.Groups). CREATE TRIGGER +-- takes SHARE ROW EXCLUSIVE only (blocks writes briefly, never reads). +-- 3. Adds owner-reassignment coverage: every guarded table also checks +-- UPDATE ... SET user_id, so a live child row cannot be re-parented +-- onto a soft-deleted user. The users-row lock is taken exactly when +-- the row starts belonging to the target user (INSERT or owner +-- reassignment); same-owner updates keep the unlocked read. +-- +-- There are deliberately no backfill DELETE statements here: unbounded deletes inside +-- the single-transaction migration would hold locks fleet-wide during the +-- upgrade. Orphaned child rows of already-soft-deleted users are removed by +-- the idempotent dbpurge reaper (PurgeSoftDeletedUserResources), which runs +-- at startup and periodically; the triggers installed here guarantee no new +-- orphans are created once this migration commits. + +-- The shared soft-delete guard. take_lock selects the locking read (INSERT +-- and owner-reassignment paths); tg_operation feeds the error message; +-- display_name and deleted_constraint parameterize the raised error, with +-- the constraint name being the stable identifier API handlers match on. +CREATE FUNCTION check_user_not_deleted( + target_user_id uuid, + take_lock boolean, + tg_operation text, + display_name text, + deleted_constraint text +) RETURNS void + LANGUAGE plpgsql +AS $$ +DECLARE + user_deleted boolean; +BEGIN + -- Serialize child-table inserts against a concurrent user soft-delete: + -- an unlocked insert can read deleted = false, lose the race to the + -- soft-delete UPDATE and its cleanup, then commit a resurrected row. + -- FOR NO KEY UPDATE conflicts with the soft-delete UPDATE but not with + -- the FOR KEY SHARE locks that foreign-key validation takes on the + -- users row for other child tables. + -- + -- The lock is taken only when the row starts belonging to + -- target_user_id (INSERT, or UPDATE that reassigns the owner). On + -- same-owner UPDATE statements the unlocked read is kept: taking the users lock + -- there deadlocks against delete_deleted_user_resources (a multi-row + -- UPDATE or ON CONFLICT path can hold one child tuple and wait on + -- users while the cleanup holds users and waits on a child tuple), and + -- would serialize routine child updates on the hot users row. An + -- existing row is cleaned up by the soft-delete either way. + -- + -- The locking path imposes an ordering contract on writers: a + -- transaction that writes a guarded child row (INSERT, UPDATE, or + -- DELETE) and later inserts a guarded row for the same user must call + -- AcquireUserSoftDeleteGuardLock first, so its lock order (users, then + -- child rows) matches delete_deleted_user_resources. The same contract + -- covers the cap triggers' advisory locks (migration 000591): without + -- the users lock first, an update-then-insert writer can cycle with a + -- concurrent insert that holds the users lock and waits on the + -- advisory lock. coderd/database/user_soft_delete_guards_test.go + -- replays each known such path as a deterministic deadlock regression, + -- and the OAuth2 token exchange is driven through its real entry point. + -- + -- Isolation: the locking read is correct at READ COMMITTED, which every + -- production writer of the guarded tables uses. Under REPEATABLE READ + -- or SERIALIZABLE the lock wait fails with a serialization error + -- (40001) whenever any transaction committed an update to the users + -- row after the snapshot (users is written on ordinary browsing to + -- bump last_seen_at), and coderd does not retry 40001; do not wrap + -- guarded inserts in database.ReadModifyUpdate or other + -- stronger-isolation transactions. + IF take_lock THEN + SELECT deleted INTO user_deleted + FROM users + WHERE id = target_user_id + FOR NO KEY UPDATE; + ELSE + SELECT deleted INTO user_deleted + FROM users + WHERE id = target_user_id; + END IF; + IF (user_deleted) THEN + RAISE EXCEPTION 'Cannot % % for deleted user', + CASE WHEN tg_operation = 'INSERT' THEN 'create' ELSE 'modify' END, + display_name + USING ERRCODE = 'check_violation', + CONSTRAINT = deleted_constraint; + END IF; +END; +$$; + +-- The generic trigger form for tables whose triggers this migration +-- creates. TG_ARGV[0] is the display name for the error message, TG_ARGV[1] +-- the stable constraint name callers match on. +CREATE FUNCTION fail_if_user_deleted() RETURNS trigger + LANGUAGE plpgsql +AS $$ +DECLARE + reassigned boolean := false; +BEGIN + IF (TG_OP = 'UPDATE') THEN + reassigned := NEW.user_id IS DISTINCT FROM OLD.user_id; + -- Same-owner updates on the tables using this function are + -- filtered out by the triggers' UPDATE OF user_id column list; + -- reassigned still guards the no-op UPDATE ... SET user_id = user_id. + END IF; + PERFORM check_user_not_deleted( + NEW.user_id, TG_OP = 'INSERT' OR reassigned, TG_OP, TG_ARGV[0], TG_ARGV[1]); + RETURN NEW; +END; +$$; + +-- Delegate the four pre-existing per-table guard functions to the shared +-- check. Their triggers are untouched; the INSERT-only tables additionally +-- get an owner-reassignment trigger below. +CREATE OR REPLACE FUNCTION insert_apikey_fail_if_user_deleted() RETURNS trigger + LANGUAGE plpgsql +AS $$ +DECLARE + reassigned boolean := false; +BEGIN + IF (TG_OP = 'UPDATE') THEN + reassigned := NEW.user_id IS DISTINCT FROM OLD.user_id; + END IF; + PERFORM check_user_not_deleted( + NEW.user_id, TG_OP = 'INSERT' OR reassigned, TG_OP, 'API key', 'api_key_user_deleted'); + RETURN NEW; +END; +$$; + +CREATE OR REPLACE FUNCTION insert_user_links_fail_if_user_deleted() RETURNS trigger + LANGUAGE plpgsql +AS $$ +DECLARE + reassigned boolean := false; +BEGIN + IF (TG_OP = 'UPDATE') THEN + reassigned := NEW.user_id IS DISTINCT FROM OLD.user_id; + END IF; + PERFORM check_user_not_deleted( + NEW.user_id, TG_OP = 'INSERT' OR reassigned, TG_OP, 'user_link', 'user_link_user_deleted'); + RETURN NEW; +END; +$$; + +CREATE OR REPLACE FUNCTION insert_user_secret_fail_if_user_deleted() RETURNS trigger + LANGUAGE plpgsql +AS $$ +DECLARE + reassigned boolean := false; +BEGIN + IF (TG_OP = 'UPDATE') THEN + reassigned := NEW.user_id IS DISTINCT FROM OLD.user_id; + END IF; + PERFORM check_user_not_deleted( + NEW.user_id, TG_OP = 'INSERT' OR reassigned, TG_OP, 'user_secret', 'user_secret_user_deleted'); + RETURN NEW; +END; +$$; + +CREATE OR REPLACE FUNCTION insert_user_skill_fail_if_user_deleted() RETURNS trigger + LANGUAGE plpgsql +AS $$ +DECLARE + reassigned boolean := false; +BEGIN + IF (TG_OP = 'UPDATE') THEN + reassigned := NEW.user_id IS DISTINCT FROM OLD.user_id; + END IF; + PERFORM check_user_not_deleted( + NEW.user_id, TG_OP = 'INSERT' OR reassigned, TG_OP, 'user_skill', 'user_skill_user_deleted'); + RETURN NEW; +END; +$$; + +-- Owner-reassignment coverage for api_keys, whose existing trigger is +-- INSERT-only. The UPDATE OF user_id column list plus the WHEN clause keep +-- every ordinary api_keys update (last_used bumps on each authenticated +-- request) out of plpgsql entirely. +CREATE TRIGGER trigger_update_apikeys_owner + BEFORE UPDATE OF user_id ON api_keys + FOR EACH ROW + WHEN (NEW.user_id IS DISTINCT FROM OLD.user_id) +EXECUTE FUNCTION insert_apikey_fail_if_user_deleted(); + +-- Guards for the directly cleaned tables that had none, plus the two whose +-- rows feed deleted-user-blind readers (see the header). +CREATE TRIGGER trigger_insert_user_ai_provider_keys + BEFORE INSERT OR UPDATE OF user_id ON user_ai_provider_keys + FOR EACH ROW +EXECUTE FUNCTION fail_if_user_deleted('user_ai_provider_key', 'user_ai_provider_key_user_deleted'); + +CREATE TRIGGER trigger_insert_organization_members + BEFORE INSERT OR UPDATE OF user_id ON organization_members + FOR EACH ROW +EXECUTE FUNCTION fail_if_user_deleted('organization_member', 'organization_member_user_deleted'); + +CREATE TRIGGER trigger_insert_user_ai_budget_overrides + BEFORE INSERT OR UPDATE OF user_id ON user_ai_budget_overrides + FOR EACH ROW +EXECUTE FUNCTION fail_if_user_deleted('user_ai_budget_override', 'user_ai_budget_override_user_deleted'); + +CREATE TRIGGER trigger_insert_group_members + BEFORE INSERT OR UPDATE OF user_id ON group_members + FOR EACH ROW +EXECUTE FUNCTION fail_if_user_deleted('group_member', 'group_member_user_deleted'); diff --git a/coderd/database/migrations/migrate_test.go b/coderd/database/migrations/migrate_test.go index 9844d8ec4bd..3d015e52eda 100644 --- a/coderd/database/migrations/migrate_test.go +++ b/coderd/database/migrations/migrate_test.go @@ -3742,3 +3742,171 @@ func TestMigration000583ChatModelOverrideOrgScope(t *testing.T) { _, err = db.ExecContext(ctx, string(upSQL)) require.NoError(t, err) } + +func TestMigration000592LockUserSoftDeleteGuards(t *testing.T) { + t.Parallel() + if testing.Short() { + t.SkipNow() + } + + // The stepping constant is the version stepped up to before applying + // the tested migration (000592). + const migrationVersion = 591 + + sqlDB := testSQLDB(t) + + // Step up to migrationVersion. + next, err := migrations.Stepper(sqlDB) + require.NoError(t, err) + for { + version, more, err := next() + require.NoError(t, err) + if !more { + t.Fatalf("migration %d not found", migrationVersion) + } + if version == migrationVersion { + break + } + } + + ctx := testutil.Context(t, testutil.WaitLong) + now := time.Now().UTC().Truncate(time.Microsecond) + + liveUser := uuid.New() + doomedUser := uuid.New() + for i, id := range []uuid.UUID{liveUser, doomedUser} { + _, err = sqlDB.ExecContext(ctx, + `INSERT INTO users (id, username, email, hashed_password, created_at, updated_at, status, rbac_roles, login_type) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`, + id, fmt.Sprintf("guards-user-%d", i), fmt.Sprintf("guards-%d@test.com", i), []byte{}, now, now, "active", pq.StringArray{}, "password", + ) + require.NoError(t, err) + } + + orgID := uuid.New() + _, err = sqlDB.ExecContext(ctx, + `INSERT INTO organizations (id, name, display_name, description, icon, created_at, updated_at, is_default, default_org_member_roles) + VALUES ($1, 'guards-org', 'Guards Org', '', '', $2, $2, false, '{}')`, + orgID, now, + ) + require.NoError(t, err) + providerID := uuid.New() + _, err = sqlDB.ExecContext(ctx, + `INSERT INTO ai_providers (id, type, name, base_url) VALUES ($1, 'openai', 'guards-provider', 'https://example.com')`, + providerID, + ) + require.NoError(t, err) + groupID := uuid.New() + _, err = sqlDB.ExecContext(ctx, + `INSERT INTO groups (id, name, organization_id) VALUES ($1, 'guards-group', $2)`, + groupID, orgID, + ) + require.NoError(t, err) + + // One row per guarded table for both users. + for _, id := range []uuid.UUID{liveUser, doomedUser} { + _, err = sqlDB.ExecContext(ctx, + `INSERT INTO api_keys (id, hashed_secret, user_id, last_used, expires_at, created_at, updated_at, login_type, scopes, allow_list) + VALUES ($1, 'hash'::bytea, $2, $3, $3, $3, $3, 'password', '{}'::api_key_scope[], ARRAY['*'])`, + "key-"+id.String()[:13], id, now, + ) + require.NoError(t, err) + _, err = sqlDB.ExecContext(ctx, + `INSERT INTO user_links (user_id, login_type, linked_id) VALUES ($1, 'github', $2)`, + id, "link-"+id.String(), + ) + require.NoError(t, err) + _, err = sqlDB.ExecContext(ctx, + `INSERT INTO user_secrets (id, user_id, name, description, value, env_name) + VALUES ($1, $2, 'seed-secret', '', 'value', 'SEED_SECRET')`, + uuid.New(), id, + ) + require.NoError(t, err) + _, err = sqlDB.ExecContext(ctx, + `INSERT INTO user_skills (id, user_id, name, description, content) + VALUES ($1, $2, 'seed-skill', '', 'content')`, + uuid.New(), id, + ) + require.NoError(t, err) + _, err = sqlDB.ExecContext(ctx, + `INSERT INTO user_ai_provider_keys (id, user_id, ai_provider_id, api_key) + VALUES ($1, $2, $3, 'seed-key')`, + uuid.New(), id, providerID, + ) + require.NoError(t, err) + _, err = sqlDB.ExecContext(ctx, + `INSERT INTO organization_members (user_id, organization_id, created_at, updated_at) + VALUES ($1, $2, $3, $3)`, + id, orgID, now, + ) + require.NoError(t, err) + _, err = sqlDB.ExecContext(ctx, + `INSERT INTO group_members (user_id, group_id) VALUES ($1, $2)`, + id, groupID, + ) + require.NoError(t, err) + _, err = sqlDB.ExecContext(ctx, + `INSERT INTO user_ai_budget_overrides (user_id, group_id, spend_limit_micros) + VALUES ($1, $2, 1000000)`, + id, groupID, + ) + require.NoError(t, err) + } + + // Reproduce the race outcome the guards close: a user soft-deleted + // while their child rows survive. Suppressing the cleanup trigger + // simulates the insert committing after the soft-delete. + dbtestutil.SoftDeleteUserKeepingRows(ctx, t, sqlDB, doomedUser) + + countRows := func(table string, userID uuid.UUID) int { + var count int + //nolint:gosec // The table name comes from the fixed list below. + err := sqlDB.QueryRowContext(ctx, + `SELECT count(*) FROM `+table+` WHERE user_id = $1`, userID).Scan(&count) + require.NoError(t, err) + return count + } + guardedTables := []string{"api_keys", "user_links", "user_secrets", "user_skills", "user_ai_provider_keys", "organization_members", "group_members", "user_ai_budget_overrides"} + for _, table := range guardedTables { + require.Equal(t, 1, countRows(table, doomedUser), "pre-migration: %s row for the doomed user must exist", table) + } + + // Apply migration 000592. + version, more, err := next() + require.NoError(t, err) + require.True(t, more) + require.EqualValues(t, migrationVersion+1, version) + + // The migration deliberately has no backfill DELETEs (they would hold + // locks fleet-wide inside the single-transaction migration): existing + // orphans survive and are removed by the dbpurge reaper + // (PurgeSoftDeletedUserResources) instead. + for _, table := range guardedTables { + require.Equal(t, 1, countRows(table, doomedUser), "post-migration: %s orphan must survive (reaper owns cleanup)", table) + require.Equal(t, 1, countRows(table, liveUser), "post-migration: %s row for the live user must survive", table) + } + + // The guards are live: inserting for the doomed user fails with the + // stable constraint name, on both a pre-existing table (delegated + // function body) and a newly guarded one. + _, err = sqlDB.ExecContext(ctx, + `INSERT INTO api_keys (id, hashed_secret, user_id, last_used, expires_at, created_at, updated_at, login_type, scopes, allow_list) + VALUES ($1, 'hash'::bytea, $2, $3, $3, $3, $3, 'password', '{}'::api_key_scope[], ARRAY['*'])`, + "key2-"+doomedUser.String()[:12], doomedUser, now, + ) + require.Error(t, err) + require.True(t, database.IsCheckViolation(err, database.CheckAPIKeyUserDeleted), "expected api_key_user_deleted, got: %v", err) + _, err = sqlDB.ExecContext(ctx, + `INSERT INTO group_members (user_id, group_id) VALUES ($1, $2)`, + doomedUser, groupID, + ) + require.Error(t, err) + require.True(t, database.IsCheckViolation(err, database.CheckGroupMemberUserDeleted), "expected group_member_user_deleted, got: %v", err) + + // The owner-reassignment leg is live: re-parenting the live user's + // api_keys row onto the doomed user fails. + _, err = sqlDB.ExecContext(ctx, + `UPDATE api_keys SET user_id = $1 WHERE user_id = $2`, doomedUser, liveUser) + require.Error(t, err) + require.True(t, database.IsCheckViolation(err, database.CheckAPIKeyUserDeleted), "expected api_key_user_deleted on reassignment, got: %v", err) +} diff --git a/coderd/database/querier.go b/coderd/database/querier.go index 88e86e97304..a60b4d8b9b0 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -40,6 +40,25 @@ type sqlcQuerier interface { // https://www.postgresql.org/docs/9.5/sql-select.html#SQL-FOR-UPDATE-SHARE AcquireProvisionerJob(ctx context.Context, arg AcquireProvisionerJobParams) (ProvisionerJob, error) AcquireStaleChatDiffStatuses(ctx context.Context, limitVal int32) ([]AcquireStaleChatDiffStatusesRow, error) + // Acquires the users-row lock that the soft-delete guard triggers take on + // child-table inserts and owner reassignments (see check_user_not_deleted in + // migration 000592). Any transaction that writes a guarded child row + // (INSERT, UPDATE, or DELETE) and later inserts a guarded row for the same + // user (for example the OAuth2 token exchange, which replaces api_keys rows) + // must call this first so its lock order (users first, then child rows) + // matches delete_deleted_user_resources and cannot deadlock with a + // concurrent user soft-delete. + // Must run inside the transaction that performs the child writes: outside + // one, the lock is released at the implicit statement commit and protects + // nothing. + // The query is dbauthz-authorized as a system primitive: callers must run + // it under a system-authorized context (dbauthz.AsSystemRestricted at the + // call site when the surrounding request runs as an ordinary user). + // Returns sql.ErrNoRows when the user does not exist. It does NOT detect a + // wrong id that belongs to some other real user: that row is locked and the + // transaction proceeds with the guard defeated, so callers must derive the + // id from the same variable the child writes use. + AcquireUserSoftDeleteGuardLock(ctx context.Context, userID uuid.UUID) (uuid.UUID, error) // Bumps the workspace deadline by the template's configured "activity_bump" // duration (default 1h). If the workspace bump will cross an autostart // threshold, then the bump is autostart + TTL. This is the deadline behavior if @@ -1367,6 +1386,19 @@ type sqlcQuerier interface { // sequence, so this is acceptable. PinChatByID(ctx context.Context, id uuid.UUID) error PopNextQueuedMessage(ctx context.Context, chatID uuid.UUID) (ChatQueuedMessage, error) + // Deletes child rows belonging to already-soft-deleted users. The guard + // triggers (migration 000592) prevent new rows from being created for + // soft-deleted users, and delete_deleted_user_resources cleans rows at + // soft-delete time; this reaper removes what predates both (legacy orphans + // from before cleanup coverage, and race products from before the guards). + // It deletes in the same table order as delete_deleted_user_resources to + // minimize deadlock exposure against a concurrent soft-delete; a lost + // deadlock surfaces as a failed purge cycle and is retried on the next one. + // group_members and user_ai_budget_overrides rows are normally wiped + // transitively (BEFORE DELETE triggers on organization_members); the direct + // deletes catch rows orphaned after the user's organization_members rows + // were already cleaned up. + PurgeSoftDeletedUserResources(ctx context.Context) error ReduceWorkspaceAgentShareLevelToAuthenticatedByTemplate(ctx context.Context, templateID uuid.UUID) error RegisterWorkspaceProxy(ctx context.Context, arg RegisterWorkspaceProxyParams) (WorkspaceProxy, error) ReindexStaleChatMessagesSearchTsv(ctx context.Context, batchSize int32) (int64, error) diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index ccbd4a5af2a..dc46525226e 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -31296,6 +31296,37 @@ func (q *sqlQuerier) UpdateUserSkillByUserIDAndName(ctx context.Context, arg Upd return i, err } +const acquireUserSoftDeleteGuardLock = `-- name: AcquireUserSoftDeleteGuardLock :one +SELECT id FROM users +WHERE id = $1 +FOR NO KEY UPDATE +` + +// Acquires the users-row lock that the soft-delete guard triggers take on +// child-table inserts and owner reassignments (see check_user_not_deleted in +// migration 000592). Any transaction that writes a guarded child row +// (INSERT, UPDATE, or DELETE) and later inserts a guarded row for the same +// user (for example the OAuth2 token exchange, which replaces api_keys rows) +// must call this first so its lock order (users first, then child rows) +// matches delete_deleted_user_resources and cannot deadlock with a +// concurrent user soft-delete. +// Must run inside the transaction that performs the child writes: outside +// one, the lock is released at the implicit statement commit and protects +// nothing. +// The query is dbauthz-authorized as a system primitive: callers must run +// it under a system-authorized context (dbauthz.AsSystemRestricted at the +// call site when the surrounding request runs as an ordinary user). +// Returns sql.ErrNoRows when the user does not exist. It does NOT detect a +// wrong id that belongs to some other real user: that row is locked and the +// transaction proceeds with the guard defeated, so callers must derive the +// id from the same variable the child writes use. +func (q *sqlQuerier) AcquireUserSoftDeleteGuardLock(ctx context.Context, userID uuid.UUID) (uuid.UUID, error) { + row := q.db.QueryRowContext(ctx, acquireUserSoftDeleteGuardLock, userID) + var id uuid.UUID + err := row.Scan(&id) + return id, err +} + const allUserIDs = `-- name: AllUserIDs :many SELECT DISTINCT id FROM USERS WHERE CASE WHEN $1::bool THEN TRUE ELSE is_system = false END @@ -32243,6 +32274,44 @@ func (q *sqlQuerier) ListUserChatCompactionThresholds(ctx context.Context, userI return items, nil } +const purgeSoftDeletedUserResources = `-- name: PurgeSoftDeletedUserResources :exec +WITH doomed_users AS ( + SELECT id FROM users WHERE deleted +), delete_api_keys AS ( + DELETE FROM api_keys WHERE user_id IN (SELECT id FROM doomed_users) +), delete_user_links AS ( + DELETE FROM user_links WHERE user_id IN (SELECT id FROM doomed_users) +), delete_user_secrets AS ( + DELETE FROM user_secrets WHERE user_id IN (SELECT id FROM doomed_users) +), delete_user_ai_provider_keys AS ( + DELETE FROM user_ai_provider_keys WHERE user_id IN (SELECT id FROM doomed_users) +), delete_organization_members AS ( + DELETE FROM organization_members WHERE user_id IN (SELECT id FROM doomed_users) +), delete_user_skills AS ( + DELETE FROM user_skills WHERE user_id IN (SELECT id FROM doomed_users) +), delete_group_members AS ( + DELETE FROM group_members WHERE user_id IN (SELECT id FROM doomed_users) +) +DELETE FROM user_ai_budget_overrides WHERE user_id IN (SELECT id FROM doomed_users) +` + +// Deletes child rows belonging to already-soft-deleted users. The guard +// triggers (migration 000592) prevent new rows from being created for +// soft-deleted users, and delete_deleted_user_resources cleans rows at +// soft-delete time; this reaper removes what predates both (legacy orphans +// from before cleanup coverage, and race products from before the guards). +// It deletes in the same table order as delete_deleted_user_resources to +// minimize deadlock exposure against a concurrent soft-delete; a lost +// deadlock surfaces as a failed purge cycle and is retried on the next one. +// group_members and user_ai_budget_overrides rows are normally wiped +// transitively (BEFORE DELETE triggers on organization_members); the direct +// deletes catch rows orphaned after the user's organization_members rows +// were already cleaned up. +func (q *sqlQuerier) PurgeSoftDeletedUserResources(ctx context.Context) error { + _, err := q.db.ExecContext(ctx, purgeSoftDeletedUserResources) + return err +} + const updateInactiveUsersToDormant = `-- name: UpdateInactiveUsersToDormant :many UPDATE users diff --git a/coderd/database/queries/users.sql b/coderd/database/queries/users.sql index fe6bbd291e6..006d68582cd 100644 --- a/coderd/database/queries/users.sql +++ b/coderd/database/queries/users.sql @@ -742,3 +742,58 @@ WHERE SELECT * FROM users WHERE id = @id::uuid; + +-- Acquires the users-row lock that the soft-delete guard triggers take on +-- child-table inserts and owner reassignments (see check_user_not_deleted in +-- migration 000592). Any transaction that writes a guarded child row +-- (INSERT, UPDATE, or DELETE) and later inserts a guarded row for the same +-- user (for example the OAuth2 token exchange, which replaces api_keys rows) +-- must call this first so its lock order (users first, then child rows) +-- matches delete_deleted_user_resources and cannot deadlock with a +-- concurrent user soft-delete. +-- Must run inside the transaction that performs the child writes: outside +-- one, the lock is released at the implicit statement commit and protects +-- nothing. +-- The query is dbauthz-authorized as a system primitive: callers must run +-- it under a system-authorized context (dbauthz.AsSystemRestricted at the +-- call site when the surrounding request runs as an ordinary user). +-- Returns sql.ErrNoRows when the user does not exist. It does NOT detect a +-- wrong id that belongs to some other real user: that row is locked and the +-- transaction proceeds with the guard defeated, so callers must derive the +-- id from the same variable the child writes use. +-- name: AcquireUserSoftDeleteGuardLock :one +SELECT id FROM users +WHERE id = @user_id +FOR NO KEY UPDATE; + +-- Deletes child rows belonging to already-soft-deleted users. The guard +-- triggers (migration 000592) prevent new rows from being created for +-- soft-deleted users, and delete_deleted_user_resources cleans rows at +-- soft-delete time; this reaper removes what predates both (legacy orphans +-- from before cleanup coverage, and race products from before the guards). +-- It deletes in the same table order as delete_deleted_user_resources to +-- minimize deadlock exposure against a concurrent soft-delete; a lost +-- deadlock surfaces as a failed purge cycle and is retried on the next one. +-- group_members and user_ai_budget_overrides rows are normally wiped +-- transitively (BEFORE DELETE triggers on organization_members); the direct +-- deletes catch rows orphaned after the user's organization_members rows +-- were already cleaned up. +-- name: PurgeSoftDeletedUserResources :exec +WITH doomed_users AS ( + SELECT id FROM users WHERE deleted +), delete_api_keys AS ( + DELETE FROM api_keys WHERE user_id IN (SELECT id FROM doomed_users) +), delete_user_links AS ( + DELETE FROM user_links WHERE user_id IN (SELECT id FROM doomed_users) +), delete_user_secrets AS ( + DELETE FROM user_secrets WHERE user_id IN (SELECT id FROM doomed_users) +), delete_user_ai_provider_keys AS ( + DELETE FROM user_ai_provider_keys WHERE user_id IN (SELECT id FROM doomed_users) +), delete_organization_members AS ( + DELETE FROM organization_members WHERE user_id IN (SELECT id FROM doomed_users) +), delete_user_skills AS ( + DELETE FROM user_skills WHERE user_id IN (SELECT id FROM doomed_users) +), delete_group_members AS ( + DELETE FROM group_members WHERE user_id IN (SELECT id FROM doomed_users) +) +DELETE FROM user_ai_budget_overrides WHERE user_id IN (SELECT id FROM doomed_users); diff --git a/coderd/database/user_caps_test.go b/coderd/database/user_caps_test.go index 882cffef496..7d3940c2cb7 100644 --- a/coderd/database/user_caps_test.go +++ b/coderd/database/user_caps_test.go @@ -82,6 +82,7 @@ func TestUserSecretsCapConcurrentUpdates(t *testing.T) { err := runLockRace(ctx, t, sqlDB, []stmt{{`UPDATE user_secrets SET value = $1 WHERE id = $2`, []any{bigValue, secretA}}}, stmt{`UPDATE user_secrets SET value = $1 WHERE id = $2`, []any{bigValue, secretB}}, + nil, ) require.Error(t, err, "the second update must not bypass the byte cap") require.True(t, database.IsCheckViolation(err, database.CheckUserSecretsPerUserTotalBytesLimit), @@ -178,6 +179,7 @@ func TestUserSkillsCapConcurrentInserts(t *testing.T) { err = runLockRace(ctx, t, sqlDB, []stmt{insert("winner-skill")}, insert("loser-skill"), + nil, ) require.Error(t, err, "the racing insert must recount and fail the cap") require.True(t, database.IsCheckViolation(err, database.CheckUserSkillsPerUserLimit), @@ -229,6 +231,7 @@ func TestUserSkillsCapConcurrentReassignment(t *testing.T) { VALUES ($1, $2, 'winner-skill', '', 'content') `, []any{uuid.New(), target.ID}}}, stmt{`UPDATE user_skills SET user_id = $1 WHERE id = $2`, []any{target.ID, movingSkill}}, + nil, ) require.Error(t, err, "the racing reassignment must recount and fail the cap") require.True(t, database.IsCheckViolation(err, database.CheckUserSkillsPerUserLimit), diff --git a/coderd/database/user_soft_delete_guards_test.go b/coderd/database/user_soft_delete_guards_test.go new file mode 100644 index 00000000000..bacd488b205 --- /dev/null +++ b/coderd/database/user_soft_delete_guards_test.go @@ -0,0 +1,690 @@ +package database_test + +import ( + "context" + "database/sql" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + + "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/database/dbgen" + "github.com/coder/coder/v2/coderd/database/dbtestutil" + "github.com/coder/coder/v2/testutil" +) + +// TestSoftDeleteGuardWinsConcurrentInsert verifies that all eight soft-delete +// guard triggers serialize against a concurrent user soft-delete via the +// parent-row lock added in migration 000592: the insert blocks on the locked +// users row and, once the soft-delete commits, fails with the guard's +// constraint instead of resurrecting a row for the deleted user. Each +// subtest also pins its database.Check* constant against the live trigger by +// matching the raised constraint name. +func TestSoftDeleteGuardWinsConcurrentInsert(t *testing.T) { + t.Parallel() + if testing.Short() { + t.SkipNow() + } + + db, _, sqlDB := dbtestutil.NewDBWithSQLDB(t) + + // Shared parents for the FK-bearing tables. + org := dbgen.Organization(t, db, database.Organization{}) + provider := dbgen.AIProvider(t, db, database.AIProvider{}) + group := dbgen.Group(t, db, database.Group{OrganizationID: org.ID}) + + testCases := []struct { + name string + table string + constraint database.CheckConstraint + // seed prepares rows the insert depends on (memberships etc.). + seed func(ctx context.Context, t *testing.T, userID uuid.UUID) + insert func(userID uuid.UUID) stmt + }{ + { + name: "APIKey", + table: "api_keys", + constraint: database.CheckAPIKeyUserDeleted, + insert: func(userID uuid.UUID) stmt { + return stmt{` + INSERT INTO api_keys (id, hashed_secret, user_id, last_used, expires_at, created_at, updated_at, login_type, scopes, allow_list) + VALUES ($1, 'race-hash'::bytea, $2, now(), now() + interval '1 hour', now(), now(), 'password', '{}'::api_key_scope[], ARRAY['*']) + `, []any{uuid.NewString(), userID}} + }, + }, + { + name: "UserLink", + table: "user_links", + constraint: database.CheckUserLinkUserDeleted, + insert: func(userID uuid.UUID) stmt { + return stmt{` + INSERT INTO user_links (user_id, login_type, linked_id) + VALUES ($1, 'github', 'race-link') + `, []any{userID}} + }, + }, + { + name: "UserSecret", + table: "user_secrets", + constraint: database.CheckUserSecretUserDeleted, + insert: func(userID uuid.UUID) stmt { + return stmt{` + INSERT INTO user_secrets (id, user_id, name, description, value, env_name) + VALUES ($1, $2, 'race-secret', '', 'value', 'RACE_SECRET') + `, []any{uuid.New(), userID}} + }, + }, + { + name: "UserSkill", + table: "user_skills", + constraint: database.CheckUserSkillUserDeleted, + insert: func(userID uuid.UUID) stmt { + return stmt{` + INSERT INTO user_skills (id, user_id, name, description, content) + VALUES ($1, $2, 'race-skill', '', 'content') + `, []any{uuid.New(), userID}} + }, + }, + { + name: "UserAIProviderKey", + table: "user_ai_provider_keys", + constraint: database.CheckUserAIProviderKeyUserDeleted, + insert: func(userID uuid.UUID) stmt { + return stmt{` + INSERT INTO user_ai_provider_keys (id, user_id, ai_provider_id, api_key) + VALUES ($1, $2, $3, 'race-key') + `, []any{uuid.New(), userID, provider.ID}} + }, + }, + { + name: "OrganizationMember", + table: "organization_members", + constraint: database.CheckOrganizationMemberUserDeleted, + insert: func(userID uuid.UUID) stmt { + return stmt{` + INSERT INTO organization_members (user_id, organization_id, created_at, updated_at) + VALUES ($1, $2, now(), now()) + `, []any{userID, org.ID}} + }, + }, + { + name: "GroupMember", + table: "group_members", + constraint: database.CheckGroupMemberUserDeleted, + insert: func(userID uuid.UUID) stmt { + return stmt{` + INSERT INTO group_members (user_id, group_id) VALUES ($1, $2) + `, []any{userID, group.ID}} + }, + }, + { + name: "UserAIBudgetOverride", + table: "user_ai_budget_overrides", + constraint: database.CheckUserAIBudgetOverrideUserDeleted, + // The membership trigger on this table fires before the guard + // (name order) and rejects non-members outright, so the racing + // user must be an org and group member for the insert to reach + // the guard's users lock. + seed: func(ctx context.Context, t *testing.T, userID uuid.UUID) { + _, err := sqlDB.ExecContext(ctx, ` + INSERT INTO organization_members (user_id, organization_id, created_at, updated_at) + VALUES ($1, $2, now(), now()) + `, userID, org.ID) + require.NoError(t, err) + _, err = sqlDB.ExecContext(ctx, + `INSERT INTO group_members (user_id, group_id) VALUES ($1, $2)`, + userID, group.ID) + require.NoError(t, err) + }, + insert: func(userID uuid.UUID) stmt { + return stmt{` + INSERT INTO user_ai_budget_overrides (user_id, group_id, spend_limit_micros) + VALUES ($1, $2, 0) + `, []any{userID, group.ID}} + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + ctx := testutil.Context(t, testutil.WaitLong) + user := dbgen.User(t, db, database.User{}) + if tc.seed != nil { + tc.seed(ctx, t, user.ID) + } + + // Hold the same lock the guard trigger takes so the insert + // blocks, then soft-delete before releasing it. + err := runLockRace(ctx, t, sqlDB, + []stmt{{`SELECT id FROM users WHERE id = $1 FOR NO KEY UPDATE`, []any{user.ID}}}, + tc.insert(user.ID), + []stmt{{`UPDATE users SET deleted = true WHERE id = $1`, []any{user.ID}}}, + ) + require.Error(t, err) + require.True(t, database.IsCheckViolation(err, tc.constraint), "expected constraint %q, got: %v", tc.constraint, err) + + var remaining int + //nolint:gosec // The table name comes from the test case definition. + err = sqlDB.QueryRowContext(ctx, + `SELECT count(*) FROM `+tc.table+` WHERE user_id = $1`, user.ID, + ).Scan(&remaining) + require.NoError(t, err) + require.Zero(t, remaining, "no rows may survive for the soft-deleted user") + }) + } +} + +// TestSoftDeleteGuardBlocksOwnerReassignment pins the UPDATE ... SET user_id +// leg added by migration 000592: re-parenting a live child row onto a +// soft-deleted user is rejected, both when the target is already deleted and +// when the soft-delete races the reassignment (the reassignment takes the +// same users-row lock as an insert). api_keys covers the dedicated +// owner-reassignment trigger on a previously INSERT-only table; +// user_ai_provider_keys covers the shared fail_if_user_deleted form. +func TestSoftDeleteGuardBlocksOwnerReassignment(t *testing.T) { + t.Parallel() + if testing.Short() { + t.SkipNow() + } + + db, _, sqlDB := dbtestutil.NewDBWithSQLDB(t) + provider := dbgen.AIProvider(t, db, database.AIProvider{}) + + insertAPIKey := func(ctx context.Context, t *testing.T, userID uuid.UUID) string { + id := uuid.NewString() + _, err := sqlDB.ExecContext(ctx, ` + INSERT INTO api_keys (id, hashed_secret, user_id, last_used, expires_at, created_at, updated_at, login_type, scopes, allow_list) + VALUES ($1, 'reassign-hash'::bytea, $2, now(), now() + interval '1 hour', now(), now(), 'password', '{}'::api_key_scope[], ARRAY['*']) + `, id, userID) + require.NoError(t, err) + return id + } + + t.Run("APIKeyOntoDeletedUser", func(t *testing.T) { + t.Parallel() + ctx := testutil.Context(t, testutil.WaitLong) + owner := dbgen.User(t, db, database.User{}) + doomed := dbgen.User(t, db, database.User{}) + keyID := insertAPIKey(ctx, t, owner.ID) + + _, err := sqlDB.ExecContext(ctx, + `UPDATE users SET deleted = true WHERE id = $1`, doomed.ID) + require.NoError(t, err) + + _, err = sqlDB.ExecContext(ctx, + `UPDATE api_keys SET user_id = $1 WHERE id = $2`, doomed.ID, keyID) + require.Error(t, err, "re-parenting a live api_keys row onto a deleted user must fail") + require.True(t, database.IsCheckViolation(err, database.CheckAPIKeyUserDeleted), + "expected constraint %q, got: %v", database.CheckAPIKeyUserDeleted, err) + + // Reassignment onto a live user succeeds. + liveTarget := dbgen.User(t, db, database.User{}) + _, err = sqlDB.ExecContext(ctx, + `UPDATE api_keys SET user_id = $1 WHERE id = $2`, liveTarget.ID, keyID) + require.NoError(t, err, "re-parenting onto a live user must succeed") + }) + + t.Run("AIProviderKeyOntoDeletedUser", func(t *testing.T) { + t.Parallel() + ctx := testutil.Context(t, testutil.WaitLong) + owner := dbgen.User(t, db, database.User{}) + doomed := dbgen.User(t, db, database.User{}) + rowID := uuid.New() + _, err := sqlDB.ExecContext(ctx, ` + INSERT INTO user_ai_provider_keys (id, user_id, ai_provider_id, api_key) + VALUES ($1, $2, $3, 'reassign-key') + `, rowID, owner.ID, provider.ID) + require.NoError(t, err) + + _, err = sqlDB.ExecContext(ctx, + `UPDATE users SET deleted = true WHERE id = $1`, doomed.ID) + require.NoError(t, err) + + _, err = sqlDB.ExecContext(ctx, + `UPDATE user_ai_provider_keys SET user_id = $1 WHERE id = $2`, doomed.ID, rowID) + require.Error(t, err, "re-parenting a provider key row onto a deleted user must fail") + require.True(t, database.IsCheckViolation(err, database.CheckUserAIProviderKeyUserDeleted), + "expected constraint %q, got: %v", database.CheckUserAIProviderKeyUserDeleted, err) + }) + + // The reassignment leg takes the same users-row lock as the INSERT leg, + // so it loses a race against a concurrent soft-delete instead of + // re-parenting onto a user whose soft-delete commits first. + t.Run("APIKeyLosesSoftDeleteRace", func(t *testing.T) { + t.Parallel() + ctx := testutil.Context(t, testutil.WaitLong) + owner := dbgen.User(t, db, database.User{}) + target := dbgen.User(t, db, database.User{}) + keyID := insertAPIKey(ctx, t, owner.ID) + + err := runLockRace(ctx, t, sqlDB, + []stmt{{`SELECT id FROM users WHERE id = $1 FOR NO KEY UPDATE`, []any{target.ID}}}, + stmt{`UPDATE api_keys SET user_id = $1 WHERE id = $2`, []any{target.ID, keyID}}, + []stmt{{`UPDATE users SET deleted = true WHERE id = $1`, []any{target.ID}}}, + ) + require.Error(t, err) + require.True(t, database.IsCheckViolation(err, database.CheckAPIKeyUserDeleted), + "expected constraint %q, got: %v", database.CheckAPIKeyUserDeleted, err) + }) +} + +// TestSoftDeleteGuardUpdatePathTakesNoUserLock pins the lock gates: no +// soft-delete guard takes a users-row lock on a same-owner UPDATE, so routine +// child-row updates proceed while the users row is locked. Without the gates +// this would block here and could deadlock in production against +// delete_deleted_user_resources. api_keys covers the hottest path (the +// last_used bump on every authenticated request), which the trigger's +// UPDATE OF user_id column list keeps out of plpgsql entirely. +func TestSoftDeleteGuardUpdatePathTakesNoUserLock(t *testing.T) { + t.Parallel() + if testing.Short() { + t.SkipNow() + } + + db, _, sqlDB := dbtestutil.NewDBWithSQLDB(t) + ctx := testutil.Context(t, testutil.WaitLong) + user := dbgen.User(t, db, database.User{}) + + keyID := uuid.NewString() + _, err := sqlDB.ExecContext(ctx, ` + INSERT INTO api_keys (id, hashed_secret, user_id, last_used, expires_at, created_at, updated_at, login_type, scopes, allow_list) + VALUES ($1, 'gate-hash'::bytea, $2, now(), now() + interval '1 hour', now(), now(), 'password', '{}'::api_key_scope[], ARRAY['*']) + `, keyID, user.ID) + require.NoError(t, err) + _, err = sqlDB.ExecContext(ctx, ` + INSERT INTO user_skills (id, user_id, name, description, content) + VALUES ($1, $2, 'gate-skill', '', 'content') + `, uuid.New(), user.ID) + require.NoError(t, err) + _, err = sqlDB.ExecContext(ctx, ` + INSERT INTO user_links (user_id, login_type, linked_id) + VALUES ($1, 'github', 'gate-link') + `, user.ID) + require.NoError(t, err) + _, err = sqlDB.ExecContext(ctx, ` + INSERT INTO user_secrets (id, user_id, name, description, value, env_name) + VALUES ($1, $2, 'gate-secret', '', 'value', 'GATE_SECRET') + `, uuid.New(), user.ID) + require.NoError(t, err) + + lockTx, err := sqlDB.BeginTx(ctx, nil) + require.NoError(t, err) + defer func() { _ = lockTx.Rollback() }() + var lockedUserID uuid.UUID + err = lockTx.QueryRowContext(ctx, + `SELECT id FROM users WHERE id = $1 FOR NO KEY UPDATE`, user.ID, + ).Scan(&lockedUserID) + require.NoError(t, err) + require.Equal(t, user.ID, lockedUserID) + + // All updates must complete while the users row is locked; blocking + // here would mean a trigger locked the parent on the same-owner UPDATE + // path, so the lock_timeout turns a missing gate into a failure. + updateConn := lockTimeoutConn(ctx, t, sqlDB, "5s") + + _, err = updateConn.ExecContext(ctx, + `UPDATE api_keys SET last_used = now() WHERE id = $1`, keyID) + require.NoError(t, err) + _, err = updateConn.ExecContext(ctx, + `UPDATE user_skills SET description = 'edited' WHERE user_id = $1`, user.ID) + require.NoError(t, err) + _, err = updateConn.ExecContext(ctx, + `UPDATE user_links SET linked_id = 'edited' WHERE user_id = $1`, user.ID) + require.NoError(t, err) + _, err = updateConn.ExecContext(ctx, + `UPDATE user_secrets SET description = 'edited' WHERE user_id = $1`, user.ID) + require.NoError(t, err) +} + +// TestSoftDeleteGuardTriggerOrder pins the trigger firing order the advisory +// cap locks depend on: BEFORE ROW triggers fire in name order, and the +// soft-delete guards (users lock) must fire before the zz_-prefixed cap +// triggers (advisory lock). A transaction that held the advisory lock while +// waiting on the users lock could cycle with an UPDATE-path advisory waiter +// and the soft-delete cleanup. Both the name order and the BEFORE ROW timing +// are asserted: an AFTER trigger sorts identically but fires after the cap +// trigger, silently inverting the real order. +func TestSoftDeleteGuardTriggerOrder(t *testing.T) { + t.Parallel() + if testing.Short() { + t.SkipNow() + } + + _, _, sqlDB := dbtestutil.NewDBWithSQLDB(t) + ctx := testutil.Context(t, testutil.WaitShort) + + for _, tc := range []struct { + table string + guard string + capName string + }{ + {"user_secrets", "trigger_upsert_user_secrets", "trigger_zz_user_secrets_per_user_limits"}, + {"user_skills", "trigger_upsert_user_skills", "trigger_zz_user_skills_per_user_limit"}, + } { + // Fetch the two triggers from the live catalog in firing (name) + // order, so the assertion derives from the database rather than + // comparing the test's own literals. tgtype bit 0 is ROW, bit 1 + // is BEFORE (an unset bit 1 with bit 6 unset means AFTER). + rows, err := sqlDB.QueryContext(ctx, ` + SELECT tgname, (tgtype::int & 1) = 1 AS is_row, (tgtype::int & 2) = 2 AS is_before + FROM pg_trigger + WHERE tgrelid = $1::regclass AND NOT tgisinternal AND tgname IN ($2, $3) + ORDER BY tgname + `, tc.table, tc.guard, tc.capName) + require.NoError(t, err) + var firingOrder []string + for rows.Next() { + var name string + var isRow, isBefore bool + require.NoError(t, rows.Scan(&name, &isRow, &isBefore)) + require.True(t, isRow, "%s: %s must be a FOR EACH ROW trigger", tc.table, name) + require.True(t, isBefore, "%s: %s must be a BEFORE trigger for name order to equal firing order", tc.table, name) + firingOrder = append(firingOrder, name) + } + require.NoError(t, rows.Err()) + require.NoError(t, rows.Close()) + require.Equal(t, []string{tc.guard, tc.capName}, firingOrder, + "%s: both triggers must exist and the guard must sort (and therefore fire) before the cap trigger", tc.table) + } +} + +// lockTimeoutConn returns a dedicated connection whose lock_timeout bounds +// every lock wait, so a statement that unexpectedly blocks fails the test +// instead of waiting for the shared context deadline to release the lock +// (which would let the statement succeed and mask the regression). The +// timeout is RESET before the connection returns to the pool: database/sql +// does not reset session state, and cleanups run LIFO, so the RESET runs +// before Close. +func lockTimeoutConn(ctx context.Context, t *testing.T, sqlDB *sql.DB, timeout string) *sql.Conn { + t.Helper() + conn, err := sqlDB.Conn(ctx) + require.NoError(t, err) + t.Cleanup(func() { _ = conn.Close() }) + _, err = conn.ExecContext(ctx, `SET lock_timeout = '`+timeout+`'`) + require.NoError(t, err) + t.Cleanup(func() { _, _ = conn.ExecContext(context.Background(), `RESET lock_timeout`) }) + return conn +} + +// runGuardedWriteRace deterministically replays a delete-or-update-then-insert +// transaction against a concurrent user soft-delete: +// +// 1. An outside transaction locks the child row the app transaction will +// touch, so the app transaction blocks mid-flight while already holding +// the users lock it took first (mirroring AcquireUserSoftDeleteGuardLock). +// 2. The soft-delete starts and queues behind the app transaction's users +// lock instead of interleaving into a lock-order inversion. +// 3. The outside lock is released; the app transaction finishes cleanly and +// the soft-delete then runs its cleanup. +// +// Remove the users-lock SELECT at the top of the app transaction below (the +// statement mirroring AcquireUserSoftDeleteGuardLock) and the replay +// deadlocks: the soft-delete's cleanup waits on the child row while the app +// transaction waits on the users row (SQLSTATE 40P01). +func runGuardedWriteRace(ctx context.Context, t *testing.T, sqlDB *sql.DB, userID uuid.UUID, childLock stmt, appStmts []stmt) { + t.Helper() + + outsideTx, err := sqlDB.BeginTx(ctx, nil) + require.NoError(t, err) + released := false + t.Cleanup(func() { + if !released { + _ = outsideTx.Rollback() + } + }) + _, err = outsideTx.ExecContext(ctx, childLock.sql, childLock.args...) + require.NoError(t, err) + + appConn, err := sqlDB.Conn(ctx) + require.NoError(t, err) + t.Cleanup(func() { _ = appConn.Close() }) + var appPID int + require.NoError(t, appConn.QueryRowContext(ctx, `SELECT pg_backend_pid()`).Scan(&appPID)) + appResult := make(chan error, 1) + go func() { + appResult <- func() error { + tx, err := appConn.BeginTx(ctx, nil) + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + var lockedID uuid.UUID + err = tx.QueryRowContext(ctx, + `SELECT id FROM users WHERE id = $1 FOR NO KEY UPDATE`, userID, + ).Scan(&lockedID) + if err != nil { + return err + } + for _, s := range appStmts { + if _, err := tx.ExecContext(ctx, s.sql, s.args...); err != nil { + return err + } + } + return tx.Commit() + }() + }() + waitForBackendBlocked(ctx, t, sqlDB, appPID) + + deleteConn, err := sqlDB.Conn(ctx) + require.NoError(t, err) + t.Cleanup(func() { _ = deleteConn.Close() }) + var deletePID int + require.NoError(t, deleteConn.QueryRowContext(ctx, `SELECT pg_backend_pid()`).Scan(&deletePID)) + deleteResult := make(chan error, 1) + go func() { + _, err := deleteConn.ExecContext(ctx, `UPDATE users SET deleted = true WHERE id = $1`, userID) + deleteResult <- err + }() + waitForBackendBlocked(ctx, t, sqlDB, deletePID) + + require.NoError(t, outsideTx.Rollback()) + released = true + + select { + case err := <-appResult: + require.NoError(t, err, "the app transaction must finish without deadlocking") + case <-ctx.Done(): + t.Fatalf("app transaction did not finish: %v", ctx.Err()) + } + select { + case err := <-deleteResult: + require.NoError(t, err, "the soft-delete must finish without deadlocking") + case <-ctx.Done(): + t.Fatalf("soft-delete did not finish: %v", ctx.Err()) + } +} + +// TestSoftDeleteGuardLockOrderPaths is the per-path deadlock-regression suite +// for the ordering contract documented on AcquireUserSoftDeleteGuardLock: +// each subtest mirrors the exact statement order of one Go transaction that +// locks a guarded child row and later inserts one, and fails with a deadlock +// if the users lock is not taken first. +func TestSoftDeleteGuardLockOrderPaths(t *testing.T) { + t.Parallel() + if testing.Short() { + t.SkipNow() + } + + db, _, sqlDB := dbtestutil.NewDBWithSQLDB(t) + org := dbgen.Organization(t, db, database.Organization{}) + + insertAPIKey := func(userID uuid.UUID) stmt { + return stmt{` + INSERT INTO api_keys (id, hashed_secret, user_id, last_used, expires_at, created_at, updated_at, login_type, scopes, allow_list) + VALUES ($1, 'lock-order-hash'::bytea, $2, now(), now() + interval '1 hour', now(), now(), 'password', '{}'::api_key_scope[], ARRAY['*']) + `, []any{uuid.NewString(), userID}} + } + + // Mirrors coderd/oauth2provider/tokens.go: both token grants delete the + // previous api_keys row and insert its replacement. + t.Run("OAuth2TokenReplacement", func(t *testing.T) { + t.Parallel() + ctx := testutil.Context(t, testutil.WaitLong) + user := dbgen.User(t, db, database.User{}) + prevKey, _ := dbgen.APIKey(t, db, database.APIKey{UserID: user.ID}) + + runGuardedWriteRace(ctx, t, sqlDB, user.ID, + stmt{`SELECT 1 FROM api_keys WHERE id = $1 FOR UPDATE`, []any{prevKey.ID}}, + []stmt{ + {`DELETE FROM api_keys WHERE id = $1`, []any{prevKey.ID}}, + insertAPIKey(user.ID), + }, + ) + }) + + // Mirrors coderd/provisionerdserver regenerateSessionToken: delete the + // workspace session token by name, insert the replacement. + t.Run("RegenerateSessionToken", func(t *testing.T) { + t.Parallel() + ctx := testutil.Context(t, testutil.WaitLong) + user := dbgen.User(t, db, database.User{}) + sessionKey, _ := dbgen.APIKey(t, db, database.APIKey{UserID: user.ID, TokenName: "session-token-lock-order"}) + + runGuardedWriteRace(ctx, t, sqlDB, user.ID, + stmt{`SELECT 1 FROM api_keys WHERE id = $1 FOR UPDATE`, []any{sessionKey.ID}}, + []stmt{ + {`DELETE FROM api_keys WHERE user_id = $1 AND token_name = $2`, []any{user.ID, "session-token-lock-order"}}, + insertAPIKey(user.ID), + }, + ) + }) + + // Mirrors coderd/userauth.go oauthLogin: update the user_links row for + // the fresh OAuth tokens, then insert organization_members via org sync. + t.Run("OAuthLoginOrgSync", func(t *testing.T) { + t.Parallel() + ctx := testutil.Context(t, testutil.WaitLong) + user := dbgen.User(t, db, database.User{}) + _, err := sqlDB.ExecContext(ctx, ` + INSERT INTO user_links (user_id, login_type, linked_id) + VALUES ($1, 'oidc', 'lock-order-link') + `, user.ID) + require.NoError(t, err) + + runGuardedWriteRace(ctx, t, sqlDB, user.ID, + stmt{`SELECT 1 FROM user_links WHERE user_id = $1 AND login_type = 'oidc' FOR UPDATE`, []any{user.ID}}, + []stmt{ + {`UPDATE user_links SET oauth_access_token = 'refreshed' WHERE user_id = $1 AND login_type = 'oidc'`, []any{user.ID}}, + {`INSERT INTO organization_members (user_id, organization_id, created_at, updated_at) VALUES ($1, $2, now(), now())`, []any{user.ID, org.ID}}, + }, + ) + }) + + // The advisory-lock leg of the ordering contract: an update-then-insert + // user_secrets writer holds the per-user advisory lock (from the + // UPDATE-path cap trigger, migration 000591) with no users lock, so a + // concurrent insert that holds the users lock and waits on the advisory + // lock would cycle with it. Taking the users lock first (as the + // contract requires) serializes the two: the concurrent insert queues + // behind the users lock and both finish. No Go path does + // update-then-insert on user_secrets today; this pins the contract the + // cap comments state. + t.Run("SecretsUpdateThenInsert", func(t *testing.T) { + t.Parallel() + ctx := testutil.Context(t, testutil.WaitLong) + user := dbgen.User(t, db, database.User{}) + secret := uuid.New() + _, err := sqlDB.ExecContext(ctx, ` + INSERT INTO user_secrets (id, user_id, name, description, value, env_name, file_path) + VALUES ($1, $2, 'update-then-insert', '', 'value', '', '/tmp/update-then-insert') + `, secret, user.ID) + require.NoError(t, err) + + // The contract-following writer: users lock, then UPDATE (advisory + // lock), then INSERT. The concurrent insert blocks on the users + // lock instead of interleaving into the advisory cycle, and + // succeeds once the writer commits. + err = runLockRace(ctx, t, sqlDB, + []stmt{ + {`SELECT id FROM users WHERE id = $1 FOR NO KEY UPDATE`, []any{user.ID}}, + {`UPDATE user_secrets SET value = 'edited' WHERE id = $1`, []any{secret}}, + }, + stmt{` + INSERT INTO user_secrets (id, user_id, name, description, value, env_name, file_path) + VALUES ($1, $2, 'concurrent-insert', '', 'value', '', '/tmp/concurrent-insert') + `, []any{uuid.New(), user.ID}}, + []stmt{ + {` + INSERT INTO user_secrets (id, user_id, name, description, value, env_name, file_path) + VALUES ($1, $2, 'writer-insert', '', 'value', '', '/tmp/writer-insert') + `, []any{uuid.New(), user.ID}}, + }, + ) + require.NoError(t, err, "with the users lock taken first, neither side may deadlock") + }) +} + +// TestSoftDeleteGuardRejectsUpdatesForDeletedUser pins the guard's UPDATE +// branch (the unlocked deleted-check) on the three upsert tables: a child +// row that survived cleanup must not keep being updated, or an orphaned +// user_links row could keep having its OAuth tokens refreshed. +func TestSoftDeleteGuardRejectsUpdatesForDeletedUser(t *testing.T) { + t.Parallel() + if testing.Short() { + t.SkipNow() + } + + db, _, sqlDB := dbtestutil.NewDBWithSQLDB(t) + + testCases := []struct { + name string + constraint database.CheckConstraint + seed func(ctx context.Context, t *testing.T, userID uuid.UUID) + update stmt + }{ + { + name: "UserLink", + constraint: database.CheckUserLinkUserDeleted, + seed: func(ctx context.Context, t *testing.T, userID uuid.UUID) { + _, err := sqlDB.ExecContext(ctx, ` + INSERT INTO user_links (user_id, login_type, linked_id) + VALUES ($1, 'oidc', 'update-branch-link') + `, userID) + require.NoError(t, err) + }, + update: stmt{`UPDATE user_links SET oauth_access_token = 'refreshed' WHERE user_id = $1`, nil}, + }, + { + name: "UserSecret", + constraint: database.CheckUserSecretUserDeleted, + seed: func(ctx context.Context, t *testing.T, userID uuid.UUID) { + _, err := sqlDB.ExecContext(ctx, ` + INSERT INTO user_secrets (id, user_id, name, description, value, env_name, file_path) + VALUES ($1, $2, 'update-branch-secret', '', 'value', '', '/tmp/update-branch-secret') + `, uuid.New(), userID) + require.NoError(t, err) + }, + update: stmt{`UPDATE user_secrets SET value = 'edited' WHERE user_id = $1`, nil}, + }, + { + name: "UserSkill", + constraint: database.CheckUserSkillUserDeleted, + seed: func(ctx context.Context, t *testing.T, userID uuid.UUID) { + _, err := sqlDB.ExecContext(ctx, ` + INSERT INTO user_skills (id, user_id, name, description, content) + VALUES ($1, $2, 'update-branch-skill', '', 'content') + `, uuid.New(), userID) + require.NoError(t, err) + }, + update: stmt{`UPDATE user_skills SET description = 'edited' WHERE user_id = $1`, nil}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + ctx := testutil.Context(t, testutil.WaitLong) + user := dbgen.User(t, db, database.User{}) + tc.seed(ctx, t, user.ID) + + dbtestutil.SoftDeleteUserKeepingRows(ctx, t, sqlDB, user.ID) + + _, err := sqlDB.ExecContext(ctx, tc.update.sql, user.ID) + require.Error(t, err, "updating a surviving child row of a deleted user must fail") + require.True(t, database.IsCheckViolation(err, tc.constraint), + "expected constraint %q, got: %v", tc.constraint, err) + }) + } +} diff --git a/coderd/database/usersoftdeleteguards.go b/coderd/database/usersoftdeleteguards.go new file mode 100644 index 00000000000..729ac666f51 --- /dev/null +++ b/coderd/database/usersoftdeleteguards.go @@ -0,0 +1,25 @@ +package database + +// Constraint names raised by the user soft-delete guard trigger functions +// installed by migration 000592 (check_user_not_deleted and the per-table +// functions delegating to it). These are raised with USING CONSTRAINT from +// plpgsql, not declared as table CHECK constraints, so dbgen does not emit +// them in check_constraint.go; they are declared once here so handlers and +// tests share one set of literals. TestSoftDeleteGuardWinsConcurrentInsert +// pins each name against the live trigger by matching the constraint a real +// failing insert raises. +const ( + // Raised by the guard triggers when inserting a child row for a + // soft-deleted user, reassigning a child row onto one, or (for the + // upsert tables user_links, user_secrets, and user_skills) updating a + // surviving child row of one. + //nolint:gosec // A trigger constraint name, not a credential. + CheckAPIKeyUserDeleted CheckConstraint = "api_key_user_deleted" + CheckUserLinkUserDeleted CheckConstraint = "user_link_user_deleted" + CheckUserSecretUserDeleted CheckConstraint = "user_secret_user_deleted" + CheckUserSkillUserDeleted CheckConstraint = "user_skill_user_deleted" + CheckUserAIProviderKeyUserDeleted CheckConstraint = "user_ai_provider_key_user_deleted" + CheckOrganizationMemberUserDeleted CheckConstraint = "organization_member_user_deleted" + CheckUserAIBudgetOverrideUserDeleted CheckConstraint = "user_ai_budget_override_user_deleted" + CheckGroupMemberUserDeleted CheckConstraint = "group_member_user_deleted" +) diff --git a/coderd/exp_chats.go b/coderd/exp_chats.go index 9f21f1c0adb..0ba6d6d382c 100644 --- a/coderd/exp_chats.go +++ b/coderd/exp_chats.go @@ -6847,6 +6847,14 @@ func (api *API) upsertUserAIProviderKey(rw http.ResponseWriter, r *http.Request) CreatedAt: now, UpdatedAt: now, }) + // 409: the request is well-formed and fails on the target's state, and + // the sibling deleted-user guard in userskills.go already uses Conflict. + if database.IsCheckViolation(err, database.CheckUserAIProviderKeyUserDeleted) { + httpapi.Write(ctx, rw, http.StatusConflict, codersdk.Response{ + Message: "Cannot store an AI provider key for a deleted user.", + }) + return + } if err != nil { api.Logger.Error(ctx, "failed to update user AI provider key", slog.Error(err), slog.F("user_id", targetUser.ID), slog.F("ai_provider_id", providerID)) httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{Message: "Failed to update user AI provider key."}) diff --git a/coderd/exp_chats_test.go b/coderd/exp_chats_test.go index 4644dab67f5..ec67c880ded 100644 --- a/coderd/exp_chats_test.go +++ b/coderd/exp_chats_test.go @@ -3399,6 +3399,28 @@ func TestUserAIProviderKeys(t *testing.T) { return nil } + t.Run("DeletedUser", func(t *testing.T) { + t.Parallel() + + ctx := testutil.Context(t, testutil.WaitLong) + adminClient := newChatClient(t) + firstUser := coderdtest.CreateFirstUser(t, adminClient.Client) + _, member := coderdtest.CreateAnotherUser(t, adminClient.Client, firstUser.OrganizationID) + provider := createOpenAIProvider(t, adminClient, "test-deleted-user-"+uuid.NewString(), true, "test-provider-api-key") + + err := adminClient.DeleteUser(ctx, member.ID) + require.NoError(t, err) + + // The {user} param resolves deleted users, so the request reaches + // the insert; the guard trigger rejects it and the handler maps the + // violation to a 409 instead of a raw pq error. + _, err = adminClient.UpsertUserAIProviderKey(ctx, member.ID.String(), provider.ID, codersdk.CreateUserAIProviderKeyRequest{APIKey: "orphan-key"}) + var apiErr *codersdk.Error + require.ErrorAs(t, err, &apiErr) + require.Equal(t, http.StatusConflict, apiErr.StatusCode()) + require.Equal(t, "Cannot store an AI provider key for a deleted user.", apiErr.Message) + }) + t.Run("SelfServiceLifecycle", func(t *testing.T) { t.Parallel() diff --git a/coderd/members.go b/coderd/members.go index 21728c065d5..ac1945ad3df 100644 --- a/coderd/members.go +++ b/coderd/members.go @@ -70,6 +70,15 @@ func (api *API) postOrganizationMember(rw http.ResponseWriter, r *http.Request) }) return } + // 409: the request is well-formed and fails on the target's state, and + // the sibling deleted-user guard in userskills.go already uses Conflict. + if database.IsCheckViolation(err, database.CheckOrganizationMemberUserDeleted) { + httpapi.Write(ctx, rw, http.StatusConflict, codersdk.Response{ + Message: "Cannot add a deleted user to an organization", + Detail: fmt.Sprintf("%s has been deleted.", user.Username), + }) + return + } if err != nil { httpapi.InternalServerError(rw, err) return diff --git a/coderd/members_test.go b/coderd/members_test.go index c2bf219c1eb..a9f0de3b9c6 100644 --- a/coderd/members_test.go +++ b/coderd/members_test.go @@ -3,6 +3,7 @@ package coderd_test import ( "context" "database/sql" + "net/http" "testing" "github.com/google/uuid" @@ -48,6 +49,26 @@ func TestAddMember(t *testing.T) { require.NoError(t, err) require.Equal(t, member.UserID, first.UserID) }) + + t.Run("DeletedUser", func(t *testing.T) { + t.Parallel() + ctx := testutil.Context(t, testutil.WaitMedium) + // The soft-delete removes the user's memberships; re-adding by ID + // reaches the insert (the {user} param resolves deleted users) and + // the guard trigger rejects it, which the handler maps to a 409. + _, deletedUser := coderdtest.CreateAnotherUser(t, owner, first.OrganizationID) + // nolint:gocritic // deleting a user requires owner permission. + err := owner.DeleteUser(ctx, deletedUser.ID) + require.NoError(t, err) + + // nolint:gocritic // adding an organization member requires owner permission. + _, err = owner.PostOrganizationMember(ctx, first.OrganizationID, deletedUser.ID.String()) + var apiErr *codersdk.Error + require.ErrorAs(t, err, &apiErr) + require.Equal(t, http.StatusConflict, apiErr.StatusCode()) + require.Equal(t, "Cannot add a deleted user to an organization", apiErr.Message) + require.Contains(t, apiErr.Detail, "has been deleted") + }) } func TestDeleteMember(t *testing.T) { diff --git a/coderd/oauth2_test.go b/coderd/oauth2_test.go index 57ddb4790d4..d47e98e3861 100644 --- a/coderd/oauth2_test.go +++ b/coderd/oauth2_test.go @@ -572,6 +572,109 @@ func TestOAuth2ProviderTokenExchange(t *testing.T) { } } +// TestOAuth2ProviderTokenExchangeLockOrder drives the token exchange through +// its real HTTP entry point as an ordinary member while the test holds the +// member's users-row lock, pinning two contracts the SQL-replay suite in +// coderd/database/user_soft_delete_guards_test.go cannot: +// 1. dbauthz authorizes AcquireUserSoftDeleteGuardLock as a system +// primitive, so a member's own token exchange passes authorization. +// 2. The real exchange transaction takes the users lock first: a backend +// blocks inside AcquireUserSoftDeleteGuardLock (identified by the sqlc +// query name in pg_stat_activity) before any api_keys or oauth2 code +// row is written. Remove the Go lock call, or weaken the query below +// FOR NO KEY UPDATE (which no longer conflicts with the held lock), +// and no backend ever blocks there: the Eventually times out red. +// "Before" is asserted directly: while the exchange is blocked, its +// backend must hold zero RowExclusiveLocks on api_keys or +// oauth2_provider_app_codes, so moving the lock call after the child +// writes (the deadlock-reintroducing inversion) also fails red. +func TestOAuth2ProviderTokenExchangeLockOrder(t *testing.T) { + t.Parallel() + + db, pubsub, sqlDB := dbtestutil.NewDBWithSQLDB(t) + ownerClient := coderdtest.New(t, &coderdtest.Options{ + Database: db, + Pubsub: pubsub, + }) + owner := coderdtest.CreateFirstUser(t, ownerClient) + ctx := testutil.Context(t, testutil.WaitLong) + apps := generateApps(ctx, t, ownerClient, "token-lock-order") + + //nolint:gocritic // OAauth2 app management requires owner permission. + secret, err := ownerClient.PostOAuth2ProviderAppSecret(ctx, apps.Default.ID) + require.NoError(t, err) + + userClient, user := coderdtest.CreateAnotherUser(t, ownerClient, owner.OrganizationID) + cfg := &oauth2.Config{ + ClientID: apps.Default.ID.String(), + ClientSecret: secret.ClientSecretFull, + Endpoint: oauth2.Endpoint{ + AuthURL: apps.Default.Endpoints.Authorization, + DeviceAuthURL: apps.Default.Endpoints.DeviceAuth, + TokenURL: apps.Default.Endpoints.Token, + AuthStyle: oauth2.AuthStyleInParams, + }, + RedirectURL: apps.Default.CallbackURL, + Scopes: []string{}, + } + code, verifier, err := authorizationFlow(ctx, userClient, cfg) + require.NoError(t, err) + + // Hold the same users-row lock a concurrent soft-delete would take. + lockTx, err := sqlDB.BeginTx(ctx, nil) + require.NoError(t, err) + defer func() { _ = lockTx.Rollback() }() + var lockedID uuid.UUID + require.NoError(t, lockTx.QueryRowContext(ctx, + `SELECT id FROM users WHERE id = $1 FOR NO KEY UPDATE`, user.ID, + ).Scan(&lockedID)) + + exchangeDone := make(chan error, 1) + var token *oauth2.Token + go func() { + tok, err := cfg.Exchange(ctx, code, oauth2.SetAuthURLParam("code_verifier", verifier)) + token = tok + exchangeDone <- err + }() + + // The exchange must block inside AcquireUserSoftDeleteGuardLock before + // writing any child row; sqlc embeds the query name in the SQL text. + var blockedPID int + testutil.Eventually(ctx, t, func(ctx context.Context) bool { + err := sqlDB.QueryRowContext(ctx, ` + SELECT pid FROM pg_stat_activity + WHERE datname = current_database() + AND wait_event_type = 'Lock' + AND query LIKE '%AcquireUserSoftDeleteGuardLock%' + `).Scan(&blockedPID) + return err == nil && blockedPID != 0 + }, testutil.IntervalFast, "the exchange must block on the users lock inside AcquireUserSoftDeleteGuardLock") + + // The lock is taken FIRST: while blocked on the users row, the exchange + // transaction must not yet hold a write lock on either child table it + // later writes. Reordering the AcquireUserSoftDeleteGuardLock call after + // DeleteOAuth2ProviderAppCodeByID / DeleteAPIKeyByID (the CRF-4 lock + // inversion) leaves a RowExclusiveLock here and fails this assertion. + var childLocks int + require.NoError(t, sqlDB.QueryRowContext(ctx, ` + SELECT count(*) FROM pg_locks + WHERE pid = $1 + AND mode = 'RowExclusiveLock' + AND relation IN ('api_keys'::regclass, 'oauth2_provider_app_codes'::regclass) + `, blockedPID).Scan(&childLocks)) + require.Zero(t, childLocks, "the exchange must take the users lock before writing any api_keys or oauth2_provider_app_codes row") + + require.NoError(t, lockTx.Rollback()) + + select { + case err := <-exchangeDone: + require.NoError(t, err, "a member's token exchange must succeed once the users lock is released") + require.NotEmpty(t, token.AccessToken) + case <-ctx.Done(): + t.Fatalf("exchange did not finish: %v", ctx.Err()) + } +} + // TestOAuth2ProviderTokenExchangeCodeBelongsToDifferentApp covers // authorizationCodeGrant's code-ownership check in isolation. The token // endpoint validates redirect_uri against the app resolved from client_id diff --git a/coderd/oauth2provider/tokens.go b/coderd/oauth2provider/tokens.go index 640f1f45659..61a4d17880f 100644 --- a/coderd/oauth2provider/tokens.go +++ b/coderd/oauth2provider/tokens.go @@ -497,6 +497,17 @@ func authorizationCodeGrant(ctx context.Context, db database.Store, logger slog. err = db.InTx(func(tx database.Store) error { ctx := dbauthz.As(ctx, actor) + + // Lock order: users before api_keys (deleted and inserted below for + // dbCode.UserID). See AcquireUserSoftDeleteGuardLock. + // The lock is a system primitive; the surrounding transaction stays + // under the end user's own actor. + //nolint:gocritic // see above + _, err = tx.AcquireUserSoftDeleteGuardLock(dbauthz.AsSystemRestricted(ctx), dbCode.UserID) + if err != nil { + return xerrors.Errorf("acquire user soft-delete guard lock: %w", err) + } + err = tx.DeleteOAuth2ProviderAppCodeByID(ctx, dbCode.ID) if err != nil { return xerrors.Errorf("delete oauth2 app code: %w", err) @@ -642,6 +653,18 @@ func refreshTokenGrant(ctx context.Context, db database.Store, app database.OAut err = db.InTx(func(tx database.Store) error { ctx := dbauthz.As(ctx, actor) + + // Lock order: users before api_keys. See AcquireUserSoftDeleteGuardLock. + // prevKey.UserID matches the rows deleted and inserted below; every + // other statement in this block derives from prevKey. + // The lock is a system primitive; the surrounding transaction stays + // under the end user's own actor. + //nolint:gocritic // see above + _, err = tx.AcquireUserSoftDeleteGuardLock(dbauthz.AsSystemRestricted(ctx), prevKey.UserID) + if err != nil { + return xerrors.Errorf("acquire user soft-delete guard lock: %w", err) + } + err = tx.DeleteAPIKeyByID(ctx, prevKey.ID) // This cascades to the token. if err != nil { return xerrors.Errorf("delete oauth2 app token: %w", err) diff --git a/coderd/provisionerdserver/provisionerdserver.go b/coderd/provisionerdserver/provisionerdserver.go index 381ab67c773..0e5e2cd2327 100644 --- a/coderd/provisionerdserver/provisionerdserver.go +++ b/coderd/provisionerdserver/provisionerdserver.go @@ -3317,7 +3317,17 @@ func (s *server) regenerateSessionToken(ctx context.Context, user database.User, } err = s.Database.InTx(func(tx database.Store) error { - err := deleteSessionToken(ctx, tx, workspace) + // Lock order: users before api_keys. See AcquireUserSoftDeleteGuardLock. + // user.ID is the same variable the inserted key derives from (the + // single caller passes the workspace owner), so the lock and the + // insert cannot diverge. + //nolint:gocritic // Session token rotation runs as the provisioner daemon, not the workspace owner. + _, err := tx.AcquireUserSoftDeleteGuardLock(dbauthz.AsSystemRestricted(ctx), user.ID) + if err != nil { + return xerrors.Errorf("acquire user soft-delete guard lock: %w", err) + } + + err = deleteSessionToken(ctx, tx, workspace) if err != nil { return xerrors.Errorf("delete session token: %w", err) } diff --git a/coderd/userauth.go b/coderd/userauth.go index fb80c2230b5..da2687df472 100644 --- a/coderd/userauth.go +++ b/coderd/userauth.go @@ -1759,6 +1759,18 @@ func (api *API) oauthLogin(r *http.Request, params *oauthLoginParams) ([]*http.C user = params.User link = params.Link + if user.ID != uuid.Nil { + // Lock order: users before user_links and organization_members. + // See AcquireUserSoftDeleteGuardLock. New signups skip this: + // their users row is created inside this transaction and is + // invisible to a concurrent soft-delete. + // nolint:gocritic // The user is not authenticated yet; the lock runs as the system. + _, err = tx.AcquireUserSoftDeleteGuardLock(dbauthz.AsSystemRestricted(ctx), user.ID) + if err != nil { + return xerrors.Errorf("acquire user soft-delete guard lock: %w", err) + } + } + // If you do a convert to OIDC and your email does not match, we need to // catch this and not make a new account. if isMergeStateString(params.State.StateString) { diff --git a/coderd/userskills.go b/coderd/userskills.go index ba0f35dd52b..9deb00ad474 100644 --- a/coderd/userskills.go +++ b/coderd/userskills.go @@ -27,12 +27,6 @@ const ( // maxPersonalSkillRequestBytes allows worst-case JSON string escaping for // otherwise valid raw skill content. maxPersonalSkillRequestBytes = skills.MaxPersonalSkillSizeBytes*personalSkillJSONEscapeExpansion + personalSkillRequestEnvelopeBytes - - // Raised by the insert_user_skill_fail_if_user_deleted trigger with - // USING CONSTRAINT. Not a table CHECK constraint, so dbgen does not - // emit it in check_constraint.go. The cap constraint lives in - // database.CheckUserSkillsPerUserLimit. - userSkillUserDeletedConstraint database.CheckConstraint = "user_skill_user_deleted" ) // @Summary Create a user skill @@ -84,7 +78,7 @@ func (api *API) postUserSkill(rw http.ResponseWriter, r *http.Request) { httpapi.Forbidden(rw) return } - if database.IsCheckViolation(err, userSkillUserDeletedConstraint) { + if database.IsCheckViolation(err, database.CheckUserSkillUserDeleted) { writeCannotCreateUserSkillForDeletedUser(ctx, rw) return } @@ -246,7 +240,7 @@ func (api *API) patchUserSkill(rw http.ResponseWriter, r *http.Request) { httpapi.Forbidden(rw) return } - if database.IsCheckViolation(err, userSkillUserDeletedConstraint) { + if database.IsCheckViolation(err, database.CheckUserSkillUserDeleted) { writeCannotModifyUserSkillForDeletedUser(ctx, rw) return } diff --git a/enterprise/cli/server_dbcrypt_test.go b/enterprise/cli/server_dbcrypt_test.go index 0915f2d3157..1fae5bc0100 100644 --- a/enterprise/cli/server_dbcrypt_test.go +++ b/enterprise/cli/server_dbcrypt_test.go @@ -43,7 +43,7 @@ func TestServerDBCrypt(t *testing.T) { // Populate the database with some unencrypted data. t.Log("Generating unencrypted data") - users := genData(t, db) + users := genData(t, db, sqlDB) // Setup an initial cipher A keyA := testutil.MustRandString(t, 32) @@ -56,7 +56,7 @@ func TestServerDBCrypt(t *testing.T) { // Populate the database with some encrypted data using cipher A. t.Log("Generating data encrypted with cipher A") - newUsers := genData(t, cryptdb) + newUsers := genData(t, cryptdb, sqlDB) // Validate that newly created users were encrypted with cipher A for _, usr := range newUsers { @@ -205,7 +205,7 @@ func TestServerDBCrypt(t *testing.T) { } } -func genData(t *testing.T, db database.Store) []database.User { +func genData(t *testing.T, db database.Store, sqlDB *sql.DB) []database.User { t.Helper() var users []database.User // Make some users @@ -213,12 +213,17 @@ func genData(t *testing.T, db database.Store) []database.User { for _, loginType := range database.AllLoginTypeValues() { for _, deleted := range []bool{false, true} { randName := testutil.MustRandString(t, 32) + // Users in the deleted lane are created live, seeded, and + // soft-deleted below with the cleanup trigger suppressed: + // the guard triggers (migration 000592) reject inserting + // child rows for already-deleted users, and the point of + // the deleted lane is encrypting the orphaned rows that + // predate them. usr := dbgen.User(t, db, database.User{ Username: randName, Email: randName + "@notcoder.com", LoginType: loginType, Status: status, - Deleted: deleted, }) _ = dbgen.ExternalAuthLink(t, db, database.ExternalAuthLink{ UserID: usr.ID, @@ -241,6 +246,9 @@ func genData(t *testing.T, db database.Store) []database.User { PrivateKey: "private-" + usr.ID.String(), PublicKey: "public-" + usr.ID.String(), }) + // Seeded for every user; the deleted lane keeps this row as + // an orphan via the trigger-suppressed soft-delete below, + // preserving deleted-user encryption coverage. now := time.Now() _, err := db.UpsertUserAIProviderKey(context.Background(), database.UpsertUserAIProviderKeyParams{ ID: uuid.New(), @@ -252,7 +260,9 @@ func genData(t *testing.T, db database.Store) []database.User { }) require.NoError(t, err) - // Deleted users cannot have user_links or user_secrets. + // The soft-delete guards reject inserting user_links or + // user_secrets for a deleted user, so seed them only for + // live users. if !deleted { // Fun fact: our schema allows _all_ login types to have // a user_link. Even though I'm not sure how it could occur @@ -272,6 +282,12 @@ func genData(t *testing.T, db database.Store) []database.User { FilePath: "", }) } + if deleted { + // Reconstructs orphaned child rows that predate the + // migration 000592 guards; rotation must handle them. + dbtestutil.SoftDeleteUserKeepingRows(context.Background(), t, sqlDB, usr.ID) + usr.Deleted = true + } users = append(users, usr) } } diff --git a/enterprise/coderd/aibridge.go b/enterprise/coderd/aibridge.go index 760f3ee176d..7cf7cedde59 100644 --- a/enterprise/coderd/aibridge.go +++ b/enterprise/coderd/aibridge.go @@ -876,6 +876,16 @@ func (api *API) upsertUserAIBudgetOverride(rw http.ResponseWriter, r *http.Reque }) return } + // The soft-delete guard rejects overrides for a user deleted after the + // middleware fetched them. 409: the request is well-formed and fails on + // the target's state, matching coderd/members.go. + if database.IsCheckViolation(err, database.CheckUserAIBudgetOverrideUserDeleted) { + httpapi.Write(ctx, rw, http.StatusConflict, codersdk.Response{ + Message: "Cannot set an AI budget override for a deleted user.", + Detail: fmt.Sprintf("%s has been deleted.", user.Username), + }) + return + } if httpapi.Is404Error(err) { httpapi.ResourceNotFound(rw) return diff --git a/enterprise/coderd/aibridge_test.go b/enterprise/coderd/aibridge_test.go index 38fcb9ad6af..9d6427b356f 100644 --- a/enterprise/coderd/aibridge_test.go +++ b/enterprise/coderd/aibridge_test.go @@ -2861,6 +2861,73 @@ func TestUserAIBudgetOverride(t *testing.T) { require.EqualValues(t, 1_000_000_000, currentOverride.SpendLimitMicros) }) + t.Run("Upsert/DeletedUserRace", func(t *testing.T) { + t.Parallel() + + // The membership trigger fires before the soft-delete guard + // (alphabetical trigger order) and a committed soft-delete removes + // the memberships it checks, so the guard's check violation + // surfaces only in the true race: a soft-delete that commits while + // the upsert's insert is parked on the locked users row. Hold that + // deletion uncommitted, let the upsert block on it, commit, and + // require the handler to map the guard violation to a 409. + db, ps, sqlDB := dbtestutil.NewDBWithSQLDB(t) + adminClient, targetUser, group := setupAICostControlTest(t, aiCostControlTestOptions{ + GroupName: "deleted-user-race-group", + Database: db, + Pubsub: ps, + }) + ctx := testutil.Context(t, testutil.WaitLong) + + deleteTx, err := sqlDB.BeginTx(ctx, nil) + require.NoError(t, err) + committed := false + defer func() { + if !committed { + _ = deleteTx.Rollback() + } + }() + _, err = deleteTx.ExecContext(ctx, + `UPDATE users SET deleted = true WHERE id = $1`, targetUser.ID) + require.NoError(t, err) + + upsertDone := make(chan error, 1) + go func() { + _, err := adminClient.UpsertUserAIBudgetOverride(ctx, targetUser.ID, codersdk.UpsertUserAIBudgetOverrideRequest{ + GroupID: group.ID, + SpendLimitMicros: 500_000_000, + }) + upsertDone <- err + }() + + // Wait until the upsert's insert is provably parked on the users + // row held by the deletion; sqlc embeds the query name in the SQL. + testutil.Eventually(ctx, t, func(ctx context.Context) bool { + var blockedPID int + err := sqlDB.QueryRowContext(ctx, ` + SELECT pid FROM pg_stat_activity + WHERE datname = current_database() + AND wait_event_type = 'Lock' + AND query LIKE '%UpsertUserAIBudgetOverride%' + `).Scan(&blockedPID) + return err == nil && blockedPID != 0 + }, testutil.IntervalFast, "the upsert must block on the users row inside the soft-delete guard") + + require.NoError(t, deleteTx.Commit()) + committed = true + + select { + case err := <-upsertDone: + var apiErr *codersdk.Error + require.ErrorAs(t, err, &apiErr) + require.Equal(t, http.StatusConflict, apiErr.StatusCode()) + require.Equal(t, "Cannot set an AI budget override for a deleted user.", apiErr.Message) + require.Contains(t, apiErr.Detail, "has been deleted") + case <-ctx.Done(): + t.Fatalf("upsert did not finish: %v", ctx.Err()) + } + }) + t.Run("Upsert/ReassignsGroup", func(t *testing.T) { t.Parallel() diff --git a/enterprise/dbcrypt/cliutil_test.go b/enterprise/dbcrypt/cliutil_test.go index fef03753191..c8eb112d230 100644 --- a/enterprise/dbcrypt/cliutil_test.go +++ b/enterprise/dbcrypt/cliutil_test.go @@ -632,11 +632,13 @@ func TestRotateUserAIProviderKeys(t *testing.T) { // alongside a live user's row purely to confirm Rotate doesn't need // one. liveUser := dbgen.User(t, f.rawDB, database.User{}) - deletedUser := dbgen.User(t, f.rawDB, database.User{Deleted: true}) + deletedUser := dbgen.User(t, f.rawDB, database.User{}) provider := dbgen.AIProvider(t, f.rawDB, database.AIProvider{}) liveKey := upsertUserAIProviderKey(f.ctx, t, f.cryptDBA, liveUser.ID, provider.ID, "user-key-live") deletedKey := upsertUserAIProviderKey(f.ctx, t, f.cryptDBA, deletedUser.ID, provider.ID, "user-key-deleted-user") + // Rotation and decryption must still handle legacy orphaned rows. + dbtestutil.SoftDeleteUserKeepingRows(f.ctx, t, f.sqlDB, deletedUser.ID) f.rotate(t) @@ -1223,11 +1225,13 @@ func TestDecryptUserAIProviderKeys(t *testing.T) { f := newDecryptFixture(t) liveUser := dbgen.User(t, f.rawDB, database.User{}) - deletedUser := dbgen.User(t, f.rawDB, database.User{Deleted: true}) + deletedUser := dbgen.User(t, f.rawDB, database.User{}) provider := dbgen.AIProvider(t, f.rawDB, database.AIProvider{}) liveKey := upsertUserAIProviderKey(f.ctx, t, f.cryptDBA, liveUser.ID, provider.ID, "user-key-live") deletedKey := upsertUserAIProviderKey(f.ctx, t, f.cryptDBA, deletedUser.ID, provider.ID, "user-key-deleted-user") + // Rotation and decryption must still handle legacy orphaned rows. + dbtestutil.SoftDeleteUserKeepingRows(f.ctx, t, f.sqlDB, deletedUser.ID) f.decrypt(t)