coder · ThomasK33 · Sep 1, 2026 · Sep 1, 2026 · Sep 1, 2026 · Sep 7, 2026
diff --git a/coderd/apikey.go b/coderd/apikey.go
@@ -176,6 +176,16 @@ func (api *API) postToken(rw http.ResponseWriter, r *http.Request) {
 			})
 			return
 		}
+		// The soft-delete guard rejects API keys for a user deleted after
+		// the middleware fetched them. 409: the request is well-formed and
+		// fails on the target's state, matching members.go.
+		if database.IsCheckViolation(err, database.CheckAPIKeyUserDeleted) {
+			httpapi.Write(ctx, rw, http.StatusConflict, codersdk.Response{
+				Message: "Cannot create a token for a deleted user.",
+				Detail:  fmt.Sprintf("%s has been deleted.", user.Username),
+			})
+			return
+		}
 		httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
 			Message: "Failed to create API key.",
 			Detail:  err.Error(),
@@ -226,6 +236,16 @@ func (api *API) postAPIKey(rw http.ResponseWriter, r *http.Request) {
 		RemoteAddr:      r.RemoteAddr,
 	})
 	if err != nil {
+		// The soft-delete guard rejects API keys for a user deleted after
+		// the middleware fetched them. 409: the request is well-formed and
+		// fails on the target's state, matching members.go.
+		if database.IsCheckViolation(err, database.CheckAPIKeyUserDeleted) {
+			httpapi.Write(ctx, rw, http.StatusConflict, codersdk.Response{
+				Message: "Cannot create an API key for a deleted user.",
+				Detail:  fmt.Sprintf("%s has been deleted.", user.Username),
+			})
+			return
+		}
 		httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
 			Message: "Failed to create API key.",
 			Detail:  err.Error(),

diff --git a/coderd/apikey_test.go b/coderd/apikey_test.go
@@ -70,6 +70,52 @@ func TestTokenCRUD(t *testing.T) {
 	require.Equal(t, database.AuditActionDelete, auditor.AuditLogs()[numLogs-1].Action)
 }
 
+// TestAPIKeysDeletedUser verifies both key-creation handlers map the
+// api_keys soft-delete guard to a 409: the {user} parameter resolves
+// deleted users by ID, so a stale ID reaches the insert and the guard
+// trigger (migration 000592) rejects it instead of surfacing a 500.
+func TestAPIKeysDeletedUser(t *testing.T) {
+	t.Parallel()
+
+	client := coderdtest.New(t, nil)
+	owner := coderdtest.CreateFirstUser(t, client)
+
+	deleteUser := func(ctx context.Context, t *testing.T) codersdk.User {
+		t.Helper()
+		_, user := coderdtest.CreateAnotherUser(t, client, owner.OrganizationID)
+		// nolint:gocritic // deleting a user requires owner permission.
+		err := client.DeleteUser(ctx, user.ID)
+		require.NoError(t, err)
+		return user
+	}
+
+	t.Run("Token", func(t *testing.T) {
+		t.Parallel()
+		ctx := testutil.Context(t, testutil.WaitMedium)
+		deletedUser := deleteUser(ctx, t)
+
+		_, err := client.CreateToken(ctx, deletedUser.ID.String(), codersdk.CreateTokenRequest{})
+		var apiErr *codersdk.Error
+		require.ErrorAs(t, err, &apiErr)
+		require.Equal(t, http.StatusConflict, apiErr.StatusCode())
+		require.Equal(t, "Cannot create a token for a deleted user.", apiErr.Message)
+		require.Contains(t, apiErr.Detail, "has been deleted")
+	})
+
+	t.Run("SessionKey", func(t *testing.T) {
+		t.Parallel()
+		ctx := testutil.Context(t, testutil.WaitMedium)
+		deletedUser := deleteUser(ctx, t)
+
+		_, err := client.CreateAPIKey(ctx, deletedUser.ID.String())
+		var apiErr *codersdk.Error
+		require.ErrorAs(t, err, &apiErr)
+		require.Equal(t, http.StatusConflict, apiErr.StatusCode())
+		require.Equal(t, "Cannot create an API key for a deleted user.", apiErr.Message)
+		require.Contains(t, apiErr.Detail, "has been deleted")
+	})
+}
+
 func TestTokensFilterExpired(t *testing.T) {
 	t.Parallel()
 

diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go
@@ -1815,6 +1815,20 @@ func (q *querier) AcquireStaleChatDiffStatuses(ctx context.Context, limitVal int
 	return q.db.AcquireStaleChatDiffStatuses(ctx, limitVal)
 }
 
+func (q *querier) AcquireUserSoftDeleteGuardLock(ctx context.Context, userID uuid.UUID) (uuid.UUID, error) {
+	// The lock is a deadlock-avoidance primitive, not a user capability: it
+	// orders a transaction's locks (users first, then child rows), granting
+	// nothing the guard triggers do not already do for free on every insert.
+	// Authorizing it on the target user would be role-dependent (owners
+	// pass, members fail) while protecting nothing, so it authorizes as a
+	// system primitive and call sites on user-scoped paths wrap only this
+	// call in dbauthz.AsSystemRestricted.
+	if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceSystem); err != nil {
+		return uuid.Nil, err
+	}
+	return q.db.AcquireUserSoftDeleteGuardLock(ctx, userID)
+}
+
 func (q *querier) ActivityBumpWorkspace(ctx context.Context, arg database.ActivityBumpWorkspaceParams) error {
 	fetch := func(ctx context.Context, arg database.ActivityBumpWorkspaceParams) (database.Workspace, error) {
 		return q.db.GetWorkspaceByID(ctx, arg.WorkspaceID)
@@ -7223,6 +7237,13 @@ func (q *querier) PopNextQueuedMessage(ctx context.Context, chatID uuid.UUID) (d
 	return q.db.PopNextQueuedMessage(ctx, chatID)
 }
 
+func (q *querier) PurgeSoftDeletedUserResources(ctx context.Context) error {
+	if err := q.authorizeContext(ctx, policy.ActionDelete, rbac.ResourceSystem); err != nil {
+		return err
+	}
+	return q.db.PurgeSoftDeletedUserResources(ctx)
+}
+
 func (q *querier) ReduceWorkspaceAgentShareLevelToAuthenticatedByTemplate(ctx context.Context, templateID uuid.UUID) error {
 	template, err := q.db.GetTemplateByID(ctx, templateID)
 	if err != nil {

diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go
@@ -3093,6 +3093,15 @@ func (s *MethodTestSuite) TestUser() {
 		dbm.EXPECT().GetUserByID(gomock.Any(), u.ID).Return(u, nil).AnyTimes()
 		check.Args(u.ID).Asserts(u, policy.ActionRead).Returns(u)
 	}))
+	s.Run("AcquireUserSoftDeleteGuardLock", s.Mocked(func(dbm *dbmock.MockStore, faker *gofakeit.Faker, check *expects) {
+		u := testutil.Fake(s.T(), faker, database.User{})
+		dbm.EXPECT().AcquireUserSoftDeleteGuardLock(gomock.Any(), u.ID).Return(u.ID, nil).AnyTimes()
+		check.Args(u.ID).Asserts(rbac.ResourceSystem, policy.ActionUpdate).Returns(u.ID)
+	}))
+	s.Run("PurgeSoftDeletedUserResources", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) {
+		dbm.EXPECT().PurgeSoftDeletedUserResources(gomock.Any()).Return(nil).AnyTimes()
+		check.Args().Asserts(rbac.ResourceSystem, policy.ActionDelete)
+	}))
 	s.Run("GetUsersByIDs", s.Mocked(func(dbm *dbmock.MockStore, faker *gofakeit.Faker, check *expects) {
 		a := testutil.Fake(s.T(), faker, database.User{CreatedAt: dbtime.Now().Add(-time.Hour)})
 		b := testutil.Fake(s.T(), faker, database.User{CreatedAt: dbtime.Now()})

diff --git a/coderd/database/dbmetrics/querymetrics.go b/coderd/database/dbmetrics/querymetrics.go
diff --git a/coderd/database/dbmock/dbmock.go b/coderd/database/dbmock/dbmock.go
diff --git a/coderd/database/dbpurge/dbpurge.go b/coderd/database/dbpurge/dbpurge.go
@@ -238,6 +238,12 @@ func (i *instance) purgeTick(ctx context.Context, db database.Store, start time.
 		if err := tx.ExpirePrebuildsAPIKeys(ctx, dbtime.Time(start)); err != nil {
 			return xerrors.Errorf("failed to expire prebuilds user api keys: %w", err)
 		}
+		// Remove child rows orphaned by a user soft-delete that predates the
+		// guard triggers and cleanup coverage (migration 000592). The guards
+		// prevent new orphans, so after the first pass this is a no-op.
+		if err := tx.PurgeSoftDeletedUserResources(ctx); err != nil {
+			return xerrors.Errorf("failed to purge soft-deleted user resources: %w", err)
+		}
 
 		var expiredAPIKeys int64
 		apiKeysRetention := i.vals.Retention.APIKeys.Value()

diff --git a/coderd/database/dbpurge/dbpurge_test.go b/coderd/database/dbpurge/dbpurge_test.go
@@ -253,6 +253,7 @@ func TestMetrics(t *testing.T) {
 		mDB.EXPECT().DeleteOldProvisionerDaemons(gomock.Any()).Return(nil).AnyTimes()
 		mDB.EXPECT().DeleteOldNotificationMessages(gomock.Any()).Return(nil).AnyTimes()
 		mDB.EXPECT().ExpirePrebuildsAPIKeys(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
+		mDB.EXPECT().PurgeSoftDeletedUserResources(gomock.Any()).Return(nil).AnyTimes()
 		mDB.EXPECT().DeleteOldTelemetryLocks(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
 		mDB.EXPECT().DeleteOldWorkspaceBuildOrchestrations(gomock.Any(), gomock.Any()).Return(int64(0), nil).AnyTimes()
 		mDB.EXPECT().DeleteOldAuditLogConnectionEvents(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
@@ -307,6 +308,7 @@ func TestMetrics(t *testing.T) {
 		mDB.EXPECT().DeleteOldProvisionerDaemons(gomock.Any()).Return(nil).AnyTimes()
 		mDB.EXPECT().DeleteOldNotificationMessages(gomock.Any()).Return(nil).AnyTimes()
 		mDB.EXPECT().ExpirePrebuildsAPIKeys(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
+		mDB.EXPECT().PurgeSoftDeletedUserResources(gomock.Any()).Return(nil).AnyTimes()
 		mDB.EXPECT().DeleteOldTelemetryLocks(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
 		mDB.EXPECT().DeleteOldWorkspaceBuildOrchestrations(gomock.Any(), gomock.Any()).Return(int64(0), nil).AnyTimes()
 		mDB.EXPECT().DeleteOldAuditLogConnectionEvents(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
@@ -3551,3 +3553,99 @@ func TestDeleteIdentifiedModuleCacheFiles(t *testing.T) {
 	assertFileExists(late.ID, "archive inserted after the one-off pass")
 	assertCacheRef(lateTV, late.ID, true, "archive inserted after the one-off pass")
 }
+
+// TestPurgeSoftDeletedUserResources verifies the reaper removes child rows
+// orphaned by a user soft-delete that predates the guard triggers and
+// cleanup coverage (migration 000592 deliberately has no backfill), while a
+// live user's rows survive.
+//
+//nolint:paralleltest // It uses LockIDDBPurge.
+func TestPurgeSoftDeletedUserResources(t *testing.T) {
+	db, _, sqlDB := dbtestutil.NewDBWithSQLDB(t)
+	logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
+	ctx := testutil.Context(t, testutil.WaitShort)
+
+	org := dbgen.Organization(t, db, database.Organization{})
+	provider := dbgen.AIProvider(t, db, database.AIProvider{})
+	group := dbgen.Group(t, db, database.Group{OrganizationID: org.ID})
+
+	liveUser := dbgen.User(t, db, database.User{})
+	doomedUser := dbgen.User(t, db, database.User{})
+
+	seed := func(userID uuid.UUID) {
+		_, err := sqlDB.ExecContext(ctx, `
+			INSERT INTO api_keys (id, hashed_secret, user_id, last_used, expires_at, created_at, updated_at, login_type, scopes, allow_list)
+			VALUES ($1, 'reap-hash'::bytea, $2, now(), now() + interval '1 hour', now(), now(), 'password', '{}'::api_key_scope[], ARRAY['*'])
+		`, uuid.NewString(), userID)
+		require.NoError(t, err)
+		_, err = sqlDB.ExecContext(ctx,
+			`INSERT INTO user_links (user_id, login_type, linked_id) VALUES ($1, 'github', $2)`,
+			userID, "reap-link-"+userID.String())
+		require.NoError(t, err)
+		_, err = sqlDB.ExecContext(ctx, `
+			INSERT INTO user_secrets (id, user_id, name, description, value, env_name)
+			VALUES ($1, $2, 'reap-secret', '', 'value', 'REAP_SECRET')
+		`, uuid.New(), userID)
+		require.NoError(t, err)
+		_, err = sqlDB.ExecContext(ctx, `
+			INSERT INTO user_skills (id, user_id, name, description, content)
+			VALUES ($1, $2, 'reap-skill', '', 'content')
+		`, uuid.New(), userID)
+		require.NoError(t, err)
+		_, err = sqlDB.ExecContext(ctx, `
+			INSERT INTO user_ai_provider_keys (id, user_id, ai_provider_id, api_key)
+			VALUES ($1, $2, $3, 'reap-key')
+		`, uuid.New(), userID, provider.ID)
+		require.NoError(t, err)
+		_, err = sqlDB.ExecContext(ctx, `
+			INSERT INTO organization_members (user_id, organization_id, created_at, updated_at)
+			VALUES ($1, $2, now(), now())
+		`, userID, org.ID)
+		require.NoError(t, err)
+		_, err = sqlDB.ExecContext(ctx,
+			`INSERT INTO group_members (user_id, group_id) VALUES ($1, $2)`,
+			userID, group.ID)
+		require.NoError(t, err)
+		_, err = sqlDB.ExecContext(ctx, `
+			INSERT INTO user_ai_budget_overrides (user_id, group_id, spend_limit_micros)
+			VALUES ($1, $2, 1000000)
+		`, userID, group.ID)
+		require.NoError(t, err)
+	}
+	seed(liveUser.ID)
+	seed(doomedUser.ID)
+
+	// Reconstruct pre-guard orphans: soft-delete with the cleanup trigger
+	// suppressed, so every child row survives.
+	dbtestutil.SoftDeleteUserKeepingRows(ctx, t, sqlDB, doomedUser.ID)
+
+	guardedTables := []string{"api_keys", "user_links", "user_secrets", "user_skills", "user_ai_provider_keys", "organization_members", "group_members", "user_ai_budget_overrides"}
+	countRows := func(table string, userID uuid.UUID) int {
+		var count int
+		//nolint:gosec // The table name comes from the fixed list above.
+		err := sqlDB.QueryRowContext(ctx,
+			`SELECT count(*) FROM `+table+` WHERE user_id = $1`, userID).Scan(&count)
+		require.NoError(t, err)
+		return count
+	}
+	for _, table := range guardedTables {
+		require.Equal(t, 1, countRows(table, doomedUser.ID), "pre-purge: %s orphan must exist", table)
+	}
+
+	// The initial tick runs the purge immediately.
+	closer := dbpurge.New(ctx, logger, db, &codersdk.DeploymentValues{}, prometheus.NewRegistry(), dbpurge.WithClock(quartz.NewReal()))
+	defer closer.Close()
+
+	require.Eventually(t, func() bool {
+		for _, table := range guardedTables {
+			if countRows(table, doomedUser.ID) != 0 {
+				return false
+			}
+		}
+		return true
+	}, testutil.WaitShort, testutil.IntervalFast, "all orphaned child rows of the soft-deleted user must be reaped")
+
+	for _, table := range guardedTables {
+		require.Equal(t, 1, countRows(table, liveUser.ID), "post-purge: %s row for the live user must survive", table)
+	}
+}
diff --git a/coderd/database/dbtestutil/softdelete.go b/coderd/database/dbtestutil/softdelete.go
@@ -0,0 +1,40 @@
+package dbtestutil
+
+import (
+	"context"
+	"database/sql"
+	"testing"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/require"
+)
+
+// SoftDeleteUserKeepingRows marks the user deleted while suppressing the
+// delete_deleted_user_resources cleanup trigger, so the user's child rows
+// (api_keys, user_links, and the other guarded tables) survive. This
+// reconstructs the orphaned-row state that could exist before migration
+// 000592 closed the insert-vs-soft-delete race (the insert guards now also
+// reject new rows for deleted users, so the state can only be constructed
+// this way). Tests use it to prove such legacy rows stay inert.
+func SoftDeleteUserKeepingRows(ctx context.Context, t testing.TB, sqlDB *sql.DB, userID uuid.UUID) {
+	t.Helper()
+	// One transaction: transactional DDL keeps the disabled trigger
+	// invisible to concurrent sessions (which may share this database under
+	// CODER_PG_CONNECTION_URL) and rolls the disable back on failure.
+	tx, err := sqlDB.BeginTx(ctx, nil)
+	require.NoError(t, err)
+	committed := false
+	defer func() {
+		if !committed {
+			_ = tx.Rollback()
+		}
+	}()
+	_, err = tx.ExecContext(ctx, `ALTER TABLE users DISABLE TRIGGER trigger_update_users`)
+	require.NoError(t, err)
+	_, err = tx.ExecContext(ctx, `UPDATE users SET deleted = true WHERE id = $1`, userID)
+	require.NoError(t, err)
+	_, err = tx.ExecContext(ctx, `ALTER TABLE users ENABLE TRIGGER trigger_update_users`)
+	require.NoError(t, err)
+	require.NoError(t, tx.Commit())
+	committed = true
+}