From 42699eb2b733d4063c20bfb207c6a1343c58bc66 Mon Sep 17 00:00:00 2001 From: Ethan Dickson Date: Thu, 9 Jul 2026 03:06:28 +0000 Subject: [PATCH 1/5] fix(coderd): retry chat model config default election on unique violation --- coderd/exp_chats.go | 29 ++++++++++++++++++++++------ coderd/exp_chats_test.go | 41 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 6 deletions(-) diff --git a/coderd/exp_chats.go b/coderd/exp_chats.go index fa4407f880e..eadaa9a60de 100644 --- a/coderd/exp_chats.go +++ b/coderd/exp_chats.go @@ -7039,6 +7039,23 @@ func validateChatModelConfigProviderModel(aiProvider database.AIProvider, model return nil } +// inChatModelConfigTx runs a default-election transaction, retrying when it +// loses the single-default unique index race. Electing a default reads the +// table then promotes a row, so concurrent writers can both self-promote and +// one trips idx_chat_model_configs_single_default. Rerunning re-reads +// committed state, so the loser sees the winner's default and no longer +// self-promotes. fn must be safe to re-run from scratch. +func (api *API) inChatModelConfigTx(fn func(tx database.Store) error) error { + var err error + for range 3 { + err = api.Database.InTx(fn, nil) + if !database.IsUniqueViolation(err, database.UniqueIndexChatModelConfigsSingleDefault) { + return err + } + } + return err +} + func (api *API) createChatModelConfig(rw http.ResponseWriter, r *http.Request) { ctx := r.Context() apiKey := httpmw.APIKey(r) @@ -7141,7 +7158,7 @@ func (api *API) createChatModelConfig(rw http.ResponseWriter, r *http.Request) { } var inserted database.ChatModelConfig - err = api.Database.InTx(func(tx database.Store) error { + err = api.inChatModelConfigTx(func(tx database.Store) error { //nolint:gocritic // The route already authorized chat model config updates. lockedAIProvider, err := tx.GetAIProviderByIDForReferenceLock(dbauthz.AsChatd(ctx), insertParams.AIProviderID.UUID) if err != nil { @@ -7193,7 +7210,7 @@ func (api *API) createChatModelConfig(rw http.ResponseWriter, r *http.Request) { } inserted = refreshedConfig return nil - }, nil) + }) if err != nil { var providerModelErr *chatModelConfigProviderModelError switch { @@ -7350,7 +7367,7 @@ func (api *API) updateChatModelConfig(rw http.ResponseWriter, r *http.Request) { // Re-derive the provider type under lock when the model or provider changes. revalidateProviderModel := updateParams.AIProviderID.Valid && (req.AIProviderID != nil || strings.TrimSpace(req.Model) != "") var updated database.ChatModelConfig - err = api.Database.InTx(func(tx database.Store) error { + err = api.inChatModelConfigTx(func(tx database.Store) error { if revalidateProviderModel { //nolint:gocritic // The route already authorized chat model config updates. aiProvider, err := tx.GetAIProviderByIDForReferenceLock(dbauthz.AsChatd(ctx), updateParams.AIProviderID.UUID) @@ -7406,7 +7423,7 @@ func (api *API) updateChatModelConfig(rw http.ResponseWriter, r *http.Request) { } updated = refreshedConfig return nil - }, nil) + }) if err != nil { var providerModelErr *chatModelConfigProviderModelError switch { @@ -7466,12 +7483,12 @@ func (api *API) deleteChatModelConfig(rw http.ResponseWriter, r *http.Request) { return } - if err := api.Database.InTx(func(tx database.Store) error { + if err := api.inChatModelConfigTx(func(tx database.Store) error { if err := tx.DeleteChatModelConfigByID(ctx, modelConfigID); err != nil { return err } return ensureDefaultChatModelConfig(ctx, tx) - }, nil); err != nil { + }); err != nil { httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ Message: "Failed to delete chat model config.", Detail: err.Error(), diff --git a/coderd/exp_chats_test.go b/coderd/exp_chats_test.go index 4b93cfa1b8a..5f7b3d11d7f 100644 --- a/coderd/exp_chats_test.go +++ b/coderd/exp_chats_test.go @@ -23,6 +23,7 @@ import ( "github.com/shopspring/decimal" "github.com/sqlc-dev/pqtype" "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" "golang.org/x/xerrors" "cdr.dev/slog/v3/sloggers/slogtest" @@ -3789,6 +3790,46 @@ func TestCreateChatModelConfig(t *testing.T) { requireChatModelPricing(t, configs[0].ModelConfig, pricing) }) + t.Run("ConcurrentCreatesElectSingleDefault", func(t *testing.T) { + t.Parallel() + + ctx := testutil.Context(t, testutil.WaitLong) + client := newChatClient(t) + _ = coderdtest.CreateFirstUser(t, client.Client) + + aiProvider := createAIProviderForTest(t, client, "openai", "test-api-key") + + // All creators race to become the default on an empty deployment. + // Without the election retry the losers hit the single-default + // unique index and surface as 409s. 10 creators mirrors Terraform's + // default parallelism, where this was hit in practice. + const creators = 10 + contextLimit := int64(4096) + var eg errgroup.Group + for i := range creators { + eg.Go(func() error { + _, err := client.CreateChatModelConfig(ctx, codersdk.CreateChatModelConfigRequest{ + AIProviderID: &aiProvider.ID, + Model: fmt.Sprintf("gpt-4o-mini-%d", i), + ContextLimit: &contextLimit, + }) + return err + }) + } + require.NoError(t, eg.Wait()) + + configs, err := client.ListChatModelConfigs(ctx) + require.NoError(t, err) + require.Len(t, configs, creators) + defaults := 0 + for _, cfg := range configs { + if cfg.IsDefault { + defaults++ + } + } + require.Equal(t, 1, defaults) + }) + t.Run("RejectsNegativePricing", func(t *testing.T) { t.Parallel() From 290aa08f89c45648d639f86dc888eca5e6d0b83c Mon Sep 17 00:00:00 2001 From: Ethan Dickson Date: Thu, 9 Jul 2026 06:33:34 +0000 Subject: [PATCH 2/5] fix(coderd): serialize chat model config default election with advisory lock Replaces the unique-violation retry loop: the election transactions now take pg_advisory_xact_lock(LockIDChatModelConfigDefault) so only one election runs at a time and the single-default index is never contended. The index and the 409 mapping stay as the schema-level backstop. --- coderd/database/lock.go | 1 + coderd/exp_chats.go | 33 +++++++++++++++------------------ coderd/exp_chats_test.go | 7 ++++--- 3 files changed, 20 insertions(+), 21 deletions(-) diff --git a/coderd/database/lock.go b/coderd/database/lock.go index 8d0894abc87..8d1d695eed8 100644 --- a/coderd/database/lock.go +++ b/coderd/database/lock.go @@ -16,6 +16,7 @@ const ( LockIDReconcileSystemRoles LockIDBoundaryUsageStats LockIDAIProvidersEnvSeed + LockIDChatModelConfigDefault ) // GenLockID generates a unique and consistent lock ID from a given string. diff --git a/coderd/exp_chats.go b/coderd/exp_chats.go index eadaa9a60de..cfd9db05b63 100644 --- a/coderd/exp_chats.go +++ b/coderd/exp_chats.go @@ -7039,21 +7039,18 @@ func validateChatModelConfigProviderModel(aiProvider database.AIProvider, model return nil } -// inChatModelConfigTx runs a default-election transaction, retrying when it -// loses the single-default unique index race. Electing a default reads the -// table then promotes a row, so concurrent writers can both self-promote and -// one trips idx_chat_model_configs_single_default. Rerunning re-reads -// committed state, so the loser sees the winner's default and no longer -// self-promotes. fn must be safe to re-run from scratch. -func (api *API) inChatModelConfigTx(fn func(tx database.Store) error) error { - var err error - for range 3 { - err = api.Database.InTx(fn, nil) - if !database.IsUniqueViolation(err, database.UniqueIndexChatModelConfigsSingleDefault) { - return err - } - } - return err +// inChatModelConfigTx runs fn in a transaction holding the advisory lock that +// serializes default elections. Electing a default reads the table before +// promoting a row, so without the lock two concurrent writers could both +// self-promote and trip idx_chat_model_configs_single_default. The lock makes +// the whole election run one at a time, so the index is never contended. +func (api *API) inChatModelConfigTx(ctx context.Context, fn func(tx database.Store) error) error { + return api.Database.InTx(func(tx database.Store) error { + if err := tx.AcquireLock(ctx, database.LockIDChatModelConfigDefault); err != nil { + return xerrors.Errorf("acquire chat model config lock: %w", err) + } + return fn(tx) + }, nil) } func (api *API) createChatModelConfig(rw http.ResponseWriter, r *http.Request) { @@ -7158,7 +7155,7 @@ func (api *API) createChatModelConfig(rw http.ResponseWriter, r *http.Request) { } var inserted database.ChatModelConfig - err = api.inChatModelConfigTx(func(tx database.Store) error { + err = api.inChatModelConfigTx(ctx, func(tx database.Store) error { //nolint:gocritic // The route already authorized chat model config updates. lockedAIProvider, err := tx.GetAIProviderByIDForReferenceLock(dbauthz.AsChatd(ctx), insertParams.AIProviderID.UUID) if err != nil { @@ -7367,7 +7364,7 @@ func (api *API) updateChatModelConfig(rw http.ResponseWriter, r *http.Request) { // Re-derive the provider type under lock when the model or provider changes. revalidateProviderModel := updateParams.AIProviderID.Valid && (req.AIProviderID != nil || strings.TrimSpace(req.Model) != "") var updated database.ChatModelConfig - err = api.inChatModelConfigTx(func(tx database.Store) error { + err = api.inChatModelConfigTx(ctx, func(tx database.Store) error { if revalidateProviderModel { //nolint:gocritic // The route already authorized chat model config updates. aiProvider, err := tx.GetAIProviderByIDForReferenceLock(dbauthz.AsChatd(ctx), updateParams.AIProviderID.UUID) @@ -7483,7 +7480,7 @@ func (api *API) deleteChatModelConfig(rw http.ResponseWriter, r *http.Request) { return } - if err := api.inChatModelConfigTx(func(tx database.Store) error { + if err := api.inChatModelConfigTx(ctx, func(tx database.Store) error { if err := tx.DeleteChatModelConfigByID(ctx, modelConfigID); err != nil { return err } diff --git a/coderd/exp_chats_test.go b/coderd/exp_chats_test.go index 5f7b3d11d7f..c8587568c69 100644 --- a/coderd/exp_chats_test.go +++ b/coderd/exp_chats_test.go @@ -3800,9 +3800,10 @@ func TestCreateChatModelConfig(t *testing.T) { aiProvider := createAIProviderForTest(t, client, "openai", "test-api-key") // All creators race to become the default on an empty deployment. - // Without the election retry the losers hit the single-default - // unique index and surface as 409s. 10 creators mirrors Terraform's - // default parallelism, where this was hit in practice. + // The advisory lock serializes the elections so only one wins; + // without it the losers hit the single-default unique index and + // surface as 409s. 10 creators mirrors Terraform's default + // parallelism, where this was hit in practice. const creators = 10 contextLimit := int64(4096) var eg errgroup.Group From fb9594b96bd3381477fc990b36fda8682dcdacb0 Mon Sep 17 00:00:00 2001 From: Ethan Dickson Date: Thu, 9 Jul 2026 12:08:19 +0000 Subject: [PATCH 3/5] review --- coderd/database/lock.go | 2 +- coderd/exp_chats.go | 21 ++++++++++----------- coderd/exp_chats_test.go | 28 ++++++++++++++++++++-------- 3 files changed, 31 insertions(+), 20 deletions(-) diff --git a/coderd/database/lock.go b/coderd/database/lock.go index 8d1d695eed8..d2ec69293dc 100644 --- a/coderd/database/lock.go +++ b/coderd/database/lock.go @@ -16,7 +16,7 @@ const ( LockIDReconcileSystemRoles LockIDBoundaryUsageStats LockIDAIProvidersEnvSeed - LockIDChatModelConfigDefault + LockIDChatModelConfigWrites ) // GenLockID generates a unique and consistent lock ID from a given string. diff --git a/coderd/exp_chats.go b/coderd/exp_chats.go index cfd9db05b63..22ef478985a 100644 --- a/coderd/exp_chats.go +++ b/coderd/exp_chats.go @@ -7039,15 +7039,14 @@ func validateChatModelConfigProviderModel(aiProvider database.AIProvider, model return nil } -// inChatModelConfigTx runs fn in a transaction holding the advisory lock that -// serializes default elections. Electing a default reads the table before -// promoting a row, so without the lock two concurrent writers could both -// self-promote and trip idx_chat_model_configs_single_default. The lock makes -// the whole election run one at a time, so the index is never contended. -func (api *API) inChatModelConfigTx(ctx context.Context, fn func(tx database.Store) error) error { +// inChatModelConfigWriteTx runs fn in a transaction that holds the advisory +// lock serializing chat model config writes. All writes to the table must go +// through this helper so concurrent writers cannot act on stale reads and +// violate the idx_chat_model_configs_single_default unique index. +func (api *API) inChatModelConfigWriteTx(ctx context.Context, fn func(tx database.Store) error) error { return api.Database.InTx(func(tx database.Store) error { - if err := tx.AcquireLock(ctx, database.LockIDChatModelConfigDefault); err != nil { - return xerrors.Errorf("acquire chat model config lock: %w", err) + if err := tx.AcquireLock(ctx, database.LockIDChatModelConfigWrites); err != nil { + return xerrors.Errorf("acquire chat model config write lock: %w", err) } return fn(tx) }, nil) @@ -7155,7 +7154,7 @@ func (api *API) createChatModelConfig(rw http.ResponseWriter, r *http.Request) { } var inserted database.ChatModelConfig - err = api.inChatModelConfigTx(ctx, func(tx database.Store) error { + err = api.inChatModelConfigWriteTx(ctx, func(tx database.Store) error { //nolint:gocritic // The route already authorized chat model config updates. lockedAIProvider, err := tx.GetAIProviderByIDForReferenceLock(dbauthz.AsChatd(ctx), insertParams.AIProviderID.UUID) if err != nil { @@ -7364,7 +7363,7 @@ func (api *API) updateChatModelConfig(rw http.ResponseWriter, r *http.Request) { // Re-derive the provider type under lock when the model or provider changes. revalidateProviderModel := updateParams.AIProviderID.Valid && (req.AIProviderID != nil || strings.TrimSpace(req.Model) != "") var updated database.ChatModelConfig - err = api.inChatModelConfigTx(ctx, func(tx database.Store) error { + err = api.inChatModelConfigWriteTx(ctx, func(tx database.Store) error { if revalidateProviderModel { //nolint:gocritic // The route already authorized chat model config updates. aiProvider, err := tx.GetAIProviderByIDForReferenceLock(dbauthz.AsChatd(ctx), updateParams.AIProviderID.UUID) @@ -7480,7 +7479,7 @@ func (api *API) deleteChatModelConfig(rw http.ResponseWriter, r *http.Request) { return } - if err := api.inChatModelConfigTx(ctx, func(tx database.Store) error { + if err := api.inChatModelConfigWriteTx(ctx, func(tx database.Store) error { if err := tx.DeleteChatModelConfigByID(ctx, modelConfigID); err != nil { return err } diff --git a/coderd/exp_chats_test.go b/coderd/exp_chats_test.go index c8587568c69..fd1207b1c0c 100644 --- a/coderd/exp_chats_test.go +++ b/coderd/exp_chats_test.go @@ -3799,15 +3799,17 @@ func TestCreateChatModelConfig(t *testing.T) { aiProvider := createAIProviderForTest(t, client, "openai", "test-api-key") - // All creators race to become the default on an empty deployment. - // The advisory lock serializes the elections so only one wins; - // without it the losers hit the single-default unique index and - // surface as 409s. 10 creators mirrors Terraform's default + // All creators race to become the default on an empty deployment, + // with one explicitly claiming it. Config writes are serialized, so + // only one self-promotes and the explicit claim demotes any interim + // winner; without that the losers hit the single-default unique + // index and surface as 409s. 10 creators mirrors Terraform's default // parallelism, where this was hit in practice. const creators = 10 contextLimit := int64(4096) + var claimed codersdk.ChatModelConfig var eg errgroup.Group - for i := range creators { + for i := range creators - 1 { eg.Go(func() error { _, err := client.CreateChatModelConfig(ctx, codersdk.CreateChatModelConfigRequest{ AIProviderID: &aiProvider.ID, @@ -3817,18 +3819,28 @@ func TestCreateChatModelConfig(t *testing.T) { return err }) } + eg.Go(func() error { + var err error + claimed, err = client.CreateChatModelConfig(ctx, codersdk.CreateChatModelConfigRequest{ + AIProviderID: &aiProvider.ID, + Model: "gpt-4o", + ContextLimit: &contextLimit, + IsDefault: ptr.Ref(true), + }) + return err + }) require.NoError(t, eg.Wait()) configs, err := client.ListChatModelConfigs(ctx) require.NoError(t, err) require.Len(t, configs, creators) - defaults := 0 + var defaults []uuid.UUID for _, cfg := range configs { if cfg.IsDefault { - defaults++ + defaults = append(defaults, cfg.ID) } } - require.Equal(t, 1, defaults) + require.Equal(t, []uuid.UUID{claimed.ID}, defaults) }) t.Run("RejectsNegativePricing", func(t *testing.T) { From 00e63d351720e6c95d60127e197458be9810020d Mon Sep 17 00:00:00 2001 From: Ethan Dickson Date: Thu, 9 Jul 2026 13:08:57 +0000 Subject: [PATCH 4/5] review --- coderd/exp_chats_test.go | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/coderd/exp_chats_test.go b/coderd/exp_chats_test.go index fd1207b1c0c..f40c155e476 100644 --- a/coderd/exp_chats_test.go +++ b/coderd/exp_chats_test.go @@ -3799,12 +3799,9 @@ func TestCreateChatModelConfig(t *testing.T) { aiProvider := createAIProviderForTest(t, client, "openai", "test-api-key") - // All creators race to become the default on an empty deployment, - // with one explicitly claiming it. Config writes are serialized, so - // only one self-promotes and the explicit claim demotes any interim - // winner; without that the losers hit the single-default unique - // index and surface as 409s. 10 creators mirrors Terraform's default - // parallelism, where this was hit in practice. + // Concurrent creators race to self-elect a default while one claims + // it via a follow-up update, mirroring a terraform apply. Unserialized, + // the losers 409 on the single-default unique index. const creators = 10 contextLimit := int64(4096) var claimed codersdk.ChatModelConfig @@ -3820,14 +3817,21 @@ func TestCreateChatModelConfig(t *testing.T) { }) } eg.Go(func() error { - var err error - claimed, err = client.CreateChatModelConfig(ctx, codersdk.CreateChatModelConfigRequest{ + created, err := client.CreateChatModelConfig(ctx, codersdk.CreateChatModelConfigRequest{ AIProviderID: &aiProvider.ID, Model: "gpt-4o", ContextLimit: &contextLimit, - IsDefault: ptr.Ref(true), }) - return err + if err != nil { + return xerrors.Errorf("create claimed config: %w", err) + } + claimed, err = client.UpdateChatModelConfig(ctx, created.ID, codersdk.UpdateChatModelConfigRequest{ + IsDefault: ptr.Ref(true), + }) + if err != nil { + return xerrors.Errorf("promote claimed config: %w", err) + } + return nil }) require.NoError(t, eg.Wait()) From c068a9307bd4fe0f31373be3f9ab9905ea0c2d41 Mon Sep 17 00:00:00 2001 From: Ethan Dickson Date: Tue, 14 Jul 2026 03:28:41 +0000 Subject: [PATCH 5/5] review --- coderd/exp_chats.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/coderd/exp_chats.go b/coderd/exp_chats.go index 22ef478985a..e5f350fa34c 100644 --- a/coderd/exp_chats.go +++ b/coderd/exp_chats.go @@ -7041,8 +7041,9 @@ func validateChatModelConfigProviderModel(aiProvider database.AIProvider, model // inChatModelConfigWriteTx runs fn in a transaction that holds the advisory // lock serializing chat model config writes. All writes to the table must go -// through this helper so concurrent writers cannot act on stale reads and -// violate the idx_chat_model_configs_single_default unique index. +// through this helper so concurrent writers cannot act on stale reads, e.g. +// two creates on an empty deployment both self-promoting to default and +// violating the idx_chat_model_configs_single_default unique index. func (api *API) inChatModelConfigWriteTx(ctx context.Context, fn func(tx database.Store) error) error { return api.Database.InTx(func(tx database.Store) error { if err := tx.AcquireLock(ctx, database.LockIDChatModelConfigWrites); err != nil {