From 3612db66951bc8ee570a16696a4cb70c2783ee76 Mon Sep 17 00:00:00 2001 From: Ethan Dickson Date: Fri, 21 Aug 2026 18:20:34 +0000 Subject: [PATCH 1/4] refactor: consolidate chat model availability --- coderd/apidoc/docs.go | 109 +--- coderd/apidoc/swagger.json | 105 +--- coderd/coderd.go | 3 +- coderd/exp_chats.go | 188 +------ coderd/exp_chats_model_config_list_test.go | 12 +- coderd/exp_chats_test.go | 483 ++++-------------- coderd/x/chatd/chatprovider/chatprovider.go | 185 ------- .../x/chatd/chatprovider/chatprovider_test.go | 294 ----------- codersdk/chats.go | 70 +-- docs/ai-coder/agents/models.md | 2 +- docs/reference/api/chats.md | 230 --------- docs/reference/api/schemas.md | 301 ++--------- site/src/api/api.test.ts | 14 +- site/src/api/api.ts | 12 - site/src/api/queries/chats.test.ts | 2 - site/src/api/queries/chats.ts | 14 - site/src/api/typesGenerated.ts | 45 +- .../modules/aiModels/providerStates.test.ts | 68 +-- site/src/modules/aiModels/providerStates.ts | 24 - .../AddModelPage/AddModelPageView.stories.tsx | 21 +- .../AISettingsPage/ModelsPage/ModelsPage.tsx | 14 +- .../OrganizationModelsLayout.stories.tsx | 28 +- .../UpdateModelPageView.stories.tsx | 11 +- .../ModelsPage/organizationModels.test.tsx | 3 +- .../AISettingsPage/ModelsPage/testFixtures.ts | 1 - .../AgentsPage/AgentChatPage.stories.tsx | 63 +-- site/src/pages/AgentsPage/AgentChatPage.tsx | 36 +- .../AgentSettingsUserAgentsPage.tsx | 21 +- .../AgentsPage/AgentsPageLayout.stories.tsx | 34 +- .../components/AgentCreateForm.stories.tsx | 161 ++++-- .../AgentsPage/components/AgentCreateForm.tsx | 41 +- .../hooks/useOrganizationChatModels.test.tsx | 1 + .../AgentsPage/utils/modelOptions.test.ts | 159 +++--- .../pages/AgentsPage/utils/modelOptions.ts | 157 ++---- site/src/testHelpers/chatModels.ts | 8 +- 35 files changed, 582 insertions(+), 2338 deletions(-) diff --git a/coderd/apidoc/docs.go b/coderd/apidoc/docs.go index 8e63c7ee70c..fc57754bc17 100644 --- a/coderd/apidoc/docs.go +++ b/coderd/apidoc/docs.go @@ -1443,41 +1443,6 @@ const docTemplate = `{ } } }, - "/api/experimental/organizations/{organization}/chats/models/available": { - "get": { - "description": "Experimental: this endpoint is subject to change.", - "produces": [ - "application/json" - ], - "tags": [ - "Chats" - ], - "summary": "List available chat models in an organization", - "operationId": "list-chat-model-availability", - "parameters": [ - { - "type": "string", - "description": "Organization name or ID", - "name": "organization", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/codersdk.ChatModelAvailabilityResponse" - } - } - }, - "security": [ - { - "CoderSessionToken": [] - } - ] - } - }, "/api/experimental/organizations/{organization}/chats/models/{model}": { "get": { "produces": [ @@ -19061,31 +19026,6 @@ const docTemplate = `{ } } }, - "codersdk.ChatModelAvailabilityResponse": { - "type": "object", - "properties": { - "models": { - "description": "Models contains the effective runtime model configs for the requested\norganization. Each config belongs to that organization.", - "type": "array", - "items": { - "$ref": "#/definitions/codersdk.ChatModel" - } - }, - "providers": { - "type": "array", - "items": { - "$ref": "#/definitions/codersdk.ChatModelProvider" - } - }, - "unsupported_providers": { - "description": "UnsupportedProviders lists configured providers the Agents harness\ncannot use, so the UI can explain the empty state.", - "type": "array", - "items": { - "$ref": "#/definitions/codersdk.ChatUnsupportedProvider" - } - } - } - }, "codersdk.ChatModelCallConfig": { "type": "object", "properties": { @@ -19118,23 +19058,6 @@ const docTemplate = `{ } } }, - "codersdk.ChatModelCatalogEntry": { - "type": "object", - "properties": { - "display_name": { - "type": "string" - }, - "id": { - "type": "string" - }, - "model": { - "type": "string" - }, - "provider": { - "type": "string" - } - } - }, "codersdk.ChatModelGoogleProviderOptions": { "type": "object", "properties": { @@ -19357,32 +19280,15 @@ const docTemplate = `{ } } }, - "codersdk.ChatModelProvider": { - "type": "object", - "properties": { - "available": { - "type": "boolean" - }, - "models": { - "type": "array", - "items": { - "$ref": "#/definitions/codersdk.ChatModelCatalogEntry" - } - }, - "provider": { - "type": "string" - }, - "unavailable_reason": { - "$ref": "#/definitions/codersdk.ChatModelProviderUnavailableReason" - } - } - }, "codersdk.ChatModelProviderDescriptor": { "type": "object", "properties": { "allow_user_api_key": { "type": "boolean" }, + "available": { + "type": "boolean" + }, "display_name": { "type": "string" }, @@ -19407,6 +19313,9 @@ const docTemplate = `{ }, "type": { "type": "string" + }, + "unavailable_reason": { + "$ref": "#/definitions/codersdk.ChatModelProviderUnavailableReason" } } }, @@ -24117,6 +24026,12 @@ const docTemplate = `{ "items": { "$ref": "#/definitions/codersdk.ChatModelProviderDescriptor" } + }, + "unsupported_providers": { + "type": "array", + "items": { + "$ref": "#/definitions/codersdk.ChatUnsupportedProvider" + } } } }, diff --git a/coderd/apidoc/swagger.json b/coderd/apidoc/swagger.json index 77033188ce8..e7b814744ce 100644 --- a/coderd/apidoc/swagger.json +++ b/coderd/apidoc/swagger.json @@ -1284,37 +1284,6 @@ } } }, - "/api/experimental/organizations/{organization}/chats/models/available": { - "get": { - "description": "Experimental: this endpoint is subject to change.", - "produces": ["application/json"], - "tags": ["Chats"], - "summary": "List available chat models in an organization", - "operationId": "list-chat-model-availability", - "parameters": [ - { - "type": "string", - "description": "Organization name or ID", - "name": "organization", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK", - "schema": { - "$ref": "#/definitions/codersdk.ChatModelAvailabilityResponse" - } - } - }, - "security": [ - { - "CoderSessionToken": [] - } - ] - } - }, "/api/experimental/organizations/{organization}/chats/models/{model}": { "get": { "produces": ["application/json"], @@ -17213,31 +17182,6 @@ } } }, - "codersdk.ChatModelAvailabilityResponse": { - "type": "object", - "properties": { - "models": { - "description": "Models contains the effective runtime model configs for the requested\norganization. Each config belongs to that organization.", - "type": "array", - "items": { - "$ref": "#/definitions/codersdk.ChatModel" - } - }, - "providers": { - "type": "array", - "items": { - "$ref": "#/definitions/codersdk.ChatModelProvider" - } - }, - "unsupported_providers": { - "description": "UnsupportedProviders lists configured providers the Agents harness\ncannot use, so the UI can explain the empty state.", - "type": "array", - "items": { - "$ref": "#/definitions/codersdk.ChatUnsupportedProvider" - } - } - } - }, "codersdk.ChatModelCallConfig": { "type": "object", "properties": { @@ -17270,23 +17214,6 @@ } } }, - "codersdk.ChatModelCatalogEntry": { - "type": "object", - "properties": { - "display_name": { - "type": "string" - }, - "id": { - "type": "string" - }, - "model": { - "type": "string" - }, - "provider": { - "type": "string" - } - } - }, "codersdk.ChatModelGoogleProviderOptions": { "type": "object", "properties": { @@ -17509,32 +17436,15 @@ } } }, - "codersdk.ChatModelProvider": { - "type": "object", - "properties": { - "available": { - "type": "boolean" - }, - "models": { - "type": "array", - "items": { - "$ref": "#/definitions/codersdk.ChatModelCatalogEntry" - } - }, - "provider": { - "type": "string" - }, - "unavailable_reason": { - "$ref": "#/definitions/codersdk.ChatModelProviderUnavailableReason" - } - } - }, "codersdk.ChatModelProviderDescriptor": { "type": "object", "properties": { "allow_user_api_key": { "type": "boolean" }, + "available": { + "type": "boolean" + }, "display_name": { "type": "string" }, @@ -17559,6 +17469,9 @@ }, "type": { "type": "string" + }, + "unavailable_reason": { + "$ref": "#/definitions/codersdk.ChatModelProviderUnavailableReason" } } }, @@ -22087,6 +22000,12 @@ "items": { "$ref": "#/definitions/codersdk.ChatModelProviderDescriptor" } + }, + "unsupported_providers": { + "type": "array", + "items": { + "$ref": "#/definitions/codersdk.ChatUnsupportedProvider" + } } } }, diff --git a/coderd/coderd.go b/coderd/coderd.go index 898d6304b00..b065c0d8824 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -1434,13 +1434,12 @@ func New(options *Options) *API { }) }, httpmw.ExtractOrganizationParam(options.Database), - ).Get("/chats/models", api.listChatModelAvailability) + ).Get("/chats/models", api.listChatModelConfigsByOrganization) r.Route("/organizations/{organization}/chats/models", func(r chi.Router) { r.Use(apiKeyMiddleware) r.With(httpmw.ExtractOrganizationParam(options.Database)).Get("/", api.listChatModelConfigsByOrganization) r.With(httpmw.ExtractOrganizationParam(options.Database)).Post("/", api.createChatModelConfig) - r.With(httpmw.ExtractOrganizationParam(options.Database)).Get("/available", api.listChatModelAvailability) r.Route("/{model}", func(r chi.Router) { r.Use( httpmw.ExtractOrganizationParam(options.Database), diff --git a/coderd/exp_chats.go b/coderd/exp_chats.go index 95d09720d1e..ff2026faa74 100644 --- a/coderd/exp_chats.go +++ b/coderd/exp_chats.go @@ -991,14 +991,10 @@ func (api *API) chatPersonalModelOverrideDeploymentDefaults( } type userChatModelAvailability struct { - configuredProviders []chatprovider.ConfiguredProvider - configuredModels []chatprovider.ConfiguredModel - enabledModels []database.ChatModelConfig - providerTypeByID map[uuid.UUID]string - providerStatus map[string]chatprovider.ProviderAvailability - providerStatusByID map[uuid.UUID]chatprovider.ProviderAvailability - enabledProviderNames map[string]struct{} - enabledProviderIDs map[uuid.UUID]struct{} + configuredProviders []chatprovider.ConfiguredProvider + enabledModels []database.ChatModelConfig + providerStatusByID map[uuid.UUID]chatprovider.ProviderAvailability + enabledProviderIDs map[uuid.UUID]struct{} } // chatModelConfigUnavailableReason reports why a model config cannot be used. @@ -1055,26 +1051,12 @@ func (api *API) getUserChatProviderAvailability( return userChatModelAvailability{}, err } availability := userChatModelAvailability{ - configuredProviders: configuredProviders, - configuredModels: make([]chatprovider.ConfiguredModel, 0, len(enabledModels)), - enabledModels: enabledModels, - providerTypeByID: make(map[uuid.UUID]string, len(enabledProviders)), - enabledProviderNames: make(map[string]struct{}, len(enabledProviders)), - enabledProviderIDs: make(map[uuid.UUID]struct{}, len(enabledProviders)), - providerStatusByID: make(map[uuid.UUID]chatprovider.ProviderAvailability, len(enabledProviders)), - } - // Model configs carry no provider type; resolve it from the enabled - // providers. A config under a disabled or deleted provider is skipped - // for status purposes, mirroring the provider join on the enabled - // models query. - for _, provider := range enabledProviders { - availability.providerTypeByID[provider.ID] = string(provider.Type) + configuredProviders: configuredProviders, + enabledModels: enabledModels, + enabledProviderIDs: make(map[uuid.UUID]struct{}, len(enabledProviders)), + providerStatusByID: make(map[uuid.UUID]chatprovider.ProviderAvailability, len(enabledProviders)), } for _, configuredProvider := range configuredProviders { - normalizedProvider := chatprovider.NormalizeProvider(configuredProvider.Provider) - if normalizedProvider != "" { - availability.enabledProviderNames[normalizedProvider] = struct{}{} - } if configuredProvider.ProviderID != uuid.Nil { availability.enabledProviderIDs[configuredProvider.ProviderID] = struct{}{} } @@ -1087,30 +1069,14 @@ func (api *API) getUserChatProviderAvailability( } userKeys = make([]chatprovider.UserProviderKey, 0, len(userKeyStatus)) for providerID, configured := range userKeyStatus { - if !configured { - continue + if configured { + userKeys = append(userKeys, chatprovider.UserProviderKey{ChatProviderID: providerID, APIKey: "configured"}) } - userKeys = append(userKeys, chatprovider.UserProviderKey{ - ChatProviderID: providerID, - APIKey: "configured", - }) } } fallbackKeys := ChatProviderAPIKeysFromDeploymentValues(api.DeploymentValues) - mergeProviderStatus := func( - statuses map[string]chatprovider.ProviderAvailability, - normalizedProvider string, - status chatprovider.ProviderAvailability, - ) { - current, ok := statuses[normalizedProvider] - if !ok || (!current.Available && status.Available) { - statuses[normalizedProvider] = status - } - } - - providerStatusByType := make(map[string]chatprovider.ProviderAvailability, len(availability.configuredProviders)) - for _, configuredProvider := range availability.configuredProviders { + for _, configuredProvider := range configuredProviders { normalizedProvider := chatprovider.NormalizeProvider(configuredProvider.Provider) if normalizedProvider == "" { continue @@ -1121,56 +1087,9 @@ func (api *API) getUserChatProviderAvailability( userKeys, ) status, ok := providerStatus[normalizedProvider] - if !ok { - continue - } - if configuredProvider.ProviderID != uuid.Nil { + if ok && configuredProvider.ProviderID != uuid.Nil { availability.providerStatusByID[configuredProvider.ProviderID] = status } - mergeProviderStatus(providerStatusByType, normalizedProvider, status) - } - - modelStatusByType := make(map[string]chatprovider.ProviderAvailability, len(enabledModels)) - for _, model := range enabledModels { - if !model.AIProviderID.Valid { - continue - } - providerID := model.AIProviderID.UUID - providerType, ok := availability.providerTypeByID[providerID] - if !ok { - continue - } - normalizedProvider := chatprovider.NormalizeProvider(providerType) - if normalizedProvider == "" { - continue - } - status, ok := availability.providerStatusByID[providerID] - if ok { - mergeProviderStatus(modelStatusByType, normalizedProvider, status) - } - } - availability.providerStatus = providerStatusByType - for provider, status := range modelStatusByType { - availability.providerStatus[provider] = status - } - - for _, model := range enabledModels { - if !model.AIProviderID.Valid { - continue - } - providerType, ok := availability.providerTypeByID[model.AIProviderID.UUID] - if !ok { - continue - } - status, ok := availability.providerStatusByID[model.AIProviderID.UUID] - if !ok || !status.Available { - continue - } - availability.configuredModels = append(availability.configuredModels, chatprovider.ConfiguredModel{ - Provider: providerType, - Model: model.Model, - DisplayName: model.DisplayName, - }) } return availability, nil } @@ -1621,70 +1540,6 @@ func (api *API) postChats(rw http.ResponseWriter, r *http.Request) { httpapi.Write(ctx, rw, http.StatusCreated, response) } -// EXPERIMENTAL: this endpoint is experimental and is subject to change. -// -// @Summary List available chat models in an organization -// @ID list-chat-model-availability -// @Security CoderSessionToken -// @Tags Chats -// @Produce json -// @Param organization path string true "Organization name or ID" -// @Success 200 {object} codersdk.ChatModelAvailabilityResponse -// @Router /api/experimental/organizations/{organization}/chats/models/available [get] -// @Description Experimental: this endpoint is subject to change. -func (api *API) listChatModelAvailability(rw http.ResponseWriter, r *http.Request) { - ctx := r.Context() - apiKey := httpmw.APIKey(r) - if !chatModelConfigReadScope(apiKey.Scopes) { - httpapi.Forbidden(rw) - return - } - organization := httpmw.OrganizationParam(r) - visible, err := api.canReadChatModelsInOrganization(ctx, r, organization) - if err != nil { - httpapi.InternalServerError(rw, err) - return - } - if !visible { - httpapi.ResourceNotFound(rw) - return - } - availability, err := api.getUserChatProviderAvailability(ctx, apiKey.UserID, organization.ID) - if err != nil { - httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ - Message: "Failed to load chat model configuration.", - Detail: err.Error(), - }) - return - } - catalog := chatprovider.NewModelCatalog() - var response codersdk.ChatModelAvailabilityResponse - if configured, ok := catalog.ListConfiguredModels( - availability.configuredProviders, - availability.configuredModels, - availability.providerStatus, - availability.enabledProviderNames, - ); ok { - response = configured - } else { - response = catalog.ListConfiguredProviderAvailability( - availability.providerStatus, - availability.enabledProviderNames, - ) - } - - response.Models = make([]codersdk.ChatModel, 0, len(availability.enabledModels)) - for _, model := range availability.enabledModels { - response.Models = append(response.Models, convertChatModelConfig(model)) - } - - // Both catalog branches drop providers the harness cannot use, so - // attach them here for the empty state. - response.UnsupportedProviders = chatprovider.UnsupportedProviders(availability.configuredProviders) - - httpapi.Write(ctx, rw, http.StatusOK, response) -} - // EXPERIMENTAL: this endpoint is experimental and is subject to change. // // @Summary Get chat by ID @@ -7006,10 +6861,10 @@ func (api *API) listChatModelConfigsByOrganization(rw http.ResponseWriter, r *ht return } - configs, err := api.Database.GetChatModelConfigs(ctx, organization.ID) + availability, err := api.getUserChatProviderAvailability(ctx, apiKey.UserID, organization.ID) if err != nil { httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ - Message: "Failed to list chat model configs.", + Message: "Failed to load chat model availability.", Detail: err.Error(), }) return @@ -7024,11 +6879,20 @@ func (api *API) listChatModelConfigsByOrganization(rw http.ResponseWriter, r *ht return } + for i := range providers { + if status, ok := availability.providerStatusByID[providers[i].ID]; ok { + providers[i].Available = status.Available + if !status.Available { + providers[i].UnavailableReason = status.UnavailableReason + } + } + } resp := codersdk.OrganizationChatModelsResponse{ - Models: make([]codersdk.ChatModel, 0, len(configs)), - Providers: providers, + Models: make([]codersdk.ChatModel, 0, len(availability.enabledModels)), + Providers: providers, + UnsupportedProviders: chatprovider.UnsupportedProviders(availability.configuredProviders), } - for _, config := range configs { + for _, config := range availability.enabledModels { resp.Models = append(resp.Models, convertChatModelConfig(config)) } diff --git a/coderd/exp_chats_model_config_list_test.go b/coderd/exp_chats_model_config_list_test.go index bbe7dbf9ff2..e3cd1e3ada6 100644 --- a/coderd/exp_chats_model_config_list_test.go +++ b/coderd/exp_chats_model_config_list_test.go @@ -107,16 +107,13 @@ func TestChatModelConfigListReadContracts(t *testing.T) { experimentalClient := codersdk.NewExperimentalClient(scopedClient) response, listErr := experimentalClient.ChatModels(ctx, defaultOrg.ID) - _, availabilityErr := experimentalClient.ChatModelAvailability(ctx, defaultOrg.ID) model, itemErr := experimentalClient.ChatModel(ctx, defaultOrg.ID, ownEnabled.ID) _, aclErr := experimentalClient.ChatModelACL(ctx, defaultOrg.ID, ownEnabled.ID) if testCase.wantCollectionStatus != 0 { requireSDKError(t, listErr, testCase.wantCollectionStatus) - requireSDKError(t, availabilityErr, testCase.wantCollectionStatus) } else { require.NoError(t, listErr) require.NotEmpty(t, response.Models) - require.NoError(t, availabilityErr) } if testCase.wantItemStatus != 0 { requireSDKError(t, itemErr, testCase.wantItemStatus) @@ -141,13 +138,6 @@ func TestChatModelConfigListReadContracts(t *testing.T) { return err }, }, - { - name: "Availability", - call: func(ctx context.Context, organizationID uuid.UUID) error { - _, err := memberClient.ChatModelAvailability(ctx, organizationID) - return err - }, - }, { name: "Item", call: func(ctx context.Context, organizationID uuid.UUID) error { @@ -247,7 +237,7 @@ func TestChatModelConfigListReadContracts(t *testing.T) { models, err := testCase.client(t, ctx).ChatModels(ctx, defaultOrg.ID) require.NoError(t, err) require.True(t, containsChatModel(models.Models, ownEnabled.ID)) - require.True(t, containsChatModel(models.Models, ownDisabled.ID)) + require.False(t, containsChatModel(models.Models, ownDisabled.ID)) require.Equal(t, testCase.seesDenied, containsChatModel(models.Models, denied.ID)) require.False(t, containsChatModel(models.Models, otherEnabled.ID)) }) diff --git a/coderd/exp_chats_test.go b/coderd/exp_chats_test.go index 34f227a8aca..0d2264f1546 100644 --- a/coderd/exp_chats_test.go +++ b/coderd/exp_chats_test.go @@ -2579,74 +2579,50 @@ func TestListChats(t *testing.T) { func TestListChatModels(t *testing.T) { t.Parallel() + providerByID := func(t *testing.T, response codersdk.OrganizationChatModelsResponse, id uuid.UUID) codersdk.ChatModelProviderDescriptor { + t.Helper() + for _, provider := range response.Providers { + if provider.ID == id { + return provider + } + } + t.Fatalf("provider %s not found", id) + return codersdk.ChatModelProviderDescriptor{} + } + containsModel := func(response codersdk.OrganizationChatModelsResponse, id uuid.UUID) bool { + return slices.ContainsFunc(response.Models, func(model codersdk.ChatModel) bool { return model.ID == id }) + } + t.Run("Success", func(t *testing.T) { t.Parallel() - ctx := testutil.Context(t, testutil.WaitLong) client := newChatClient(t) firstUser := coderdtest.CreateFirstUser(t, client.Client) - modelConfig := createChatModel(t, client) - - models, err := client.ChatModelAvailability(ctx, firstUser.OrganizationID) + model := createChatModel(t, client) + response, err := client.ChatModels(ctx, firstUser.OrganizationID) require.NoError(t, err) - - var openAIProvider *codersdk.ChatModelProvider - for i := range models.Providers { - if models.Providers[i].Provider == coderdtest.TestChatProviderOpenAICompat { - openAIProvider = &models.Providers[i] - break - } - } - require.NotNil(t, openAIProvider) - require.True(t, openAIProvider.Available) - - foundModel := false - for _, model := range openAIProvider.Models { - if model.Provider == coderdtest.TestChatProviderOpenAICompat && model.Model == modelConfig.Model { - foundModel = true - break - } - } - require.True(t, foundModel) + require.True(t, containsModel(response, model.ID)) + require.True(t, providerByID(t, response, model.AIProviderID).Available) }) t.Run("DeniedSameOrganizationEnabledModelIsHidden", func(t *testing.T) { t.Parallel() - ctx := testutil.Context(t, testutil.WaitLong) client, db := newChatClientWithDatabase(t) firstUser := coderdtest.CreateFirstUser(t, client.Client) provider := createAIProviderForTest(t, client, "openai-compat", "test-api-key") privateConfig := dbgen.ChatModelConfig(t, db, database.ChatModelConfig{ - AIProviderID: uuid.NullUUID{UUID: provider.ID, Valid: true}, - Model: "private-discovery-" + uuid.NewString(), - Enabled: true, - OrganizationID: firstUser.OrganizationID, - GroupACL: database.ChatACL{}, + AIProviderID: uuid.NullUUID{UUID: provider.ID, Valid: true}, OrganizationID: firstUser.OrganizationID, + Model: "private-" + uuid.NewString(), Enabled: true, GroupACL: database.ChatACL{}, }) - memberClientRaw, _ := coderdtest.CreateAnotherUser( - t, - client.Client, - firstUser.OrganizationID, - rbac.ScopedRoleAgentsAccess(firstUser.OrganizationID), - ) - memberClient := codersdk.NewExperimentalClient(memberClientRaw) - - models, err := memberClient.ChatModelAvailability(ctx, firstUser.OrganizationID) + memberRaw, _ := coderdtest.CreateAnotherUser(t, client.Client, firstUser.OrganizationID, rbac.ScopedRoleAgentsAccess(firstUser.OrganizationID)) + response, err := codersdk.NewExperimentalClient(memberRaw).ChatModels(ctx, firstUser.OrganizationID) require.NoError(t, err) - for _, model := range models.Models { - require.NotEqual(t, privateConfig.ID, model.ID) - } - for _, availableProvider := range models.Providers { - for _, model := range availableProvider.Models { - require.NotEqual(t, privateConfig.Model, model.Model) - } - } + require.False(t, containsModel(response, privateConfig.ID)) }) t.Run("NonDefaultOrgIncludesOnlyLocalModels", func(t *testing.T) { t.Parallel() - ctx := testutil.Context(t, testutil.WaitLong) client, db := newChatClientWithDatabase(t) _ = coderdtest.CreateFirstUser(t, client.Client) @@ -2655,402 +2631,169 @@ func TestListChatModels(t *testing.T) { chatOrg := dbgen.Organization(t, db, database.Organization{IsDefault: false}) thirdOrg := dbgen.Organization(t, db, database.Organization{IsDefault: false}) contextLimit := int64(4096) - localConfig, err := client.CreateChatModel(ctx, chatOrg.ID, codersdk.CreateChatModelRequest{ - AIProviderID: &provider.ID, - Model: "local-" + uuid.NewString(), - ContextLimit: &contextLimit, - }) + local, err := client.CreateChatModel(ctx, chatOrg.ID, codersdk.CreateChatModelRequest{AIProviderID: &provider.ID, Model: "local-" + uuid.NewString(), ContextLimit: &contextLimit}) require.NoError(t, err) - localConfig, err = client.UpdateChatModel(ctx, localConfig.OrganizationID, localConfig.ID, codersdk.UpdateChatModelRequest{ - IsDefault: ptr.Ref(true), - }) - require.NoError(t, err) - thirdConfig, err := client.CreateChatModel(ctx, thirdOrg.ID, codersdk.CreateChatModelRequest{ - AIProviderID: &provider.ID, - Model: "third-" + uuid.NewString(), - ContextLimit: &contextLimit, - }) + third, err := client.CreateChatModel(ctx, thirdOrg.ID, codersdk.CreateChatModelRequest{AIProviderID: &provider.ID, Model: "third-" + uuid.NewString(), ContextLimit: &contextLimit}) require.NoError(t, err) - - memberClientRaw, _ := coderdtest.CreateAnotherUser(t, client.Client, chatOrg.ID, rbac.ScopedRoleAgentsAccess(chatOrg.ID)) - memberClient := codersdk.NewExperimentalClient(memberClientRaw) - models, err := memberClient.ChatModelAvailability(ctx, chatOrg.ID) + memberRaw, _ := coderdtest.CreateAnotherUser(t, client.Client, chatOrg.ID, rbac.ScopedRoleAgentsAccess(chatOrg.ID)) + response, err := codersdk.NewExperimentalClient(memberRaw).ChatModels(ctx, chatOrg.ID) require.NoError(t, err) - var modelNames []string - for _, availableProvider := range models.Providers { - for _, model := range availableProvider.Models { - modelNames = append(modelNames, model.Model) - } - } - require.Contains(t, modelNames, localConfig.Model) - require.NotContains(t, modelNames, defaultConfig.Model) - require.NotContains(t, modelNames, thirdConfig.Model) - - modelConfigIDs := make([]uuid.UUID, 0, len(models.Models)) - for _, model := range models.Models { - modelConfigIDs = append(modelConfigIDs, model.ID) - } - require.Contains(t, modelConfigIDs, localConfig.ID) - require.NotContains(t, modelConfigIDs, defaultConfig.ID) - require.NotContains(t, modelConfigIDs, thirdConfig.ID) - require.Len(t, models.Models, 1) - require.True(t, models.Models[0].IsDefault) + require.True(t, containsModel(response, local.ID)) + require.False(t, containsModel(response, defaultConfig.ID)) + require.False(t, containsModel(response, third.ID)) }) t.Run("Unauthenticated", func(t *testing.T) { t.Parallel() - ctx := testutil.Context(t, testutil.WaitLong) client := newChatClient(t) firstUser := coderdtest.CreateFirstUser(t, client.Client) - - unauthenticatedClient := codersdk.NewExperimentalClient(codersdk.New(client.URL)) - _, err := unauthenticatedClient.ChatModelAvailability(ctx, firstUser.OrganizationID) + _, err := codersdk.NewExperimentalClient(codersdk.New(client.URL)).ChatModels(ctx, firstUser.OrganizationID) requireSDKError(t, err, http.StatusUnauthorized) }) t.Run("CopilotOnlyUnsupported", func(t *testing.T) { t.Parallel() - ctx := testutil.Context(t, testutil.WaitLong) client := newChatClient(t) firstUser := coderdtest.CreateFirstUser(t, client.Client) - - // Copilot is a valid AI Gateway provider but the Agents harness - // cannot use it. It must surface as an unsupported provider rather - // than vanish, so the empty state can explain why. - _ = createAIProviderForTest(t, client, string(codersdk.AIProviderTypeCopilot), "") - - models, err := client.ChatModelAvailability(ctx, firstUser.OrganizationID) + provider := createAIProviderForTest(t, client, string(codersdk.AIProviderTypeCopilot), "") + response, err := client.ChatModels(ctx, firstUser.OrganizationID) require.NoError(t, err) - - require.False(t, slices.ContainsFunc(models.Providers, func(p codersdk.ChatModelProvider) bool { - return p.Provider == string(codersdk.AIProviderTypeCopilot) - }), "copilot must not appear in the supported model picker") - - require.Equal(t, []codersdk.ChatUnsupportedProvider{ - { - Provider: "copilot", - DisplayName: "GitHub Copilot", - }, - }, models.UnsupportedProviders) + require.False(t, providerByID(t, response, provider.ID).Available) + require.Equal(t, []codersdk.ChatUnsupportedProvider{{Provider: "copilot", DisplayName: "GitHub Copilot"}}, response.UnsupportedProviders) }) t.Run("SupportedProviderHasNoUnsupportedEntry", func(t *testing.T) { t.Parallel() - ctx := testutil.Context(t, testutil.WaitLong) client := newChatClient(t) firstUser := coderdtest.CreateFirstUser(t, client.Client) _ = createChatModel(t, client) - - models, err := client.ChatModelAvailability(ctx, firstUser.OrganizationID) + response, err := client.ChatModels(ctx, firstUser.OrganizationID) require.NoError(t, err) - require.Empty(t, models.UnsupportedProviders) + require.Empty(t, response.UnsupportedProviders) }) t.Run("CentralOnlyProviderAvailable", func(t *testing.T) { t.Parallel() - ctx := testutil.Context(t, testutil.WaitLong) client := newChatClient(t) firstUser := coderdtest.CreateFirstUser(t, client.Client) - _ = createChatModel(t, client) - - models, err := client.ChatModelAvailability(ctx, firstUser.OrganizationID) + model := createChatModel(t, client) + response, err := client.ChatModels(ctx, firstUser.OrganizationID) require.NoError(t, err) - - var openAIProvider *codersdk.ChatModelProvider - for i := range models.Providers { - if models.Providers[i].Provider == coderdtest.TestChatProviderOpenAICompat { - openAIProvider = &models.Providers[i] - break - } - } - require.NotNil(t, openAIProvider) - require.True(t, openAIProvider.Available) + require.True(t, providerByID(t, response, model.AIProviderID).Available) }) t.Run("UserOnlyProviderRequiresUserKey", func(t *testing.T) { t.Parallel() - ctx := testutil.Context(t, testutil.WaitLong) client := newChatClient(t) firstUser := coderdtest.CreateFirstUser(t, client.Client) - - providerType := database.AIProviderTypeAnthropic - provider := createAIProviderForTest(t, client, string(providerType), "") - - contextLimit := int64(4096) - _, err := client.CreateChatModel(ctx, firstUser.OrganizationID, codersdk.CreateChatModelRequest{ - AIProviderID: &provider.ID, - Model: "claude-sonnet", - ContextLimit: &contextLimit, - }) + provider := createAIProviderForTest(t, client, "anthropic", "") + response, err := client.ChatModels(ctx, firstUser.OrganizationID) require.NoError(t, err) - - models, err := client.ChatModelAvailability(ctx, firstUser.OrganizationID) + descriptor := providerByID(t, response, provider.ID) + require.False(t, descriptor.Available) + require.Equal(t, codersdk.ChatModelProviderUnavailableReasonUserAPIKeyRequired, descriptor.UnavailableReason) + _, err = client.UpsertUserAIProviderKey(ctx, "me", provider.ID, codersdk.CreateUserAIProviderKeyRequest{APIKey: "user-key"}) require.NoError(t, err) - - var anthropicProvider *codersdk.ChatModelProvider - for i := range models.Providers { - if models.Providers[i].Provider == string(providerType) { - anthropicProvider = &models.Providers[i] - break - } - } - require.NotNil(t, anthropicProvider) - require.False(t, anthropicProvider.Available) - require.Equal(t, codersdk.ChatModelProviderUnavailableReasonUserAPIKeyRequired, anthropicProvider.UnavailableReason) - - _, err = client.UpsertUserAIProviderKey(ctx, "me", provider.ID, codersdk.CreateUserAIProviderKeyRequest{ - APIKey: "user-api-key", - }) - require.NoError(t, err) - - models, err = client.ChatModelAvailability(ctx, firstUser.OrganizationID) + response, err = client.ChatModels(ctx, firstUser.OrganizationID) require.NoError(t, err) - - anthropicProvider = nil - for i := range models.Providers { - if models.Providers[i].Provider == "anthropic" { - anthropicProvider = &models.Providers[i] - break - } - } - require.NotNil(t, anthropicProvider) - require.True(t, anthropicProvider.Available) + require.True(t, providerByID(t, response, provider.ID).Available) }) t.Run("CentralAndUserWithFallback", func(t *testing.T) { t.Parallel() - ctx := testutil.Context(t, testutil.WaitLong) client := newChatClient(t) firstUser := coderdtest.CreateFirstUser(t, client.Client) - - provider := createAIProviderForTest(t, client, "google", "provider-api-key") - - contextLimit := int64(4096) - _, err := client.CreateChatModel(ctx, firstUser.OrganizationID, codersdk.CreateChatModelRequest{ - AIProviderID: &provider.ID, - Model: "gemini-1.5-pro", - ContextLimit: &contextLimit, - }) + provider := createAIProviderForTest(t, client, "google", "provider-key") + response, err := client.ChatModels(ctx, firstUser.OrganizationID) require.NoError(t, err) - - models, err := client.ChatModelAvailability(ctx, firstUser.OrganizationID) + require.True(t, providerByID(t, response, provider.ID).Available) + _, err = client.UpsertUserAIProviderKey(ctx, "me", provider.ID, codersdk.CreateUserAIProviderKeyRequest{APIKey: "user-key"}) require.NoError(t, err) - - var googleProvider *codersdk.ChatModelProvider - for i := range models.Providers { - if models.Providers[i].Provider == "google" { - googleProvider = &models.Providers[i] - break - } - } - require.NotNil(t, googleProvider) - require.True(t, googleProvider.Available) - - _, err = client.UpsertUserAIProviderKey(ctx, "me", provider.ID, codersdk.CreateUserAIProviderKeyRequest{ - APIKey: "user-api-key", - }) + response, err = client.ChatModels(ctx, firstUser.OrganizationID) require.NoError(t, err) - - models, err = client.ChatModelAvailability(ctx, firstUser.OrganizationID) - require.NoError(t, err) - - googleProvider = nil - for i := range models.Providers { - if models.Providers[i].Provider == "google" { - googleProvider = &models.Providers[i] - break - } - } - require.NotNil(t, googleProvider) - require.True(t, googleProvider.Available) + require.True(t, providerByID(t, response, provider.ID).Available) }) t.Run("SameTypeProvidersUseExactCredentialStatus", func(t *testing.T) { t.Parallel() - ctx := testutil.Context(t, testutil.WaitLong) client := newChatClient(t) firstUser := coderdtest.CreateFirstUser(t, client.Client) availableProvider := createAIProviderForTest(t, client, "openai", "available-key") unavailableProvider := createAIProviderForTest(t, client, "openai", "") - contextLimit := int64(4096) - - availableModel, err := client.CreateChatModel(ctx, firstUser.OrganizationID, codersdk.CreateChatModelRequest{ - AIProviderID: &availableProvider.ID, - Model: "available-" + uuid.NewString(), - ContextLimit: &contextLimit, - }) + response, err := client.ChatModels(ctx, firstUser.OrganizationID) require.NoError(t, err) - unavailableModel, err := client.CreateChatModel(ctx, firstUser.OrganizationID, codersdk.CreateChatModelRequest{ - AIProviderID: &unavailableProvider.ID, - Model: "unavailable-" + uuid.NewString(), - ContextLimit: &contextLimit, - }) - require.NoError(t, err) - - models, err := client.ChatModelAvailability(ctx, firstUser.OrganizationID) - require.NoError(t, err) - var catalogModels []string - for _, provider := range models.Providers { - for _, model := range provider.Models { - catalogModels = append(catalogModels, model.Model) - } - } - require.Contains(t, catalogModels, availableModel.Model) - require.NotContains(t, catalogModels, unavailableModel.Model) + require.True(t, providerByID(t, response, availableProvider.ID).Available) + unavailable := providerByID(t, response, unavailableProvider.ID) + require.False(t, unavailable.Available) + require.NotEmpty(t, unavailable.UnavailableReason) }) t.Run("UserCredentialsAreCallerIsolated", func(t *testing.T) { t.Parallel() - ctx := testutil.Context(t, testutil.WaitLong) client := newChatClient(t) firstUser := coderdtest.CreateFirstUser(t, client.Client) provider := createAIProviderForTest(t, client, "anthropic", "") - contextLimit := int64(4096) - modelConfig, err := client.CreateChatModel(ctx, firstUser.OrganizationID, codersdk.CreateChatModelRequest{ - AIProviderID: &provider.ID, - Model: "caller-isolated-" + uuid.NewString(), - ContextLimit: &contextLimit, - }) - require.NoError(t, err) - otherClientRaw, _ := coderdtest.CreateAnotherUser( - t, client.Client, firstUser.OrganizationID, rbac.ScopedRoleAgentsAccess(firstUser.OrganizationID), - ) - otherClient := codersdk.NewExperimentalClient(otherClientRaw) - - _, err = client.UpsertUserAIProviderKey(ctx, "me", provider.ID, codersdk.CreateUserAIProviderKeyRequest{APIKey: "owner-key"}) - require.NoError(t, err) - - ownerModels, err := client.ChatModelAvailability(ctx, firstUser.OrganizationID) - require.NoError(t, err) - otherModels, err := otherClient.ChatModelAvailability(ctx, firstUser.OrganizationID) + otherRaw, _ := coderdtest.CreateAnotherUser(t, client.Client, firstUser.OrganizationID, rbac.ScopedRoleAgentsAccess(firstUser.OrganizationID)) + other := codersdk.NewExperimentalClient(otherRaw) + _, err := client.UpsertUserAIProviderKey(ctx, "me", provider.ID, codersdk.CreateUserAIProviderKeyRequest{APIKey: "owner-key"}) require.NoError(t, err) - ownerConfigs, err := client.ChatModels(ctx, firstUser.OrganizationID) + ownerResponse, err := client.ChatModels(ctx, firstUser.OrganizationID) require.NoError(t, err) - otherConfigs, err := otherClient.ChatModels(ctx, firstUser.OrganizationID) + otherResponse, err := other.ChatModels(ctx, firstUser.OrganizationID) require.NoError(t, err) - - hasUserAPIKey := func(response codersdk.OrganizationChatModelsResponse) bool { - for _, descriptor := range response.Providers { - if descriptor.ID == provider.ID { - return descriptor.HasUserAPIKey - } - } - t.Fatal("provider descriptor not found") - return false - } - require.True(t, hasUserAPIKey(ownerConfigs)) - require.False(t, hasUserAPIKey(otherConfigs)) - - containsModel := func(response codersdk.ChatModelAvailabilityResponse) bool { - for _, availableProvider := range response.Providers { - for _, model := range availableProvider.Models { - if model.Model == modelConfig.Model { - return true - } - } - } - return false - } - require.True(t, containsModel(ownerModels)) - require.False(t, containsModel(otherModels)) + require.True(t, providerByID(t, ownerResponse, provider.ID).Available) + require.True(t, providerByID(t, ownerResponse, provider.ID).HasUserAPIKey) + require.False(t, providerByID(t, otherResponse, provider.ID).Available) + require.False(t, providerByID(t, otherResponse, provider.ID).HasUserAPIKey) }) t.Run("IncludesOnlyDefaultOrganizationModels", func(t *testing.T) { t.Parallel() - ctx := testutil.Context(t, testutil.WaitLong) client, db := newChatClientWithDatabase(t) firstUser := coderdtest.CreateFirstUser(t, client.Client) provider := createAIProviderForTest(t, client, "openai-compat", "test-api-key") contextLimit := int64(4096) - defaultConfig, err := client.CreateChatModel(ctx, firstUser.OrganizationID, codersdk.CreateChatModelRequest{ - AIProviderID: &provider.ID, - Model: "default-" + uuid.NewString(), - ContextLimit: &contextLimit, - }) + defaultConfig, err := client.CreateChatModel(ctx, firstUser.OrganizationID, codersdk.CreateChatModelRequest{AIProviderID: &provider.ID, Model: "default-" + uuid.NewString(), ContextLimit: &contextLimit}) require.NoError(t, err) - otherOrganization := dbgen.Organization(t, db, database.Organization{IsDefault: false}) - otherConfig := dbgen.ChatModelConfig(t, db, database.ChatModelConfig{ - AIProviderID: uuid.NullUUID{UUID: provider.ID, Valid: true}, - Model: "other-" + uuid.NewString(), - OrganizationID: otherOrganization.ID, - }) - - models, err := client.ChatModelAvailability(ctx, firstUser.OrganizationID) + otherOrg := dbgen.Organization(t, db, database.Organization{IsDefault: false}) + other := dbgen.ChatModelConfig(t, db, database.ChatModelConfig{AIProviderID: uuid.NullUUID{UUID: provider.ID, Valid: true}, Model: "other-" + uuid.NewString(), OrganizationID: otherOrg.ID}) + response, err := client.ChatModels(ctx, firstUser.OrganizationID) require.NoError(t, err) - - var catalogModels []string - for _, availableProvider := range models.Providers { - for _, model := range availableProvider.Models { - catalogModels = append(catalogModels, model.Model) - } - } - require.Contains(t, catalogModels, defaultConfig.Model) - require.NotContains(t, catalogModels, otherConfig.Model) + require.True(t, containsModel(response, defaultConfig.ID)) + require.False(t, containsModel(response, other.ID)) }) t.Run("DisabledProvidersAndModelsAreFilteredOut", func(t *testing.T) { t.Parallel() - ctx := testutil.Context(t, testutil.WaitLong) - values := coderdtest.DeploymentValues(t) - values.AI.BridgeConfig.LegacyOpenAI.Key = serpent.String("deployment-openai-key") - client := newChatClientWithDeploymentValues(t, values) + client := newChatClient(t) firstUser := coderdtest.CreateFirstUser(t, client.Client) - provider := createAIProviderForTest(t, client, "openai", "test-key") - contextLimit := int64(4096) - _, err := client.CreateChatModel(ctx, firstUser.OrganizationID, codersdk.CreateChatModelRequest{ - AIProviderID: &provider.ID, - Model: "gpt-4o-mini", - ContextLimit: &contextLimit, - }) - require.NoError(t, err) - - models, err := client.ChatModelAvailability(ctx, firstUser.OrganizationID) + enabledModel, err := client.CreateChatModel(ctx, firstUser.OrganizationID, codersdk.CreateChatModelRequest{AIProviderID: &provider.ID, Model: "enabled-" + uuid.NewString(), ContextLimit: &contextLimit}) require.NoError(t, err) - require.Len(t, models.Providers, 1) - require.Equal(t, "openai", models.Providers[0].Provider) - require.Len(t, models.Providers[0].Models, 1) - require.Equal(t, "gpt-4o-mini", models.Providers[0].Models[0].Model) - - // A disabled model under an enabled provider must not appear in - // availability, even though the management list still returns it. - disabledModel, err := client.CreateChatModel(ctx, firstUser.OrganizationID, codersdk.CreateChatModelRequest{ - AIProviderID: &provider.ID, - Model: "gpt-4o", - ContextLimit: &contextLimit, - }) + disabledModel, err := client.CreateChatModel(ctx, firstUser.OrganizationID, codersdk.CreateChatModelRequest{AIProviderID: &provider.ID, Model: "disabled-" + uuid.NewString(), ContextLimit: &contextLimit}) require.NoError(t, err) - enabled := false - _, err = client.UpdateChatModel(ctx, disabledModel.OrganizationID, disabledModel.ID, codersdk.UpdateChatModelRequest{ - Enabled: &enabled, - }) + _, err = client.UpdateChatModel(ctx, firstUser.OrganizationID, disabledModel.ID, codersdk.UpdateChatModelRequest{Enabled: ptr.Ref(false)}) require.NoError(t, err) - - models, err = client.ChatModelAvailability(ctx, firstUser.OrganizationID) + response, err := client.ChatModels(ctx, firstUser.OrganizationID) require.NoError(t, err) - require.Len(t, models.Providers, 1) - require.Len(t, models.Providers[0].Models, 1) - require.Equal(t, "gpt-4o-mini", models.Providers[0].Models[0].Model) - - _, err = client.UpdateAIProvider(ctx, provider.ID.String(), codersdk.UpdateAIProviderRequest{ - Enabled: &enabled, - }) + require.True(t, containsModel(response, enabledModel.ID)) + require.False(t, containsModel(response, disabledModel.ID)) + _, err = client.UpdateAIProvider(ctx, provider.ID.String(), codersdk.UpdateAIProviderRequest{Enabled: ptr.Ref(false)}) require.NoError(t, err) - - models, err = client.ChatModelAvailability(ctx, firstUser.OrganizationID) + response, err = client.ChatModels(ctx, firstUser.OrganizationID) require.NoError(t, err) - require.Empty(t, models.Providers) + require.Empty(t, response.Models) }) } @@ -4837,6 +4580,8 @@ func TestListChatModelConfigs(t *testing.T) { configs, err := client.ChatModels(ctx, modelConfig.OrganizationID) require.NoError(t, err) require.NotEmpty(t, configs.Models) + require.NotEmpty(t, configs.Providers) + require.Empty(t, configs.UnsupportedProviders) found := false for _, config := range configs.Models { @@ -4866,13 +4611,13 @@ func TestListChatModelConfigs(t *testing.T) { require.NoError(t, codersdk.ReadBodyAsJSON(res, &configs)) require.Contains(t, configs, modelConfig) - availabilityRes, err := client.Request(ctx, http.MethodGet, "/api/experimental/chats/models", nil) + collectionRes, err := client.Request(ctx, http.MethodGet, "/api/experimental/chats/models", nil) require.NoError(t, err) - defer availabilityRes.Body.Close() - require.Equal(t, http.StatusOK, availabilityRes.StatusCode) - var availability codersdk.ChatModelAvailabilityResponse - require.NoError(t, codersdk.ReadBodyAsJSON(availabilityRes, &availability)) - require.Contains(t, availability.Models, modelConfig) + defer collectionRes.Body.Close() + require.Equal(t, http.StatusOK, collectionRes.StatusCode) + var collection codersdk.OrganizationChatModelsResponse + require.NoError(t, codersdk.ReadBodyAsJSON(collectionRes, &collection)) + require.Contains(t, collection.Models, modelConfig) contextLimit := int64(8192) createdRes, err := client.Request(ctx, http.MethodPost, "/api/experimental/chats/model-configs", codersdk.CreateChatModelRequest{ @@ -4893,7 +4638,7 @@ func TestListChatModelConfigs(t *testing.T) { require.NoError(t, client.DeleteChatModel(ctx, created.OrganizationID, created.ID)) }) - t.Run("AdminIncludesDisabledModelConfigs", func(t *testing.T) { + t.Run("CollectionFiltersDisabledModelConfigs", func(t *testing.T) { t.Parallel() ctx := testutil.Context(t, testutil.WaitLong) @@ -4916,16 +4661,9 @@ func TestListChatModelConfigs(t *testing.T) { configs, err := client.ChatModels(ctx, firstUser.OrganizationID) require.NoError(t, err) - - found := false - for _, config := range configs.Models { - if config.ID == disabledConfig.ID { - found = true - require.False(t, config.Enabled) - require.Equal(t, disabledConfig.DisplayName, config.DisplayName) - } - } - require.True(t, found) + require.False(t, slices.ContainsFunc(configs.Models, func(config codersdk.ChatModel) bool { + return config.ID == disabledConfig.ID + })) }) t.Run("OrganizationMemberSeesReadableModels", func(t *testing.T) { @@ -4972,6 +4710,8 @@ func TestListChatModelConfigs(t *testing.T) { require.False(t, descriptor.HasAPIKey) require.False(t, descriptor.HasUserAPIKey) require.True(t, descriptor.HasEffectiveAPIKey) + require.True(t, descriptor.Available) + require.Empty(t, descriptor.UnavailableReason) return } } @@ -5945,19 +5685,14 @@ func TestUpdateChatModel(t *testing.T) { adminConfigs, err := adminClient.ChatModels(ctx, firstUser.OrganizationID) require.NoError(t, err) - foundForAdmin := false - for _, config := range adminConfigs.Models { - if config.ID == modelConfig.ID { - foundForAdmin = true - require.False(t, config.Enabled) - } - } - require.True(t, foundForAdmin) + require.False(t, slices.ContainsFunc(adminConfigs.Models, func(config codersdk.ChatModel) bool { + return config.ID == modelConfig.ID + })) - availability, err := memberClient.ChatModelAvailability(ctx, firstUser.OrganizationID) + collection, err := memberClient.ChatModels(ctx, firstUser.OrganizationID) require.NoError(t, err) - runtimeIDs := make([]uuid.UUID, 0, len(availability.Models)) - for _, config := range availability.Models { + runtimeIDs := make([]uuid.UUID, 0, len(collection.Models)) + for _, config := range collection.Models { runtimeIDs = append(runtimeIDs, config.ID) } require.NotContains(t, runtimeIDs, modelConfig.ID) @@ -5997,10 +5732,10 @@ func TestUpdateChatModel(t *testing.T) { require.NoError(t, err) require.False(t, modelConfig.Enabled) - availability, err := memberClient.ChatModelAvailability(ctx, firstUser.OrganizationID) + collection, err := memberClient.ChatModels(ctx, firstUser.OrganizationID) require.NoError(t, err) - runtimeIDs := make([]uuid.UUID, 0, len(availability.Models)) - for _, config := range availability.Models { + runtimeIDs := make([]uuid.UUID, 0, len(collection.Models)) + for _, config := range collection.Models { runtimeIDs = append(runtimeIDs, config.ID) } require.NotContains(t, runtimeIDs, modelConfig.ID) @@ -6013,10 +5748,10 @@ func TestUpdateChatModel(t *testing.T) { require.Equal(t, modelConfig.ID, updated.ID) require.True(t, updated.Enabled) - availability, err = memberClient.ChatModelAvailability(ctx, firstUser.OrganizationID) + collection, err = memberClient.ChatModels(ctx, firstUser.OrganizationID) require.NoError(t, err) runtimeIDs = runtimeIDs[:0] - for _, config := range availability.Models { + for _, config := range collection.Models { runtimeIDs = append(runtimeIDs, config.ID) } require.Contains(t, runtimeIDs, modelConfig.ID) diff --git a/coderd/x/chatd/chatprovider/chatprovider.go b/coderd/x/chatd/chatprovider/chatprovider.go index 76de22ae289..9e507a02cd6 100644 --- a/coderd/x/chatd/chatprovider/chatprovider.go +++ b/coderd/x/chatd/chatprovider/chatprovider.go @@ -26,17 +26,6 @@ import ( "github.com/coder/coder/v2/codersdk" ) -var supportedProviderNames = []string{ - fantasyanthropic.Name, - fantasyazure.Name, - fantasybedrock.Name, - fantasygoogle.Name, - fantasyopenai.Name, - fantasyopenaicompat.Name, - fantasyopenrouter.Name, - fantasyvercel.Name, -} - var providerDisplayNameByName = map[string]string{ fantasyanthropic.Name: "Anthropic", fantasyazure.Name: "Azure OpenAI", @@ -205,13 +194,6 @@ type ConfiguredProvider struct { AllowCentralAPIKeyFallback bool } -// ConfiguredModel is an enabled model loaded from database config. -type ConfiguredModel struct { - Provider string - Model string - DisplayName string -} - // APIKey returns the effective API key for a provider. func (k ProviderAPIKeys) APIKey(provider string) string { normalized := NormalizeProvider(provider) @@ -487,131 +469,6 @@ func setResolvedProviderAPIKey(keys *ProviderAPIKeys, provider string, apiKey st } } -type ModelCatalog struct{} - -func NewModelCatalog() *ModelCatalog { - return &ModelCatalog{} -} - -// ListConfiguredModels returns a model catalog from enabled DB-backed model -// configs. The second return value reports whether DB-backed models were used. -func (*ModelCatalog) ListConfiguredModels( - configuredProviders []ConfiguredProvider, - configuredModels []ConfiguredModel, - availabilityByProvider map[string]ProviderAvailability, - enabledProviders map[string]struct{}, -) (codersdk.ChatModelAvailabilityResponse, bool) { - if len(configuredModels) == 0 { - return codersdk.ChatModelAvailabilityResponse{}, false - } - - modelsByProvider := make(map[string][]codersdk.ChatModelCatalogEntry) - seenByProvider := make(map[string]map[string]struct{}) - providerSet := make(map[string]struct{}) - - for _, provider := range configuredProviders { - normalized := NormalizeProvider(provider.Provider) - if normalized == "" { - continue - } - providerSet[normalized] = struct{}{} - } - - for _, model := range configuredModels { - provider, modelID, err := ResolveModelWithProviderHint(model.Model, model.Provider) - if err != nil { - continue - } - - providerSet[provider] = struct{}{} - if seenByProvider[provider] == nil { - seenByProvider[provider] = make(map[string]struct{}) - } - normalizedModelID := strings.ToLower(strings.TrimSpace(modelID)) - if _, ok := seenByProvider[provider][normalizedModelID]; ok { - continue - } - seenByProvider[provider][normalizedModelID] = struct{}{} - modelsByProvider[provider] = append( - modelsByProvider[provider], - newChatModelCatalogEntry(provider, modelID, model.DisplayName), - ) - } - - providers := orderProviders(providerSet) - if len(providers) == 0 { - return codersdk.ChatModelAvailabilityResponse{}, false - } - - response := codersdk.ChatModelAvailabilityResponse{ - Providers: make([]codersdk.ChatModelProvider, 0, len(providers)), - } - for _, provider := range providers { - if _, ok := enabledProviders[provider]; !ok { - continue - } - - models := modelsByProvider[provider] - if models == nil { - models = []codersdk.ChatModelCatalogEntry{} - } - sortChatModelCatalogEntries(models) - - result := codersdk.ChatModelProvider{ - Provider: provider, - Models: models, - } - if avail, ok := availabilityByProvider[provider]; ok { - result.Available = avail.Available - if !avail.Available { - result.UnavailableReason = avail.UnavailableReason - } - } else { - result.Available = false - result.UnavailableReason = codersdk.ChatModelProviderUnavailableMissingAPIKey - } - - response.Providers = append(response.Providers, result) - } - - return response, true -} - -// ListConfiguredProviderAvailability returns provider availability derived from -// the policy-aware availability map for enabled providers. -func (*ModelCatalog) ListConfiguredProviderAvailability( - availabilityByProvider map[string]ProviderAvailability, - enabledProviders map[string]struct{}, -) codersdk.ChatModelAvailabilityResponse { - response := codersdk.ChatModelAvailabilityResponse{ - Providers: make([]codersdk.ChatModelProvider, 0, len(supportedProviderNames)), - } - - for _, provider := range supportedProviderNames { - if _, ok := enabledProviders[provider]; !ok { - continue - } - - result := codersdk.ChatModelProvider{ - Provider: provider, - Models: []codersdk.ChatModelCatalogEntry{}, - } - if avail, ok := availabilityByProvider[provider]; ok { - result.Available = avail.Available - if !avail.Available { - result.UnavailableReason = avail.UnavailableReason - } - } else { - result.Available = false - result.UnavailableReason = codersdk.ChatModelProviderUnavailableMissingAPIKey - } - - response.Providers = append(response.Providers, result) - } - - return response -} - // PruneDisabledProviderKeys removes entries from keys that do not // belong to an enabled provider. It clears ByProvider, // BaseURLByProvider, and RegionByProvider entries for disabled @@ -644,48 +501,6 @@ func PruneDisabledProviderKeys(keys *ProviderAPIKeys, enabledProviders map[strin } } -func newChatModelCatalogEntry(provider, modelID, displayName string) codersdk.ChatModelCatalogEntry { - name := strings.TrimSpace(displayName) - if name == "" { - name = modelID - } - - return codersdk.ChatModelCatalogEntry{ - ID: canonicalModelID(provider, modelID), - Provider: provider, - Model: modelID, - DisplayName: name, - } -} - -func sortChatModelCatalogEntries(models []codersdk.ChatModelCatalogEntry) { - slices.SortFunc(models, func(a, b codersdk.ChatModelCatalogEntry) int { - return strings.Compare(a.Model, b.Model) - }) -} - -func canonicalModelID(provider, modelID string) string { - return NormalizeProvider(provider) + ":" + strings.TrimSpace(modelID) -} - -func orderProviders(providerSet map[string]struct{}) []string { - if len(providerSet) == 0 { - return nil - } - - ordered := make([]string, 0, len(providerSet)) - for _, provider := range supportedProviderNames { - if _, ok := providerSet[provider]; ok { - ordered = append(ordered, provider) - } - } - - // Unknown providers are dropped. The providerSet keys are - // already normalized, so any provider not in - // supportedProviderNames is silently excluded. - return ordered -} - // NormalizeProvider canonicalizes a provider name. func NormalizeProvider(provider string) string { switch strings.ToLower(strings.TrimSpace(provider)) { diff --git a/coderd/x/chatd/chatprovider/chatprovider_test.go b/coderd/x/chatd/chatprovider/chatprovider_test.go index 9bf7d07c4bc..b6ce235c3f0 100644 --- a/coderd/x/chatd/chatprovider/chatprovider_test.go +++ b/coderd/x/chatd/chatprovider/chatprovider_test.go @@ -654,300 +654,6 @@ func TestResolveUserProviderKeys_UnavailableReason(t *testing.T) { } } -func TestListConfiguredModels_PolicyAwareAvailability(t *testing.T) { - t.Parallel() - - configuredProvider := func(provider string, apiKey string) chatprovider.ConfiguredProvider { - return chatprovider.ConfiguredProvider{ - ProviderID: uuid.New(), - Provider: provider, - APIKey: apiKey, - } - } - enabledProviders := func(providers ...string) map[string]struct{} { - result := make(map[string]struct{}, len(providers)) - for _, provider := range providers { - result[chatprovider.NormalizeProvider(provider)] = struct{}{} - } - return result - } - - catalog := chatprovider.NewModelCatalog() - tests := []struct { - name string - configuredProviders []chatprovider.ConfiguredProvider - configuredModels []chatprovider.ConfiguredModel - availabilityByProvider map[string]chatprovider.ProviderAvailability - enabledProviders map[string]struct{} - want codersdk.ChatModelAvailabilityResponse - }{ - { - name: "PolicyUnavailableOverridesConfiguredKey", - configuredProviders: []chatprovider.ConfiguredProvider{ - configuredProvider(fantasyopenai.Name, "sk-central"), - }, - configuredModels: []chatprovider.ConfiguredModel{{ - Provider: fantasyopenai.Name, - Model: "gpt-4", - }}, - availabilityByProvider: map[string]chatprovider.ProviderAvailability{ - fantasyopenai.Name: { - Available: false, - UnavailableReason: codersdk.ChatModelProviderUnavailableReasonUserAPIKeyRequired, - }, - }, - enabledProviders: enabledProviders(fantasyopenai.Name), - want: codersdk.ChatModelAvailabilityResponse{Providers: []codersdk.ChatModelProvider{{ - Provider: fantasyopenai.Name, - Available: false, - UnavailableReason: codersdk.ChatModelProviderUnavailableReasonUserAPIKeyRequired, - Models: []codersdk.ChatModelCatalogEntry{{ - ID: fantasyopenai.Name + ":gpt-4", - Provider: fantasyopenai.Name, - Model: "gpt-4", - DisplayName: "gpt-4", - }}, - }}}, - }, - { - name: "PolicyAvailableMarksProviderAvailable", - configuredProviders: []chatprovider.ConfiguredProvider{ - configuredProvider(fantasyanthropic.Name, "sk-central"), - }, - configuredModels: []chatprovider.ConfiguredModel{{ - Provider: fantasyanthropic.Name, - Model: "claude-3-5-sonnet", - }}, - availabilityByProvider: map[string]chatprovider.ProviderAvailability{ - fantasyanthropic.Name: {Available: true}, - }, - enabledProviders: enabledProviders(fantasyanthropic.Name), - want: codersdk.ChatModelAvailabilityResponse{Providers: []codersdk.ChatModelProvider{{ - Provider: fantasyanthropic.Name, - Available: true, - Models: []codersdk.ChatModelCatalogEntry{{ - ID: fantasyanthropic.Name + ":claude-3-5-sonnet", - Provider: fantasyanthropic.Name, - Model: "claude-3-5-sonnet", - DisplayName: "claude-3-5-sonnet", - }}, - }}}, - }, - { - name: "DisabledProviderOmitted", - configuredProviders: []chatprovider.ConfiguredProvider{ - configuredProvider(fantasyanthropic.Name, "sk-anthropic"), - configuredProvider(fantasyopenai.Name, "sk-openai"), - }, - configuredModels: []chatprovider.ConfiguredModel{ - {Provider: fantasyanthropic.Name, Model: "claude-3-5-sonnet"}, - {Provider: fantasyopenai.Name, Model: "gpt-4"}, - }, - availabilityByProvider: map[string]chatprovider.ProviderAvailability{ - fantasyanthropic.Name: {Available: true}, - fantasyopenai.Name: {Available: true}, - }, - enabledProviders: enabledProviders(fantasyopenai.Name), - want: codersdk.ChatModelAvailabilityResponse{Providers: []codersdk.ChatModelProvider{{ - Provider: fantasyopenai.Name, - Available: true, - Models: []codersdk.ChatModelCatalogEntry{{ - ID: fantasyopenai.Name + ":gpt-4", - Provider: fantasyopenai.Name, - Model: "gpt-4", - DisplayName: "gpt-4", - }}, - }}}, - }, - { - name: "ProviderWithoutConfiguredModelUsesEmptyArray", - configuredProviders: []chatprovider.ConfiguredProvider{ - configuredProvider(fantasyanthropic.Name, "sk-anthropic"), - configuredProvider(fantasyopenai.Name, "sk-openai"), - }, - configuredModels: []chatprovider.ConfiguredModel{{ - Provider: fantasyanthropic.Name, - Model: "claude-3-5-sonnet", - }}, - availabilityByProvider: map[string]chatprovider.ProviderAvailability{ - fantasyanthropic.Name: {Available: true}, - fantasyopenai.Name: {Available: true}, - }, - enabledProviders: enabledProviders(fantasyanthropic.Name, fantasyopenai.Name), - want: codersdk.ChatModelAvailabilityResponse{Providers: []codersdk.ChatModelProvider{ - { - Provider: fantasyanthropic.Name, - Available: true, - Models: []codersdk.ChatModelCatalogEntry{{ - ID: fantasyanthropic.Name + ":claude-3-5-sonnet", - Provider: fantasyanthropic.Name, - Model: "claude-3-5-sonnet", - DisplayName: "claude-3-5-sonnet", - }}, - }, - { - Provider: fantasyopenai.Name, - Available: true, - Models: []codersdk.ChatModelCatalogEntry{}, - }, - }}, - }, - { - name: "MissingAvailabilityDefaultsToMissingAPIKey", - configuredProviders: []chatprovider.ConfiguredProvider{ - configuredProvider(fantasyopenai.Name, "sk-central"), - }, - configuredModels: []chatprovider.ConfiguredModel{{ - Provider: fantasyopenai.Name, - Model: "gpt-4o", - }}, - enabledProviders: enabledProviders(fantasyopenai.Name), - want: codersdk.ChatModelAvailabilityResponse{Providers: []codersdk.ChatModelProvider{{ - Provider: fantasyopenai.Name, - Available: false, - UnavailableReason: codersdk.ChatModelProviderUnavailableMissingAPIKey, - Models: []codersdk.ChatModelCatalogEntry{{ - ID: fantasyopenai.Name + ":gpt-4o", - Provider: fantasyopenai.Name, - Model: "gpt-4o", - DisplayName: "gpt-4o", - }}, - }}}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - got, ok := catalog.ListConfiguredModels( - tt.configuredProviders, - tt.configuredModels, - tt.availabilityByProvider, - tt.enabledProviders, - ) - require.True(t, ok) - require.Equal(t, tt.want, got) - }) - } -} - -func TestListConfiguredModels_EmptyModelsJSON(t *testing.T) { - t.Parallel() - - catalog := chatprovider.NewModelCatalog() - response, ok := catalog.ListConfiguredModels( - []chatprovider.ConfiguredProvider{ - {ProviderID: uuid.New(), Provider: fantasyanthropic.Name, APIKey: "anthropic-key"}, - {ProviderID: uuid.New(), Provider: fantasyopenai.Name, APIKey: "openai-key"}, - }, - []chatprovider.ConfiguredModel{{Provider: fantasyanthropic.Name, Model: "claude-3-5-sonnet"}}, - map[string]chatprovider.ProviderAvailability{ - fantasyanthropic.Name: {Available: true}, - fantasyopenai.Name: {Available: true}, - }, - map[string]struct{}{fantasyanthropic.Name: {}, fantasyopenai.Name: {}}, - ) - require.True(t, ok) - - encoded, err := json.Marshal(response) - require.NoError(t, err) - var wire struct { - Providers []struct { - Provider string `json:"provider"` - Models json.RawMessage `json:"models"` - } `json:"providers"` - } - require.NoError(t, json.Unmarshal(encoded, &wire)) - for _, provider := range wire.Providers { - if provider.Provider == fantasyopenai.Name { - require.JSONEq(t, `[]`, string(provider.Models)) - return - } - } - t.Fatal("openai provider not found") -} - -func TestListConfiguredProviderAvailability_PolicyAwareFiltering(t *testing.T) { - t.Parallel() - - enabledProviders := func(providers ...string) map[string]struct{} { - result := make(map[string]struct{}, len(providers)) - for _, provider := range providers { - result[chatprovider.NormalizeProvider(provider)] = struct{}{} - } - return result - } - - catalog := chatprovider.NewModelCatalog() - tests := []struct { - name string - availabilityByProvider map[string]chatprovider.ProviderAvailability - enabledProviders map[string]struct{} - want codersdk.ChatModelAvailabilityResponse - }{ - { - name: "EnabledProvidersUsePolicyAvailability", - availabilityByProvider: map[string]chatprovider.ProviderAvailability{ - fantasyanthropic.Name: { - Available: false, - UnavailableReason: codersdk.ChatModelProviderUnavailableReasonUserAPIKeyRequired, - }, - fantasyopenai.Name: {Available: true}, - }, - enabledProviders: enabledProviders(fantasyanthropic.Name, fantasyopenai.Name), - want: codersdk.ChatModelAvailabilityResponse{Providers: []codersdk.ChatModelProvider{ - { - Provider: fantasyanthropic.Name, - Available: false, - UnavailableReason: codersdk.ChatModelProviderUnavailableReasonUserAPIKeyRequired, - Models: []codersdk.ChatModelCatalogEntry{}, - }, - { - Provider: fantasyopenai.Name, - Available: true, - Models: []codersdk.ChatModelCatalogEntry{}, - }, - }}, - }, - { - name: "DisabledSupportedProviderOmitted", - availabilityByProvider: map[string]chatprovider.ProviderAvailability{ - fantasyanthropic.Name: {Available: true}, - fantasyopenai.Name: {Available: true}, - }, - enabledProviders: enabledProviders(fantasyopenai.Name), - want: codersdk.ChatModelAvailabilityResponse{Providers: []codersdk.ChatModelProvider{{ - Provider: fantasyopenai.Name, - Available: true, - Models: []codersdk.ChatModelCatalogEntry{}, - }}}, - }, - { - name: "MissingAvailabilityDefaultsToMissingAPIKey", - enabledProviders: enabledProviders(fantasyopenai.Name), - want: codersdk.ChatModelAvailabilityResponse{Providers: []codersdk.ChatModelProvider{{ - Provider: fantasyopenai.Name, - Available: false, - UnavailableReason: codersdk.ChatModelProviderUnavailableMissingAPIKey, - Models: []codersdk.ChatModelCatalogEntry{}, - }}}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - got := catalog.ListConfiguredProviderAvailability( - tt.availabilityByProvider, - tt.enabledProviders, - ) - require.Equal(t, tt.want, got) - }) - } -} - func TestPruneDisabledProviderKeys(t *testing.T) { t.Parallel() diff --git a/codersdk/chats.go b/codersdk/chats.go index 3e80ced483f..7df91c96cc9 100644 --- a/codersdk/chats.go +++ b/codersdk/chats.go @@ -722,35 +722,6 @@ const ( ChatModelProviderUnavailableReasonUserAPIKeyRequired ChatModelProviderUnavailableReason = "user_api_key_required" ) -// ChatModelCatalogEntry is the runtime catalog view of a model. Its ID is the -// synthetic catalog identity `provider:model` built by canonicalModelID, not -// the chat_model_configs row UUID that ChatModel.ID carries. -type ChatModelCatalogEntry struct { - ID string `json:"id"` - Provider string `json:"provider"` - Model string `json:"model"` - DisplayName string `json:"display_name"` -} - -// ChatModelProvider represents provider availability and model results. -type ChatModelProvider struct { - Provider string `json:"provider"` - Available bool `json:"available"` - UnavailableReason ChatModelProviderUnavailableReason `json:"unavailable_reason,omitempty"` - Models []ChatModelCatalogEntry `json:"models"` -} - -// ChatModelAvailabilityResponse is the catalog returned from chat model discovery. -type ChatModelAvailabilityResponse struct { - // Models contains the effective runtime model configs for the requested - // organization. Each config belongs to that organization. - Models []ChatModel `json:"models,omitempty"` - Providers []ChatModelProvider `json:"providers"` - // UnsupportedProviders lists configured providers the Agents harness - // cannot use, so the UI can explain the empty state. - UnsupportedProviders []ChatUnsupportedProvider `json:"unsupported_providers"` -} - // ChatUnsupportedProvider is a configured provider the Agents harness cannot // use. type ChatUnsupportedProvider struct { @@ -2047,22 +2018,6 @@ func (c *ExperimentalClient) ListChats(ctx context.Context, opts *ListChatsOptio return chats, ReadBodyAsJSON(res, &chats) } -// ChatModelAvailability returns the provider-grouped, per-caller -// availability view of one organization's chat models. -func (c *ExperimentalClient) ChatModelAvailability(ctx context.Context, organizationID uuid.UUID) (ChatModelAvailabilityResponse, error) { - res, err := c.Request(ctx, http.MethodGet, fmt.Sprintf("/api/experimental/organizations/%s/chats/models/available", organizationID), nil) - if err != nil { - return ChatModelAvailabilityResponse{}, err - } - defer res.Body.Close() - if res.StatusCode != http.StatusOK { - return ChatModelAvailabilityResponse{}, ReadBodyAsError(res) - } - - var catalog ChatModelAvailabilityResponse - return catalog, ReadBodyAsJSON(res, &catalog) -} - // ListChatProviders returns admin-managed chat provider configs. func (c *ExperimentalClient) ListChatProviders(ctx context.Context) ([]ChatProviderConfig, error) { res, err := c.Request(ctx, http.MethodGet, "/api/experimental/chats/providers", nil) @@ -2318,23 +2273,26 @@ func (c *ExperimentalClient) DeleteChatModel(ctx context.Context, organizationID // never exposed. The fields mirror what /api/experimental/chats/models // already discloses to any authenticated caller. type ChatModelProviderDescriptor struct { - ID uuid.UUID `json:"id" format:"uuid"` - Type string `json:"type"` - DisplayName string `json:"display_name"` - Icon string `json:"icon"` - Enabled bool `json:"enabled"` - HasAPIKey bool `json:"has_api_key"` - HasUserAPIKey bool `json:"has_user_api_key"` - HasEffectiveAPIKey bool `json:"has_effective_api_key"` - AllowUserAPIKey bool `json:"allow_user_api_key"` + ID uuid.UUID `json:"id" format:"uuid"` + Type string `json:"type"` + DisplayName string `json:"display_name"` + Icon string `json:"icon"` + Enabled bool `json:"enabled"` + HasAPIKey bool `json:"has_api_key"` + HasUserAPIKey bool `json:"has_user_api_key"` + HasEffectiveAPIKey bool `json:"has_effective_api_key"` + AllowUserAPIKey bool `json:"allow_user_api_key"` + Available bool `json:"available"` + UnavailableReason ChatModelProviderUnavailableReason `json:"unavailable_reason,omitempty"` } // OrganizationChatModelsResponse is the org chat model config collection: // the caller-readable configs plus the redacted provider descriptors the // authoring page needs. type OrganizationChatModelsResponse struct { - Models []ChatModel `json:"models"` - Providers []ChatModelProviderDescriptor `json:"providers"` + Models []ChatModel `json:"models"` + Providers []ChatModelProviderDescriptor `json:"providers"` + UnsupportedProviders []ChatUnsupportedProvider `json:"unsupported_providers"` } // GetChatCost returns the AI Gateway cost for the whole chat tree that diff --git a/docs/ai-coder/agents/models.md b/docs/ai-coder/agents/models.md index 5227260da9f..fd92f9981b4 100644 --- a/docs/ai-coder/agents/models.md +++ b/docs/ai-coder/agents/models.md @@ -148,7 +148,7 @@ Members with model share permission can grant model read access to members and g 5. Add or remove organization members and groups. 6. Select **Save**. -Coder applies the complete member and group list when you save. +Coder applies the member and group changes when you save. Removing all entries clears the model's access list, so members without another read grant lose access on their next request. ### Model visibility and runtime availability diff --git a/docs/reference/api/chats.md b/docs/reference/api/chats.md index d8fe3468d00..0db7af7b819 100644 --- a/docs/reference/api/chats.md +++ b/docs/reference/api/chats.md @@ -3215,233 +3215,3 @@ Experimental: this endpoint is subject to change. | 200 | [OK](https://tools.ietf.org/html/rfc7231#section-6.3.1) | OK | [codersdk.Chat](schemas.md#codersdkchat) | To perform this operation, you must be authenticated. [Learn more](authentication.md). - -## List available chat models in an organization - -### Code samples - -```sh -# Example request using curl -curl -X GET http://coder-server:8080/api/experimental/organizations/{organization}/chats/models/available \ - -H 'Accept: application/json' \ - -H 'Coder-Session-Token: API_KEY' -``` - -`GET /api/experimental/organizations/{organization}/chats/models/available` - -Experimental: this endpoint is subject to change. - -### Parameters - -| Name | In | Type | Required | Description | -|----------------|------|--------|----------|-------------------------| -| `organization` | path | string | true | Organization name or ID | - -### Example responses - -> 200 Response - -```json -{ - "models": [ - { - "ai_provider_id": "5a3b8ff9-20e7-4c37-ba1a-5b433e355819", - "compression_threshold": 0, - "context_limit": 0, - "created_at": "2019-08-24T14:15:22Z", - "display_name": "string", - "enabled": true, - "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08", - "is_default": true, - "model": "string", - "model_config": { - "frequency_penalty": 0, - "max_output_tokens": 0, - "openai_config": { - "use_responses_api": true - }, - "presence_penalty": 0, - "provider_options": { - "anthropic": { - "allowed_domains": [ - "string" - ], - "blocked_domains": [ - "string" - ], - "context_1m_enabled": true, - "disable_parallel_tool_use": true, - "send_reasoning": true, - "thinking": { - "budget_tokens": 0 - }, - "thinking_display": "string", - "web_search_enabled": true - }, - "google": { - "cached_content": "string", - "safety_settings": [ - { - "category": "string", - "threshold": "string" - } - ], - "thinking_config": { - "include_thoughts": true, - "thinking_budget": 0, - "thinking_level": "string" - }, - "threshold": "string", - "web_search_enabled": true - }, - "openai": { - "allowed_domains": [ - "string" - ], - "include": [ - "string" - ], - "instructions": "string", - "log_probs": true, - "logit_bias": { - "property1": 0, - "property2": 0 - }, - "max_completion_tokens": 0, - "max_tool_calls": 0, - "metadata": { - "property1": null, - "property2": null - }, - "parallel_tool_calls": true, - "prediction": { - "property1": null, - "property2": null - }, - "prompt_cache_key": "string", - "reasoning_summary": "string", - "safety_identifier": "string", - "search_context_size": "string", - "service_tier": "string", - "store": true, - "strict_json_schema": true, - "structured_outputs": true, - "text_verbosity": "string", - "top_log_probs": 0, - "user": "string", - "web_search_enabled": true - }, - "openaicompat": { - "user": "string" - }, - "openrouter": { - "extra_body": { - "property1": null, - "property2": null - }, - "include_usage": true, - "log_probs": true, - "logit_bias": { - "property1": 0, - "property2": 0 - }, - "parallel_tool_calls": true, - "provider": { - "allow_fallbacks": true, - "data_collection": "string", - "ignore": [ - "string" - ], - "only": [ - "string" - ], - "order": [ - "string" - ], - "quantizations": [ - "string" - ], - "require_parameters": true, - "sort": "string" - }, - "reasoning": { - "enabled": true, - "exclude": true, - "max_tokens": 0 - }, - "user": "string" - }, - "vercel": { - "extra_body": { - "property1": null, - "property2": null - }, - "logit_bias": { - "property1": 0, - "property2": 0 - }, - "logprobs": true, - "parallel_tool_calls": true, - "providerOptions": { - "models": [ - "string" - ], - "order": [ - "string" - ] - }, - "reasoning": { - "enabled": true, - "exclude": true, - "max_tokens": 0 - }, - "top_logprobs": 0, - "user": "string" - } - }, - "reasoning_effort": { - "default": "string", - "max": "string" - }, - "temperature": 0, - "top_k": 0, - "top_p": 0 - }, - "organization_id": "7c60d51f-b44e-4682-87d6-449835ea4de6", - "reasoning_efforts": [ - "string" - ], - "updated_at": "2019-08-24T14:15:22Z" - } - ], - "providers": [ - { - "available": true, - "models": [ - { - "display_name": "string", - "id": "string", - "model": "string", - "provider": "string" - } - ], - "provider": "string", - "unavailable_reason": "missing_api_key" - } - ], - "unsupported_providers": [ - { - "display_name": "string", - "provider": "string" - } - ] -} -``` - -### Responses - -| Status | Meaning | Description | Schema | -|--------|---------------------------------------------------------|-------------|--------------------------------------------------------------------------------------------| -| 200 | [OK](https://tools.ietf.org/html/rfc7231#section-6.3.1) | OK | [codersdk.ChatModelAvailabilityResponse](schemas.md#codersdkchatmodelavailabilityresponse) | - -To perform this operation, you must be authenticated. [Learn more](authentication.md). diff --git a/docs/reference/api/schemas.md b/docs/reference/api/schemas.md index 4259672f371..b7ac8f58506 100644 --- a/docs/reference/api/schemas.md +++ b/docs/reference/api/schemas.md @@ -3657,213 +3657,6 @@ AuthorizationObject can represent a "set" of objects, such as: all workspaces in |-----------------|---------|----------|--------------|-------------| | `budget_tokens` | integer | false | | | -## codersdk.ChatModelAvailabilityResponse - -```json -{ - "models": [ - { - "ai_provider_id": "5a3b8ff9-20e7-4c37-ba1a-5b433e355819", - "compression_threshold": 0, - "context_limit": 0, - "created_at": "2019-08-24T14:15:22Z", - "display_name": "string", - "enabled": true, - "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08", - "is_default": true, - "model": "string", - "model_config": { - "frequency_penalty": 0, - "max_output_tokens": 0, - "openai_config": { - "use_responses_api": true - }, - "presence_penalty": 0, - "provider_options": { - "anthropic": { - "allowed_domains": [ - "string" - ], - "blocked_domains": [ - "string" - ], - "context_1m_enabled": true, - "disable_parallel_tool_use": true, - "send_reasoning": true, - "thinking": { - "budget_tokens": 0 - }, - "thinking_display": "string", - "web_search_enabled": true - }, - "google": { - "cached_content": "string", - "safety_settings": [ - { - "category": "string", - "threshold": "string" - } - ], - "thinking_config": { - "include_thoughts": true, - "thinking_budget": 0, - "thinking_level": "string" - }, - "threshold": "string", - "web_search_enabled": true - }, - "openai": { - "allowed_domains": [ - "string" - ], - "include": [ - "string" - ], - "instructions": "string", - "log_probs": true, - "logit_bias": { - "property1": 0, - "property2": 0 - }, - "max_completion_tokens": 0, - "max_tool_calls": 0, - "metadata": { - "property1": null, - "property2": null - }, - "parallel_tool_calls": true, - "prediction": { - "property1": null, - "property2": null - }, - "prompt_cache_key": "string", - "reasoning_summary": "string", - "safety_identifier": "string", - "search_context_size": "string", - "service_tier": "string", - "store": true, - "strict_json_schema": true, - "structured_outputs": true, - "text_verbosity": "string", - "top_log_probs": 0, - "user": "string", - "web_search_enabled": true - }, - "openaicompat": { - "user": "string" - }, - "openrouter": { - "extra_body": { - "property1": null, - "property2": null - }, - "include_usage": true, - "log_probs": true, - "logit_bias": { - "property1": 0, - "property2": 0 - }, - "parallel_tool_calls": true, - "provider": { - "allow_fallbacks": true, - "data_collection": "string", - "ignore": [ - "string" - ], - "only": [ - "string" - ], - "order": [ - "string" - ], - "quantizations": [ - "string" - ], - "require_parameters": true, - "sort": "string" - }, - "reasoning": { - "enabled": true, - "exclude": true, - "max_tokens": 0 - }, - "user": "string" - }, - "vercel": { - "extra_body": { - "property1": null, - "property2": null - }, - "logit_bias": { - "property1": 0, - "property2": 0 - }, - "logprobs": true, - "parallel_tool_calls": true, - "providerOptions": { - "models": [ - "string" - ], - "order": [ - "string" - ] - }, - "reasoning": { - "enabled": true, - "exclude": true, - "max_tokens": 0 - }, - "top_logprobs": 0, - "user": "string" - } - }, - "reasoning_effort": { - "default": "string", - "max": "string" - }, - "temperature": 0, - "top_k": 0, - "top_p": 0 - }, - "organization_id": "7c60d51f-b44e-4682-87d6-449835ea4de6", - "reasoning_efforts": [ - "string" - ], - "updated_at": "2019-08-24T14:15:22Z" - } - ], - "providers": [ - { - "available": true, - "models": [ - { - "display_name": "string", - "id": "string", - "model": "string", - "provider": "string" - } - ], - "provider": "string", - "unavailable_reason": "missing_api_key" - } - ], - "unsupported_providers": [ - { - "display_name": "string", - "provider": "string" - } - ] -} -``` - -### Properties - -| Name | Type | Required | Restrictions | Description | -|-------------------------|-------------------------------------------------------------------------------|----------|--------------|-------------------------------------------------------------------------------------------------------------------------------| -| `models` | array of [codersdk.ChatModel](#codersdkchatmodel) | false | | Models contains the effective runtime model configs for the requested organization. Each config belongs to that organization. | -| `providers` | array of [codersdk.ChatModelProvider](#codersdkchatmodelprovider) | false | | | -| `unsupported_providers` | array of [codersdk.ChatUnsupportedProvider](#codersdkchatunsupportedprovider) | false | | Unsupported providers lists configured providers the Agents harness cannot use, so the UI can explain the empty state. | - ## codersdk.ChatModelCallConfig ```json @@ -4036,26 +3829,6 @@ AuthorizationObject can represent a "set" of objects, such as: all workspaces in | `top_k` | integer | false | | | | `top_p` | number | false | | | -## codersdk.ChatModelCatalogEntry - -```json -{ - "display_name": "string", - "id": "string", - "model": "string", - "provider": "string" -} -``` - -### Properties - -| Name | Type | Required | Restrictions | Description | -|----------------|--------|----------|--------------|-------------| -| `display_name` | string | false | | | -| `id` | string | false | | | -| `model` | string | false | | | -| `provider` | string | false | | | - ## codersdk.ChatModelGoogleProviderOptions ```json @@ -4314,38 +4087,12 @@ AuthorizationObject can represent a "set" of objects, such as: all workspaces in | `reasoning` | [codersdk.ChatModelReasoningOptions](#codersdkchatmodelreasoningoptions) | false | | | | `user` | string | false | | | -## codersdk.ChatModelProvider - -```json -{ - "available": true, - "models": [ - { - "display_name": "string", - "id": "string", - "model": "string", - "provider": "string" - } - ], - "provider": "string", - "unavailable_reason": "missing_api_key" -} -``` - -### Properties - -| Name | Type | Required | Restrictions | Description | -|----------------------|--------------------------------------------------------------------------------------------|----------|--------------|-------------| -| `available` | boolean | false | | | -| `models` | array of [codersdk.ChatModelCatalogEntry](#codersdkchatmodelcatalogentry) | false | | | -| `provider` | string | false | | | -| `unavailable_reason` | [codersdk.ChatModelProviderUnavailableReason](#codersdkchatmodelproviderunavailablereason) | false | | | - ## codersdk.ChatModelProviderDescriptor ```json { "allow_user_api_key": true, + "available": true, "display_name": "string", "enabled": true, "has_api_key": true, @@ -4353,23 +4100,26 @@ AuthorizationObject can represent a "set" of objects, such as: all workspaces in "has_user_api_key": true, "icon": "string", "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08", - "type": "string" + "type": "string", + "unavailable_reason": "missing_api_key" } ``` ### Properties -| Name | Type | Required | Restrictions | Description | -|-------------------------|---------|----------|--------------|-------------| -| `allow_user_api_key` | boolean | false | | | -| `display_name` | string | false | | | -| `enabled` | boolean | false | | | -| `has_api_key` | boolean | false | | | -| `has_effective_api_key` | boolean | false | | | -| `has_user_api_key` | boolean | false | | | -| `icon` | string | false | | | -| `id` | string | false | | | -| `type` | string | false | | | +| Name | Type | Required | Restrictions | Description | +|-------------------------|--------------------------------------------------------------------------------------------|----------|--------------|-------------| +| `allow_user_api_key` | boolean | false | | | +| `available` | boolean | false | | | +| `display_name` | string | false | | | +| `enabled` | boolean | false | | | +| `has_api_key` | boolean | false | | | +| `has_effective_api_key` | boolean | false | | | +| `has_user_api_key` | boolean | false | | | +| `icon` | string | false | | | +| `id` | string | false | | | +| `type` | string | false | | | +| `unavailable_reason` | [codersdk.ChatModelProviderUnavailableReason](#codersdkchatmodelproviderunavailablereason) | false | | | ## codersdk.ChatModelProviderOptions @@ -11560,6 +11310,7 @@ Git clone makes use of this by parsing the URL from: 'Username for "https://gith "providers": [ { "allow_user_api_key": true, + "available": true, "display_name": "string", "enabled": true, "has_api_key": true, @@ -11567,7 +11318,14 @@ Git clone makes use of this by parsing the URL from: 'Username for "https://gith "has_user_api_key": true, "icon": "string", "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08", - "type": "string" + "type": "string", + "unavailable_reason": "missing_api_key" + } + ], + "unsupported_providers": [ + { + "display_name": "string", + "provider": "string" } ] } @@ -11575,10 +11333,11 @@ Git clone makes use of this by parsing the URL from: 'Username for "https://gith ### Properties -| Name | Type | Required | Restrictions | Description | -|-------------|---------------------------------------------------------------------------------------|----------|--------------|-------------| -| `models` | array of [codersdk.ChatModel](#codersdkchatmodel) | false | | | -| `providers` | array of [codersdk.ChatModelProviderDescriptor](#codersdkchatmodelproviderdescriptor) | false | | | +| Name | Type | Required | Restrictions | Description | +|-------------------------|---------------------------------------------------------------------------------------|----------|--------------|-------------| +| `models` | array of [codersdk.ChatModel](#codersdkchatmodel) | false | | | +| `providers` | array of [codersdk.ChatModelProviderDescriptor](#codersdkchatmodelproviderdescriptor) | false | | | +| `unsupported_providers` | array of [codersdk.ChatUnsupportedProvider](#codersdkchatunsupportedprovider) | false | | | ## codersdk.OrganizationGroupAISpend diff --git a/site/src/api/api.test.ts b/site/src/api/api.test.ts index 5c6e56fbf82..b4e6413360e 100644 --- a/site/src/api/api.test.ts +++ b/site/src/api/api.test.ts @@ -435,18 +435,10 @@ describe("api.ts", () => { const organizationId = "organization/id"; it.each<[string, () => Promise, unknown]>([ - [ - "/api/experimental/organizations/organization%2Fid/chats/models/available", - () => API.experimental.getChatModelAvailability(organizationId), - { - providers: [], - unsupported_providers: [], - }, - ], [ "/api/experimental/organizations/organization%2Fid/chats/models", () => API.experimental.getChatModels(organizationId), - { models: [], providers: [] }, + { models: [], providers: [], unsupported_providers: [] }, ], ])("returns response data for %s", async (path, request, responseData) => { vi.spyOn(axiosInstance, "get").mockResolvedValueOnce({ @@ -460,10 +452,6 @@ describe("api.ts", () => { }); it.each<[string, () => Promise]>([ - [ - "/api/experimental/organizations/organization%2Fid/chats/models/available", - () => API.experimental.getChatModelAvailability(organizationId), - ], [ "/api/experimental/organizations/organization%2Fid/chats/models", () => API.experimental.getChatModels(organizationId), diff --git a/site/src/api/api.ts b/site/src/api/api.ts index 4bc5b25ada5..fb0a0139ef9 100644 --- a/site/src/api/api.ts +++ b/site/src/api/api.ts @@ -358,8 +358,6 @@ const chatModelPath = (organizationId: string, modelId: string) => `${chatModelsPath(organizationId)}/${encodeURIComponent(modelId)}`; const chatModelACLPath = (organizationId: string, modelId: string) => `${chatModelPath(organizationId, modelId)}/acl`; -const chatModelAvailabilityPath = (organizationId: string) => - `${chatModelsPath(organizationId)}/available`; const userSkillsPath = (user: string) => `/api/experimental/users/${encodeURIComponent(user)}/skills`; const userSkillPath = (user: string, name: string) => @@ -3497,16 +3495,6 @@ class ExperimentalApiMethods { return response.data; }; - getChatModelAvailability = async ( - organizationId: string, - ): Promise => { - const response = - await this.axios.get( - chatModelAvailabilityPath(organizationId), - ); - return response.data; - }; - getAIModelPrices = async (filter: { provider?: string; model?: string; diff --git a/site/src/api/queries/chats.test.ts b/site/src/api/queries/chats.test.ts index ad612c940e3..5264ab284cd 100644 --- a/site/src/api/queries/chats.test.ts +++ b/site/src/api/queries/chats.test.ts @@ -40,7 +40,6 @@ import { chatModel, chatModelACL, chatModelACLKey, - chatModelAvailabilityKey, chatModelKey, chatPromptsKey, chatSearch, @@ -261,7 +260,6 @@ describe("chat model query factories", () => { chatModelACLKey(organizationId, modelId), chatModelKey(organizationId, modelId), organizationChatModelsKey(organizationId), - chatModelAvailabilityKey(organizationId), ["authorization", "models"], ["organizations", [organizationId], "permissions"], ] as const; diff --git a/site/src/api/queries/chats.ts b/site/src/api/queries/chats.ts index 0e9d9ad8917..cfae7eecc02 100644 --- a/site/src/api/queries/chats.ts +++ b/site/src/api/queries/chats.ts @@ -2189,16 +2189,6 @@ export const chatModels = (organizationId: string) => ({ staleTime: CHAT_MODELS_STALE_MS, }); -export const chatModelAvailabilityKey = (organizationId: string) => - [...organizationChatModelsKey(organizationId), "available"] as const; - -export const chatModelAvailability = (organizationId: string) => ({ - queryKey: chatModelAvailabilityKey(organizationId), - queryFn: (): Promise => - API.experimental.getChatModelAvailability(organizationId), - enabled: organizationId !== "", -}); - export const userChatProviderConfigsKey = [ "ai", "provider-keys", @@ -2354,10 +2344,6 @@ export const updateChatModelACL = (queryClient: QueryClient) => ({ queryKey: organizationChatModelsKey(organizationId), exact: true, }), - queryClient.invalidateQueries({ - queryKey: chatModelAvailabilityKey(organizationId), - exact: true, - }), queryClient.invalidateQueries({ queryKey: authorizationKey }), queryClient.invalidateQueries({ predicate: ({ queryKey }) => diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts index 0b426ba5a84..8d903ee2a25 100644 --- a/site/src/api/typesGenerated.ts +++ b/site/src/api/typesGenerated.ts @@ -2885,24 +2885,6 @@ export interface ChatModelAnthropicThinkingOptions { readonly budget_tokens?: number; } -// From codersdk/chats.go -/** - * ChatModelAvailabilityResponse is the catalog returned from chat model discovery. - */ -export interface ChatModelAvailabilityResponse { - /** - * Models contains the effective runtime model configs for the requested - * organization. Each config belongs to that organization. - */ - readonly models?: readonly ChatModel[]; - readonly providers: readonly ChatModelProvider[]; - /** - * UnsupportedProviders lists configured providers the Agents harness - * cannot use, so the UI can explain the empty state. - */ - readonly unsupported_providers: readonly ChatUnsupportedProvider[]; -} - // From codersdk/chats.go /** * ChatModelCallConfig configures per-call model behavior defaults. @@ -2919,19 +2901,6 @@ export interface ChatModelCallConfig { readonly provider_options?: ChatModelProviderOptions; } -// From codersdk/chats.go -/** - * ChatModelCatalogEntry is the runtime catalog view of a model. Its ID is the - * synthetic catalog identity `provider:model` built by canonicalModelID, not - * the chat_model_configs row UUID that ChatModel.ID carries. - */ -export interface ChatModelCatalogEntry { - readonly id: string; - readonly provider: string; - readonly model: string; - readonly display_name: string; -} - // From codersdk/chats.go /** * ChatModelGoogleProviderOptions configures Google provider behavior. @@ -3068,17 +3037,6 @@ export interface ChatModelOverrideResponse { readonly is_malformed: boolean; } -// From codersdk/chats.go -/** - * ChatModelProvider represents provider availability and model results. - */ -export interface ChatModelProvider { - readonly provider: string; - readonly available: boolean; - readonly unavailable_reason?: ChatModelProviderUnavailableReason; - readonly models: readonly ChatModelCatalogEntry[]; -} - // From codersdk/chats.go /** * ChatModelProviderDescriptor is the redacted view of an AI provider carried @@ -3097,6 +3055,8 @@ export interface ChatModelProviderDescriptor { readonly has_user_api_key: boolean; readonly has_effective_api_key: boolean; readonly allow_user_api_key: boolean; + readonly available: boolean; + readonly unavailable_reason?: ChatModelProviderUnavailableReason; } // From codersdk/chats.go @@ -7062,6 +7022,7 @@ export interface Organization extends MinimalOrganization { export interface OrganizationChatModelsResponse { readonly models: readonly ChatModel[]; readonly providers: readonly ChatModelProviderDescriptor[]; + readonly unsupported_providers: readonly ChatUnsupportedProvider[]; } // From codersdk/aibridge.go diff --git a/site/src/modules/aiModels/providerStates.test.ts b/site/src/modules/aiModels/providerStates.test.ts index f561bc226c4..5df3bd06572 100644 --- a/site/src/modules/aiModels/providerStates.test.ts +++ b/site/src/modules/aiModels/providerStates.test.ts @@ -2,7 +2,6 @@ import { describe, expect, it } from "vitest"; import type * as TypesGen from "#/api/typesGenerated"; import { MockChatModel, - MockChatModelProvider, MockChatModelProviderDescriptor, } from "#/testHelpers/chatModels"; import { @@ -17,7 +16,6 @@ const baseProviderState: ProviderState = { label: MockChatModelProviderDescriptor.display_name, providerDescriptor: MockChatModelProviderDescriptor, models: [], - catalogModelCount: 0, hasEffectiveAPIKey: true, allowUserAPIKey: false, }; @@ -39,7 +37,7 @@ describe("deriveProviderStates", () => { }, ]; - const states = deriveProviderStates([], descriptors, undefined); + const states = deriveProviderStates([], descriptors); expect(states.map((state) => state.provider)).toEqual([ "anthropic", @@ -53,49 +51,37 @@ describe("deriveProviderStates", () => { { ...MockChatModel, id: "m2" }, ]; - const states = deriveProviderStates( - modelConfigs, - [MockChatModelProviderDescriptor], - undefined, - ); + const states = deriveProviderStates(modelConfigs, [ + MockChatModelProviderDescriptor, + ]); expect(states[0].models.map((model) => model.id)).toEqual(["m1", "m2"]); }); - it("uses availability only for catalog model counts", () => { - const availability: TypesGen.ChatModelAvailabilityResponse = { - providers: [ - { - ...MockChatModelProvider, - models: [ - { - id: "openai:gpt-x", - provider: "openai", - model: "gpt-x", - display_name: "GPT-X", - }, - ], - }, - ], - unsupported_providers: [], - }; - - const states = deriveProviderStates( - [], - [MockChatModelProviderDescriptor], - availability, - ); - - expect(states[0].catalogModelCount).toBe(1); - }); + it("keeps same-type provider availability independent by UUID", () => { + const descriptors: TypesGen.ChatModelProviderDescriptor[] = [ + { + ...MockChatModelProviderDescriptor, + id: "openai-available", + available: true, + }, + { + ...MockChatModelProviderDescriptor, + id: "openai-unavailable", + display_name: "OpenAI Secondary", + available: false, + unavailable_reason: "missing_api_key", + }, + ]; - it("treats null provider models as an empty catalog", () => { - const states = deriveProviderStates([], [MockChatModelProviderDescriptor], { - providers: [{ ...MockChatModelProvider, models: null }], - unsupported_providers: [], - }); + const states = deriveProviderStates([], descriptors); - expect(states[0].catalogModelCount).toBe(0); + expect( + states.map((state) => [state.key, state.providerDescriptor.available]), + ).toEqual([ + ["openai-available", true], + ["openai-unavailable", false], + ]); }); it.each([ @@ -119,7 +105,7 @@ describe("deriveProviderStates", () => { has_effective_api_key: hasEffectiveAPIKey, }; - const states = deriveProviderStates([], [descriptor], undefined); + const states = deriveProviderStates([], [descriptor]); expect(states[0].hasEffectiveAPIKey).toBe(hasEffectiveAPIKey); }); diff --git a/site/src/modules/aiModels/providerStates.ts b/site/src/modules/aiModels/providerStates.ts index 6bfcd793b22..d43ce3e712f 100644 --- a/site/src/modules/aiModels/providerStates.ts +++ b/site/src/modules/aiModels/providerStates.ts @@ -8,36 +8,14 @@ export type ProviderState = { label: string; providerDescriptor: TypesGen.ChatModelProviderDescriptor; models: readonly TypesGen.ChatModel[]; - catalogModelCount: number; hasEffectiveAPIKey: boolean; allowUserAPIKey: boolean; }; -type GeneratedAvailableProvider = - TypesGen.ChatModelAvailabilityResponse["providers"][number]; -type AvailableProvider = Omit & { - models: GeneratedAvailableProvider["models"] | null; -}; -type ChatModelAvailability = Omit< - TypesGen.ChatModelAvailabilityResponse, - "providers" -> & { - providers: readonly AvailableProvider[]; -}; - export const deriveProviderStates = ( modelConfigs: readonly TypesGen.ChatModel[], providerDescriptors: readonly TypesGen.ChatModelProviderDescriptor[], - availability?: ChatModelAvailability | null, ): readonly ProviderState[] => { - const availableProvidersByType = new Map(); - for (const availableProvider of availability?.providers ?? []) { - availableProvidersByType.set( - normalizeProvider(availableProvider.provider), - availableProvider, - ); - } - const modelConfigsByProviderID = new Map(); for (const modelConfig of modelConfigs) { const existing = modelConfigsByProviderID.get(modelConfig.ai_provider_id); @@ -51,7 +29,6 @@ export const deriveProviderStates = ( return providerDescriptors .map((providerDescriptor) => { const provider = normalizeProvider(providerDescriptor.type); - const availableProvider = availableProvidersByType.get(provider); return { key: providerDescriptor.id, provider, @@ -60,7 +37,6 @@ export const deriveProviderStates = ( formatProviderLabel(provider), providerDescriptor, models: modelConfigsByProviderID.get(providerDescriptor.id) ?? [], - catalogModelCount: availableProvider?.models?.length ?? 0, hasEffectiveAPIKey: providerDescriptor.has_effective_api_key, allowUserAPIKey: providerDescriptor.allow_user_api_key, }; diff --git a/site/src/pages/AISettingsPage/ModelsPage/AddModelPage/AddModelPageView.stories.tsx b/site/src/pages/AISettingsPage/ModelsPage/AddModelPage/AddModelPageView.stories.tsx index 5704054f15c..b0036a4d651 100644 --- a/site/src/pages/AISettingsPage/ModelsPage/AddModelPage/AddModelPageView.stories.tsx +++ b/site/src/pages/AISettingsPage/ModelsPage/AddModelPage/AddModelPageView.stories.tsx @@ -1,10 +1,7 @@ import type { Meta, StoryObj } from "@storybook/react-vite"; import { expect, fn, userEvent, within } from "storybook/test"; import { deriveProviderStates } from "#/modules/aiModels/providerStates"; -import { - MockChatModelProvider, - MockChatModelProviderDescriptor, -} from "#/testHelpers/chatModels"; +import { MockChatModelProviderDescriptor } from "#/testHelpers/chatModels"; import { MockDefaultOrganization, MockOrganizationPermissions, @@ -94,21 +91,11 @@ export const NoProviderConfigurationFields: Story = { }, }; -export const NullAvailabilityModelsUsesEmptyCatalog: Story = { +export const ProviderWithoutConfiguredModels: Story = { args: { - providerStates: deriveProviderStates( - [], - [MockChatModelProviderDescriptor], - { - providers: [{ ...MockChatModelProvider, models: null }], - unsupported_providers: [], - }, - ), + providerStates: deriveProviderStates([], [MockChatModelProviderDescriptor]), selectedProviderState: - deriveProviderStates([], [MockChatModelProviderDescriptor], { - providers: [{ ...MockChatModelProvider, models: null }], - unsupported_providers: [], - })[0] ?? null, + deriveProviderStates([], [MockChatModelProviderDescriptor])[0] ?? null, }, play: async ({ canvasElement }) => { const canvas = within(canvasElement); diff --git a/site/src/pages/AISettingsPage/ModelsPage/ModelsPage.tsx b/site/src/pages/AISettingsPage/ModelsPage/ModelsPage.tsx index 1cf5239f543..7a599ff8c39 100644 --- a/site/src/pages/AISettingsPage/ModelsPage/ModelsPage.tsx +++ b/site/src/pages/AISettingsPage/ModelsPage/ModelsPage.tsx @@ -1,6 +1,6 @@ import type { FC } from "react"; import { useQuery } from "react-query"; -import { chatModelAvailability, chatModels } from "#/api/queries/chats"; +import { chatModels } from "#/api/queries/chats"; import { deriveProviderStates } from "#/modules/aiModels/providerStates"; import { pageTitle } from "#/utils/page"; import ModelsPageView from "./ModelsPageView"; @@ -12,7 +12,6 @@ import { const ModelsPage: FC = () => { const { organization, permissions } = useOrganizationModels(); const organizationModelsQuery = useQuery(chatModels(organization.id)); - const availableModelsQuery = useQuery(chatModelAvailability(organization.id)); const providers = organizationModelsQuery.data?.providers ?? []; const providerTypeByID = new Map( providers.map((provider) => [provider.id, provider.type]), @@ -25,14 +24,9 @@ const ModelsPage: FC = () => { return cmp !== 0 ? cmp : a.model.localeCompare(b.model); }, ); - const providerStates = deriveProviderStates( - models, - providers, - availableModelsQuery.data, - ); + const providerStates = deriveProviderStates(models, providers); const { loadError, refetchError } = splitModelQueryErrors( organizationModelsQuery, - availableModelsQuery, ); return ( @@ -41,9 +35,7 @@ const ModelsPage: FC = () => { = { queries: [ { key: chatModels(MockDefaultOrganization.id).queryKey, - data: { models: [], providers: [] }, + data: { models: [], providers: [], unsupported_providers: [] }, }, { key: chatModels(MockOrganization2.id).queryKey, - data: { models: [], providers: [] }, + data: { models: [], providers: [], unsupported_providers: [] }, }, { key: organizationsPermissions([MockDefaultOrganization.id]).queryKey, @@ -152,15 +152,11 @@ export const InvalidRequestedOrganizationDeniesAdd: Story = { queries: [ { key: chatModels(MockDefaultOrganization.id).queryKey, - data: { models: [], providers: [] }, + data: { models: [], providers: [], unsupported_providers: [] }, }, { key: chatModels(MockOrganization2.id).queryKey, - data: { models: [], providers: [] }, - }, - { - key: chatModelAvailability(MockDefaultOrganization.id).queryKey, - data: { providers: [], unsupported_providers: [] }, + data: { models: [], providers: [], unsupported_providers: [] }, }, { key: organizationsPermissions([MockDefaultOrganization.id]).queryKey, @@ -192,7 +188,7 @@ export const SingleAccessibleOrganizationHidesPicker: Story = { queries: [ { key: chatModels(MockDefaultOrganization.id).queryKey, - data: { models: [], providers: [] }, + data: { models: [], providers: [], unsupported_providers: [] }, }, { key: organizationsPermissions([MockDefaultOrganization.id]).queryKey, @@ -221,11 +217,11 @@ export const DuplicateDisplayNamesAreDisambiguated: Story = { queries: [ { key: chatModels(MockDefaultOrganization.id).queryKey, - data: { models: [], providers: [] }, + data: { models: [], providers: [], unsupported_providers: [] }, }, { key: chatModels(duplicateNameOrganization.id).queryKey, - data: { models: [], providers: [] }, + data: { models: [], providers: [], unsupported_providers: [] }, }, { key: organizationsPermissions([MockDefaultOrganization.id]).queryKey, @@ -260,11 +256,11 @@ export const PermissionLoadingShowsLoader: Story = { queries: [ { key: chatModels(MockDefaultOrganization.id).queryKey, - data: { models: [], providers: [] }, + data: { models: [], providers: [], unsupported_providers: [] }, }, { key: chatModels(MockOrganization2.id).queryKey, - data: { models: [], providers: [] }, + data: { models: [], providers: [], unsupported_providers: [] }, }, ], }, @@ -287,11 +283,11 @@ export const PermissionLoadErrorShowsAlert: Story = { queries: [ { key: chatModels(MockDefaultOrganization.id).queryKey, - data: { models: [], providers: [] }, + data: { models: [], providers: [], unsupported_providers: [] }, }, { key: chatModels(MockOrganization2.id).queryKey, - data: { models: [], providers: [] }, + data: { models: [], providers: [], unsupported_providers: [] }, }, ], }, diff --git a/site/src/pages/AISettingsPage/ModelsPage/UpdateModelPage/UpdateModelPageView.stories.tsx b/site/src/pages/AISettingsPage/ModelsPage/UpdateModelPage/UpdateModelPageView.stories.tsx index 2646de037f0..81338947902 100644 --- a/site/src/pages/AISettingsPage/ModelsPage/UpdateModelPage/UpdateModelPageView.stories.tsx +++ b/site/src/pages/AISettingsPage/ModelsPage/UpdateModelPage/UpdateModelPageView.stories.tsx @@ -2,11 +2,7 @@ import type { Meta, StoryObj } from "@storybook/react-vite"; import { useLocation } from "react-router"; import { expect, fn, screen, userEvent, waitFor, within } from "storybook/test"; import { reactRouterParameters } from "storybook-addon-remix-react-router"; -import { - chatModel, - chatModelAvailability, - chatModels, -} from "#/api/queries/chats"; +import { chatModel, chatModels } from "#/api/queries/chats"; import { MockDefaultOrganization, MockOrganizationPermissions, @@ -223,12 +219,9 @@ export const DuplicateNavigatesToStructuralAddPath: Story = { data: { models: [mockGPT5], providers: [MockOpenAIProviderState.providerDescriptor], + unsupported_providers: [], }, }, - { - key: chatModelAvailability(MockDefaultOrganization.id).queryKey, - data: { providers: [], unsupported_providers: [] }, - }, ], }, play: async ({ canvasElement }) => { diff --git a/site/src/pages/AISettingsPage/ModelsPage/organizationModels.test.tsx b/site/src/pages/AISettingsPage/ModelsPage/organizationModels.test.tsx index 7076d58351b..1566494d662 100644 --- a/site/src/pages/AISettingsPage/ModelsPage/organizationModels.test.tsx +++ b/site/src/pages/AISettingsPage/ModelsPage/organizationModels.test.tsx @@ -87,6 +87,7 @@ describe("useAccessibleModelOrganizations", () => { .mockImplementation(async (_organizationId) => ({ models: [], providers: [], + unsupported_providers: [], })); const { queryClient, wrapper } = createQueryWrapper(); const { result, unmount } = renderHook( @@ -173,7 +174,7 @@ describe("useAccessibleModelOrganizations", () => { if (organizationId === MockOrganization2.id) { throw new Error("Failed to load organization"); } - return { models: [], providers: [] }; + return { models: [], providers: [], unsupported_providers: [] }; }, ); const { queryClient, wrapper } = createQueryWrapper(); diff --git a/site/src/pages/AISettingsPage/ModelsPage/testFixtures.ts b/site/src/pages/AISettingsPage/ModelsPage/testFixtures.ts index b45bb49641a..35c51263fbd 100644 --- a/site/src/pages/AISettingsPage/ModelsPage/testFixtures.ts +++ b/site/src/pages/AISettingsPage/ModelsPage/testFixtures.ts @@ -58,7 +58,6 @@ export const MockOpenAIProviderState: ProviderState = { label: "OpenAI", providerDescriptor: MockOpenAIProviderDescriptor, models: [mockGPT5, mockDisabledModel], - catalogModelCount: 0, hasEffectiveAPIKey: true, allowUserAPIKey: false, }; diff --git a/site/src/pages/AgentsPage/AgentChatPage.stories.tsx b/site/src/pages/AgentsPage/AgentChatPage.stories.tsx index fbf47c5cb13..dcdc77ced22 100644 --- a/site/src/pages/AgentsPage/AgentChatPage.stories.tsx +++ b/site/src/pages/AgentsPage/AgentChatPage.stories.tsx @@ -14,9 +14,9 @@ import { chatEntityKey, chatListKey, chatMessagesKey, - chatModelAvailabilityKey, chatPromptsKey, mcpServerConfigsKey, + organizationChatModelsKey, toChatListParams, userChatProviderConfigsKey, } from "#/api/queries/chats"; @@ -28,7 +28,10 @@ import { MockChatQueuedMessage, MockMCPServerConfig, } from "#/testHelpers/chatEntities"; -import { MockChatModel } from "#/testHelpers/chatModels"; +import { + MockChatModel, + MockChatModelProviderDescriptor, +} from "#/testHelpers/chatModels"; import { MockGroup, MockOrganizationMember, @@ -128,7 +131,7 @@ const mockWorkspace: TypesGen.Workspace = { }, }; -const mockModelCatalog: TypesGen.ChatModelAvailabilityResponse = { +const mockModelCatalog: TypesGen.OrganizationChatModelsResponse = { models: [ { ...MockChatModel, @@ -145,24 +148,11 @@ const mockModelCatalog: TypesGen.ChatModelAvailabilityResponse = { updated_at: "2026-02-18T00:00:00.000Z", }, ], - providers: [ - { - provider: "openai", - available: true, - models: [ - { - id: "openai:gpt-4o", - provider: "openai", - model: "gpt-4o", - display_name: "GPT-4o", - }, - ], - }, - ], + providers: [MockChatModelProviderDescriptor], unsupported_providers: [], }; -const foreignOnlyModelCatalog: TypesGen.ChatModelAvailabilityResponse = { +const foreignOnlyModelCatalog: TypesGen.OrganizationChatModelsResponse = { ...mockModelCatalog, models: [ { @@ -176,14 +166,15 @@ const foreignOnlyModelCatalog: TypesGen.ChatModelAvailabilityResponse = { ], }; -const userApiKeyRequiredModelCatalog: TypesGen.ChatModelAvailabilityResponse = { - ...mockModelCatalog, - providers: mockModelCatalog.providers.map((provider) => ({ - ...provider, - available: false, - unavailable_reason: "user_api_key_required", - })), -}; +const userApiKeyRequiredModelCatalog: TypesGen.OrganizationChatModelsResponse = + { + ...mockModelCatalog, + providers: mockModelCatalog.providers.map((provider) => ({ + ...provider, + available: false, + unavailable_reason: "user_api_key_required", + })), + }; const baseChatFields = { organization_id: "test-org-id", @@ -206,7 +197,7 @@ const baseChatFields = { children: [], } as const; -const recoveryModelCatalog: TypesGen.ChatModelAvailabilityResponse = { +const recoveryModelCatalog: TypesGen.OrganizationChatModelsResponse = { ...mockModelCatalog, models: [ { @@ -358,7 +349,7 @@ const buildQueries = ( data: mockWorkspace, }, { - key: chatModelAvailabilityKey(chat.organization_id), + key: organizationChatModelsKey(chat.organization_id), data: mockModelCatalog, }, { @@ -1403,10 +1394,10 @@ export const StaleEditedModelUsesUsableLocalModel: Story = { has_more: false, }, ), - chatModelAvailabilityKey(baseChatFields.organization_id), + organizationChatModelsKey(baseChatFields.organization_id), ), { - key: chatModelAvailabilityKey(baseChatFields.organization_id), + key: organizationChatModelsKey(baseChatFields.organization_id), data: recoveryModelCatalog, }, ], @@ -1526,10 +1517,10 @@ export const NoLocalModelDisablesGeneration: Story = { }, { messages: [], queued_messages: [], has_more: false }, ), - chatModelAvailabilityKey(baseChatFields.organization_id), + organizationChatModelsKey(baseChatFields.organization_id), ), { - key: chatModelAvailabilityKey(baseChatFields.organization_id), + key: organizationChatModelsKey(baseChatFields.organization_id), data: foreignOnlyModelCatalog, }, ], @@ -3923,11 +3914,11 @@ export const ModelEndpointFailureKeepsHistoryReadable: Story = { has_more: false, }, ), - chatModelAvailabilityKey(baseChatFields.organization_id), + organizationChatModelsKey(baseChatFields.organization_id), ), }, beforeEach: () => { - spyOn(API.experimental, "getChatModelAvailability").mockRejectedValue( + spyOn(API.experimental, "getChatModels").mockRejectedValueOnce( mockServerError, ); }, @@ -3959,10 +3950,10 @@ export const ProviderRequiresUserApiKey: Story = { has_more: false, }, ), - chatModelAvailabilityKey(baseChatFields.organization_id), + organizationChatModelsKey(baseChatFields.organization_id), ), { - key: chatModelAvailabilityKey(baseChatFields.organization_id), + key: organizationChatModelsKey(baseChatFields.organization_id), data: userApiKeyRequiredModelCatalog, }, ], diff --git a/site/src/pages/AgentsPage/AgentChatPage.tsx b/site/src/pages/AgentsPage/AgentChatPage.tsx index e80a774cc50..0adeda7a70a 100644 --- a/site/src/pages/AgentsPage/AgentChatPage.tsx +++ b/site/src/pages/AgentsPage/AgentChatPage.tsx @@ -33,7 +33,7 @@ import { buildOptimisticEditedMessage } from "#/api/queries/chatMessageEdits"; import { chat, chatMessagesForInfiniteScroll, - chatModelAvailability, + chatModels, chatQueueConvergence, compactChat, createChatMessage, @@ -50,7 +50,6 @@ import { updateChatWorkspace, updateInfiniteChatsCache, userChatDebugLogging, - userChatProviderConfigs, userCompactionThresholds, } from "#/api/queries/chats"; import { deploymentSSHConfig } from "#/api/queries/deployment"; @@ -943,15 +942,12 @@ const AgentChatPage: FC = () => { }); const workspace = workspaceQuery.data; - const availableModelsQuery = useQuery( - chatModelAvailability(chatOrganizationId), - ); - const models = availableModelsQuery.data?.models ?? []; + const modelsQuery = useQuery(chatModels(chatOrganizationId)); + const models = modelsQuery.data?.models ?? []; const chatProviderConfigsQuery = useQuery({ ...chatProviderConfigs(), enabled: permissions.editDeploymentConfig, }); - const userProviderConfigsQuery = useQuery(userChatProviderConfigs()); const userThresholdsQuery = useQuery(userCompactionThresholds()); const preferencesQuery = useQuery(preferenceSettings()); const userDebugLoggingQuery = useQuery(userChatDebugLogging()); @@ -995,26 +991,20 @@ const AgentChatPage: FC = () => { isModelCatalogLoading, modelCatalog, hasConfiguredModels, - } = resolveModelSelector( - chatOrganizationId, - availableModelsQuery, - userProviderConfigsQuery, - ); + } = resolveModelSelector(chatOrganizationId, modelsQuery); const isModelDataPending = chatOrganizationId === "" || isModelCatalogLoading; const providerCount = permissions.editDeploymentConfig && chatProviderConfigsQuery.data && - availableModelsQuery.data + modelsQuery.data ? countConfiguredProviderConfigs( chatProviderConfigsQuery.data, - availableModelsQuery.data, + modelsQuery.data, ) : undefined; - const modelCount = availableModelsQuery.data - ? modelOptions.length - : undefined; + const modelCount = modelsQuery.data ? modelOptions.length : undefined; const unsupportedProviderNames = getUnsupportedProviderNames( - availableModelsQuery.data, + modelsQuery.data, ); const agentBindingRefetchKeyRef = useRef(undefined); @@ -1369,10 +1359,8 @@ const AgentChatPage: FC = () => { const hasModelOptions = modelOptions.length > 0; const hasResolvedModelData = !isModelDataPending && - availableModelsQuery.data !== undefined && - availableModelsQuery.error == null && - userProviderConfigsQuery.data !== undefined && - userProviderConfigsQuery.error == null; + modelsQuery.data !== undefined && + modelsQuery.error == null; const hasUnavailableHistoricalModel = hasResolvedModelData && isUnavailableHistoricalModelID(chatLastModelConfigID, modelOptions); @@ -2042,9 +2030,7 @@ const AgentChatPage: FC = () => { modelOptions={modelOptions} modelSelectorPlaceholder={modelSelectorPlaceholder} modelSelectorHelp={modelSelectorHelp} - modelCatalogError={ - availableModelsQuery.error ?? userProviderConfigsQuery.error - } + modelCatalogError={modelsQuery.error} unavailableModelNotice={unavailableModelNotice} reasoningEffort={effectiveReasoningEffort} onReasoningEffortChange={(value) => { diff --git a/site/src/pages/AgentsPage/AgentSettingsUserAgentsPage.tsx b/site/src/pages/AgentsPage/AgentSettingsUserAgentsPage.tsx index 4223ff652d6..e5eacbd084f 100644 --- a/site/src/pages/AgentsPage/AgentSettingsUserAgentsPage.tsx +++ b/site/src/pages/AgentsPage/AgentSettingsUserAgentsPage.tsx @@ -1,10 +1,9 @@ import type { FC } from "react"; import { useMutation, useQuery, useQueryClient } from "react-query"; import { - chatModelAvailability, + chatModels, updateUserChatPersonalModelOverride, userChatPersonalModelOverrides, - userChatProviderConfigs, } from "#/api/queries/chats"; import type * as TypesGen from "#/api/typesGenerated"; import { @@ -19,10 +18,7 @@ const AgentSettingsUserAgentsPage: FC = () => { const { organizations } = useDashboard(); const overridesQuery = useQuery(userChatPersonalModelOverrides()); const defaultOrganizationId = getDefaultOrganizationId(organizations); - const availableModelsQuery = useQuery( - chatModelAvailability(defaultOrganizationId), - ); - const providerConfigsQuery = useQuery(userChatProviderConfigs()); + const modelsQuery = useQuery(chatModels(defaultOrganizationId)); const saveRootModelOverrideMutation = useMutation( updateUserChatPersonalModelOverride(queryClient), ); @@ -33,13 +29,12 @@ const AgentSettingsUserAgentsPage: FC = () => { updateUserChatPersonalModelOverride(queryClient), ); - const defaultOrgModelConfigs = availableModelsQuery.data?.models ?? []; + const defaultOrgModelConfigs = modelsQuery.data?.models ?? []; const hasDefaultOrgModels = defaultOrgModelConfigs.length > 0; const { options: modelOptions, isModelCatalogLoading } = resolveModelSelector( defaultOrganizationId, - availableModelsQuery, - providerConfigsQuery, + modelsQuery, ); const saveModelOverride = ( @@ -65,14 +60,14 @@ const AgentSettingsUserAgentsPage: FC = () => { isLoadingOverrides={overridesQuery.isLoading} modelOptions={modelOptions} models={defaultOrgModelConfigs} - modelsError={availableModelsQuery.error} + modelsError={modelsQuery.error} isLoadingModels={isModelCatalogLoading} isDefaultOrganizationUnresolved={defaultOrganizationId === ""} hasNoDefaultOrgModels={ defaultOrganizationId !== "" && - !availableModelsQuery.isLoading && - availableModelsQuery.error === null && - availableModelsQuery.data !== undefined && + !modelsQuery.isLoading && + modelsQuery.error === null && + modelsQuery.data !== undefined && !hasDefaultOrgModels } onSaveRootModelOverride={saveModelOverride( diff --git a/site/src/pages/AgentsPage/AgentsPageLayout.stories.tsx b/site/src/pages/AgentsPage/AgentsPageLayout.stories.tsx index 544a7f03b91..71536340b18 100644 --- a/site/src/pages/AgentsPage/AgentsPageLayout.stories.tsx +++ b/site/src/pages/AgentsPage/AgentsPageLayout.stories.tsx @@ -348,24 +348,6 @@ const meta: Meta = { custom_prompt: "", }); // Mocks for child route pages that fetch their own data. - spyOn(API.experimental, "getChatModelAvailability").mockResolvedValue({ - models: defaultModels, - providers: [ - { - provider: "openai", - available: true, - models: [ - { - id: "openai:gpt-4o", - provider: "openai", - model: "gpt-4o", - display_name: "GPT-4o", - }, - ], - }, - ], - unsupported_providers: [], - }); spyOn(API.experimental, "getChatModels").mockImplementation( async (organizationId) => ({ models: [ @@ -378,7 +360,21 @@ const meta: Meta = { organization_id: organizationId, }, ], - providers: [], + providers: [ + { + id: defaultModels[0].ai_provider_id, + type: "openai", + display_name: "OpenAI", + icon: "", + enabled: true, + has_api_key: true, + has_user_api_key: false, + has_effective_api_key: true, + allow_user_api_key: false, + available: true, + }, + ], + unsupported_providers: [], }), ); spyOn(API.experimental, "getUserAIProviderKeyConfigs").mockResolvedValue([ diff --git a/site/src/pages/AgentsPage/components/AgentCreateForm.stories.tsx b/site/src/pages/AgentsPage/components/AgentCreateForm.stories.tsx index f9c19e392e8..54f05973003 100644 --- a/site/src/pages/AgentsPage/components/AgentCreateForm.stories.tsx +++ b/site/src/pages/AgentsPage/components/AgentCreateForm.stories.tsx @@ -14,13 +14,16 @@ import { import { API } from "#/api/api"; import { aiProvidersListKey } from "#/api/queries/aiProviders"; import { - chatModelAvailabilityKey, + organizationChatModelsKey, userChatProviderConfigsKey, } from "#/api/queries/chats"; import { permittedOrganizationsKey } from "#/api/queries/organizations"; import type * as TypesGen from "#/api/typesGenerated"; import { ConfirmDialog } from "#/components/Dialog/ConfirmDialog/ConfirmDialog"; -import { MockChatModel } from "#/testHelpers/chatModels"; +import { + MockChatModel, + MockChatModelProviderDescriptor, +} from "#/testHelpers/chatModels"; import { MockDefaultOrganization, MockOrganization2, @@ -77,32 +80,37 @@ const organization2ModelConfig = buildModelConfig({ is_default: true, }); -const defaultModelCatalog: TypesGen.ChatModelAvailabilityResponse = { +const defaultModelCatalog: TypesGen.OrganizationChatModelsResponse = { models: defaultModelConfigs, providers: [ - { provider: "openai", available: true, models: [] }, - { provider: "anthropic", available: true, models: [] }, + MockChatModelProviderDescriptor, + { + ...MockChatModelProviderDescriptor, + id: "provider-anthropic", + type: "anthropic", + display_name: "Anthropic", + }, ], unsupported_providers: [], }; -const organization2RuntimeCatalog: TypesGen.ChatModelAvailabilityResponse = { +const organization2RuntimeCatalog: TypesGen.OrganizationChatModelsResponse = { ...defaultModelCatalog, models: [organization2ModelConfig, ...defaultModelConfigs], }; -const organization2LocalCatalog: TypesGen.ChatModelAvailabilityResponse = { +const organization2LocalCatalog: TypesGen.OrganizationChatModelsResponse = { ...defaultModelCatalog, models: [organization2ModelConfig], }; -const organization2ForeignOnlyCatalog: TypesGen.ChatModelAvailabilityResponse = +const organization2ForeignOnlyCatalog: TypesGen.OrganizationChatModelsResponse = { ...defaultModelCatalog, models: defaultModelConfigs, }; -const userApiKeyRequiredCatalog: TypesGen.ChatModelAvailabilityResponse = { +const userApiKeyRequiredCatalog: TypesGen.OrganizationChatModelsResponse = { ...defaultModelCatalog, providers: defaultModelCatalog.providers.map((provider) => ({ ...provider, @@ -111,6 +119,32 @@ const userApiKeyRequiredCatalog: TypesGen.ChatModelAvailabilityResponse = { })), }; +const missingAPIKeyCatalog: TypesGen.OrganizationChatModelsResponse = { + ...defaultModelCatalog, + providers: defaultModelCatalog.providers.map((provider) => ({ + ...provider, + available: false, + unavailable_reason: "missing_api_key", + })), +}; + +const fetchFailedCatalog: TypesGen.OrganizationChatModelsResponse = { + ...defaultModelCatalog, + providers: defaultModelCatalog.providers.map((provider) => ({ + ...provider, + available: false, + unavailable_reason: "fetch_failed", + })), +}; + +const unsupportedProviderCatalog: TypesGen.OrganizationChatModelsResponse = { + models: [], + providers: [], + unsupported_providers: [ + { provider: "copilot", display_name: "GitHub Copilot" }, + ], +}; + const defaultUserProviderConfigs: TypesGen.UserChatProviderConfig[] = [ { provider_id: "provider-1", @@ -182,7 +216,7 @@ const meta: Meta = { parameters: { queries: [ { - key: chatModelAvailabilityKey(MockDefaultOrganization.id), + key: organizationChatModelsKey(MockDefaultOrganization.id), data: defaultModelCatalog, }, { @@ -411,12 +445,12 @@ const effortModelConfigs: TypesGen.ChatModel[] = [ }), ]; -const effortModelCatalog: TypesGen.ChatModelAvailabilityResponse = { +const effortModelCatalog: TypesGen.OrganizationChatModelsResponse = { ...defaultModelCatalog, models: effortModelConfigs, }; -const limitedEffortModelCatalog: TypesGen.ChatModelAvailabilityResponse = { +const limitedEffortModelCatalog: TypesGen.OrganizationChatModelsResponse = { ...defaultModelCatalog, models: [ buildModelConfig({ @@ -434,7 +468,7 @@ export const RemembersReasoningEffortByModel: Story = { parameters: { queries: [ { - key: chatModelAvailabilityKey(MockDefaultOrganization.id), + key: organizationChatModelsKey(MockDefaultOrganization.id), data: effortModelCatalog, }, { @@ -496,7 +530,7 @@ export const PersistedReasoningEffortOutranksRootOverride: Story = { parameters: { queries: [ { - key: chatModelAvailabilityKey(MockDefaultOrganization.id), + key: organizationChatModelsKey(MockDefaultOrganization.id), data: effortModelCatalog, }, { @@ -541,7 +575,7 @@ export const ManualReselectKeepsRootOverrideEffort: Story = { parameters: { queries: [ { - key: chatModelAvailabilityKey(MockDefaultOrganization.id), + key: organizationChatModelsKey(MockDefaultOrganization.id), data: effortModelCatalog, }, { @@ -579,7 +613,7 @@ export const StalePersistedEffortFallsThroughToRootOverride: Story = { parameters: { queries: [ { - key: chatModelAvailabilityKey(MockDefaultOrganization.id), + key: organizationChatModelsKey(MockDefaultOrganization.id), data: limitedEffortModelCatalog, }, { @@ -619,7 +653,7 @@ export const SubmitsReasoningEffort: Story = { pixel: { exclude: true }, queries: [ { - key: chatModelAvailabilityKey(MockDefaultOrganization.id), + key: organizationChatModelsKey(MockDefaultOrganization.id), data: effortModelCatalog, }, { @@ -807,7 +841,7 @@ export const CachedModelsWithRefetchError: Story = { const queryClient = useQueryClient(); useEffect(() => { void queryClient.invalidateQueries({ - queryKey: chatModelAvailabilityKey(MockDefaultOrganization.id), + queryKey: organizationChatModelsKey(MockDefaultOrganization.id), exact: true, }); }, [queryClient]); @@ -815,7 +849,7 @@ export const CachedModelsWithRefetchError: Story = { }, ], beforeEach: () => { - spyOn(API.experimental, "getChatModelAvailability").mockRejectedValue( + spyOn(API.experimental, "getChatModels").mockRejectedValueOnce( new Error("Failed to refresh available models."), ); }, @@ -842,7 +876,8 @@ export const LoadingPersonalModelOverrides: Story = { }, }; -const emptyModelCatalog: TypesGen.ChatModelAvailabilityResponse = { +const emptyModelCatalog: TypesGen.OrganizationChatModelsResponse = { + models: [], providers: [], unsupported_providers: [], }; @@ -851,7 +886,7 @@ export const NoModelsConfigured: Story = { parameters: { queries: [ { - key: chatModelAvailabilityKey(MockDefaultOrganization.id), + key: organizationChatModelsKey(MockDefaultOrganization.id), data: emptyModelCatalog, }, { @@ -869,7 +904,7 @@ export const ProviderRequiresUserApiKey: Story = { parameters: { queries: [ { - key: chatModelAvailabilityKey(MockDefaultOrganization.id), + key: organizationChatModelsKey(MockDefaultOrganization.id), data: userApiKeyRequiredCatalog, }, { @@ -892,11 +927,69 @@ export const ProviderRequiresUserApiKey: Story = { }, }; +export const ProviderMissingAPIKey: Story = { + parameters: { + queries: [ + { + key: organizationChatModelsKey(MockDefaultOrganization.id), + data: missingAPIKeyCatalog, + }, + ], + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + expect( + canvas.getByText( + "No chat model is currently available for this organization.", + ), + ).toBeVisible(); + }, +}; + +export const ProviderFetchFailed: Story = { + parameters: { + queries: [ + { + key: organizationChatModelsKey(MockDefaultOrganization.id), + data: fetchFailedCatalog, + }, + ], + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + expect( + canvas.getByText( + "No chat model is currently available for this organization.", + ), + ).toBeVisible(); + }, +}; + +export const UnsupportedProviderOnly: Story = { + args: { ...defaultArgs, canConfigureAgentSetup: true }, + parameters: { + queries: [ + { + key: organizationChatModelsKey(MockDefaultOrganization.id), + data: unsupportedProviderCatalog, + }, + { key: aiProvidersListKey, data: [] }, + ], + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + expect(canvas.getByText(/GitHub Copilot is configured but/i)).toBeVisible(); + expect( + canvas.getByRole("link", { name: "not supported by Coder Agents" }), + ).toBeVisible(); + }, +}; + export const MissingProviderAndModelSetup: Story = { parameters: { queries: [ { - key: chatModelAvailabilityKey(MockDefaultOrganization.id), + key: organizationChatModelsKey(MockDefaultOrganization.id), data: emptyModelCatalog, }, { @@ -1113,11 +1206,11 @@ export const WithOrganizationPicker: Story = { data: [MockOrganization2, MockDefaultOrganization], }, { - key: chatModelAvailabilityKey(MockDefaultOrganization.id), + key: organizationChatModelsKey(MockDefaultOrganization.id), data: defaultModelCatalog, }, { - key: chatModelAvailabilityKey(MockOrganization2.id), + key: organizationChatModelsKey(MockOrganization2.id), data: organization2RuntimeCatalog, }, { @@ -1186,11 +1279,11 @@ export const DelayedAuthorizationPreservesForeignPersistedModel: Story = { organizations: [MockDefaultOrganization, MockOrganization2], queries: [ { - key: chatModelAvailabilityKey(MockDefaultOrganization.id), + key: organizationChatModelsKey(MockDefaultOrganization.id), data: defaultModelCatalog, }, { - key: chatModelAvailabilityKey(MockOrganization2.id), + key: organizationChatModelsKey(MockOrganization2.id), data: organization2LocalCatalog, }, { @@ -1227,7 +1320,7 @@ export const RestrictedMultiOrganizationUser: Story = { organizations: [MockDefaultOrganization, MockOrganization2], queries: [ { - key: chatModelAvailabilityKey(MockOrganization2.id), + key: organizationChatModelsKey(MockOrganization2.id), data: organization2LocalCatalog, }, { @@ -1274,7 +1367,7 @@ export const RestrictedUserKeepsPersistedWorkspace: Story = { organizations: [MockDefaultOrganization, MockOrganization2], queries: [ { - key: chatModelAvailabilityKey(MockOrganization2.id), + key: organizationChatModelsKey(MockOrganization2.id), data: organization2LocalCatalog, }, { @@ -1646,7 +1739,7 @@ export const SingleOrgIgnoresStalePermittedCache: Story = { data: [MockOrganization2], }, { - key: chatModelAvailabilityKey(MockDefaultOrganization.id), + key: organizationChatModelsKey(MockDefaultOrganization.id), data: defaultModelCatalog, }, { @@ -1730,7 +1823,7 @@ export const LocalOrganizationModels: Story = { data: [MockOrganization2], }, { - key: chatModelAvailabilityKey(MockOrganization2.id), + key: organizationChatModelsKey(MockOrganization2.id), data: organization2LocalCatalog, }, { @@ -1762,7 +1855,7 @@ export const ForeignOnlyModelsDisableGeneration: Story = { data: [MockOrganization2], }, { - key: chatModelAvailabilityKey(MockOrganization2.id), + key: organizationChatModelsKey(MockOrganization2.id), data: organization2ForeignOnlyCatalog, }, { @@ -1929,7 +2022,7 @@ export const PermittedOrgsResolvesToSubset: Story = { organizations: [MockDefaultOrganization, MockOrganization2], queries: [ { - key: chatModelAvailabilityKey(MockDefaultOrganization.id), + key: organizationChatModelsKey(MockDefaultOrganization.id), data: defaultModelCatalog, }, { @@ -1937,7 +2030,7 @@ export const PermittedOrgsResolvesToSubset: Story = { data: defaultUserProviderConfigs, }, { - key: chatModelAvailabilityKey(MockOrganization2.id), + key: organizationChatModelsKey(MockOrganization2.id), data: organization2LocalCatalog, }, ], diff --git a/site/src/pages/AgentsPage/components/AgentCreateForm.tsx b/site/src/pages/AgentsPage/components/AgentCreateForm.tsx index e2a68059279..70daf1e2fb3 100644 --- a/site/src/pages/AgentsPage/components/AgentCreateForm.tsx +++ b/site/src/pages/AgentsPage/components/AgentCreateForm.tsx @@ -3,11 +3,7 @@ import { useQuery } from "react-query"; import { toast } from "sonner"; import { isApiError } from "#/api/errors"; import { chatProviderConfigs } from "#/api/queries/aiProviders"; -import { - chatModelAvailability, - mcpServerConfigs, - userChatProviderConfigs, -} from "#/api/queries/chats"; +import { chatModels, mcpServerConfigs } from "#/api/queries/chats"; import { permittedOrganizations } from "#/api/queries/organizations"; import type * as TypesGen from "#/api/typesGenerated"; import type { AgentChatSendShortcut } from "#/api/typesGenerated"; @@ -275,23 +271,18 @@ export const AgentCreateForm: FC = ({ localStorage.removeItem(selectedWorkspaceIdStorageKey); } }, [selectedWorkspaceId]); - const availableModelsQuery = useQuery(chatModelAvailability(organizationId)); - const availableModelConfigs = availableModelsQuery.data?.models ?? []; + const modelsQuery = useQuery(chatModels(organizationId)); + const availableModelConfigs = modelsQuery.data?.models ?? []; const chatProviderConfigsQuery = useQuery({ ...chatProviderConfigs(), enabled: canConfigureAgentSetup, }); - const userProviderConfigsQuery = useQuery(userChatProviderConfigs()); const { options: modelOptions, isModelCatalogLoading, modelCatalog, hasConfiguredModels, - } = resolveModelSelector( - organizationId, - availableModelsQuery, - userProviderConfigsQuery, - ); + } = resolveModelSelector(organizationId, modelsQuery); const modelConfigs = availableModelConfigs; /* * Model precedence: user click > root override (specific model) > root @@ -393,19 +384,15 @@ export const AgentCreateForm: FC = ({ hasUserFixableModelProviders, }); const providerCount = - canConfigureAgentSetup && - chatProviderConfigsQuery.data && - availableModelsQuery.data + canConfigureAgentSetup && chatProviderConfigsQuery.data && modelsQuery.data ? countConfiguredProviderConfigs( chatProviderConfigsQuery.data, - availableModelsQuery.data, + modelsQuery.data, ) : undefined; - const modelCount = availableModelsQuery.data - ? modelOptions.length - : undefined; + const modelCount = modelsQuery.data ? modelOptions.length : undefined; const unsupportedProviderNames = getUnsupportedProviderNames( - availableModelsQuery.data, + modelsQuery.data, ); const effectiveMCPServerIds = (() => { @@ -588,16 +575,12 @@ export const AgentCreateForm: FC = ({ {permittedOrgsQuery.error != null && ( )} - {availableModelsQuery.error != null && ( - - )} - {userProviderConfigsQuery.error != null && ( - + {modelsQuery.error != null && ( + )} {organizationId !== "" && - availableModelsQuery.data !== undefined && - availableModelsQuery.error == null && - userProviderConfigsQuery.error == null && + modelsQuery.data !== undefined && + modelsQuery.error == null && !isModelCatalogLoading && !hasModelOptions && ( diff --git a/site/src/pages/AgentsPage/hooks/useOrganizationChatModels.test.tsx b/site/src/pages/AgentsPage/hooks/useOrganizationChatModels.test.tsx index b91ecdd29c3..97f2e18ec6b 100644 --- a/site/src/pages/AgentsPage/hooks/useOrganizationChatModels.test.tsx +++ b/site/src/pages/AgentsPage/hooks/useOrganizationChatModels.test.tsx @@ -28,6 +28,7 @@ const modelsResponse = ( }, ], providers: [], + unsupported_providers: [], }); const apiError = (status: number, message: string) => ({ diff --git a/site/src/pages/AgentsPage/utils/modelOptions.test.ts b/site/src/pages/AgentsPage/utils/modelOptions.test.ts index 4ebbcd2fd81..df6982cf4ad 100644 --- a/site/src/pages/AgentsPage/utils/modelOptions.test.ts +++ b/site/src/pages/AgentsPage/utils/modelOptions.test.ts @@ -1,11 +1,13 @@ import { describe, expect, it } from "vitest"; import type { ChatModel, - ChatModelAvailabilityResponse, + ChatModelProviderDescriptor, ChatProviderConfig, + OrganizationChatModelsResponse, } from "#/api/typesGenerated"; import { MockChatModel, + MockChatModelProviderDescriptor, MockChatProviderConfig, } from "#/testHelpers/chatModels"; import { @@ -53,11 +55,30 @@ const providerInfoByID = new Map([ ], ]); +type TestProvider = Pick< + ChatModelProviderDescriptor, + "available" | "unavailable_reason" +> & { + id?: string; + provider: string; +}; + const createCatalog = ( - providers: ChatModelAvailabilityResponse["providers"], - unsupportedProviders: ChatModelAvailabilityResponse["unsupported_providers"] = [], -): ChatModelAvailabilityResponse => ({ - providers, + providers: readonly TestProvider[], + unsupportedProviders: OrganizationChatModelsResponse["unsupported_providers"] = [], + models: readonly ChatModel[] = [], +): OrganizationChatModelsResponse => ({ + models, + providers: providers.map(({ id, provider, ...status }) => ({ + ...MockChatModelProviderDescriptor, + id: id ?? `prov-${provider}`, + type: provider, + display_name: + provider === MockChatModelProviderDescriptor.type + ? MockChatModelProviderDescriptor.display_name + : provider, + ...status, + })), unsupported_providers: unsupportedProviders, }); @@ -66,7 +87,7 @@ const createProviderConfig = ( Partial, ): ChatProviderConfig => ({ ...MockChatProviderConfig, - id: "provider-config-1", + id: `prov-${overrides.provider}`, display_name: overrides.provider, has_api_key: false, central_api_key_enabled: true, @@ -82,7 +103,6 @@ describe("hasUserFixableProviders", () => { provider: "openai", available: false, unavailable_reason: "user_api_key_required", - models: [], }, ]); @@ -95,7 +115,6 @@ describe("hasUserFixableProviders", () => { provider: "openai", available: false, unavailable_reason: "missing_api_key", - models: [], }, ]); @@ -105,9 +124,7 @@ describe("hasUserFixableProviders", () => { describe("hasConfiguredProviderConfigs", () => { it("ignores supported provider placeholders", () => { - const catalog = createCatalog([ - { provider: "openai", available: true, models: [] }, - ]); + const catalog = createCatalog([{ provider: "openai", available: true }]); expect( hasConfiguredProviderConfigs( @@ -118,9 +135,7 @@ describe("hasConfiguredProviderConfigs", () => { }); it("returns true for database and env preset provider models", () => { - const catalog = createCatalog([ - { provider: "openai", available: true, models: [] }, - ]); + const catalog = createCatalog([{ provider: "openai", available: true }]); expect( hasConfiguredProviderConfigs( @@ -138,12 +153,11 @@ describe("hasConfiguredProviderConfigs", () => { it("excludes disabled and unavailable provider models", () => { const catalog = createCatalog([ - { provider: "openai", available: true, models: [] }, + { provider: "openai", available: true }, { provider: "anthropic", available: false, unavailable_reason: "missing_api_key", - models: [], }, ]); @@ -169,15 +183,14 @@ describe("hasConfiguredProviderConfigs", () => { describe("countConfiguredProviderConfigs", () => { it("counts only enabled provider models available in the catalog", () => { const catalog = createCatalog([ - { provider: "openai", available: true, models: [] }, - { provider: "anthropic", available: true, models: [] }, - { provider: "google", available: true, models: [] }, - { provider: "azure", available: true, models: [] }, + { provider: "openai", available: true }, + { provider: "anthropic", available: true }, + { provider: "google", available: true }, + { provider: "azure", available: true }, { provider: "bedrock", available: false, unavailable_reason: "missing_api_key", - models: [], }, ]); @@ -223,7 +236,6 @@ describe("getModelSelectorPlaceholder", () => { provider: "openai", available: false, unavailable_reason: "user_api_key_required", - models: [], }, ]); @@ -238,7 +250,6 @@ describe("getModelSelectorPlaceholder", () => { provider: "openai", available: false, unavailable_reason: "missing_api_key", - models: [], }, ]); @@ -397,9 +408,7 @@ describe("getModelOptionsFromModels", () => { model: "gpt-4o", }), ]; - const catalog = createCatalog([ - { provider: "openai", available: true, models: [] }, - ]); + const catalog = createCatalog([{ provider: "openai", available: true }]); expect( getModelOptionsFromModels( @@ -432,7 +441,6 @@ describe("getModelOptionsFromModels", () => { { provider: "openai", available: true, - models: [], }, ]); @@ -478,9 +486,7 @@ describe("getModelOptionsFromModels", () => { model_config: {}, }), ]; - const catalog = createCatalog([ - { provider: "openai", available: true, models: [] }, - ]); + const catalog = createCatalog([{ provider: "openai", available: true }]); expect( getModelOptionsFromModels( @@ -529,7 +535,6 @@ describe("getModelOptionsFromModels", () => { { provider: "anthropic", available: false, - models: [], }, ]); @@ -565,7 +570,6 @@ describe("getModelOptionsFromModels", () => { { provider: "openai", available: true, - models: [], }, ]); @@ -593,7 +597,6 @@ describe("getModelOptionsFromModels", () => { { provider: "openai", available: true, - models: [], }, ]); @@ -660,12 +663,10 @@ describe("getModelOptionsFromModels", () => { { provider: "openai", available: true, - models: [], }, { provider: "anthropic", available: true, - models: [], }, ]); @@ -704,7 +705,6 @@ describe("getModelOptionsFromModels", () => { { provider: "openrouter", available: true, - models: [], }, ]); @@ -728,9 +728,7 @@ describe("getModelOptionsFromModels", () => { context_limit: 128_000, }), ]; - const catalog = createCatalog([ - { provider: "openai", available: true, models: [] }, - ]); + const catalog = createCatalog([{ provider: "openai", available: true }]); expect( getModelOptionsFromModels(models, catalog, new Map(), testOrganizationID), @@ -755,8 +753,8 @@ describe("getModelOptionsFromModels", () => { }), ]; const catalog = createCatalog([ - { provider: "openai", available: true, models: [] }, - { provider: "anthropic", available: true, models: [] }, + { provider: "openai", available: true }, + { provider: "anthropic", available: true }, ]); const partialMap = new Map([ ["prov-openai", { provider: "openai", displayName: "OpenAI", icon: "" }], @@ -786,7 +784,16 @@ describe("getModelOptionsFromModels", () => { }), ]; const catalog = createCatalog([ - { provider: "anthropic", available: true, models: [] }, + { + id: "prov-anthropic-primary", + provider: "anthropic", + available: true, + }, + { + id: "prov-anthropic-hyper", + provider: "anthropic", + available: true, + }, ]); const sameTypeProviders = new Map([ [ @@ -839,7 +846,8 @@ describe("getModelOptionsFromModels", () => { }), ]; const catalog = createCatalog([ - { provider: "openai", available: true, models: [] }, + { id: "prov-enabled", provider: "openai", available: true }, + { id: "prov-disabled", provider: "openai", available: true }, ]); const providers = new Map([ [ @@ -875,9 +883,7 @@ describe("getModelOptionsFromModels", () => { model: "gpt-4o", }), ]; - const catalog = createCatalog([ - { provider: "openai", available: true, models: [] }, - ]); + const catalog = createCatalog([{ provider: "openai", available: true }]); expect( getModelOptionsFromModels( @@ -889,9 +895,7 @@ describe("getModelOptionsFromModels", () => { ).toEqual(["config-openai"]); }); - it("excludes only the disabled instance for same-type providers", () => { - // The catalog marks the type as available because of the enabled - // instance, so only the per-row flag can exclude the disabled one. + it("uses exact UUID availability for same-type providers", () => { const models = [ createConfig({ id: "config-primary", @@ -905,7 +909,17 @@ describe("getModelOptionsFromModels", () => { }), ]; const catalog = createCatalog([ - { provider: "anthropic", available: true, models: [] }, + { + id: "prov-anthropic-primary", + provider: "anthropic", + available: true, + }, + { + id: "prov-anthropic-secondary", + provider: "anthropic", + available: false, + unavailable_reason: "missing_api_key", + }, ]); const sameTypeProviders = new Map([ [ @@ -923,7 +937,7 @@ describe("getModelOptionsFromModels", () => { provider: "anthropic", displayName: "Anthropic Secondary", icon: "", - enabled: false, + enabled: true, }, ], ]); @@ -1045,7 +1059,7 @@ describe("providerTypeByIDFromUserConfigs", () => { }); describe("getUnsupportedProviderNames", () => { - const unsupportedCopilot: ChatModelAvailabilityResponse["unsupported_providers"] = + const unsupportedCopilot: OrganizationChatModelsResponse["unsupported_providers"] = [ { provider: "copilot", @@ -1060,7 +1074,7 @@ describe("getUnsupportedProviderNames", () => { it("returns empty when a supported provider is also configured", () => { const catalog = createCatalog( - [{ provider: "anthropic", available: false, models: [] }], + [{ provider: "anthropic", available: false }], unsupportedCopilot, ); expect(getUnsupportedProviderNames(catalog)).toEqual([]); @@ -1097,31 +1111,13 @@ describe("resolveModelSelector", () => { display_name: "GPT-4o", context_limit: 128_000, }); - const catalog = createCatalog([ - { provider: "openai", available: true, models: [] }, - ]); - const userProviderModels = [ - { - provider_id: "prov-openai", - provider: "openai", - display_name: "OpenAI", - icon: "", - enabled: true, - has_user_api_key: false, - has_central_api_key_fallback: true, - byok_enabled: true, - }, - ]; + const catalog = createCatalog([{ provider: "openai", available: true }]); - it("stays loading and drops options while the provider query is pending", () => { - // Catalog + models have resolved, but provider identity has not, so - // the provider map is empty. Options must be dropped and the flag must - // stay loading rather than flashing "No Models". - const state = resolveModelSelector( - testOrganizationID, - { data: { ...catalog, models: [config] }, isLoading: false }, - { data: undefined, isLoading: true }, - ); + it("stays loading and drops options while the collection query is pending", () => { + const state = resolveModelSelector(testOrganizationID, { + data: undefined, + isLoading: true, + }); expect(state.isModelCatalogLoading).toBe(true); expect(state.options).toEqual([]); @@ -1129,11 +1125,10 @@ describe("resolveModelSelector", () => { it("resolves options once every query settles", () => { const runtimeCatalog = { ...catalog, models: [config] }; - const state = resolveModelSelector( - testOrganizationID, - { data: runtimeCatalog, isLoading: false }, - { data: userProviderModels, isLoading: false }, - ); + const state = resolveModelSelector(testOrganizationID, { + data: runtimeCatalog, + isLoading: false, + }); expect(state.isModelCatalogLoading).toBe(false); expect(state.modelCatalog).toBe(runtimeCatalog); diff --git a/site/src/pages/AgentsPage/utils/modelOptions.ts b/site/src/pages/AgentsPage/utils/modelOptions.ts index 5ca610fa766..c352ed61532 100644 --- a/site/src/pages/AgentsPage/utils/modelOptions.ts +++ b/site/src/pages/AgentsPage/utils/modelOptions.ts @@ -5,97 +5,40 @@ import { asString, } from "../components/ChatElements/runtimeTypeUtils"; -type CatalogModelLike = - | TypesGen.ChatModel - | { - readonly id?: unknown; - readonly display_name?: unknown; - }; - -type CatalogProviderLike = Omit & { - readonly models?: readonly CatalogModelLike[]; -}; - -type ModelCatalogLike = { - readonly providers?: readonly CatalogProviderLike[]; -}; - export const hasConfiguredProviderConfigs = ( providerConfigs: readonly TypesGen.ChatProviderConfig[] | null | undefined, - catalog: TypesGen.ChatModelAvailabilityResponse | null | undefined, -): boolean => { - return countConfiguredProviderConfigs(providerConfigs, catalog) > 0; -}; + catalog: TypesGen.OrganizationChatModelsResponse | null | undefined, +): boolean => countConfiguredProviderConfigs(providerConfigs, catalog) > 0; export const countConfiguredProviderConfigs = ( providerConfigs: readonly TypesGen.ChatProviderConfig[] | null | undefined, - catalog: TypesGen.ChatModelAvailabilityResponse | null | undefined, + catalog: TypesGen.OrganizationChatModelsResponse | null | undefined, ): number => { - const availableProviders = getAvailableProviders(catalog); + const availableProviderIDs = new Set( + (catalog?.providers ?? []) + .filter((provider) => provider.available) + .map((provider) => provider.id), + ); return ( - providerConfigs?.filter((providerConfig) => { - if ( - providerConfig.source === "supported" || - providerConfig.enabled !== true - ) { - return false; - } - const provider = asString(providerConfig.provider).trim().toLowerCase(); - return provider !== "" && availableProviders.has(provider); - }).length ?? 0 + providerConfigs?.filter( + (providerConfig) => + providerConfig.source !== "supported" && + providerConfig.enabled && + availableProviderIDs.has(providerConfig.id), + ).length ?? 0 ); }; -const getCatalogProviders = ( - catalog: ModelCatalogLike | null | undefined, -): readonly CatalogProviderLike[] => { - const providers = catalog?.providers; - return Array.isArray(providers) ? providers : []; -}; - -const getProviderModels = ( - provider: CatalogProviderLike, -): readonly CatalogModelLike[] => { - const models = provider.models; - return Array.isArray(models) ? models : []; -}; - -const isProviderConfiguredInCatalog = ( - provider: CatalogProviderLike, -): boolean => { - if (getProviderModels(provider).length > 0) { - return true; - } - if (provider.available === true) { - return true; - } - const unavailableReason = asString(provider.unavailable_reason).trim(); - return unavailableReason !== "" && unavailableReason !== "missing_api_key"; -}; - const hasConfiguredModelsInCatalog = ( - catalog: ModelCatalogLike | null | undefined, -): boolean => { - return getCatalogProviders(catalog).some(isProviderConfiguredInCatalog); -}; + catalog: TypesGen.OrganizationChatModelsResponse | null | undefined, +): boolean => (catalog?.models.length ?? 0) > 0; export const hasUserFixableProviders = ( - catalog: TypesGen.ChatModelAvailabilityResponse | null | undefined, -): boolean => { - if (!catalog?.providers) { - return false; - } - return catalog.providers.some( + catalog: TypesGen.OrganizationChatModelsResponse | null | undefined, +): boolean => + catalog?.providers.some( (provider) => provider.unavailable_reason === "user_api_key_required", - ); -}; - -const getCatalogUnsupportedProviders = ( - catalog: TypesGen.ChatModelAvailabilityResponse | null | undefined, -): readonly TypesGen.ChatUnsupportedProvider[] => { - const unsupported = catalog?.unsupported_providers; - return Array.isArray(unsupported) ? unsupported : []; -}; + ) ?? false; /** * Display names of configured providers the Agents harness cannot serve, @@ -103,13 +46,10 @@ const getCatalogUnsupportedProviders = ( * missing its API key returns an empty list, keeping normal setup guidance. */ export const getUnsupportedProviderNames = ( - catalog: TypesGen.ChatModelAvailabilityResponse | null | undefined, + catalog: TypesGen.OrganizationChatModelsResponse | null | undefined, ): readonly string[] => { - const unsupported = getCatalogUnsupportedProviders(catalog); - if (unsupported.length === 0) { - return []; - } - if (getCatalogProviders(catalog).length > 0) { + const unsupported = catalog?.unsupported_providers ?? []; + if (unsupported.length === 0 || (catalog?.providers.length ?? 0) > 0) { return []; } return unsupported.map( @@ -119,22 +59,6 @@ export const getUnsupportedProviderNames = ( ); }; -const getAvailableProviders = ( - catalog: TypesGen.ChatModelAvailabilityResponse | null | undefined, -): ReadonlySet => { - const availableProviders = new Set(); - for (const provider of getCatalogProviders(catalog)) { - if (provider.available !== true) { - continue; - } - const providerName = asString(provider.provider).trim().toLowerCase(); - if (providerName) { - availableProviders.add(providerName); - } - } - return availableProviders; -}; - /** * The nil UUID (all zeros) is the backend sentinel for an unset stored * model reference. Exported so all model-selection surfaces share one value. @@ -208,6 +132,8 @@ export type ProviderInfo = { readonly icon: string; // Absent is treated as enabled. readonly enabled?: boolean; + readonly available?: boolean; + readonly unavailableReason?: TypesGen.ChatModelProviderUnavailableReason; }; // Provider descriptors use `id`; personal provider configs use `provider_id`. @@ -276,7 +202,7 @@ export const filterModelsWithEnabledProvider = ( export const getModelOptionsFromModels = ( models: readonly TypesGen.ChatModel[] | null | undefined, - catalog: TypesGen.ChatModelAvailabilityResponse | null | undefined, + catalog: TypesGen.OrganizationChatModelsResponse | null | undefined, providerInfoByID: ReadonlyMap, organizationID: string, ): readonly ModelSelectorOption[] => { @@ -284,11 +210,12 @@ export const getModelOptionsFromModels = ( return []; } - const availableProviders = getAvailableProviders(catalog); + const availableProviderIDs = new Set( + catalog.providers + .filter((provider) => provider.available) + .map((provider) => provider.id), + ); const options: ModelSelectorOption[] = []; - - // The catalog check below is keyed by provider type, so it cannot - // exclude a disabled provider when another of the same type is enabled. for (const model of filterModelsWithEnabledProvider( models.filter((model) => model.organization_id === organizationID), providerInfoByID, @@ -301,10 +228,13 @@ export const getModelOptionsFromModels = ( const providerInfo = providerInfoByID.get(model.ai_provider_id); const provider = asString(providerInfo?.provider).trim().toLowerCase(); const modelName = model.model.trim(); - if (!modelID || !providerInfo || !provider || !modelName) { - continue; - } - if (!availableProviders.has(provider)) { + if ( + !modelID || + !providerInfo || + !availableProviderIDs.has(model.ai_provider_id) || + !provider || + !modelName + ) { continue; } @@ -346,7 +276,7 @@ type SelectorQuery = { interface ModelSelectorState { readonly options: readonly ModelSelectorOption[]; readonly isModelCatalogLoading: boolean; - readonly modelCatalog: TypesGen.ChatModelAvailabilityResponse | undefined; + readonly modelCatalog: TypesGen.OrganizationChatModelsResponse | undefined; readonly hasConfiguredModels: boolean; } @@ -356,16 +286,15 @@ interface ModelSelectorState { // build an empty provider map, drop every option, and flash "No Models". export const resolveModelSelector = ( organizationID: string, - catalog: SelectorQuery, - userProviderModels: SelectorQuery, + catalog: SelectorQuery, ): ModelSelectorState => ({ options: getModelOptionsFromModels( catalog.data?.models, catalog.data, - providerInfoByIDFromUserConfigs(userProviderModels.data), + providerInfoByIDFromDescriptors(catalog.data?.providers), organizationID, ), - isModelCatalogLoading: catalog.isLoading || userProviderModels.isLoading, + isModelCatalogLoading: catalog.isLoading, modelCatalog: catalog.data, hasConfiguredModels: hasConfiguredModelsInCatalog(catalog.data), }); @@ -386,7 +315,7 @@ export const getModelSelectorPlaceholder = ( modelOptions: readonly ModelSelectorOption[], isModelCatalogLoading: boolean, hasConfiguredModels: boolean, - catalog?: TypesGen.ChatModelAvailabilityResponse | null, + catalog?: TypesGen.OrganizationChatModelsResponse | null, ): string => { if (modelOptions.length > 0) { return "Select model"; diff --git a/site/src/testHelpers/chatModels.ts b/site/src/testHelpers/chatModels.ts index fb047e73a32..eeb96d8244b 100644 --- a/site/src/testHelpers/chatModels.ts +++ b/site/src/testHelpers/chatModels.ts @@ -1,7 +1,6 @@ import type { AIModelPrice, ChatModel, - ChatModelProvider, ChatModelProviderDescriptor, ChatProviderConfig, } from "#/api/typesGenerated"; @@ -37,12 +36,6 @@ export const MockChatProviderConfig: ChatProviderConfig = { updated_at: MOCK_TIMESTAMP, }; -export const MockChatModelProvider: ChatModelProvider = { - provider: "openai", - available: true, - models: [], -}; - export const MockChatModelProviderDescriptor: ChatModelProviderDescriptor = { id: "provider-1", type: "openai", @@ -53,6 +46,7 @@ export const MockChatModelProviderDescriptor: ChatModelProviderDescriptor = { has_user_api_key: false, has_effective_api_key: true, allow_user_api_key: false, + available: true, }; // Prices are micro-units per million tokens. From a5550d1bc1301b7fd3df9f99f5aaa29374dc1145 Mon Sep 17 00:00:00 2001 From: Ethan Dickson Date: Fri, 21 Aug 2026 20:47:43 +0000 Subject: [PATCH 2/4] test(site): complete model response fixtures --- .../pages/AISettingsPage/ModelsPage/organizationModels.test.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/site/src/pages/AISettingsPage/ModelsPage/organizationModels.test.tsx b/site/src/pages/AISettingsPage/ModelsPage/organizationModels.test.tsx index 1566494d662..b27116e4469 100644 --- a/site/src/pages/AISettingsPage/ModelsPage/organizationModels.test.tsx +++ b/site/src/pages/AISettingsPage/ModelsPage/organizationModels.test.tsx @@ -114,6 +114,7 @@ describe("useAccessibleModelOrganizations", () => { vi.spyOn(API.experimental, "getChatModels").mockResolvedValue({ models: [], providers: [], + unsupported_providers: [], }); const { queryClient, wrapper } = createQueryWrapper(); const { result, unmount } = renderHook( @@ -146,6 +147,7 @@ describe("useAccessibleModelOrganizations", () => { ] : [], providers: [], + unsupported_providers: [], }), ); const { queryClient, wrapper } = createQueryWrapper(); From e95a212f0638c94b29627d0586a863525658a533 Mon Sep 17 00:00:00 2001 From: Ethan Dickson Date: Sun, 23 Aug 2026 10:35:15 +0000 Subject: [PATCH 3/4] fix: surface disabled chat models and unsupported provider guidance --- coderd/exp_chats.go | 20 +++++++--------- coderd/exp_chats_model_config_list_test.go | 24 ++++++++++++++++++- .../components/AgentCreateForm.stories.tsx | 12 +++++++++- .../AgentsPage/utils/modelOptions.test.ts | 18 ++++++++++++-- .../pages/AgentsPage/utils/modelOptions.ts | 11 ++++++++- 5 files changed, 69 insertions(+), 16 deletions(-) diff --git a/coderd/exp_chats.go b/coderd/exp_chats.go index ff2026faa74..433ece3d461 100644 --- a/coderd/exp_chats.go +++ b/coderd/exp_chats.go @@ -6851,7 +6851,7 @@ func (api *API) listChatModelConfigsByOrganization(rw http.ResponseWriter, r *ht httpapi.Forbidden(rw) return } - visible, err := api.canReadChatModelsInOrganization(ctx, r, organization) + configs, visible, err := api.readableChatModelsInOrganization(ctx, r, organization) if err != nil { httpapi.InternalServerError(rw, err) return @@ -6888,32 +6888,30 @@ func (api *API) listChatModelConfigsByOrganization(rw http.ResponseWriter, r *ht } } resp := codersdk.OrganizationChatModelsResponse{ - Models: make([]codersdk.ChatModel, 0, len(availability.enabledModels)), + Models: make([]codersdk.ChatModel, 0, len(configs)), Providers: providers, UnsupportedProviders: chatprovider.UnsupportedProviders(availability.configuredProviders), } - for _, config := range availability.enabledModels { + for _, config := range configs { resp.Models = append(resp.Models, convertChatModelConfig(config)) } httpapi.Write(ctx, rw, http.StatusOK, resp) } -func (api *API) canReadChatModelsInOrganization( +func (api *API) readableChatModelsInOrganization( ctx context.Context, r *http.Request, organization database.Organization, -) (bool, error) { +) ([]database.ChatModelConfig, bool, error) { configs, err := api.Database.GetChatModelConfigs(ctx, organization.ID) if err != nil { - return false, err + return nil, false, err } - for _, config := range configs { - if api.Authorize(r, policy.ActionRead, chatModelConfigRBACObject(config)) { - return true, nil - } + if len(configs) > 0 { + return configs, true, nil } - return api.Authorize(r, policy.ActionRead, organization.RBACObject()), nil + return configs, api.Authorize(r, policy.ActionRead, organization.RBACObject()), nil } func chatModelConfigReadScope(scopes database.APIKeyScopes) bool { diff --git a/coderd/exp_chats_model_config_list_test.go b/coderd/exp_chats_model_config_list_test.go index e3cd1e3ada6..d746f9b0bdf 100644 --- a/coderd/exp_chats_model_config_list_test.go +++ b/coderd/exp_chats_model_config_list_test.go @@ -51,6 +51,18 @@ func TestChatModelConfigListReadContracts(t *testing.T) { }, func(params *database.InsertChatModelConfigParams) { params.Enabled = false }) + disabledProvider := dbgen.AIProvider(t, rawDB, database.AIProvider{ + Type: database.AIProviderTypeAnthropic, + }, func(params *database.InsertAIProviderParams) { + params.Enabled = false + }) + providerDisabled := dbgen.ChatModelConfig(t, rawDB, database.ChatModelConfig{ + AIProviderID: uuid.NullUUID{UUID: disabledProvider.ID, Valid: true}, + OrganizationID: defaultOrg.ID, + GroupACL: database.ChatACL{ + defaultOrg.ID.String(): {Permissions: []policy.Action{policy.ActionRead}}, + }, + }) denied := dbgen.ChatModelConfig(t, rawDB, database.ChatModelConfig{ OrganizationID: defaultOrg.ID, GroupACL: database.ChatACL{}, @@ -64,6 +76,8 @@ func TestChatModelConfigListReadContracts(t *testing.T) { }) require.True(t, ownEnabled.Enabled) require.False(t, ownDisabled.Enabled) + require.False(t, disabledProvider.Enabled) + require.True(t, providerDisabled.Enabled) require.True(t, denied.Enabled) require.True(t, otherEnabled.Enabled) @@ -237,9 +251,17 @@ func TestChatModelConfigListReadContracts(t *testing.T) { models, err := testCase.client(t, ctx).ChatModels(ctx, defaultOrg.ID) require.NoError(t, err) require.True(t, containsChatModel(models.Models, ownEnabled.ID)) - require.False(t, containsChatModel(models.Models, ownDisabled.ID)) + require.True(t, containsChatModel(models.Models, ownDisabled.ID)) + require.True(t, containsChatModel(models.Models, providerDisabled.ID)) require.Equal(t, testCase.seesDenied, containsChatModel(models.Models, denied.ID)) require.False(t, containsChatModel(models.Models, otherEnabled.ID)) + + disabledProviderIndex := slices.IndexFunc(models.Providers, func(provider codersdk.ChatModelProviderDescriptor) bool { + return provider.ID == disabledProvider.ID + }) + require.NotEqual(t, -1, disabledProviderIndex) + require.False(t, models.Providers[disabledProviderIndex].Enabled) + require.False(t, models.Providers[disabledProviderIndex].Available) }) } } diff --git a/site/src/pages/AgentsPage/components/AgentCreateForm.stories.tsx b/site/src/pages/AgentsPage/components/AgentCreateForm.stories.tsx index 54f05973003..3318b7abd03 100644 --- a/site/src/pages/AgentsPage/components/AgentCreateForm.stories.tsx +++ b/site/src/pages/AgentsPage/components/AgentCreateForm.stories.tsx @@ -139,7 +139,17 @@ const fetchFailedCatalog: TypesGen.OrganizationChatModelsResponse = { const unsupportedProviderCatalog: TypesGen.OrganizationChatModelsResponse = { models: [], - providers: [], + providers: [ + { + ...MockChatModelProviderDescriptor, + id: "provider-copilot", + type: "copilot", + display_name: "GitHub Copilot", + has_api_key: false, + has_effective_api_key: false, + available: false, + }, + ], unsupported_providers: [ { provider: "copilot", display_name: "GitHub Copilot" }, ], diff --git a/site/src/pages/AgentsPage/utils/modelOptions.test.ts b/site/src/pages/AgentsPage/utils/modelOptions.test.ts index df6982cf4ad..d490b9057f9 100644 --- a/site/src/pages/AgentsPage/utils/modelOptions.test.ts +++ b/site/src/pages/AgentsPage/utils/modelOptions.test.ts @@ -1068,13 +1068,27 @@ describe("getUnsupportedProviderNames", () => { ]; it("returns names when no supported provider is configured", () => { - const catalog = createCatalog([], unsupportedCopilot); + const catalog = createCatalog( + [{ provider: "copilot", available: false }], + unsupportedCopilot, + ); + expect(getUnsupportedProviderNames(catalog)).toEqual(["GitHub Copilot"]); + }); + + it("normalizes provider types when identifying unsupported descriptors", () => { + const catalog = createCatalog( + [{ provider: " COPILOT ", available: false }], + unsupportedCopilot, + ); expect(getUnsupportedProviderNames(catalog)).toEqual(["GitHub Copilot"]); }); it("returns empty when a supported provider is also configured", () => { const catalog = createCatalog( - [{ provider: "anthropic", available: false }], + [ + { provider: "copilot", available: false }, + { provider: "anthropic", available: false }, + ], unsupportedCopilot, ); expect(getUnsupportedProviderNames(catalog)).toEqual([]); diff --git a/site/src/pages/AgentsPage/utils/modelOptions.ts b/site/src/pages/AgentsPage/utils/modelOptions.ts index c352ed61532..f51017a9e1c 100644 --- a/site/src/pages/AgentsPage/utils/modelOptions.ts +++ b/site/src/pages/AgentsPage/utils/modelOptions.ts @@ -1,4 +1,5 @@ import type * as TypesGen from "#/api/typesGenerated"; +import { normalizeProvider } from "#/modules/aiModels/helpers"; import type { ModelSelectorOption } from "../components/ChatElements"; import { asNumber, @@ -49,7 +50,15 @@ export const getUnsupportedProviderNames = ( catalog: TypesGen.OrganizationChatModelsResponse | null | undefined, ): readonly string[] => { const unsupported = catalog?.unsupported_providers ?? []; - if (unsupported.length === 0 || (catalog?.providers.length ?? 0) > 0) { + const unsupportedProviderTypes = new Set( + unsupported.map((provider) => normalizeProvider(provider.provider)), + ); + const hasSupportedProvider = + catalog?.providers.some( + (provider) => + !unsupportedProviderTypes.has(normalizeProvider(provider.type)), + ) ?? false; + if (unsupported.length === 0 || hasSupportedProvider) { return []; } return unsupported.map( From 4c58630fbec9e2934d33d98ea08c43c71c8445a7 Mon Sep 17 00:00:00 2001 From: Ethan Dickson Date: Mon, 24 Aug 2026 08:37:13 +0000 Subject: [PATCH 4/4] fix: align disabled chat model tests and settings gating with collection contract --- coderd/exp_chats_test.go | 48 +++++++++---------- .../AgentSettingsUserAgentsPage.tsx | 15 +++--- ...gentSettingsUserAgentsPageView.stories.tsx | 27 ++++++++--- .../AgentSettingsUserAgentsPageView.tsx | 8 ++-- .../components/AgentCreateForm.stories.tsx | 36 ++++++++++++++ .../AgentsPage/utils/modelOptions.test.ts | 12 +++++ .../pages/AgentsPage/utils/modelOptions.ts | 7 +-- 7 files changed, 107 insertions(+), 46 deletions(-) diff --git a/coderd/exp_chats_test.go b/coderd/exp_chats_test.go index 0d2264f1546..a7363263f61 100644 --- a/coderd/exp_chats_test.go +++ b/coderd/exp_chats_test.go @@ -2772,7 +2772,7 @@ func TestListChatModels(t *testing.T) { require.False(t, containsModel(response, other.ID)) }) - t.Run("DisabledProvidersAndModelsAreFilteredOut", func(t *testing.T) { + t.Run("DisabledProvidersAndModelsRemainVisible", func(t *testing.T) { t.Parallel() ctx := testutil.Context(t, testutil.WaitLong) client := newChatClient(t) @@ -2788,12 +2788,18 @@ func TestListChatModels(t *testing.T) { response, err := client.ChatModels(ctx, firstUser.OrganizationID) require.NoError(t, err) require.True(t, containsModel(response, enabledModel.ID)) - require.False(t, containsModel(response, disabledModel.ID)) + require.True(t, slices.ContainsFunc(response.Models, func(model codersdk.ChatModel) bool { + return model.ID == disabledModel.ID && !model.Enabled + })) _, err = client.UpdateAIProvider(ctx, provider.ID.String(), codersdk.UpdateAIProviderRequest{Enabled: ptr.Ref(false)}) require.NoError(t, err) response, err = client.ChatModels(ctx, firstUser.OrganizationID) require.NoError(t, err) - require.Empty(t, response.Models) + require.True(t, containsModel(response, enabledModel.ID)) + require.True(t, containsModel(response, disabledModel.ID)) + disabledProvider := providerByID(t, response, provider.ID) + require.False(t, disabledProvider.Enabled) + require.False(t, disabledProvider.Available) }) } @@ -4638,7 +4644,7 @@ func TestListChatModelConfigs(t *testing.T) { require.NoError(t, client.DeleteChatModel(ctx, created.OrganizationID, created.ID)) }) - t.Run("CollectionFiltersDisabledModelConfigs", func(t *testing.T) { + t.Run("CollectionIncludesDisabledModelConfigs", func(t *testing.T) { t.Parallel() ctx := testutil.Context(t, testutil.WaitLong) @@ -4661,8 +4667,8 @@ func TestListChatModelConfigs(t *testing.T) { configs, err := client.ChatModels(ctx, firstUser.OrganizationID) require.NoError(t, err) - require.False(t, slices.ContainsFunc(configs.Models, func(config codersdk.ChatModel) bool { - return config.ID == disabledConfig.ID + require.True(t, slices.ContainsFunc(configs.Models, func(config codersdk.ChatModel) bool { + return config.ID == disabledConfig.ID && !config.Enabled })) }) @@ -5685,17 +5691,15 @@ func TestUpdateChatModel(t *testing.T) { adminConfigs, err := adminClient.ChatModels(ctx, firstUser.OrganizationID) require.NoError(t, err) - require.False(t, slices.ContainsFunc(adminConfigs.Models, func(config codersdk.ChatModel) bool { - return config.ID == modelConfig.ID + require.True(t, slices.ContainsFunc(adminConfigs.Models, func(config codersdk.ChatModel) bool { + return config.ID == modelConfig.ID && !config.Enabled })) collection, err := memberClient.ChatModels(ctx, firstUser.OrganizationID) require.NoError(t, err) - runtimeIDs := make([]uuid.UUID, 0, len(collection.Models)) - for _, config := range collection.Models { - runtimeIDs = append(runtimeIDs, config.ID) - } - require.NotContains(t, runtimeIDs, modelConfig.ID) + require.True(t, slices.ContainsFunc(collection.Models, func(config codersdk.ChatModel) bool { + return config.ID == modelConfig.ID && !config.Enabled + })) _, err = memberClient.CreateChat(ctx, codersdk.CreateChatRequest{ OrganizationID: firstUser.OrganizationID, @@ -5709,7 +5713,7 @@ func TestUpdateChatModel(t *testing.T) { require.Equal(t, "Invalid model_config_id: model config not found or disabled.", sdkErr.Message) }) - t.Run("ReEnableUpdatesRuntimeAvailability", func(t *testing.T) { + t.Run("ReEnableUpdatesCollectionDescriptor", func(t *testing.T) { t.Parallel() ctx := testutil.Context(t, testutil.WaitLong) @@ -5734,11 +5738,9 @@ func TestUpdateChatModel(t *testing.T) { collection, err := memberClient.ChatModels(ctx, firstUser.OrganizationID) require.NoError(t, err) - runtimeIDs := make([]uuid.UUID, 0, len(collection.Models)) - for _, config := range collection.Models { - runtimeIDs = append(runtimeIDs, config.ID) - } - require.NotContains(t, runtimeIDs, modelConfig.ID) + require.True(t, slices.ContainsFunc(collection.Models, func(config codersdk.ChatModel) bool { + return config.ID == modelConfig.ID && !config.Enabled + })) enabled = true updated, err := adminClient.UpdateChatModel(ctx, modelConfig.OrganizationID, modelConfig.ID, codersdk.UpdateChatModelRequest{ @@ -5750,11 +5752,9 @@ func TestUpdateChatModel(t *testing.T) { collection, err = memberClient.ChatModels(ctx, firstUser.OrganizationID) require.NoError(t, err) - runtimeIDs = runtimeIDs[:0] - for _, config := range collection.Models { - runtimeIDs = append(runtimeIDs, config.ID) - } - require.Contains(t, runtimeIDs, modelConfig.ID) + require.True(t, slices.ContainsFunc(collection.Models, func(config codersdk.ChatModel) bool { + return config.ID == modelConfig.ID && config.Enabled + })) }) t.Run("UpdateAIProviderID", func(t *testing.T) { diff --git a/site/src/pages/AgentsPage/AgentSettingsUserAgentsPage.tsx b/site/src/pages/AgentsPage/AgentSettingsUserAgentsPage.tsx index e5eacbd084f..c302468dfe8 100644 --- a/site/src/pages/AgentsPage/AgentSettingsUserAgentsPage.tsx +++ b/site/src/pages/AgentsPage/AgentSettingsUserAgentsPage.tsx @@ -30,12 +30,17 @@ const AgentSettingsUserAgentsPage: FC = () => { ); const defaultOrgModelConfigs = modelsQuery.data?.models ?? []; - const hasDefaultOrgModels = defaultOrgModelConfigs.length > 0; const { options: modelOptions, isModelCatalogLoading } = resolveModelSelector( defaultOrganizationId, modelsQuery, ); + const hasNoAvailableDefaultOrgModels = + defaultOrganizationId !== "" && + !modelsQuery.isLoading && + modelsQuery.error === null && + modelsQuery.data !== undefined && + modelOptions.length === 0; const saveModelOverride = ( context: TypesGen.ChatPersonalModelOverrideContext, @@ -63,13 +68,7 @@ const AgentSettingsUserAgentsPage: FC = () => { modelsError={modelsQuery.error} isLoadingModels={isModelCatalogLoading} isDefaultOrganizationUnresolved={defaultOrganizationId === ""} - hasNoDefaultOrgModels={ - defaultOrganizationId !== "" && - !modelsQuery.isLoading && - modelsQuery.error === null && - modelsQuery.data !== undefined && - !hasDefaultOrgModels - } + hasNoAvailableDefaultOrgModels={hasNoAvailableDefaultOrgModels} onSaveRootModelOverride={saveModelOverride( "root", saveRootModelOverrideMutation, diff --git a/site/src/pages/AgentsPage/AgentSettingsUserAgentsPageView.stories.tsx b/site/src/pages/AgentsPage/AgentSettingsUserAgentsPageView.stories.tsx index 3f7afbb881f..5f76db380fa 100644 --- a/site/src/pages/AgentsPage/AgentSettingsUserAgentsPageView.stories.tsx +++ b/site/src/pages/AgentsPage/AgentSettingsUserAgentsPageView.stories.tsx @@ -161,7 +161,7 @@ const buildArgs = ( modelsError: undefined, isLoadingModels: false, isDefaultOrganizationUnresolved: false, - hasNoDefaultOrgModels: false, + hasNoAvailableDefaultOrgModels: false, onSaveRootModelOverride: fn(), isSavingRootModelOverride: false, isSaveRootModelOverrideError: false, @@ -700,11 +700,11 @@ export const SaveErrorState: Story = { }, }; -export const NoDefaultOrgModels: Story = { +export const NoAvailableDefaultOrgModels: Story = { args: buildArgs({ - hasNoDefaultOrgModels: true, + hasNoAvailableDefaultOrgModels: true, modelOptions: [], - models: [], + models: [disabledModelConfig], }), play: async ({ canvasElement }) => { const canvas = within(canvasElement); @@ -712,9 +712,22 @@ export const NoDefaultOrgModels: Story = { canvas.getByText(/default organization has no available chat models/i), ).toBeInTheDocument(); const rootSection = await getSection(canvasElement, "Root agent model"); - expect( - within(rootSection).getByRole("button", { name: "Save" }), - ).toBeDisabled(); + const generalSection = await getSection( + canvasElement, + "General subagent model", + ); + const exploreSection = await getSection( + canvasElement, + "Explore subagent model", + ); + for (const section of [rootSection, generalSection, exploreSection]) { + expect( + within(section).getByRole("combobox", { name: /behavior/i }), + ).toBeDisabled(); + expect( + within(section).getByRole("button", { name: "Save" }), + ).toBeDisabled(); + } }, }; diff --git a/site/src/pages/AgentsPage/AgentSettingsUserAgentsPageView.tsx b/site/src/pages/AgentsPage/AgentSettingsUserAgentsPageView.tsx index abeb2e06cc4..36c4bfef8b0 100644 --- a/site/src/pages/AgentsPage/AgentSettingsUserAgentsPageView.tsx +++ b/site/src/pages/AgentsPage/AgentSettingsUserAgentsPageView.tsx @@ -21,7 +21,7 @@ export interface AgentSettingsUserAgentsPageViewProps { modelsError: unknown; isLoadingModels: boolean; isDefaultOrganizationUnresolved: boolean; - hasNoDefaultOrgModels: boolean; + hasNoAvailableDefaultOrgModels: boolean; onSaveRootModelOverride: SavePersonalOverride; isSavingRootModelOverride: boolean; isSaveRootModelOverrideError: boolean; @@ -46,7 +46,7 @@ export const AgentSettingsUserAgentsPageView: FC< modelsError, isLoadingModels, isDefaultOrganizationUnresolved, - hasNoDefaultOrgModels, + hasNoAvailableDefaultOrgModels, onSaveRootModelOverride, isSavingRootModelOverride, isSaveRootModelOverrideError, @@ -63,7 +63,7 @@ export const AgentSettingsUserAgentsPageView: FC< isLoading || !personalOverridesEnabled || isDefaultOrganizationUnresolved || - hasNoDefaultOrgModels; + hasNoAvailableDefaultOrgModels; return (
@@ -103,7 +103,7 @@ export const AgentSettingsUserAgentsPageView: FC< )} - {hasNoDefaultOrgModels && ( + {hasNoAvailableDefaultOrgModels && ( Your default organization has no available chat models. Ask an diff --git a/site/src/pages/AgentsPage/components/AgentCreateForm.stories.tsx b/site/src/pages/AgentsPage/components/AgentCreateForm.stories.tsx index 3318b7abd03..f3cce23be39 100644 --- a/site/src/pages/AgentsPage/components/AgentCreateForm.stories.tsx +++ b/site/src/pages/AgentsPage/components/AgentCreateForm.stories.tsx @@ -155,6 +155,22 @@ const unsupportedProviderCatalog: TypesGen.OrganizationChatModelsResponse = { ], }; +const unsupportedProviderWithDisabledSupportedCatalog: TypesGen.OrganizationChatModelsResponse = + { + ...unsupportedProviderCatalog, + providers: [ + ...unsupportedProviderCatalog.providers, + { + ...MockChatModelProviderDescriptor, + id: "provider-anthropic", + type: "anthropic", + display_name: "Anthropic", + enabled: false, + available: false, + }, + ], + }; + const defaultUserProviderConfigs: TypesGen.UserChatProviderConfig[] = [ { provider_id: "provider-1", @@ -995,6 +1011,26 @@ export const UnsupportedProviderOnly: Story = { }, }; +export const UnsupportedProviderAndDisabledSupportedProvider: Story = { + args: { ...defaultArgs, canConfigureAgentSetup: true }, + parameters: { + queries: [ + { + key: organizationChatModelsKey(MockDefaultOrganization.id), + data: unsupportedProviderWithDisabledSupportedCatalog, + }, + { key: aiProvidersListKey, data: [] }, + ], + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + expect(canvas.getByText(/GitHub Copilot is configured but/i)).toBeVisible(); + expect( + canvas.getByRole("link", { name: "not supported by Coder Agents" }), + ).toBeVisible(); + }, +}; + export const MissingProviderAndModelSetup: Story = { parameters: { queries: [ diff --git a/site/src/pages/AgentsPage/utils/modelOptions.test.ts b/site/src/pages/AgentsPage/utils/modelOptions.test.ts index d490b9057f9..29ed6f0d610 100644 --- a/site/src/pages/AgentsPage/utils/modelOptions.test.ts +++ b/site/src/pages/AgentsPage/utils/modelOptions.test.ts @@ -59,6 +59,7 @@ type TestProvider = Pick< ChatModelProviderDescriptor, "available" | "unavailable_reason" > & { + enabled?: boolean; id?: string; provider: string; }; @@ -1094,6 +1095,17 @@ describe("getUnsupportedProviderNames", () => { expect(getUnsupportedProviderNames(catalog)).toEqual([]); }); + it("returns names when the only supported provider is disabled", () => { + const catalog = createCatalog( + [ + { provider: "copilot", available: false }, + { provider: "anthropic", available: false, enabled: false }, + ], + unsupportedCopilot, + ); + expect(getUnsupportedProviderNames(catalog)).toEqual(["GitHub Copilot"]); + }); + it("returns empty when there are no unsupported providers", () => { expect(getUnsupportedProviderNames(createCatalog([]))).toEqual([]); }); diff --git a/site/src/pages/AgentsPage/utils/modelOptions.ts b/site/src/pages/AgentsPage/utils/modelOptions.ts index f51017a9e1c..0dce1296c28 100644 --- a/site/src/pages/AgentsPage/utils/modelOptions.ts +++ b/site/src/pages/AgentsPage/utils/modelOptions.ts @@ -56,6 +56,7 @@ export const getUnsupportedProviderNames = ( const hasSupportedProvider = catalog?.providers.some( (provider) => + provider.enabled && !unsupportedProviderTypes.has(normalizeProvider(provider.type)), ) ?? false; if (unsupported.length === 0 || hasSupportedProvider) { @@ -196,9 +197,9 @@ export const providerTypeByIDFromUserConfigs = ( ); /** - * Drops models whose provider row is disabled or missing. Both - * provider-info sources include every enabled provider, so a missing row - * means the provider is disabled or deleted. + * Drops models whose provider row is disabled or missing. Provider descriptors + * include disabled providers, while user provider configs may omit inaccessible + * providers. */ export const filterModelsWithEnabledProvider = ( models: readonly TypesGen.ChatModel[],