diff --git a/site/src/pages/AISettingsPage/ModelsPage/components/ModelForm.stories.tsx b/site/src/pages/AISettingsPage/ModelsPage/components/ModelForm.stories.tsx index 7c1c3bf54cac5..d7eeea19fbc6a 100644 --- a/site/src/pages/AISettingsPage/ModelsPage/components/ModelForm.stories.tsx +++ b/site/src/pages/AISettingsPage/ModelsPage/components/ModelForm.stories.tsx @@ -1,9 +1,22 @@ import type { Meta, StoryObj } from "@storybook/react-vite"; -import { expect, fn, screen, spyOn, userEvent, within } from "storybook/test"; +import { + expect, + fn, + mocked, + screen, + spyOn, + userEvent, + waitFor, + within, +} from "storybook/test"; import { reactRouterParameters } from "storybook-addon-remix-react-router"; import { API } from "#/api/api"; import { aiModelPrices } from "#/api/queries/aiProviders"; import type * as TypesGen from "#/api/typesGenerated"; +import { + MockGPT5BelowThresholdModelPrice, + MockGPT5ModelPrice, +} from "#/testHelpers/chatModels"; import { withDashboardProvider, withToaster } from "#/testHelpers/storybook"; import { MockAnthropicProviderState, @@ -23,6 +36,15 @@ const onUpdateModel = fn( ): Promise => undefined, ); +// The price loading placeholder is transient: it disappears once the +// debounce settles and the lookup resolves, so poll for it instead of +// asserting synchronously. +const waitForPriceLoading = async (canvas: ReturnType) => { + await waitFor(() => + expect(canvas.getByLabelText("Input price loading")).toBeInTheDocument(), + ); +}; + const meta: Meta = { title: "pages/AISettingsPage/ModelsPage/ModelForm", component: ModelForm, @@ -484,18 +506,7 @@ export const CostEstimateFromLivePriceBook: Story = { queries: [ { key: aiModelPrices("openai", "gpt-5").queryKey, - data: [ - { - provider: "openai", - model: "gpt-5", - input_price: 1250000, - output_price: 10000000, - cache_read_price: 125000, - cache_write_price: null, - created_at: "2026-02-18T12:00:00.000Z", - updated_at: "2026-02-18T12:00:00.000Z", - }, - ], + data: [MockGPT5ModelPrice], }, ], }, @@ -511,6 +522,123 @@ export const CostEstimateFromLivePriceBook: Story = { }, }; +// A live price below $0.0001 per million tokens would render as $0.00 and +// read as free, so the field shows a threshold instead. +export const CostEstimateBelowThresholdPrice: Story = { + args: { + editingModel: mockGPT5, + onDeleteModel: fn(async () => undefined), + }, + parameters: { + queries: [ + { + key: aiModelPrices("openai", "gpt-5").queryKey, + data: [MockGPT5BelowThresholdModelPrice], + }, + ], + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await userEvent.click( + canvas.getByRole("button", { name: /cost estimate/i }), + ); + const inputField = canvas.getByLabelText(/^input$/i); + await expect(inputField).toHaveValue("0.0001"); + await expect(canvas.getByText("<$")).toBeInTheDocument(); + // The threshold marker is part of the input's accessible description + // so screen readers announce "less than $0.0001" rather than an exact + // price. + await expect(inputField).toHaveAccessibleDescription( + "less than $0.0001 USD per million tokens", + ); + }, +}; + +// Editing the model identifier fires a live price lookup per settled value +// rather than per keystroke, so typing a full identifier costs one request. +export const CostEstimateDebouncesLivePriceLookup: Story = { + args: { + editingModel: mockGPT5, + onDeleteModel: fn(async () => undefined), + }, + beforeEach: () => { + spyOn(API.experimental, "getAIModelPrices").mockResolvedValue([]); + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await userEvent.click( + canvas.getByRole("button", { name: /cost estimate/i }), + ); + const modelInput = canvas.getByLabelText(/model identifier/i); + await userEvent.clear(modelInput); + await userEvent.type(modelInput, "gpt-5.5"); + await waitFor(() => + expect(API.experimental.getAIModelPrices).toHaveBeenCalledWith({ + provider: "openai", + model: "gpt-5.5", + }), + ); + const calls = mocked(API.experimental.getAIModelPrices).mock.calls.map( + ([params]) => params.model, + ); + expect(calls).toStrictEqual(["gpt-5", "gpt-5.5"]); + }, +}; + +// On an entitled form the live lookup may override the catalog, so typing +// into an empty identifier shows the loading placeholder immediately rather +// than briefly flashing catalog prices before the lookup fires. +export const CostEstimateShowsLoadingWhileDebouncePending: Story = { + args: { + selectedProviderState: MockAnthropicProviderState, + }, + beforeEach: () => { + spyOn(API.experimental, "getAIModelPrices").mockResolvedValue([]); + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await userEvent.click( + canvas.getByRole("button", { name: /cost estimate/i }), + ); + await userEvent.type( + canvas.getByLabelText(/model identifier/i), + "claude-haiku-4-5", + ); + await waitForPriceLoading(canvas); + // The catalog price must not render while the lookup is pending. + expect(canvas.queryByDisplayValue("1")).not.toBeInTheDocument(); + }, +}; + +// Clearing the identifier discards the settled live price immediately: +// until the debounce settles the section shows the loading placeholder +// rather than the previous model's prices. +export const CostEstimateClearingModelDiscardsStalePrices: Story = { + args: { + editingModel: mockGPT5, + onDeleteModel: fn(async () => undefined), + }, + parameters: { + queries: [ + { + key: aiModelPrices("openai", "gpt-5").queryKey, + data: [MockGPT5ModelPrice], + }, + ], + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await userEvent.click( + canvas.getByRole("button", { name: /cost estimate/i }), + ); + await expect(canvas.getByLabelText(/^input$/i)).toHaveValue("1.25"); + + await userEvent.clear(canvas.getByLabelText(/model identifier/i)); + await waitForPriceLoading(canvas); + expect(canvas.queryByDisplayValue("1.25")).not.toBeInTheDocument(); + }, +}; + // A price book row is the deployment's own pricing, so it wins outright. A // null category on the row means the model is unpriced there and bills as // zero, so the field stays blank instead of falling back to the catalog. @@ -602,6 +730,63 @@ export const CostEstimateCatalogOnlyWhenNotEntitled: Story = { }, }; +// Without the entitlement the live price query is disabled, so changing the +// model identifier must not flash loading placeholders: catalog prices are +// synchronously available and update immediately. +export const CostEstimateCatalogNotHiddenByDebounce: Story = { + args: { + editingModel: mockClaude, + selectedProviderState: MockAnthropicProviderState, + onDeleteModel: fn(async () => undefined), + }, + parameters: { + features: [], + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await userEvent.click( + canvas.getByRole("button", { name: /cost estimate/i }), + ); + const modelInput = canvas.getByLabelText(/model identifier/i); + await userEvent.clear(modelInput); + await userEvent.type(modelInput, "claude-haiku-4-5"); + await expect(canvas.getByLabelText(/^input$/i)).toHaveValue("1"); + expect(canvas.queryByLabelText(/price loading/i)).not.toBeInTheDocument(); + }, +}; + +// A failed lookup must not linger while the debounce settles for a new +// model: the section shows the loading placeholder until the new request +// has actually fired. +export const CostEstimateErrorClearsWhileDebouncePending: Story = { + args: { + editingModel: mockClaude, + selectedProviderState: MockAnthropicProviderState, + onDeleteModel: fn(async () => undefined), + }, + beforeEach: () => { + spyOn(API.experimental, "getAIModelPrices").mockRejectedValue( + new Error("request failed"), + ); + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await userEvent.click( + canvas.getByRole("button", { name: /cost estimate/i }), + ); + await expect( + canvas.getByText("Couldn't load pricing."), + ).toBeInTheDocument(); + + const modelInput = canvas.getByLabelText(/model identifier/i); + await userEvent.type(modelInput, "-20251001"); + await waitForPriceLoading(canvas); + expect( + canvas.queryByText("Couldn't load pricing."), + ).not.toBeInTheDocument(); + }, +}; + // While the price book lookup is in flight, the four boxes stay rendered // with a loading placeholder in each instead of a catalog price. The // catalog must not appear because the model may have a deployment override. diff --git a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/ModelConfigFields.tsx b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/ModelConfigFields.tsx index d0456815f2c7a..5a537951e684e 100644 --- a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/ModelConfigFields.tsx +++ b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/ModelConfigFields.tsx @@ -32,6 +32,7 @@ import { TooltipContent, TooltipTrigger, } from "#/components/Tooltip/Tooltip"; +import { useDebouncedValue } from "#/hooks/debounce"; import { normalizeProvider } from "#/modules/aiModels/helpers"; import { useFeatureVisibility } from "#/modules/dashboard/useFeatureVisibility"; import { cn } from "#/utils/cn"; @@ -683,14 +684,25 @@ export const PricingEstimateFields: FC<{ const aibridgeEntitled = Boolean(useFeatureVisibility().aibridge); const normalizedProvider = normalizeProvider(provider); const trimmedModel = model.trim(); + // Debounce the pair so the lookup never mixes a new provider with the + // previous model identifier. + const debouncedLookup = useDebouncedValue( + { provider: normalizedProvider, model: trimmedModel }, + 500, + ); + const entitledWithProvider = aibridgeEntitled && normalizedProvider !== ""; const livePricesQuery = useQuery({ - ...aiModelPrices(normalizedProvider, trimmedModel), - enabled: - aibridgeEntitled && normalizedProvider !== "" && trimmedModel !== "", + ...aiModelPrices(debouncedLookup.provider, debouncedLookup.model), + enabled: entitledWithProvider && debouncedLookup.model !== "", }); + const debouncePending = + entitledWithProvider && + (debouncedLookup.provider !== normalizedProvider || + debouncedLookup.model !== trimmedModel); const livePriceLoading = - livePricesQuery.fetchStatus !== "idle" && !livePricesQuery.isSuccess; - const livePrice = livePricesQuery.data?.[0]; + debouncePending || + (livePricesQuery.fetchStatus !== "idle" && !livePricesQuery.isSuccess); + const livePrice = debouncePending ? undefined : livePricesQuery.data?.[0]; const knownModel = findKnownModelByCanonicalId(normalizedProvider, trimmedModel) ?? @@ -709,7 +721,7 @@ export const PricingEstimateFields: FC<{ } : knownModel; - if (livePricesQuery.isError) { + if (!debouncePending && livePricesQuery.isError) { return (

@@ -735,13 +747,15 @@ export const PricingEstimateFields: FC<{ {priceEstimateFields.map(([label, key]) => { const cost = costs?.[key]; const fieldId = `${fieldIdPrefix}-${label.toLowerCase().replace(/\s+/g, "-")}`; - const displayValue = - cost === undefined ? "" : formatPricePerMillionTokens(cost).slice(1); + const price = + cost === undefined ? undefined : formatPricePerMillionTokens(cost); return (

- $ + + {price?.belowThreshold ? "<$" : "$"} + {livePriceLoading ? ( )} + {price?.belowThreshold && !livePriceLoading && ( + + {`less than $${price.value} USD per million tokens`} + + )} USD/1M tokens diff --git a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/index.test.ts b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/index.test.ts index b49c3567322b1..b45f9c7d7cacc 100644 --- a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/index.test.ts +++ b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/index.test.ts @@ -126,29 +126,38 @@ describe("findKnownModelByExactAlias", () => { describe("formatPricePerMillionTokens", () => { it("formats whole-dollar prices", () => { - expect(formatPricePerMillionTokens(10)).toBe("$10"); + expect(formatPricePerMillionTokens(10)).toStrictEqual({ + belowThreshold: false, + value: "10", + }); }); it("formats fractional prices without dropping precision", () => { - expect(formatPricePerMillionTokens(1.25)).toBe("$1.25"); - expect(formatPricePerMillionTokens(0.1)).toBe("$0.10"); - expect(formatPricePerMillionTokens(0.3)).toBe("$0.30"); + expect(formatPricePerMillionTokens(1.25).value).toBe("1.25"); + expect(formatPricePerMillionTokens(0.1).value).toBe("0.10"); + expect(formatPricePerMillionTokens(0.3).value).toBe("0.30"); }); it("keeps sub-cent prices visible", () => { - expect(formatPricePerMillionTokens(0.075)).toBe("$0.075"); - expect(formatPricePerMillionTokens(0.003625)).toBe("$0.0036"); - expect(formatPricePerMillionTokens(0.125)).toBe("$0.125"); + expect(formatPricePerMillionTokens(0.075).value).toBe("0.075"); + expect(formatPricePerMillionTokens(0.003625).value).toBe("0.0036"); + expect(formatPricePerMillionTokens(0.125).value).toBe("0.125"); }); - it("shows a threshold for positive prices below four decimals", () => { - expect(formatPricePerMillionTokens(0.000001)).toBe("<$0.0001"); - expect(formatPricePerMillionTokens(0.000049)).toBe("<$0.0001"); - expect(formatPricePerMillionTokens(0.0001)).toBe("$0.0001"); + it("flags positive prices below four decimals as a threshold", () => { + expect(formatPricePerMillionTokens(0.000001)).toStrictEqual({ + belowThreshold: true, + value: "0.0001", + }); + expect(formatPricePerMillionTokens(0.000049).belowThreshold).toBe(true); + expect(formatPricePerMillionTokens(0.0001)).toStrictEqual({ + belowThreshold: false, + value: "0.0001", + }); }); it("formats zero", () => { - expect(formatPricePerMillionTokens(0)).toBe("$0"); + expect(formatPricePerMillionTokens(0).value).toBe("0"); }); it("rejects non-finite values", () => { diff --git a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/index.ts b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/index.ts index 765face1b66e4..1e918138e3306 100644 --- a/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/index.ts +++ b/site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/index.ts @@ -101,21 +101,23 @@ export const formatContextBadge = (contextLimit: number): string => { return `${formatCompactNumber(contextLimit / 1_000_000)}M context`; }; -export const formatPricePerMillionTokens = (value: number): string => { +export const formatPricePerMillionTokens = ( + value: number, +): { belowThreshold: boolean; value: string } => { if (!Number.isFinite(value)) { throw new Error("price must be a finite number"); } if (Number.isInteger(value)) { - return `$${value}`; + return { belowThreshold: false, value: String(value) }; } // A positive price too small to show at four decimals would otherwise // render as $0.00 and read as free, so show it as a threshold instead. if (value > 0 && value < 0.0001) { - return "<$0.0001"; + return { belowThreshold: true, value: "0.0001" }; } - // Keep two decimals so cents read as cents ($0.10, not $0.1). Keep up to - // four so sub-cent prices stay visible ($0.075, $0.0036). + // Keep two decimals so cents read as cents (0.10, not 0.1). Keep up to + // four so sub-cent prices stay visible (0.075, 0.0036). const [whole, decimals = ""] = value.toFixed(4).split("."); const trimmed = decimals.slice(0, 2) + decimals.slice(2).replace(/0+$/, ""); - return `$${whole}.${trimmed}`; + return { belowThreshold: false, value: `${whole}.${trimmed}` }; }; diff --git a/site/src/testHelpers/chatModels.ts b/site/src/testHelpers/chatModels.ts index 0a3c021fd9bad..e1ad922911267 100644 --- a/site/src/testHelpers/chatModels.ts +++ b/site/src/testHelpers/chatModels.ts @@ -1,4 +1,5 @@ import type { + AIModelPrice, ChatModelConfig, ChatModelProvider, ChatProviderConfig, @@ -39,3 +40,24 @@ export const MockChatModelProvider: ChatModelProvider = { available: true, models: [], }; + +// Prices are micro-units per million tokens. +export const MockGPT5ModelPrice: AIModelPrice = { + provider: "openai", + model: "gpt-5", + input_price: 1250000, + output_price: 10000000, + cache_read_price: 125000, + cache_write_price: null, + created_at: MOCK_TIMESTAMP, + updated_at: MOCK_TIMESTAMP, +}; + +// An input price below $0.0001 per million tokens renders as a threshold +// rather than an exact value. +export const MockGPT5BelowThresholdModelPrice: AIModelPrice = { + ...MockGPT5ModelPrice, + input_price: 50, + output_price: null, + cache_read_price: null, +};