diff --git a/.github/workflows/aigateway-prices-refresh.yaml b/.github/workflows/aigateway-prices-refresh.yaml new file mode 100644 index 0000000000000..d189145e36300 --- /dev/null +++ b/.github/workflows/aigateway-prices-refresh.yaml @@ -0,0 +1,178 @@ +# Refreshes the AI Gateway price book from live upstream data (models.dev) +# once a week and opens a pull request when the generated artifacts change. +# +# The price book seeds customer-visible cost numbers, so the refresh is never +# merged automatically. The workflow only ever proposes a change; a human +# reviews and merges it. +# +# Behavior: +# - Runs every Thursday. If regeneration produces no diff, the run ends +# without opening anything. +# - Reuses a single branch and pull request, force-pushing each week, so at +# most one refresh PR is open and it always carries the newest snapshot. +# - Fails loudly when the generator refuses to run, which it does by design +# when upstream drops a model pinned in overrides.jq or curated in +# curation.json. Failures are announced in Slack. +name: aigateway-prices-refresh + +on: + schedule: + # 09:00 UTC every Thursday, leaving three business days before Tuesday releases. + - cron: "0 9 * * 4" + workflow_dispatch: # allows manual runs for testing + +permissions: {} + +concurrency: + group: aigateway-prices-refresh + +env: + REFRESH_BRANCH: bot/aigateway-prices-refresh + PRICES_FILE: coderd/aibridge/prices/data/prices.json + CATALOG_FILE: site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.json + +jobs: + refresh: + name: Refresh price book + runs-on: ubuntu-latest + permissions: {} + steps: + - name: Harden Runner + uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1 + with: + egress-policy: audit + + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up mise tools + uses: ./.github/actions/setup-mise + with: + install-args: "go node pnpm" + + # Needed by catalog generation, which formats its output with biome. + - name: Install pnpm dependencies + uses: ./.github/actions/pnpm-install + + - name: Snapshot the current price book + run: cp "${PRICES_FILE}" "${RUNNER_TEMP}/prices-before.json" + + - name: Regenerate price book and model catalog + run: make gen/aibridge-prices + + - name: Detect changes + id: detect + run: | + set -euo pipefail + if git diff --quiet -- "${PRICES_FILE}" "${CATALOG_FILE}"; then + # exit 0 => NO differences => nothing to propose + echo "Price book already matches upstream; nothing to propose." + echo "changed=false" >> "$GITHUB_OUTPUT" + else + # exit 1 => differences found + git diff --stat -- "${PRICES_FILE}" "${CATALOG_FILE}" + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + + - name: Build pull request body + if: steps.detect.outputs.changed == 'true' + run: | + set -euo pipefail + go run ./scripts/aibridgepricesdiff \ + -old "${RUNNER_TEMP}/prices-before.json" \ + -new "${PRICES_FILE}" > "${RUNNER_TEMP}/summary.md" + + { + cat "${RUNNER_TEMP}/summary.md" + echo + echo "## Review notes" + echo + echo "Regenerated by \`make gen/aibridge-prices\` from the live [models.dev](https://models.dev) catalog. Both artifacts come from one snapshot, so they ship together:" + echo + echo "- \`${PRICES_FILE}\`" + echo "- \`${CATALOG_FILE}\`" + echo + echo "These are customer-visible cost numbers taken from upstream data, so this PR is never merged automatically. The summary above lists what moved; check the diff for exact figures before approving." + echo + echo "Opened automatically by the [aigateway-prices-refresh workflow](${RUN_URL})." + } > "${RUNNER_TEMP}/body.md" + + cat "${RUNNER_TEMP}/body.md" + env: + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + - name: Open or update the refresh pull request + if: steps.detect.outputs.changed == 'true' + env: + # Use cdrci's token instead of the default GITHUB_TOKEN: PRs opened + # with GITHUB_TOKEN do not trigger workflow runs, so the refresh + # would arrive without CI signal. + GH_TOKEN: ${{ secrets.CDRCI_GITHUB_TOKEN }} + PR_TITLE: "chore: refresh AI model price book" + run: | + set -euo pipefail + + # persist-credentials is disabled on checkout, so authenticate the + # push explicitly. + git remote set-url origin "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + + git checkout -b "${REFRESH_BRANCH}" + git add -- "${PRICES_FILE}" "${CATALOG_FILE}" + git commit -m "${PR_TITLE}" + + # Force-push: the branch is regenerated from the newest upstream + # snapshot each week, so the previous contents are always stale. + git push --force origin "refs/heads/${REFRESH_BRANCH}" + + # REST, not `gh pr`: those go through GraphQL, which needs a read:org + # scope that cdrci's token lacks. + owner="${GITHUB_REPOSITORY%%/*}" + pr_number="$(gh api "repos/${GITHUB_REPOSITORY}/pulls?state=open&base=main&head=${owner}:${REFRESH_BRANCH}" --jq '.[0].number // empty')" + if [ -n "${pr_number}" ]; then + gh api --method PATCH "repos/${GITHUB_REPOSITORY}/pulls/${pr_number}" \ + -f title="${PR_TITLE}" \ + -f body="$(cat "${RUNNER_TEMP}/body.md")" \ + --silent + echo "Updated existing PR #${pr_number}." + else + gh api --method POST "repos/${GITHUB_REPOSITORY}/pulls" \ + -f title="${PR_TITLE}" \ + -f head="${REFRESH_BRANCH}" \ + -f base=main \ + -f body="$(cat "${RUNNER_TEMP}/body.md")" \ + --jq '.html_url' + fi + + - name: Send Slack notification + if: failure() || steps.detect.outputs.changed == 'false' + env: + JOB_STATUS: ${{ job.status }} + PRICE_BOOK_CHANGED: ${{ steps.detect.outputs.changed }} + SLACK_WEBHOOK: ${{ secrets.AIGATEWAY_PRICES_SLACK_WEBHOOK }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + if [ -z "${SLACK_WEBHOOK}" ]; then + echo "::error::AIGATEWAY_PRICES_SLACK_WEBHOOK is not set; the notification could not be sent." + exit 1 + fi + + if [ "${JOB_STATUS}" = "failure" ]; then + # printf, not a double-quoted literal: bash leaves \n as two + # characters, and jq --arg then escapes the backslash, so Slack + # would print \n as text instead of breaking the line. + text="$(printf ':warning: *AI model price book refresh failed.*\nThe generator fails by design when upstream drops a model pinned in scripts/aibridgepricesgen/overrides.jq or curated in curation.json. Logs: %s' "${RUN_URL}")" + elif [ "${PRICE_BOOK_CHANGED}" = "false" ]; then + text=":white_check_mark: *AI Gateway price book refresh completed.* No generated changes were found. Run: ${RUN_URL}" + else + echo "::error::Unexpected notification state: status=${JOB_STATUS}, changed=${PRICE_BOOK_CHANGED}" + exit 1 + fi + + payload="$(jq -nc --arg text "${text}" '{text: $text}')" + curl -fsSL -X POST -H 'Content-type: application/json' -d "${payload}" "${SLACK_WEBHOOK}" + echo "Sent Slack notification" diff --git a/CODEOWNERS b/CODEOWNERS index 513013605a2e5..2c50db3b79b90 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -40,3 +40,10 @@ coderd/database/queries/ai_provider_keys.sql @ibetitsmike @johnstcn coderd/database/queries/aicostcontrol.sql @ibetitsmike @johnstcn codersdk/aiproviders.go @ibetitsmike @johnstcn codersdk/aiproviders_bedrock.go @ibetitsmike @johnstcn + +# Generated price book and frontend model catalog. The +# aigateway-prices-refresh workflow proposes updates to both from live +# models.dev data, and they carry customer-visible cost numbers, so every +# refresh needs a review from someone who owns them. +coderd/aibridge/prices/data/prices.json @evgeniy-scherbina @ssncferreira @johnstcn +site/src/pages/AgentsPage/components/ChatModelAdminPanel/knownModels/knownModelsGenerated.json @evgeniy-scherbina @ssncferreira @johnstcn diff --git a/scripts/aibridgepricesdiff/main.go b/scripts/aibridgepricesdiff/main.go new file mode 100644 index 0000000000000..c2167fb809dae --- /dev/null +++ b/scripts/aibridgepricesdiff/main.go @@ -0,0 +1,198 @@ +// aibridgepricesdiff renders a human-readable Markdown summary of the +// difference between two AI Gateway price seed files (the prices.json produced +// by aibridgepricesgen). +// +// The price refresh workflow uses it to fill the pull request body, so +// a reviewer sees at a glance which models appeared, which disappeared, and +// which repriced. Exact figures are deliberately left to the pull request +// diff, which is the source of truth. +// +// Usage: +// +// aibridgepricesdiff -old -new +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "os" + "sort" + "strings" + + "golang.org/x/xerrors" +) + +// priceRow mirrors the seed file schema written by aibridgepricesgen. Pointer +// fields preserve the distinction between "not populated by upstream" (null) +// and "explicitly zero" (0). +type priceRow struct { + Provider string `json:"provider"` + Model string `json:"model"` + InputPrice *int64 `json:"input_price"` + OutputPrice *int64 `json:"output_price"` + CacheReadPrice *int64 `json:"cache_read_price"` + CacheWritePrice *int64 `json:"cache_write_price"` +} + +// modelKey identifies a model across the two snapshots. +type modelKey struct { + provider string + model string +} + +func (r priceRow) key() modelKey { + return modelKey{provider: r.Provider, model: r.Model} +} + +func (k modelKey) String() string { + return k.provider + "/" + k.model +} + +func less(a, b modelKey) bool { + if a.provider != b.provider { + return a.provider < b.provider + } + return a.model < b.model +} + +// diff is the full comparison between two snapshots +type diff struct { + added []modelKey + removed []modelKey + changed []modelKey +} + +func (d diff) empty() bool { + return len(d.added) == 0 && len(d.removed) == 0 && len(d.changed) == 0 +} + +func main() { + oldPath := flag.String("old", "", "path to the previous prices.json (required)") + newPath := flag.String("new", "", "path to the refreshed prices.json (required)") + flag.Parse() + if err := run(*oldPath, *newPath, os.Stdout); err != nil { + _, _ = fmt.Fprintf(os.Stderr, "aibridgepricesdiff: %v\n", err) + os.Exit(1) + } +} + +func run(oldPath, newPath string, w io.Writer) error { + if oldPath == "" || newPath == "" { + return xerrors.New("-old and -new are both required") + } + oldRows, err := readRows(oldPath) + if err != nil { + return xerrors.Errorf("read %s: %w", oldPath, err) + } + newRows, err := readRows(newPath) + if err != nil { + return xerrors.Errorf("read %s: %w", newPath, err) + } + _, err = io.WriteString(w, render(compare(oldRows, newRows))) + return err +} + +func readRows(path string) ([]priceRow, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var rows []priceRow + if err := json.Unmarshal(data, &rows); err != nil { + return nil, xerrors.Errorf("parse: %w", err) + } + return rows, nil +} + +// compare classifies every model as added, removed, or changed. +func compare(oldRows, newRows []priceRow) diff { + oldByKey := make(map[modelKey]priceRow, len(oldRows)) + for _, r := range oldRows { + oldByKey[r.key()] = r + } + + var d diff + seen := make(map[modelKey]struct{}, len(newRows)) + for _, r := range newRows { + seen[r.key()] = struct{}{} + prev, ok := oldByKey[r.key()] + switch { + case !ok: + d.added = append(d.added, r.key()) + case !samePrices(prev, r): + d.changed = append(d.changed, r.key()) + } + } + for _, r := range oldRows { + if _, ok := seen[r.key()]; !ok { + d.removed = append(d.removed, r.key()) + } + } + + for _, keys := range [][]modelKey{d.added, d.removed, d.changed} { + sort.Slice(keys, func(i, j int) bool { return less(keys[i], keys[j]) }) + } + return d +} + +// samePrices reports whether two rows for the same model carry identical +// prices. A price moving to or from null counts as a change. +func samePrices(a, b priceRow) bool { + return equalPrice(a.InputPrice, b.InputPrice) && + equalPrice(a.OutputPrice, b.OutputPrice) && + equalPrice(a.CacheReadPrice, b.CacheReadPrice) && + equalPrice(a.CacheWritePrice, b.CacheWritePrice) +} + +func equalPrice(a, b *int64) bool { + if a == nil || b == nil { + return a == nil && b == nil + } + return *a == *b +} + +// render writes the Markdown summary: counts, then the models in each +// category. Empty categories are omitted. +func render(d diff) string { + var b strings.Builder + write := func(format string, args ...any) { + _, _ = fmt.Fprintf(&b, format, args...) + } + + write("## Price book changes\n\n") + if d.empty() { + write("No price changes.\n") + return b.String() + } + + write("%s added, %s removed, %s changed.\n", + plural(len(d.added), "model"), + plural(len(d.removed), "model"), + plural(len(d.changed), "model"), + ) + + renderList := func(heading string, keys []modelKey) { + if len(keys) == 0 { + return + } + write("\n
\n%s\n\n", heading) + for _, k := range keys { + write("- %s\n", k) + } + write("
\n") + } + renderList("Added", d.added) + renderList("Removed", d.removed) + renderList("Changed", d.changed) + + return b.String() +} + +func plural(n int, noun string) string { + if n == 1 { + return fmt.Sprintf("%d %s", n, noun) + } + return fmt.Sprintf("%d %ss", n, noun) +} diff --git a/scripts/aibridgepricesdiff/main_test.go b/scripts/aibridgepricesdiff/main_test.go new file mode 100644 index 0000000000000..0b334ce131dee --- /dev/null +++ b/scripts/aibridgepricesdiff/main_test.go @@ -0,0 +1,313 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCompare(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + old []priceRow + new []priceRow + wantAdded []string + wantRemoved []string + wantChanged []string + }{ + { + name: "identical", + old: []priceRow{row("anthropic", "claude", 1, 2)}, + new: []priceRow{row("anthropic", "claude", 1, 2)}, + }, + { + name: "added", + old: []priceRow{ + row("anthropic", "claude", 1, 2), + }, + new: []priceRow{ + row("anthropic", "claude", 1, 2), + row("openai", "gpt", 3, 4), + }, + wantAdded: []string{"openai/gpt"}, + }, + { + name: "removed", + old: []priceRow{ + row("anthropic", "claude", 1, 2), + row("openai", "gpt", 3, 4), + }, + new: []priceRow{ + row("anthropic", "claude", 1, 2), + }, + wantRemoved: []string{"openai/gpt"}, + }, + { + name: "changed", + old: []priceRow{row("anthropic", "claude", 1, 2)}, + new: []priceRow{row("anthropic", "claude", 1, 5)}, + wantChanged: []string{"anthropic/claude"}, + }, + { + // Every price field participates, not just input and output. + name: "cache price changed", + old: []priceRow{{ + Provider: "anthropic", + Model: "claude", + CacheReadPrice: int64Ptr(1), + }}, + new: []priceRow{{ + Provider: "anthropic", + Model: "claude", + CacheReadPrice: int64Ptr(2), + }}, + wantChanged: []string{"anthropic/claude"}, + }, + { + // A previously missing price becoming populated is a change. + name: "price set", + old: []priceRow{{ + Provider: "anthropic", + Model: "claude", + }}, + new: []priceRow{{ + Provider: "anthropic", + Model: "claude", + InputPrice: int64Ptr(1), + }}, + wantChanged: []string{"anthropic/claude"}, + }, + { + // A model whose price becomes null is a change, not a removal. + name: "price unset", + old: []priceRow{ + row("anthropic", "claude", 1, 2), + }, + new: []priceRow{{ + Provider: "anthropic", + Model: "claude", + InputPrice: int64Ptr(1), + }}, + wantChanged: []string{"anthropic/claude"}, + }, + { + // Zero is a populated price, distinct from an absent one. + name: "null becomes zero", + old: []priceRow{{ + Provider: "openai", + Model: "gpt", + }}, + new: []priceRow{{ + Provider: "openai", + Model: "gpt", + InputPrice: int64Ptr(0), + }}, + wantChanged: []string{"openai/gpt"}, + }, + { + // Zero is a real price, distinct from an absent one. + name: "zero is not null", + old: []priceRow{{ + Provider: "openai", + Model: "gpt", + InputPrice: int64Ptr(0), + }}, + new: []priceRow{{ + Provider: "openai", + Model: "gpt", + }}, + wantChanged: []string{"openai/gpt"}, + }, + { + // Same model identifier under two providers must not collide. + name: "same model different providers", + old: []priceRow{ + row("anthropic", "shared", 1, 2), + row("openai", "shared", 1, 2), + }, + new: []priceRow{ + row("anthropic", "shared", 1, 2), + row("openai", "shared", 9, 2), + }, + wantChanged: []string{"openai/shared"}, + }, + { + name: "added removed and changed together", + old: []priceRow{ + row("anthropic", "old-model", 1, 2), + row("openai", "gpt", 3, 4), + }, + new: []priceRow{ + row("anthropic", "new-model", 5, 6), + row("openai", "gpt", 3, 7), + }, + wantAdded: []string{"anthropic/new-model"}, + wantRemoved: []string{"anthropic/old-model"}, + wantChanged: []string{"openai/gpt"}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + got := compare(tc.old, tc.new) + require.Equal(t, tc.wantAdded, names(got.added)) + require.Equal(t, tc.wantRemoved, names(got.removed)) + require.Equal(t, tc.wantChanged, names(got.changed)) + }) + } +} + +func TestCompareSortsDeterministically(t *testing.T) { + t.Parallel() + + newRows := []priceRow{ + row("openai", "b", 1, 1), + row("anthropic", "z", 1, 1), + row("anthropic", "a", 1, 1), + } + + got := compare(nil, newRows) + require.Equal(t, []string{"anthropic/a", "anthropic/z", "openai/b"}, names(got.added)) +} + +func TestRender(t *testing.T) { + t.Parallel() + + t.Run("no changes", func(t *testing.T) { + t.Parallel() + + out := render(compare( + []priceRow{row("anthropic", "claude", 1, 2)}, + []priceRow{row("anthropic", "claude", 1, 2)}, + )) + require.Equal(t, `## Price book changes + +No price changes. +`, out) + }) + + t.Run("full summary", func(t *testing.T) { + t.Parallel() + + out := render(compare( + []priceRow{ + row("anthropic", "gone", 1, 2), + row("openai", "gpt", 1, 2), + }, + []priceRow{ + row("anthropic", "fresh", 3, 4), + row("openai", "gpt", 2, 3), + }, + )) + + // Asserted whole rather than by substring so section order, spacing, + // and the absence of stray content are covered too. + require.Equal(t, `## Price book changes + +1 model added, 1 model removed, 1 model changed. + +
+Added + +- anthropic/fresh +
+ +
+Removed + +- anthropic/gone +
+ +
+Changed + +- openai/gpt +
+`, out) + }) +} + +func TestRun(t *testing.T) { + t.Parallel() + + t.Run("missing flags", func(t *testing.T) { + t.Parallel() + + var out strings.Builder + require.ErrorContains(t, run("", "", &out), "-old and -new are both required") + }) + + t.Run("reads files", func(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + oldPath := filepath.Join(dir, "old.json") + newPath := filepath.Join(dir, "new.json") + writeFile(t, oldPath, `[{"provider":"openai","model":"gpt","input_price":1000000,"output_price":2000000,"cache_read_price":null,"cache_write_price":null}]`) + writeFile(t, newPath, `[{"provider":"openai","model":"gpt","input_price":1500000,"output_price":2000000,"cache_read_price":null,"cache_write_price":null}]`) + + var out strings.Builder + require.NoError(t, run(oldPath, newPath, &out)) + require.Contains(t, out.String(), "Changed\n\n- openai/gpt\n") + }) + + t.Run("invalid json", func(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + path := filepath.Join(dir, "bad.json") + writeFile(t, path, "{") + + var out strings.Builder + err := run(path, path, &out) + // The path is part of the message so a failed refresh names the file + // that could not be read. + require.ErrorContains(t, err, path) + require.ErrorContains(t, err, "parse:") + }) + + t.Run("missing file", func(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "absent.json") + + var out strings.Builder + err := run(path, path, &out) + require.ErrorIs(t, err, os.ErrNotExist) + require.ErrorContains(t, err, path) + }) +} + +func row(provider, model string, input, output int64) priceRow { + return priceRow{ + Provider: provider, + Model: model, + InputPrice: int64Ptr(input), + OutputPrice: int64Ptr(output), + } +} + +func names(keys []modelKey) []string { + if len(keys) == 0 { + return nil + } + out := make([]string, 0, len(keys)) + for _, k := range keys { + out = append(out, k.String()) + } + return out +} + +func writeFile(t *testing.T, path, content string) { + t.Helper() + require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) +} + +func int64Ptr(v int64) *int64 { return &v }