From 1f45d5bd02a17f51594cebfddb7dfccde5b5239c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B6=85=E8=B6=85=E8=B6=85=E8=B6=85=E8=B6=85=E7=BA=A7?= =?UTF-8?q?=E5=96=9C=E6=AC=A2=E4=BD=A0=E7=9A=84=E8=BE=BE=E5=A6=AE=E5=A8=85?= <176143450+My-Denia@users.noreply.github.com> Date: Sun, 13 Sep 2026 17:34:28 +0800 Subject: [PATCH 01/12] feat(workbench): keep versioned measurements and offline replay Measurement packages bind a baseline, record usage and replay Chat Completions and Responses transports offline so a result can be checked without repeating the live call. --- .gitattributes | 3 + biome.json | 4 + electron/ai-edition/chat-service.ts | 3 + .../ai-edition/deep-agent/chat-model.test.ts | 27 + electron/ai-edition/deep-agent/chat-model.ts | 3 + package.json | 1 + workbench/README.md | 163 +- workbench/cli.ts | 280 +- workbench/l0/baseline.wb.ts | 40 + workbench/l0/measurements.wb.ts | 623 ++++ workbench/l0/persist.wb.ts | 25 +- workbench/l0/provenance.wb.ts | 395 +++ workbench/l0/responses-wire.wb.ts | 316 ++ workbench/l0/score.wb.ts | 2 + workbench/l0/stats-report.wb.ts | 3 + workbench/l0/transport-policy.wb.ts | 354 +++ workbench/l0/wire.wb.ts | 66 +- workbench/l1/cassette.wb.ts | 338 ++- workbench/l1/measurements.wb.ts | 496 +++ workbench/l1/responses-cli.wb.ts | 113 + workbench/l1/responses-judge.wb.ts | 68 + workbench/l1/responses-record-replay.wb.ts | 386 +++ workbench/lib/baseline.ts | 20 + workbench/lib/cassette.ts | 902 ++++-- workbench/lib/env.ts | 41 +- workbench/lib/harness.ts | 7 +- workbench/lib/judge.ts | 47 +- workbench/lib/measurement.ts | 1483 +++++++++ workbench/lib/model-server.ts | 212 +- workbench/lib/persist.ts | 16 +- workbench/lib/provenance.ts | 479 +++ workbench/lib/real-fixture.ts | 38 +- workbench/lib/report.ts | 17 +- workbench/lib/runner.ts | 49 +- workbench/lib/transport.ts | 193 ++ workbench/lib/wire.ts | 159 +- workbench/measurement-cli.ts | 181 ++ workbench/measurement-entry.ts | 5 + .../input-fixture.json | 8 + .../main-cassette-rep-0.json | 60 + .../measurement-report.json | 272 ++ .../measurement.json | 2665 +++++++++++++++++ .../recorded-checks.json | 89 + .../review.json | 8 + 44 files changed, 10413 insertions(+), 247 deletions(-) create mode 100644 workbench/l0/baseline.wb.ts create mode 100644 workbench/l0/measurements.wb.ts create mode 100644 workbench/l0/provenance.wb.ts create mode 100644 workbench/l0/responses-wire.wb.ts create mode 100644 workbench/l0/transport-policy.wb.ts create mode 100644 workbench/l1/measurements.wb.ts create mode 100644 workbench/l1/responses-cli.wb.ts create mode 100644 workbench/l1/responses-judge.wb.ts create mode 100644 workbench/l1/responses-record-replay.wb.ts create mode 100644 workbench/lib/measurement.ts create mode 100644 workbench/lib/provenance.ts create mode 100644 workbench/lib/transport.ts create mode 100644 workbench/measurement-cli.ts create mode 100644 workbench/measurement-entry.ts create mode 100644 workbench/measurements/local-integrated-control-20260912-v4/input-fixture.json create mode 100644 workbench/measurements/local-integrated-control-20260912-v4/main-cassette-rep-0.json create mode 100644 workbench/measurements/local-integrated-control-20260912-v4/measurement-report.json create mode 100644 workbench/measurements/local-integrated-control-20260912-v4/measurement.json create mode 100644 workbench/measurements/local-integrated-control-20260912-v4/recorded-checks.json create mode 100644 workbench/measurements/local-integrated-control-20260912-v4/review.json diff --git a/.gitattributes b/.gitattributes index 60897224b..80a112dcd 100644 --- a/.gitattributes +++ b/.gitattributes @@ -12,3 +12,6 @@ # Same drift, caught on the native helper's build file: an edit from Windows # rewrote all 67 lines as CRLF and buried a 22-line change in a 156-line diff. CMakeLists.txt text eol=lf + +# Adopted measurement manifests hash exact bytes; keep their artifacts unchanged. +workbench/measurements/** -text diff --git a/biome.json b/biome.json index 8954e61bf..bb51ce78e 100644 --- a/biome.json +++ b/biome.json @@ -107,6 +107,10 @@ }, "javascript": { "formatter": { "quoteStyle": "double" } }, "overrides": [ + { + "includes": ["workbench/measurements/**/*.json"], + "formatter": { "enabled": false } + }, { "includes": ["*.ts", "*.tsx", "*.mts", "*.cts"], "linter": { diff --git a/electron/ai-edition/chat-service.ts b/electron/ai-edition/chat-service.ts index 20e635062..067b957ed 100644 --- a/electron/ai-edition/chat-service.ts +++ b/electron/ai-edition/chat-service.ts @@ -270,6 +270,8 @@ export interface ChatRunEnv { /** Reads recorded cursor telemetry for an asset. Built in `electron/ipc/ * handlers.ts`, where the path allow-list lives. */ cursor?: CursorTelemetryReader; + /** Runtime-only model retry override used by bounded measurement harnesses. */ + maxRetries?: number; } // ponytail: zero-config noop for sink callbacks that the caller did not provide. @@ -405,6 +407,7 @@ export async function runChat( apiKey: apiKey ?? undefined, baseUrl: config.baseUrl, reasoningEffort: config.reasoningEffort, + maxRetries: env.maxRetries, }, history, userMessage: message, diff --git a/electron/ai-edition/deep-agent/chat-model.test.ts b/electron/ai-edition/deep-agent/chat-model.test.ts index 477f5bece..2b917ca5b 100644 --- a/electron/ai-edition/deep-agent/chat-model.test.ts +++ b/electron/ai-edition/deep-agent/chat-model.test.ts @@ -91,6 +91,33 @@ describe("createOpenScreenChatModel — Anthropic-wire output budget", () => { }); }); +describe("createOpenScreenChatModel — runtime retry override", () => { + function retries(model: unknown): number | undefined { + return (model as { caller?: { maxRetries?: number } }).caller?.maxRetries; + } + + it("passes an explicit zero to LangChain", async () => { + const model = await createOpenScreenChatModel({ + provider: "openai-compatible", + model: "gpt-5-test", + apiKey: "test-key", + baseUrl: "http://127.0.0.1:1/v1", + maxRetries: 0, + }); + expect(retries(model)).toBe(0); + }); + + it("keeps LangChain's default when the override is omitted", async () => { + const model = await createOpenScreenChatModel({ + provider: "openai-compatible", + model: "gpt-5-test", + apiKey: "test-key", + baseUrl: "http://127.0.0.1:1/v1", + }); + expect(retries(model)).toBe(6); + }); +}); + describe("messageContentToText", () => { it("passes a plain string through", () => { expect(messageContentToText("hello")).toBe("hello"); diff --git a/electron/ai-edition/deep-agent/chat-model.ts b/electron/ai-edition/deep-agent/chat-model.ts index 037343262..3b57f379e 100644 --- a/electron/ai-edition/deep-agent/chat-model.ts +++ b/electron/ai-edition/deep-agent/chat-model.ts @@ -283,6 +283,8 @@ export interface OpenScreenChatModelConfig { apiKey?: string; baseUrl?: string; reasoningEffort?: string; + /** Runtime-only retry override. Omitted callers keep LangChain's default. */ + maxRetries?: number; } // ponytail: placeholder API key for self-hosted OpenAI-compatible endpoints @@ -422,6 +424,7 @@ export async function createOpenScreenChatModel( ...(reasoningOptions.reasoning ? { reasoning: reasoningOptions.reasoning } : {}), ...(reasoningOptions.useResponsesApi ? { useResponsesApi: true } : {}), ...(reasoningOptions.modelKwargs ? { modelKwargs: reasoningOptions.modelKwargs } : {}), + ...(config.maxRetries !== undefined ? { maxRetries: config.maxRetries } : {}), // ponytail: Gemini's OpenAI-compat path can't stream + tool-call at once // — disable streaming so the ChatOpenAI compat layer buffers and returns // cleanly. axcut does the same. diff --git a/package.json b/package.json index 94a1401ec..b8bec6e01 100644 --- a/package.json +++ b/package.json @@ -66,6 +66,7 @@ "wb:judge": "npm run wb:build --silent && node --env-file=.env.workbench workbench/.build/cli.cjs judge", "wb:judge:replay": "npm run wb:build --silent && node workbench/.build/cli.cjs judge --replay", "wb:compare": "npm run wb:build --silent && node workbench/.build/cli.cjs compare", + "wb:measurement": "esbuild workbench/measurement-entry.ts --bundle --platform=node --format=cjs --packages=external --outfile=workbench/.build/measurement-cli.cjs --log-level=warning && node workbench/.build/measurement-cli.cjs", "test:watch": "vitest", "test:cursor-native:win": "node scripts/test-windows-native-cursor.mjs", "test:wgc-helper:win": "node scripts/test-windows-wgc-helper.mjs", diff --git a/workbench/README.md b/workbench/README.md index 32131b47e..1ff4d8ebd 100644 --- a/workbench/README.md +++ b/workbench/README.md @@ -66,8 +66,36 @@ par `node --env-file`. Aucun parseur maison, jamais `dotenv`. OPENSCREEN_WORKBENCH_API_KEY=… OPENSCREEN_WORKBENCH_BASE_URL=… OPENSCREEN_WORKBENCH_MODEL=… +OPENSCREEN_WORKBENCH_WIRE_API=responses +OPENSCREEN_WORKBENCH_USER_AGENT=codex_exec/… +OPENSCREEN_WORKBENCH_ORIGINATOR=codex_exec ``` +`WIRE_API` vaut `chat-completions` par défaut ou `responses`. Les deux en-têtes sont facultatifs, +validés comme valeurs publiques ordinaires, et leur profil normalisé est lié à l'identité de +transport sans enregistrer la clé. En mode Responses, le modèle configuré doit réellement +sélectionner le client Responses natif ; un désaccord échoue avant tout transfert. + +Un run Responses borné nomme toujours ses deux plafonds : + +```bash +node workbench/.build/cli.cjs run --scenario target-right-clip --reps 1 \ + --label