import { describe, test, expect, beforeAll, afterAll } from 'bun:test';

import { runSkillTest } from './helpers/session-runner';

import type { SkillTestResult } from './helpers/session-runner';

import { outcomeJudge, callJudge } from './helpers/llm-judge';

import { EvalCollector, judgePassed } from './helpers/eval-store';

import type { EvalTestEntry } from './helpers/eval-store';

import { startTestServer } from '../browse/test/test-server';

import { selectTests, detectBaseBranch, getChangedFiles, E2E_TOUCHFILES, GLOBAL_TOUCHFILES } from './helpers/touchfiles';

import { spawnSync } from 'child_process';

import * as fs from 'fs';

import * as path from 'path';

import * as os from 'os';

const ROOT = path.resolve(import.meta.dir, '..');

// Skip unless EVALS=1. Session runner strips CLAUDE* env vars to avoid nested session issues.

//

// BLAME PROTOCOL: When an eval fails, do NOT claim "pre-existing" or "not related

// to our changes" without proof. Run the same eval on main to verify. These tests

// have invisible couplings — preamble text, SKILL.md content, and timing all affect

// agent behavior. See CLAUDE.md "E2E eval failure blame protocol" for details.

const evalsEnabled = !!process.env.EVALS;

const describeE2E = evalsEnabled ? describe : describe.skip;

// --- Diff-based test selection ---

// When EVALS_ALL is not set, only run tests whose touchfiles were modified.

// Set EVALS_ALL=1 to force all tests. Set EVALS_BASE to override base branch.

let selectedTests: string[] | null = null; // null = run all

if (evalsEnabled && !process.env.EVALS_ALL) {

const baseBranch = process.env.EVALS_BASE

|| detectBaseBranch(ROOT)

|| 'main';

const changedFiles = getChangedFiles(baseBranch, ROOT);

if (changedFiles.length > 0) {

const selection = selectTests(changedFiles, E2E_TOUCHFILES, GLOBAL_TOUCHFILES);

selectedTests = selection.selected;

process.stderr.write(`\nE2E selection (${selection.reason}): ${selection.selected.length}/${Object.keys(E2E_TOUCHFILES).length} tests\n`);

if (selection.skipped.length > 0) {

process.stderr.write(` Skipped: ${selection.skipped.join(', ')}\n`);

}

process.stderr.write('\n');

}

// If changedFiles is empty (e.g., on main branch), selectedTests stays null → run all

}

/** Wrap a describe block to skip entirely if none of its tests are selected. */

function describeIfSelected(name: string, testNames: string[], fn: () => void) {

const anySelected = selectedTests === null || testNames.some(t => selectedTests!.includes(t));

(anySelected ? describeE2E : describe.skip)(name, fn);

}

/** Skip an individual test if not selected (for multi-test describe blocks). */

function testIfSelected(testName: string, fn: () => Promise<void>, timeout: number) {

const shouldRun = selectedTests === null || selectedTests.includes(testName);

(shouldRun ? test : test.skip)(testName, fn, timeout);

}

// Eval result collector — accumulates test results, writes to ~/.gstack-dev/evals/ on finalize

const evalCollector = evalsEnabled ? new EvalCollector('e2e') : null;

// Unique run ID for this E2E session — used for heartbeat + per-run log directory

const runId = new Date().toISOString().replace(/[:.]/g, '').replace('T', '-').slice(0, 15);

/** DRY helper to record an E2E test result into the eval collector. */

function recordE2E(name: string, suite: string, result: SkillTestResult, extra?: Partial<EvalTestEntry>) {

// Derive last tool call from transcript for machine-readable diagnostics

const lastTool = result.toolCalls.length > 0

? `${result.toolCalls[result.toolCalls.length - 1].tool}(${JSON.stringify(result.toolCalls[result.toolCalls.length - 1].input).slice(0, 60)})`

: undefined;

evalCollector?.addTest({

name, suite, tier: 'e2e',

passed: result.exitReason === 'success' && result.browseErrors.length === 0,

duration_ms: result.duration,

cost_usd: result.costEstimate.estimatedCost,

transcript: result.transcript,

output: result.output?.slice(0, 2000),

turns_used: result.costEstimate.turnsUsed,

browse_errors: result.browseErrors,

exit_reason: result.exitReason,

timeout_at_turn: result.exitReason === 'timeout' ? result.costEstimate.turnsUsed : undefined,

last_tool_call: lastTool,

...extra,

});

}

let testServer: ReturnType<typeof startTestServer>;

let tmpDir: string;

const browseBin = path.resolve(ROOT, 'browse', 'dist', 'browse');

/**

* Copy a directory tree recursively (files only, follows structure).

*/

function copyDirSync(src: string, dest: string) {

fs.mkdirSync(dest, { recursive: true });

for (const entry of fs.readdirSync(src, { withFileTypes: true })) {

const srcPath = path.join(src, entry.name);

const destPath = path.join(dest, entry.name);

if (entry.isDirectory()) {

copyDirSync(srcPath, destPath);

} else {

fs.copyFileSync(srcPath, destPath);

}

}

}

/**

* Set up browse shims (binary symlink, find-browse, remote-slug) in a tmpDir.

*/

function setupBrowseShims(dir: string) {

// Symlink browse binary

const binDir = path.join(dir, 'browse', 'dist');

fs.mkdirSync(binDir, { recursive: true });

if (fs.existsSync(browseBin)) {

fs.symlinkSync(browseBin, path.join(binDir, 'browse'));

}

// find-browse shim

const findBrowseDir = path.join(dir, 'browse', 'bin');

fs.mkdirSync(findBrowseDir, { recursive: true });

fs.writeFileSync(

path.join(findBrowseDir, 'find-browse'),

`#!/bin/bash\necho "${browseBin}"\n`,

{ mode: 0o755 },

);

// remote-slug shim (returns test-project)

fs.writeFileSync(

path.join(findBrowseDir, 'remote-slug'),

`#!/bin/bash\necho "test-project"\n`,

{ mode: 0o755 },

);

}

/**

* Print cost summary after an E2E test.

*/

function logCost(label: string, result: { costEstimate: { turnsUsed: number; estimatedTokens: number; estimatedCost: number }; duration: number }) {

const { turnsUsed, estimatedTokens, estimatedCost } = result.costEstimate;

const durationSec = Math.round(result.duration / 1000);

console.log(`${label}: $${estimatedCost.toFixed(2)} (${turnsUsed} turns, ${(estimatedTokens / 1000).toFixed(1)}k tokens, ${durationSec}s)`);

}

/**

* Dump diagnostic info on planted-bug outcome failure (decision 1C).

*/

function dumpOutcomeDiagnostic(dir: string, label: string, report: string, judgeResult: any) {

try {

const transcriptDir = path.join(dir, '.gstack', 'test-transcripts');

fs.mkdirSync(transcriptDir, { recursive: true });

const timestamp = new Date().toISOString().replace(/[:.]/g, '-');

fs.writeFileSync(

path.join(transcriptDir, `${label}-outcome-${timestamp}.json`),

JSON.stringify({ label, report, judgeResult }, null, 2),

);

} catch { /* non-fatal */ }

}

// Fail fast if Anthropic API is unreachable — don't burn through 13 tests getting ConnectionRefused

if (evalsEnabled) {

const check = spawnSync('sh', ['-c', 'echo "ping" | claude -p --max-turns 1 --output-format stream-json --verbose --dangerously-skip-permissions'], {

stdio: 'pipe', timeout: 30_000,

});

const output = check.stdout?.toString() || '';

if (output.includes('ConnectionRefused') || output.includes('Unable to connect')) {

throw new Error('Anthropic API unreachable — aborting E2E suite. Fix connectivity and retry.');

}

}

describeIfSelected('Skill E2E tests', [

'browse-basic', 'browse-snapshot', 'skillmd-setup-discovery',

'skillmd-no-local-binary', 'skillmd-outside-git', 'contributor-mode', 'session-awareness',

], () => {

beforeAll(() => {

testServer = startTestServer();

tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-'));

setupBrowseShims(tmpDir);

});

afterAll(() => {

testServer?.server?.stop();

try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch {}

});

testIfSelected('browse-basic', async () => {

const result = await runSkillTest({

prompt: `You have a browse binary at ${browseBin}. Assign it to B variable and run these commands in sequence:

1. $B goto ${testServer.url}

2. $B snapshot -i

3. $B text

4. $B screenshot /tmp/skill-e2e-test.png

Report the results of each command.`,

workingDirectory: tmpDir,

maxTurns: 10,

timeout: 60_000,

testName: 'browse-basic',

runId,

});

logCost('browse basic', result);

recordE2E('browse basic commands', 'Skill E2E tests', result);

expect(result.browseErrors).toHaveLength(0);

expect(result.exitReason).toBe('success');

}, 90_000);

testIfSelected('browse-snapshot', async () => {

const result = await runSkillTest({

prompt: `You have a browse binary at ${browseBin}. Assign it to B variable and run:

1. $B goto ${testServer.url}

2. $B snapshot -i

3. $B snapshot -c

4. $B snapshot -D

5. $B snapshot -i -a -o /tmp/skill-e2e-annotated.png

Report what each command returned.`,

workingDirectory: tmpDir,

maxTurns: 10,

timeout: 60_000,

testName: 'browse-snapshot',

runId,

});

logCost('browse snapshot', result);

recordE2E('browse snapshot flags', 'Skill E2E tests', result);

// browseErrors can include false positives from hallucinated paths (e.g. "baltimore" vs "bangalore")

if (result.browseErrors.length > 0) {

console.warn('Browse errors (non-fatal):', result.browseErrors);

}

expect(result.exitReason).toBe('success');

}, 90_000);

testIfSelected('skillmd-setup-discovery', async () => {

const skillMd = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');

const setupStart = skillMd.indexOf('## SETUP');

const setupEnd = skillMd.indexOf('## IMPORTANT');

const setupBlock = skillMd.slice(setupStart, setupEnd);

// Guard: verify we extracted a valid setup block

expect(setupBlock).toContain('browse/dist/browse');

const result = await runSkillTest({

prompt: `Follow these instructions to find the browse binary and run a basic command.

${setupBlock}

After finding the binary, run: $B goto ${testServer.url}

Then run: $B text

Report whether it worked.`,

workingDirectory: tmpDir,

maxTurns: 10,

timeout: 60_000,

testName: 'skillmd-setup-discovery',

runId,

});

recordE2E('SKILL.md setup block discovery', 'Skill E2E tests', result);

expect(result.browseErrors).toHaveLength(0);

expect(result.exitReason).toBe('success');

}, 90_000);

testIfSelected('skillmd-no-local-binary', async () => {

// Create a tmpdir with no browse binary — no local .claude/skills/gstack/browse/dist/browse

const emptyDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-empty-'));

const skillMd = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');

const setupStart = skillMd.indexOf('## SETUP');

const setupEnd = skillMd.indexOf('## IMPORTANT');

const setupBlock = skillMd.slice(setupStart, setupEnd);

const result = await runSkillTest({

prompt: `Follow these instructions exactly. Run the bash code block below and report what it outputs.

${setupBlock}

Report the exact output. Do NOT try to fix or install anything — just report what you see.`,

workingDirectory: emptyDir,

maxTurns: 5,

timeout: 30_000,

testName: 'skillmd-no-local-binary',

runId,

});

// Setup block should either find the global binary (READY) or show NEEDS_SETUP.

// On dev machines with gstack installed globally, the fallback path

// ~/.claude/skills/gstack/browse/dist/browse exists, so we get READY.

// The important thing is it doesn't crash or give a confusing error.

const allText = result.output || '';

recordE2E('SKILL.md setup block (no local binary)', 'Skill E2E tests', result);

expect(allText).toMatch(/READY|NEEDS_SETUP/);

expect(result.exitReason).toBe('success');

// Clean up

try { fs.rmSync(emptyDir, { recursive: true, force: true }); } catch {}

}, 60_000);

testIfSelected('skillmd-outside-git', async () => {

// Create a tmpdir outside any git repo

const nonGitDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-nogit-'));

const skillMd = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');

const setupStart = skillMd.indexOf('## SETUP');

const setupEnd = skillMd.indexOf('## IMPORTANT');

const setupBlock = skillMd.slice(setupStart, setupEnd);

const result = await runSkillTest({

prompt: `Follow these instructions exactly. Run the bash code block below and report what it outputs.

${setupBlock}

Report the exact output — either "READY: <path>" or "NEEDS_SETUP".`,

workingDirectory: nonGitDir,

maxTurns: 5,

timeout: 30_000,

testName: 'skillmd-outside-git',

runId,

});

// Should either find global binary (READY) or show NEEDS_SETUP — not crash

const allText = result.output || '';

recordE2E('SKILL.md outside git repo', 'Skill E2E tests', result);

expect(allText).toMatch(/READY|NEEDS_SETUP/);

// Clean up

try { fs.rmSync(nonGitDir, { recursive: true, force: true }); } catch {}

}, 60_000);

testIfSelected('contributor-mode', async () => {

const contribDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-contrib-'));

const logsDir = path.join(contribDir, 'contributor-logs');

fs.mkdirSync(logsDir, { recursive: true });

// Extract contributor mode instructions from generated SKILL.md

const skillMd = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');

const contribStart = skillMd.indexOf('## Contributor Mode');

const contribEnd = skillMd.indexOf('\n## ', contribStart + 1);

const contribBlock = skillMd.slice(contribStart, contribEnd > 0 ? contribEnd : undefined);

const result = await runSkillTest({

prompt: `You are in contributor mode (_CONTRIB=true).

${contribBlock}

OVERRIDE: Write contributor logs to ${logsDir}/ instead of ~/.gstack/contributor-logs/

Now try this browse command (it will fail — there is no binary at this path):

/nonexistent/path/browse goto https://example.com

This is a gstack issue (the browse binary is missing/misconfigured).

File a contributor report about this issue. Then tell me what you filed.`,

workingDirectory: contribDir,

maxTurns: 8,

timeout: 60_000,

testName: 'contributor-mode',

runId,

});

logCost('contributor mode', result);

// Override passed: this test intentionally triggers a browse error (nonexistent binary)

// so browseErrors will be non-empty — that's expected, not a failure

recordE2E('contributor mode report', 'Skill E2E tests', result, {

passed: result.exitReason === 'success',

});

// Verify a contributor log was created with expected format

const logFiles = fs.readdirSync(logsDir).filter(f => f.endsWith('.md'));

expect(logFiles.length).toBeGreaterThan(0);

// Verify new reflection-based format

const logContent = fs.readFileSync(path.join(logsDir, logFiles[0]), 'utf-8');

expect(logContent).toContain('Hey gstack team');

expect(logContent).toContain('What I was trying to do');

expect(logContent).toContain('What happened instead');

expect(logContent).toMatch(/rating/i);

// Verify report has repro steps (agent may use "Steps to reproduce", "Repro Steps", etc.)

expect(logContent).toMatch(/repro|steps to reproduce|how to reproduce/i);

// Verify report has date/version footer (agent may format differently)

expect(logContent).toMatch(/date.*2026|2026.*date/i);

// Clean up

try { fs.rmSync(contribDir, { recursive: true, force: true }); } catch {}

}, 90_000);

testIfSelected('session-awareness', async () => {

const sessionDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-session-'));

// Set up a git repo so there's project/branch context to reference

const run = (cmd: string, args: string[]) =>

spawnSync(cmd, args, { cwd: sessionDir, stdio: 'pipe', timeout: 5000 });

run('git', ['init', '-b', 'main']);

run('git', ['config', 'user.email', '[email protected]']);

run('git', ['config', 'user.name', 'Test']);

fs.writeFileSync(path.join(sessionDir, 'app.rb'), '# my app\n');

run('git', ['add', '.']);

run('git', ['commit', '-m', 'init']);

run('git', ['checkout', '-b', 'feature/add-payments']);

// Add a remote so the agent can derive a project name

run('git', ['remote', 'add', 'origin', 'https://github.com/acme/billing-app.git']);

// Extract AskUserQuestion format instructions from generated SKILL.md

const skillMd = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');

const aqStart = skillMd.indexOf('## AskUserQuestion Format');

const aqEnd = skillMd.indexOf('\n## ', aqStart + 1);

const aqBlock = skillMd.slice(aqStart, aqEnd > 0 ? aqEnd : undefined);

const outputPath = path.join(sessionDir, 'question-output.md');

const result = await runSkillTest({

prompt: `You are running a gstack skill. The session preamble detected _SESSIONS=4 (the user has 4 gstack windows open).

${aqBlock}

You are on branch feature/add-payments in the billing-app project. You were reviewing a plan to add Stripe integration.

You've hit a decision point: the plan doesn't specify whether to use Stripe Checkout (hosted) or Stripe Elements (embedded). You need to ask the user which approach to use.

Since this is non-interactive, DO NOT actually call AskUserQuestion. Instead, write the EXACT text you would display to the user (the full AskUserQuestion content) to the file: ${outputPath}

Remember: _SESSIONS=4, so ELI16 mode is active. The user is juggling multiple windows and may not remember what this conversation is about. Re-ground them.`,

workingDirectory: sessionDir,

maxTurns: 8,

timeout: 60_000,

testName: 'session-awareness',

runId,

});

logCost('session awareness', result);

recordE2E('session awareness ELI16', 'Skill E2E tests', result);

// Verify the output contains ELI16 re-grounding context

if (fs.existsSync(outputPath)) {

const output = fs.readFileSync(outputPath, 'utf-8');

const lower = output.toLowerCase();

// Must mention project name

expect(lower.includes('billing') || lower.includes('acme')).toBe(true);

// Must mention branch

expect(lower.includes('payment') || lower.includes('feature')).toBe(true);

// Must mention what we're working on

expect(lower.includes('stripe') || lower.includes('checkout') || lower.includes('payment')).toBe(true);

// Must have a RECOMMENDATION

expect(output).toContain('RECOMMENDATION');

} else {

// Check agent output as fallback

const output = result.output || '';

expect(output).toContain('RECOMMENDATION');

}

// Clean up

try { fs.rmSync(sessionDir, { recursive: true, force: true }); } catch {}

}, 90_000);

});

// --- B4: QA skill E2E ---

describeIfSelected('QA skill E2E', ['qa-quick'], () => {

let qaDir: string;

beforeAll(() => {

testServer = testServer || startTestServer();

qaDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-qa-'));

setupBrowseShims(qaDir);

// Copy qa skill files into tmpDir

copyDirSync(path.join(ROOT, 'qa'), path.join(qaDir, 'qa'));

// Create report directory

fs.mkdirSync(path.join(qaDir, 'qa-reports'), { recursive: true });

});

afterAll(() => {

testServer?.server?.stop();

try { fs.rmSync(qaDir, { recursive: true, force: true }); } catch {}

});

test('/qa quick completes without browse errors', async () => {

const result = await runSkillTest({

prompt: `B="${browseBin}"

The test server is already running at: ${testServer.url}

Target page: ${testServer.url}/basic.html

Read the file qa/SKILL.md for the QA workflow instructions.

Run a Quick-depth QA test on ${testServer.url}/basic.html

Do NOT use AskUserQuestion — run Quick tier directly.

Do NOT try to start a server or discover ports — the URL above is ready.

Write your report to ${qaDir}/qa-reports/qa-report.md`,

workingDirectory: qaDir,

maxTurns: 35,

timeout: 240_000,

testName: 'qa-quick',

runId,

});

logCost('/qa quick', result);

recordE2E('/qa quick', 'QA skill E2E', result, {

passed: ['success', 'error_max_turns'].includes(result.exitReason),

});

// browseErrors can include false positives from hallucinated paths

if (result.browseErrors.length > 0) {

console.warn('/qa quick browse errors (non-fatal):', result.browseErrors);

}

// Accept error_max_turns — the agent doing thorough QA work is not a failure

expect(['success', 'error_max_turns']).toContain(result.exitReason);

}, 300_000);

});

// --- B5: Review skill E2E ---

describeIfSelected('Review skill E2E', ['review-sql-injection'], () => {

let reviewDir: string;

beforeAll(() => {

reviewDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-review-'));

// Pre-build a git repo with a vulnerable file on a feature branch (decision 5A)

const { spawnSync } = require('child_process');

const run = (cmd: string, args: string[]) =>

spawnSync(cmd, args, { cwd: reviewDir, stdio: 'pipe', timeout: 5000 });

run('git', ['init', '-b', 'main']);

run('git', ['config', 'user.email', '[email protected]']);

run('git', ['config', 'user.name', 'Test']);

// Commit a clean base on main

fs.writeFileSync(path.join(reviewDir, 'app.rb'), '# clean base\nclass App\nend\n');

run('git', ['add', 'app.rb']);

run('git', ['commit', '-m', 'initial commit']);

// Create feature branch with vulnerable code

run('git', ['checkout', '-b', 'feature/add-user-controller']);

const vulnContent = fs.readFileSync(path.join(ROOT, 'test', 'fixtures', 'review-eval-vuln.rb'), 'utf-8');

fs.writeFileSync(path.join(reviewDir, 'user_controller.rb'), vulnContent);

run('git', ['add', 'user_controller.rb']);

run('git', ['commit', '-m', 'add user controller']);

// Copy review skill files

fs.copyFileSync(path.join(ROOT, 'review', 'SKILL.md'), path.join(reviewDir, 'review-SKILL.md'));

fs.copyFileSync(path.join(ROOT, 'review', 'checklist.md'), path.join(reviewDir, 'review-checklist.md'));

fs.copyFileSync(path.join(ROOT, 'review', 'greptile-triage.md'), path.join(reviewDir, 'review-greptile-triage.md'));

});

afterAll(() => {

try { fs.rmSync(reviewDir, { recursive: true, force: true }); } catch {}

});

test('/review produces findings on SQL injection branch', async () => {

const result = await runSkillTest({

prompt: `You are in a git repo on a feature branch with changes against main.

Read review-SKILL.md for the review workflow instructions.

Also read review-checklist.md and apply it.

Run /review on the current diff (git diff main...HEAD).

Write your review findings to ${reviewDir}/review-output.md`,

workingDirectory: reviewDir,

maxTurns: 15,

timeout: 90_000,

testName: 'review-sql-injection',

runId,

});

logCost('/review', result);

recordE2E('/review SQL injection', 'Review skill E2E', result);

expect(result.exitReason).toBe('success');

}, 120_000);

});

// --- Review: Enum completeness E2E ---

describeIfSelected('Review enum completeness E2E', ['review-enum-completeness'], () => {

let enumDir: string;

beforeAll(() => {

enumDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-enum-'));

const run = (cmd: string, args: string[]) =>

spawnSync(cmd, args, { cwd: enumDir, stdio: 'pipe', timeout: 5000 });

run('git', ['init', '-b', 'main']);

run('git', ['config', 'user.email', '[email protected]']);

run('git', ['config', 'user.name', 'Test']);

// Commit baseline on main — order model with 4 statuses

const baseContent = fs.readFileSync(path.join(ROOT, 'test', 'fixtures', 'review-eval-enum.rb'), 'utf-8');

fs.writeFileSync(path.join(enumDir, 'order.rb'), baseContent);

run('git', ['add', 'order.rb']);

run('git', ['commit', '-m', 'initial order model']);

// Feature branch adds "returned" status but misses handlers

run('git', ['checkout', '-b', 'feature/add-returned-status']);

const diffContent = fs.readFileSync(path.join(ROOT, 'test', 'fixtures', 'review-eval-enum-diff.rb'), 'utf-8');

fs.writeFileSync(path.join(enumDir, 'order.rb'), diffContent);

run('git', ['add', 'order.rb']);

run('git', ['commit', '-m', 'add returned status']);

// Copy review skill files

fs.copyFileSync(path.join(ROOT, 'review', 'SKILL.md'), path.join(enumDir, 'review-SKILL.md'));

fs.copyFileSync(path.join(ROOT, 'review', 'checklist.md'), path.join(enumDir, 'review-checklist.md'));

fs.copyFileSync(path.join(ROOT, 'review', 'greptile-triage.md'), path.join(enumDir, 'review-greptile-triage.md'));

});

afterAll(() => {

try { fs.rmSync(enumDir, { recursive: true, force: true }); } catch {}

});

test('/review catches missing enum handlers for new status value', async () => {

const result = await runSkillTest({

prompt: `You are in a git repo on branch feature/add-returned-status with changes against main.

Read review-SKILL.md for the review workflow instructions.

Also read review-checklist.md and apply it — pay special attention to the Enum & Value Completeness section.

Run /review on the current diff (git diff main...HEAD).

Write your review findings to ${enumDir}/review-output.md

The diff adds a new "returned" status to the Order model. Your job is to check if all consumers handle it.`,

workingDirectory: enumDir,

maxTurns: 15,

timeout: 90_000,

testName: 'review-enum-completeness',

runId,

});

logCost('/review enum', result);

recordE2E('/review enum completeness', 'Review enum completeness E2E', result);

expect(result.exitReason).toBe('success');

// Verify the review caught the missing enum handlers

const reviewPath = path.join(enumDir, 'review-output.md');

if (fs.existsSync(reviewPath)) {

const review = fs.readFileSync(reviewPath, 'utf-8');

// Should mention the missing "returned" handling in at least one of the methods

const mentionsReturned = review.toLowerCase().includes('returned');

const mentionsEnum = review.toLowerCase().includes('enum') || review.toLowerCase().includes('status');

const mentionsCritical = review.toLowerCase().includes('critical');

expect(mentionsReturned).toBe(true);

expect(mentionsEnum || mentionsCritical).toBe(true);

}

}, 120_000);

});

// --- Review: Design review lite E2E ---

describeE2E('Review design lite E2E', () => {

let designDir: string;

beforeAll(() => {

designDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-design-lite-'));

const run = (cmd: string, args: string[]) =>

spawnSync(cmd, args, { cwd: designDir, stdio: 'pipe', timeout: 5000 });

run('git', ['init', '-b', 'main']);

run('git', ['config', 'user.email', '[email protected]']);

run('git', ['config', 'user.name', 'Test']);

// Commit clean base on main

fs.writeFileSync(path.join(designDir, 'index.html'), '<h1>Clean</h1>\n');

fs.writeFileSync(path.join(designDir, 'styles.css'), 'body { font-size: 16px; }\n');

run('git', ['add', '.']);

run('git', ['commit', '-m', 'initial']);

// Feature branch adds AI slop CSS + HTML

run('git', ['checkout', '-b', 'feature/add-landing-page']);

const slopCss = fs.readFileSync(path.join(ROOT, 'test', 'fixtures', 'review-eval-design-slop.css'), 'utf-8');

const slopHtml = fs.readFileSync(path.join(ROOT, 'test', 'fixtures', 'review-eval-design-slop.html'), 'utf-8');

fs.writeFileSync(path.join(designDir, 'styles.css'), slopCss);

fs.writeFileSync(path.join(designDir, 'landing.html'), slopHtml);

run('git', ['add', '.']);

run('git', ['commit', '-m', 'add landing page']);

// Copy review skill files

fs.copyFileSync(path.join(ROOT, 'review', 'SKILL.md'), path.join(designDir, 'review-SKILL.md'));

fs.copyFileSync(path.join(ROOT, 'review', 'checklist.md'), path.join(designDir, 'review-checklist.md'));

fs.copyFileSync(path.join(ROOT, 'review', 'design-checklist.md'), path.join(designDir, 'review-design-checklist.md'));

fs.copyFileSync(path.join(ROOT, 'review', 'greptile-triage.md'), path.join(designDir, 'review-greptile-triage.md'));

});

afterAll(() => {

try { fs.rmSync(designDir, { recursive: true, force: true }); } catch {}

});

test('/review catches design anti-patterns in CSS/HTML diff', async () => {

const result = await runSkillTest({

prompt: `You are in a git repo on branch feature/add-landing-page with changes against main.

Read review-SKILL.md for the review workflow instructions.

Read review-checklist.md for the code review checklist.

Read review-design-checklist.md for the design review checklist.

Run /review on the current diff (git diff main...HEAD).

The diff adds a landing page with CSS and HTML. Check for both code issues AND design anti-patterns.

Write your review findings to ${designDir}/review-output.md

Important: The design checklist should catch issues like blacklisted fonts, small font sizes, outline:none, !important, AI slop patterns (purple gradients, generic hero copy, 3-column feature grid), etc.`,

workingDirectory: designDir,

maxTurns: 15,

timeout: 120_000,

testName: 'review-design-lite',

runId,

});

logCost('/review design lite', result);

recordE2E('/review design lite', 'Review design lite E2E', result);

expect(result.exitReason).toBe('success');

// Verify the review caught at least 4 of 7 planted design issues

const reviewPath = path.join(designDir, 'review-output.md');

if (fs.existsSync(reviewPath)) {

const review = fs.readFileSync(reviewPath, 'utf-8').toLowerCase();

let detected = 0;

// Issue 1: Blacklisted font (Papyrus) — HIGH

if (review.includes('papyrus') || review.includes('blacklisted font') || review.includes('font family')) detected++;

// Issue 2: Body text < 16px — HIGH

if (review.includes('14px') || review.includes('font-size') || review.includes('font size') || review.includes('body text')) detected++;

// Issue 3: outline: none — HIGH

if (review.includes('outline') || review.includes('focus')) detected++;

// Issue 4: !important — HIGH

if (review.includes('!important') || review.includes('important')) detected++;

// Issue 5: Purple gradient — MEDIUM

if (review.includes('gradient') || review.includes('purple') || review.includes('violet') || review.includes('#6366f1') || review.includes('#8b5cf6')) detected++;

// Issue 6: Generic hero copy — MEDIUM

if (review.includes('welcome to') || review.includes('all-in-one') || review.includes('generic') || review.includes('hero copy') || review.includes('ai slop')) detected++;

// Issue 7: 3-column feature grid — LOW

if (review.includes('3-column') || review.includes('three-column') || review.includes('feature grid') || review.includes('icon') || review.includes('circle')) detected++;

console.log(`Design review detected ${detected}/7 planted issues`);

expect(detected).toBeGreaterThanOrEqual(4);

}

}, 150_000);

});

// --- B6/B7/B8: Planted-bug outcome evals ---

// Outcome evals also need ANTHROPIC_API_KEY for the LLM judge

const hasApiKey = !!process.env.ANTHROPIC_API_KEY;

const describeOutcome = (evalsEnabled && hasApiKey) ? describe : describe.skip;

// Wrap describeOutcome with selection — skip if no planted-bug tests are selected

const outcomeTestNames = ['qa-b6-static', 'qa-b7-spa', 'qa-b8-checkout'];

const anyOutcomeSelected = selectedTests === null || outcomeTestNames.some(t => selectedTests!.includes(t));

(anyOutcomeSelected ? describeOutcome : describe.skip)('Planted-bug outcome evals', () => {

let outcomeDir: string;

beforeAll(() => {

// Always start fresh — previous tests' agents may have killed the shared server

try { testServer?.server?.stop(); } catch {}

testServer = startTestServer();

outcomeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-outcome-'));

setupBrowseShims(outcomeDir);

// Copy qa skill files

copyDirSync(path.join(ROOT, 'qa'), path.join(outcomeDir, 'qa'));

});

afterAll(() => {

testServer?.server?.stop();

try { fs.rmSync(outcomeDir, { recursive: true, force: true }); } catch {}

});

/**

* Shared planted-bug eval runner.

* Gives the agent concise bug-finding instructions (not the full QA workflow),

* then scores the report with an LLM outcome judge.

*/

async function runPlantedBugEval(fixture: string, groundTruthFile: string, label: string) {

// Each test gets its own isolated working directory to prevent cross-contamination

// (agents reading previous tests' reports and hallucinating those bugs)

const testWorkDir = fs.mkdtempSync(path.join(os.tmpdir(), `skill-e2e-${label}-`));

setupBrowseShims(testWorkDir);

const reportDir = path.join(testWorkDir, 'reports');

fs.mkdirSync(path.join(reportDir, 'screenshots'), { recursive: true });

const reportPath = path.join(reportDir, 'qa-report.md');

// Direct bug-finding with browse. Keep prompt concise — no reading long SKILL.md docs.

// "Write early, update later" pattern ensures report exists even if agent hits max turns.

const targetUrl = `${testServer.url}/${fixture}`;

const result = await runSkillTest({

prompt: `Find bugs on this page: ${targetUrl}

Browser binary: B="${browseBin}"

PHASE 1 — Quick scan (5 commands max):

$B goto ${targetUrl}

$B console --errors

$B snapshot -i

$B snapshot -c

$B accessibility

PHASE 2 — Write initial report to ${reportPath}:

Write every bug you found so far. Format each as:

- Category: functional / visual / accessibility / console

- Severity: high / medium / low

- Evidence: what you observed

PHASE 3 — Interactive testing (targeted — max 15 commands):

- Test email: type "user@" (no domain) and blur — does it validate?

- Test quantity: clear the field entirely — check the total display

- Test credit card: type a 25-character string — check for overflow

- Submit the form with zip code empty — does it require zip?

- Submit a valid form and run $B console --errors

- After finding more bugs, UPDATE ${reportPath} with new findings

PHASE 4 — Finalize report:

- UPDATE ${reportPath} with ALL bugs found across all phases

- Include console errors, form validation issues, visual overflow, missing attributes

CRITICAL RULES:

- ONLY test the page at ${targetUrl} — do not navigate to other sites

- Write the report file in PHASE 2 before doing interactive testing

- The report MUST exist at ${reportPath} when you finish`,

workingDirectory: testWorkDir,

maxTurns: 50,

timeout: 300_000,

testName: `qa-${label}`,

runId,

});

logCost(`/qa ${label}`, result);

// Phase 1: browse mechanics. Accept error_max_turns — agent may have written

// a partial report before running out of turns. What matters is detection rate.

if (result.browseErrors.length > 0) {

console.warn(`${label} browse errors:`, result.browseErrors);

}

if (result.exitReason !== 'success' && result.exitReason !== 'error_max_turns') {

throw new Error(`${label}: unexpected exit reason: ${result.exitReason}`);

}

// Phase 2: Outcome evaluation via LLM judge

const groundTruth = JSON.parse(

fs.readFileSync(path.join(ROOT, 'test', 'fixtures', groundTruthFile), 'utf-8'),

);

// Read the generated report (try expected path, then glob for any .md in reportDir or workDir)

let report: string | null = null;

if (fs.existsSync(reportPath)) {

report = fs.readFileSync(reportPath, 'utf-8');

} else {

// Agent may have named it differently — find any .md in reportDir or testWorkDir

for (const searchDir of [reportDir, testWorkDir]) {

try {

const mdFiles = fs.readdirSync(searchDir).filter(f => f.endsWith('.md'));

if (mdFiles.length > 0) {

report = fs.readFileSync(path.join(searchDir, mdFiles[0]), 'utf-8');

break;

}

} catch { /* dir may not exist if agent hit max_turns early */ }

}

// Also check the agent's final output for inline report content

if (!report && result.output && result.output.length > 100) {

report = result.output;

}

}

if (!report) {

dumpOutcomeDiagnostic(testWorkDir, label, '(no report file found)', { error: 'missing report' });

recordE2E(`/qa ${label}`, 'Planted-bug outcome evals', result, { error: 'no report generated' });

throw new Error(`No report file found in ${reportDir}`);

}

const judgeResult = await outcomeJudge(groundTruth, report);

console.log(`${label} outcome:`, JSON.stringify(judgeResult, null, 2));

// Record to eval collector with outcome judge results

recordE2E(`/qa ${label}`, 'Planted-bug outcome evals', result, {

passed: judgePassed(judgeResult, groundTruth),

detection_rate: judgeResult.detection_rate,

false_positives: judgeResult.false_positives,

evidence_quality: judgeResult.evidence_quality,

detected_bugs: judgeResult.detected,

missed_bugs: judgeResult.missed,

});

// Diagnostic dump on failure (decision 1C)

if (judgeResult.detection_rate < groundTruth.minimum_detection || judgeResult.false_positives > groundTruth.max_false_positives) {

dumpOutcomeDiagnostic(testWorkDir, label, report, judgeResult);

}

// Phase 2 assertions

expect(judgeResult.detection_rate).toBeGreaterThanOrEqual(groundTruth.minimum_detection);

expect(judgeResult.false_positives).toBeLessThanOrEqual(groundTruth.max_false_positives);

expect(judgeResult.evidence_quality).toBeGreaterThanOrEqual(2);

}

// B6: Static dashboard — broken link, disabled submit, overflow, missing alt, console error

test('/qa finds >= 2 of 5 planted bugs (static)', async () => {

await runPlantedBugEval('qa-eval.html', 'qa-eval-ground-truth.json', 'b6-static');

}, 360_000);

// B7: SPA — broken route, stale state, async race, missing aria, console warning

test('/qa finds >= 2 of 5 planted SPA bugs', async () => {

await runPlantedBugEval('qa-eval-spa.html', 'qa-eval-spa-ground-truth.json', 'b7-spa');

}, 360_000);

// B8: Checkout — email regex, NaN total, CC overflow, missing required, stripe error

test('/qa finds >= 2 of 5 planted checkout bugs', async () => {

await runPlantedBugEval('qa-eval-checkout.html', 'qa-eval-checkout-ground-truth.json', 'b8-checkout');

}, 360_000);

});

// --- Plan CEO Review E2E ---

describeIfSelected('Plan CEO Review E2E', ['plan-ceo-review'], () => {

let planDir: string;

beforeAll(() => {

planDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-plan-ceo-'));

const { spawnSync } = require('child_process');

const run = (cmd: string, args: string[]) =>

spawnSync(cmd, args, { cwd: planDir, stdio: 'pipe', timeout: 5000 });

// Init git repo (CEO review SKILL.md has a "System Audit" step that runs git)

run('git', ['init', '-b', 'main']);

run('git', ['config', 'user.email', '[email protected]']);

run('git', ['config', 'user.name', 'Test']);

// Create a simple plan document for the agent to review

fs.writeFileSync(path.join(planDir, 'plan.md'), `# Plan: Add User Dashboard

## Context

We're building a new user dashboard that shows recent activity, notifications, and quick actions.

## Changes

1. New React component \`UserDashboard\` in \`src/components/\`

2. REST API endpoint \`GET /api/dashboard\` returning user stats

3. PostgreSQL query for activity aggregation

4. Redis cache layer for dashboard data (5min TTL)

## Architecture

- Frontend: React + TailwindCSS

- Backend: Express.js REST API

- Database: PostgreSQL with existing user/activity tables

- Cache: Redis for dashboard aggregates

## Open questions

- Should we use WebSocket for real-time updates?

- How do we handle users with 100k+ activity records?

`);

run('git', ['add', '.']);

run('git', ['commit', '-m', 'add plan']);

// Copy plan-ceo-review skill

fs.mkdirSync(path.join(planDir, 'plan-ceo-review'), { recursive: true });

fs.copyFileSync(

path.join(ROOT, 'plan-ceo-review', 'SKILL.md'),

path.join(planDir, 'plan-ceo-review', 'SKILL.md'),

);

});

afterAll(() => {

try { fs.rmSync(planDir, { recursive: true, force: true }); } catch {}

});

test('/plan-ceo-review produces structured review output', async () => {

const result = await runSkillTest({

prompt: `Read plan-ceo-review/SKILL.md for the review workflow.

Read plan.md — that's the plan to review. This is a standalone plan document, not a codebase — skip any codebase exploration or system audit steps.

Choose HOLD SCOPE mode. Skip any AskUserQuestion calls — this is non-interactive.

Write your complete review directly to ${planDir}/review-output.md

Focus on reviewing the plan content: architecture, error handling, security, and performance.`,

workingDirectory: planDir,

maxTurns: 15,

timeout: 360_000,

testName: 'plan-ceo-review',

runId,

});

logCost('/plan-ceo-review', result);

recordE2E('/plan-ceo-review', 'Plan CEO Review E2E', result, {

passed: ['success', 'error_max_turns'].includes(result.exitReason),

});

// Accept error_max_turns — the CEO review is very thorough and may exceed turns

expect(['success', 'error_max_turns']).toContain(result.exitReason);

// Verify the review was written

const reviewPath = path.join(planDir, 'review-output.md');

if (fs.existsSync(reviewPath)) {

const review = fs.readFileSync(reviewPath, 'utf-8');

expect(review.length).toBeGreaterThan(200);

}

}, 420_000);

});

// --- Plan CEO Review (SELECTIVE EXPANSION) E2E ---

describeIfSelected('Plan CEO Review SELECTIVE EXPANSION E2E', ['plan-ceo-review-selective'], () => {

let planDir: string;

beforeAll(() => {

planDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-plan-ceo-sel-'));

const { spawnSync } = require('child_process');

const run = (cmd: string, args: string[]) =>

spawnSync(cmd, args, { cwd: planDir, stdio: 'pipe', timeout: 5000 });

run('git', ['init', '-b', 'main']);

run('git', ['config', 'user.email', '[email protected]']);