v1.90.0.0 feat: make browser cookie imports explicit and safe (#2964)

* fix(browse): prepare reliable cookie import wave for validation

* ci: sequence quality and behavior for validation branch

* fix(browse): isolate Windows qualification and preserve native diagnostics

* test(browse): cover cookie workflow quality and isolate Windows user paths

* test(browse): trace native member startup and initialize fresh folders

* fix(browse): keep Windows member stdin alive through EOF

* fix(browse): latch native timeouts and compare contained Edge startup

* test(browse): verify native version metadata and actual Windows argv

* test(browse): qualify Dia import on isolated macOS CI

* fix(browse): require picker origin for session mutations

* fix(browse): bound credential reads through stream completion

* test(browse): inspect owned Windows process arguments natively

* test(evals): preserve passing coverage during cookie repair reruns

* test(browse): isolate Dia qualification in a fresh macOS account

* test(browse): pass bounded integer timeouts to native Mac probes

* test(browse): distinguish Windows profile initialization from containment

* test(browse): await descendant pipe readiness before parent exit

* test(browse): initialize and restore isolated macOS Keychain state

* test(browse): initialize Windows fixture folders before qualification

* test(ci): pin the same Node runtime across Windows checks

* test(browse): distinguish native macOS browser preflight stages

* test(browse): isolate Windows descendant console lifetime

* test(browse): preserve native receipts and identify fixture lock holders

* test(browse): prepare dependency resolution before native Mac worker startup

* test(ci): include lock and close checks in native diagnostics

* test(browse): preserve native owner probe stages and subprocess deadlines

* fix(browse): classify Chromium profile-in-use exit precisely

* test(browse): retain Mac qualification evidence through cleanup failures

* test(browse): bound Mac fixture paths and retire its owned user domain

* test(browse): accept vanished fixture entries without weakening cleanup

* test(browse): identify probe-created macOS user domains safely

* test(browse): observe Mac user domains without targeting them first

* test(browse): use passive fresh-user ownership throughout Mac qualification

* test(browse): distinguish profile and registered-home Keychain lookups

* test(browse): qualify Dia under one registered account home

* test(browse): identify Dia startup and owned process-group failures

* test(browse): classify bounded Dia startup diagnostics without leaking output

* fix(test): preserve native Mac sandboxing and reap owned browser children

* fix(browse): preserve Chromium sandboxing for native profile imports

* test(browse): inspect signed Mach-O architecture without launching Xcode tools

* test(browse): sample pending Dia startup and reap on all cleanup paths

* test(browse): compare protected Dia launches in fresh Bun and Node accounts

* test(browse): inspect isolated Mac GUI readiness without browser access

* v1.90.0.0 fix: bind cookie picker actions to their document

* test: validate cookie guards and fit nested launch fixtures

* ci: configure the bundled Chromium sandbox helper

* fix(browse): classify Playwright authentication timeouts

* test: retain bounded Windows lifecycle diagnostics

* test(cso): reuse bounded NTFS precision candidates

* test(review): handle explicit preservation choices safely

* test(browse): remove owned fixture directories with explicit primitives

* test(review): distinguish descriptive reuse from edit commitments

* test: admit only the approved unscored cookie workflow refusal

* test: keep the Office Hours judge mock export-complete

* fix: keep dependency-free CI planners independent of the model SDK

* test: observe the exact holder after a native fixture unlink failure

* fix: start seeded PTY observations at owned readiness

* test: acquire identity-bound Windows deletion admission before profile resets

* test: preserve qualified Git index bits without authorizing mutations
This commit is contained in:
Garry Tan authored and GitHub committed 2026-09-25 12:06:45 -04:00
1 parent 730a1017d1
commit a84b0b5b6d
111 files changed
+14996 -1057

No files matched your search

+77
View File
@@ -0,0 +1,77 @@
import { afterEach, expect, test } from 'bun:test';
import { spawnSync } from 'node:child_process';
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import path from 'node:path';
const root = path.resolve(import.meta.dir, '..');
const workflow = Bun.YAML.parse(readFileSync(path.join(root, '.github/workflows/free-tests.yml'), 'utf8')) as any;
const steps = workflow.jobs['free-suite'].steps;
const index = steps.findIndex((step: any) => step.name === 'Configure the bundled Chromium sandbox helper');
const command = steps[index]?.run;
const fixtures: string[] = [];
afterEach(() => {
for (const fixture of fixtures.splice(0)) rmSync(fixture, { recursive: true, force: true });
});
function run(options: { outside?: boolean; link?: boolean; mode?: string; corrupt?: boolean } = {}) {
const fixture = mkdtempSync(path.join(tmpdir(), 'chromium-ci-'));
fixtures.push(fixture);
const bin = path.join(fixture, 'bin');
const home = path.join(fixture, 'home');
const directory = options.outside ? path.join(fixture, 'outside') : path.join(home, '.cache/ms-playwright/chromium-1234/chrome-linux64');
mkdirSync(bin, { recursive: true });
mkdirSync(directory, { recursive: true });
const executable = path.join(directory, 'chrome');
const helper = path.join(directory, 'chrome_sandbox');
const installed = path.join(directory, 'chrome-sandbox');
const receipt = path.join(fixture, 'install.json');
writeFileSync(executable, 'synthetic browser');
if (options.link) symlinkSync(executable, helper);
else writeFileSync(helper, 'synthetic matching helper');
writeFileSync(path.join(bin, 'bun'), '#!/bin/sh\nprintf "%s\\n" "$FIXTURE_CHROME"\n', { mode: 0o755 });
writeFileSync(path.join(bin, 'stat'), '#!/bin/sh\nprintf "%s\\n" "$FIXTURE_STAT"\n', { mode: 0o755 });
writeFileSync(path.join(bin, 'sudo'), `#!/usr/bin/env node
const fs = require('node:fs');
const args = process.argv.slice(2);
fs.writeFileSync(process.env.FIXTURE_RECEIPT, JSON.stringify(args));
if (JSON.stringify(args.slice(0, 8)) !== JSON.stringify(['install', '-T', '-o', 'root', '-g', 'root', '-m', '4755']) || args.length !== 10) process.exit(2);
fs.copyFileSync(args[8], args[9]);
if (process.env.FIXTURE_CORRUPT === '1') fs.appendFileSync(args[9], 'changed');
`, { mode: 0o755 });
const result = spawnSync('/bin/bash', ['-c', command], {
cwd: root, encoding: 'utf8', timeout: 10_000,
env: { ...process.env, HOME: home, PATH: bin + path.delimiter + process.env.PATH,
FIXTURE_CHROME: executable, FIXTURE_STAT: options.mode ?? '0:4755',
FIXTURE_RECEIPT: receipt, FIXTURE_CORRUPT: options.corrupt ? '1' : '0' },
});
return { result, helper, installed, calls: existsSync(receipt) ? JSON.parse(readFileSync(receipt, 'utf8')) : null };
}
test('the actual CI sandbox setup uses the resolved bundled helper before tests without disabling protections', () => {
expect(typeof command).toBe('string');
expect(index).toBeGreaterThan(steps.findIndex((step: any) => step.name === 'Install Playwright Chromium'));
expect(index).toBeLessThan(steps.findIndex((step: any) => step.name === 'Run free suite'));
expect(command).not.toMatch(/--no-sandbox|chromiumSandbox:\s*false|sysctl|apparmor_restrict/);
const { result, helper, installed, calls } = run();
expect(result.status).toBe(0);
expect(calls).toEqual(['install', '-T', '-o', 'root', '-g', 'root', '-m', '4755', helper, installed]);
expect(readFileSync(installed)).toEqual(readFileSync(helper));
});
test('unexpected paths and linked helpers are refused before privileged installation', () => {
for (const options of [{ outside: true }, { link: true }]) {
const { result, calls } = run(options);
expect(result.status).not.toBe(0);
expect(calls).toBeNull();
}
});
test('the actual CI setup requires root ownership, setuid mode, and unchanged helper bytes', () => {
for (const options of [{ mode: '1000:4755' }, { mode: '0:755' }, { corrupt: true }]) {
const { result, calls } = run(options);
expect(result.status).not.toBe(0);
expect(calls).not.toBeNull();
}
});
+64 -1
View File
@@ -1,5 +1,5 @@
import { expect, test } from 'bun:test';
import { readFileSync, writeFileSync, mkdtempSync, mkdirSync, rmSync } from 'node:fs';
import { readFileSync, writeFileSync, mkdtempSync, mkdirSync, rmSync, chmodSync } from 'node:fs';
import { resolve, join } from 'node:path';
import { tmpdir } from 'node:os';
import { spawnSync } from 'node:child_process';
@@ -102,3 +102,66 @@ test.skipIf(!Bun.which('jq'))('the actual comment separates reused evidence, ret
expect(text).toContain('receive no PR-pass credit');
expect(comment).not.toContain('diff-selected gate census');
});
test.skipIf(!Bun.which('jq') || !Bun.which('bash'))('comment consumes verified final counts without running repository code and fails closed without them', () => {
const job = paid.jobs['slices-comment'];
expect(job.permissions).toMatchObject({ 'pull-requests': 'write' });
expect(JSON.stringify(job.steps)).not.toMatch(/actions\/checkout|setup-bun|bun run|npm |node /);
const upload = paid.jobs['slices-report'].steps.find((step: any) => step.with?.name === 'report-verdict');
expect(upload.with.path.trim().split('\n')).toEqual(['/tmp/report.txt', '/tmp/paid-report/collector-outcomes.json']);
expect(job.steps.find((step: any) => step.with?.name === 'report-verdict').with.path).toBe('/tmp/verdict');
const root = mkdtempSync(join(tmpdir(), 'ci-comment-'));
const paidDir = join(root, 'paid-report');
const verdictDir = join(root, 'verdict');
const binDir = join(root, 'bin');
mkdirSync(paidDir); mkdirSync(verdictDir); mkdirSync(binDir);
writeFileSync(join(binDir, 'gh'), '#!/bin/sh\ncase "$*" in *--jq*) exit 0;; esac\nfor arg do case "$arg" in body=*) printf "%s\\n" "${arg#body=}";; esac; done\n');
chmodSync(join(binDir, 'gh'), 0o755);
writeFileSync(join(binDir, 'bc'), '#!/bin/sh\nread -r expression\n[ "$expression" = "0 + 0" ] && printf "0\\n"\n');
chmodSync(join(binDir, 'bc'), 0o755);
writeFileSync(join(paidDir, 'manifest.json'), JSON.stringify({ profile: 'pr', selection: { e2e: [], judges: [] } }));
writeFileSync(join(paidDir, 'judge.json'), JSON.stringify({ total_tests: 2, tier: 'llm-judge', shard: 1,
tests: [{ name: 'manual', passed: false, manual_review: { unverified: true } },
{ name: 'reused', passed: true, execution: 'reused' }], flaky_retries: [] }));
const summary = { version: 1, files: [{ file: 'judge.json', tier: 'llm-judge', shard: 1, cost: 0,
total: 2, passed: 1, failed: 0, manual_accepted: 1, executed: 1, reused: 1, attempts: 2, flaky: 0 }],
totals: { total: 2, passed: 1, failed: 0, manual_accepted: 1, executed: 1, reused: 1, attempts: 2, flaky: 0 } };
mkdirSync(join(verdictDir, 'paid-report'));
const summaryPath = join(verdictDir, 'paid-report/collector-outcomes.json');
const script = (job.steps.find((step: any) => step.name === 'Post PR comment').run as string)
.replaceAll('/tmp/paid-report', paidDir).replaceAll('/tmp/verdict', verdictDir)
.replaceAll('${{ github.repository }}', 'garrytan/gstack')
.replaceAll('${{ github.event.pull_request.number }}', '123');
const check = spawnSync('bash', ['-n', '-c', script], { cwd: root, encoding: 'utf8', timeout: 5000 });
expect(check.status, check.stderr).toBe(0);
const run = () => spawnSync('bash', ['-e', '-c', script], { cwd: root,
env: { ...process.env, PATH: `${binDir}:${process.env.PATH}`, RECONCILE_EXIT: '0' },
encoding: 'utf8', timeout: 5000 });
try {
writeFileSync(summaryPath, JSON.stringify(summary));
const verified = run();
expect(verified.status, verified.stderr).toBe(0);
expect(verified.stdout).toContain('⚠ MANUAL ACCEPTED (unscored)');
expect(verified.stdout).toContain('1 automated passed / 2 final results');
expect(verified.stdout).toContain('0 failed, 1 manual accepted');
const unrelatedFailure = { ...summary, files: [{ ...summary.files[0], total: 3, failed: 1,
executed: 2, attempts: 3 }], totals: { ...summary.totals, total: 3, failed: 1,
executed: 2, attempts: 3 } };
writeFileSync(summaryPath, JSON.stringify(unrelatedFailure));
const red = run();
expect(red.status, red.stderr).toBe(0);
expect(red.stdout).toContain('❌ FAIL');
expect(red.stdout).toContain('1 failed, 1 manual accepted');
writeFileSync(summaryPath, JSON.stringify({ ...summary, totals: { ...summary.totals, manual_accepted: 2 } }));
const tampered = run();
expect(tampered.status, tampered.stderr).toBe(0);
expect(tampered.stdout).toContain('manual acceptance unavailable/unverified');
expect(tampered.stdout).not.toContain('⚠ MANUAL ACCEPTED (unscored)');
rmSync(summaryPath);
const absent = run();
expect(absent.status, absent.stderr).toBe(0);
expect(absent.stdout).toContain('manual acceptance unavailable/unverified');
} finally { rmSync(root, { recursive: true, force: true }); }
});
+168 -3
View File
@@ -5,6 +5,7 @@ import * as path from 'node:path';
import { spawnSync } from 'node:child_process';
import { buildRunManifest, collectPaidTestFiles, type PaidRunManifest, type SliceResult } from '../scripts/test-paid-shards';
import { STRICT_RETRY_CASE_BUDGETS } from './helpers/eval-budgets';
import { manualReviewFixture } from './helpers/manual-judge-review-fixture';
const ROOT = path.resolve(import.meta.dir, '..');
type Step = { uses?: string; run?: string; if?: string; with?: Record<string, unknown> };
@@ -23,6 +24,48 @@ const workflows = ['evals.yml', 'evals-periodic.yml'].map(name => ({
}));
describe('paid CI coordination stays off the eval image', () => {
test('the actual planner and reporter load from a checkout without installed packages', () => {
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'paid-offline-'));
const listed = spawnSync('git', ['ls-files', '-z'], { cwd: ROOT, encoding: 'utf8', timeout: 5_000 });
expect(listed.status, listed.stderr).toBe(0);
try {
for (const relative of listed.stdout.split('\0').filter(Boolean)) {
const source = path.join(ROOT, relative);
const destination = path.join(directory, relative);
fs.mkdirSync(path.dirname(destination), { recursive: true });
expect(fs.realpathSync(path.dirname(destination)).startsWith(fs.realpathSync(directory) + path.sep)
|| fs.realpathSync(path.dirname(destination)) === fs.realpathSync(directory)).toBe(true);
if (fs.lstatSync(source).isSymbolicLink()) fs.symlinkSync(fs.readlinkSync(source), destination);
else fs.copyFileSync(source, destination);
}
const env = { ...process.env, ANTHROPIC_API_KEY: undefined, EVALS: undefined, EVALS_ALL: undefined,
EVALS_PROFILE: 'pr', EVALS_TIER: 'gate' };
for (const args of [['init', '-b', 'main'], ['add', '-A'], ['commit', '-m', 'Seed offline planner fixture'], ['checkout', '-b', 'fixture-head']]) {
const git = spawnSync('git', args, { cwd: directory, env, encoding: 'utf8', timeout: 15_000 });
expect(git.status, git.stderr).toBe(0);
}
fs.appendFileSync(path.join(directory, '.github/workflows/evals.yml'), '\n');
const report = path.join(directory, 'report');
const plan = spawnSync(process.execPath, ['--no-install', 'run', 'scripts/test-paid-shards.ts', '--tier', 'gate',
'--emit-plan', path.join(report, 'manifest.json'), '--slices', '6'], { cwd: directory, env, encoding: 'utf8', timeout: 15_000 });
expect(plan.status, plan.stdout + plan.stderr).toBe(0);
const manifest = JSON.parse(fs.readFileSync(path.join(report, 'manifest.json'), 'utf8'));
expect(manifest.sliceCount).toBe(6);
expect(manifest.entries.some((entry: any) => entry.status === 'planned')).toBe(true);
const reconcile = spawnSync(process.execPath, ['--no-install', 'run', 'scripts/test-paid-shards.ts', '--tier', 'gate',
'--report', report], { cwd: directory, env, encoding: 'utf8', timeout: 15_000 });
expect(reconcile.status, reconcile.stdout + reconcile.stderr).toBe(1);
expect(reconcile.stdout).toContain('report: 0/6 slices');
expect(reconcile.stderr.match(/slice \d\/6 reported NO result/g)).toHaveLength(6);
expect(reconcile.stderr).not.toContain('Cannot find module');
expect(fs.existsSync(path.join(directory, 'node_modules'))).toBe(false);
const provider = spawnSync(process.execPath, ['--no-install', '-e', 'import "./test/helpers/llm-judge.ts"'],
{ cwd: directory, env, encoding: 'utf8', timeout: 15_000 });
expect(provider.status).toBe(1);
expect(provider.stderr).toContain("Cannot find module '@anthropic-ai/sdk'");
} finally { fs.rmSync(directory, { recursive: true, force: true }); }
}, 60_000);
for (const { name, jobs } of workflows) {
test(`${name}: planning is independent of image startup and has no dependency install`, () => {
const planner = jobs['plan-slices'];
@@ -88,8 +131,18 @@ describe('dependency-free CI planner and report execution', () => {
beforeAll(() => {
fixture = fs.mkdtempSync(path.join(os.tmpdir(), 'ci-paid-coordination-'));
fs.cpSync(path.join(ROOT, 'scripts'), path.join(fixture, 'scripts'), { recursive: true });
fs.cpSync(path.join(ROOT, 'test/helpers'), path.join(fixture, 'test/helpers'), { recursive: true });
const sourceOnly = { recursive: true, filter: (file: string) => path.basename(file) !== 'node_modules' };
fs.cpSync(path.join(ROOT, 'scripts'), path.join(fixture, 'scripts'), sourceOnly);
fs.cpSync(path.join(ROOT, 'test/helpers'), path.join(fixture, 'test/helpers'), sourceOnly);
expect(fs.readFileSync(path.join(fixture, 'test/helpers/llm-judge.ts'), 'utf8'))
.toBe(fs.readFileSync(path.join(ROOT, 'test/helpers/llm-judge.ts'), 'utf8'));
fs.cpSync(path.join(ROOT, 'lib'), path.join(fixture, 'lib'), sourceOnly);
expect(fs.existsSync(path.join(fixture, 'lib/diagram-render/node_modules'))).toBe(false);
fs.mkdirSync(path.join(fixture, '.github'), { recursive: true });
for (const file of ['.github/cookie-workflow-manual-review.json', 'setup-browser-cookies/SKILL.md', 'BROWSER.md']) {
fs.mkdirSync(path.dirname(path.join(fixture, file)), { recursive: true });
fs.copyFileSync(path.join(ROOT, file), path.join(fixture, file));
}
for (const file of collectPaidTestFiles()) {
fs.copyFileSync(path.join(ROOT, file), path.join(fixture, file));
}
@@ -183,7 +236,7 @@ describe('dependency-free CI planner and report execution', () => {
const red = run(['--report', reportDir], tier);
expect(red.status).toBe(1);
expect(red.stderr).toContain(`${failed.outcomes[0].files[0]}: failed`);
expect(red.stdout).toContain('3 executed, 0 reused; 1 passed, 2 failed (6 attempt records from 1 collectors)');
expect(red.stdout).toContain('3 executed, 0 reused; 1 passed, 2 failed, 0 manual accepted (unscored; no score-cache credit) (6 attempt records from 1 collectors)');
expect(red.stdout).toContain('3 cases with multiple attempts this run:');
expect(red.stdout).not.toMatch(/passed only on retry|not blocking/);
@@ -192,4 +245,116 @@ describe('dependency-free CI planner and report execution', () => {
expect(corrupt.status).toBe(1);
});
}
test('report verifies every manual claim against current source, preserves attempts, and never masks a failed shard', () => {
const reportDir = path.join(fixture, 'manual-report');
const manifestPath = path.join(reportDir, 'manifest.json');
const planned = run(['--emit-plan', manifestPath, '--slices', '1'], 'gate');
expect(planned.status, planned.stderr).toBe(0);
const manifest: PaidRunManifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
const slice: SliceResult = {
version: 1, tier: 'gate', sliceIndex: 1, sliceCount: 1,
outcomes: manifest.entries.filter(entry => entry.status === 'planned').map(entry => ({
files: [entry.file], status: 'passed', exitCode: 0, elapsedMs: 1,
executedTests: STRICT_RETRY_CASE_BUDGETS.find(budget => budget.file === entry.file)?.cases ?? 1,
skippedTests: 0, ...(entry.budget ? { budget: entry.budget } : {}),
})),
};
const slicePath = path.join(reportDir, 'slice-1.json');
const collectorPath = path.join(reportDir, 'judge-results.json');
const summaryPath = path.join(reportDir, 'collector-outcomes.json');
const receipt = manualReviewFixture(ROOT);
const write = (tests: unknown[]) => fs.writeFileSync(collectorPath, JSON.stringify({
total_tests: tests.length, tier: 'llm-judge', shard: 1, total_cost_usd: 0,
tests, flaky_retries: [{ name: receipt.name, attempts: tests.length }],
}));
fs.writeFileSync(slicePath, JSON.stringify(slice));
write([{ ...receipt, passed: true }, receipt]);
const historical = run(['--report', reportDir], 'gate');
expect(historical.status).toBe(1);
expect(historical.stderr).toContain('attempt 1: Malformed manual-review claim');
expect(fs.existsSync(summaryPath)).toBe(false);
write([{ ...receipt, manual_review: { ...receipt.manual_review, refusal: {
...receipt.manual_review!.refusal, response_id: '',
} } }, receipt]);
const malformed = run(['--report', reportDir], 'gate');
expect(malformed.status).toBe(1);
expect(malformed.stderr).toContain('attempt 1: Malformed manual-review claim');
write([{ ...receipt, manual_review: undefined, passed: false }, receipt]);
const forgedFirstAttempt = run(['--report', reportDir], 'gate');
expect(forgedFirstAttempt.status).toBe(1);
expect(forgedFirstAttempt.stderr).toContain('attempt 2: manual review is only valid on the first case attempt');
expect(fs.existsSync(summaryPath)).toBe(false);
write([receipt, { ...receipt, attempt: 2 }]);
const retriedManual = run(['--report', reportDir], 'gate');
expect(retriedManual.status).toBe(1);
expect(retriedManual.stderr).toContain('attempt 2: Malformed manual-review claim');
expect(fs.existsSync(summaryPath)).toBe(false);
write([receipt]);
const secondCollector = path.join(reportDir, 'other-results.json');
fs.writeFileSync(secondCollector, JSON.stringify({ total_tests: 1, tests: [receipt] }));
const duplicateCollector = run(['--report', reportDir], 'gate');
expect(duplicateCollector.status).toBe(1);
expect(duplicateCollector.stderr).toContain('duplicate manual-review claim');
expect(fs.existsSync(summaryPath)).toBe(false);
fs.rmSync(secondCollector);
write([receipt, { name: 'automated', suite: 'other', passed: true, execution: 'reused' }]);
const clean = run(['--report', reportDir], 'gate');
expect(clean.status, clean.stderr).toBe(0);
expect(clean.stdout).toContain('1 passed, 0 failed, 1 manual accepted (unscored; no score-cache credit) (2 attempt records');
const summary = JSON.parse(fs.readFileSync(summaryPath, 'utf8'));
expect(summary.totals).toEqual({ executed: 1, reused: 1, passed: 1, failed: 0,
manual_accepted: 1, attempts: 2, total: 2, flaky: 1 });
expect(summary.files[0]).toMatchObject({ file: 'judge-results.json', total: 2, manual_accepted: 1, passed: 1 });
expect(JSON.parse(fs.readFileSync(collectorPath, 'utf8')).tests[0]).toEqual(receipt);
const browserPath = path.join(fixture, 'BROWSER.md');
const browserSource = fs.readFileSync(browserPath, 'utf8');
fs.writeFileSync(browserPath, browserSource.replace('#### Choosing a source and checking sign-in',
'#### Choosing a source and checking sign-in\nchanged approved source'));
const sourceDrift = run(['--report', reportDir], 'gate');
expect(sourceDrift.status).toBe(1);
expect(sourceDrift.stderr).toContain('does not match current source and approval');
expect(fs.existsSync(summaryPath)).toBe(false);
fs.writeFileSync(browserPath, browserSource);
write([receipt, { name: 'unapproved', passed: false, execution: 'executed' }]);
const failedCollector = run(['--report', reportDir], 'gate');
expect(failedCollector.status).toBe(1);
expect(failedCollector.stderr).toContain('1 unapproved final collector failure(s)');
expect(JSON.parse(fs.readFileSync(summaryPath, 'utf8')).totals).toMatchObject({ failed: 1, manual_accepted: 1 });
write([receipt, { passed: true }]);
const missingName = run(['--report', reportDir], 'gate');
expect(missingName.status).toBe(1);
expect(missingName.stderr).toContain('attempt 2: malformed collector entry (name/passed required)');
expect(fs.existsSync(summaryPath)).toBe(false);
write([receipt, { name: 'malformed', passed: 'true' }]);
const malformedPassed = run(['--report', reportDir], 'gate');
expect(malformedPassed.status).toBe(1);
expect(malformedPassed.stderr).toContain('attempt 2: malformed collector entry (name/passed required)');
expect(fs.existsSync(summaryPath)).toBe(false);
write([receipt, { name: 'automated', suite: 'other', passed: true, execution: 'reused' }]);
slice.outcomes[0].status = 'failed';
slice.outcomes[0].exitCode = 1;
fs.writeFileSync(slicePath, JSON.stringify(slice));
const failedShard = run(['--report', reportDir], 'gate');
expect(failedShard.status).toBe(1);
expect(failedShard.stderr).toContain(`${slice.outcomes[0].files[0]}: failed`);
expect(JSON.parse(fs.readFileSync(summaryPath, 'utf8')).totals.manual_accepted).toBe(1);
const stale = structuredClone(receipt);
stale.manual_review!.approval.prompt_sha256 = '0'.repeat(64);
write([stale]);
const mismatched = run(['--report', reportDir], 'gate');
expect(mismatched.status).toBe(1);
expect(mismatched.stderr).toContain('attempt 1: Malformed manual-review claim');
expect(fs.existsSync(summaryPath)).toBe(false);
});
});
+75 -4
View File
@@ -1,5 +1,5 @@
import { afterAll, describe, expect, spyOn, test } from 'bun:test';
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { existsSync, lstatSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import path from 'node:path';
import { spawnSync } from 'node:child_process';
@@ -11,22 +11,42 @@ const FAKE = path.join(DIR, 'fake claude.ts');
const DESCENDANT = path.join(DIR, 'pipe holder.ts');
const CAPTURE = path.join(DIR, 'capture.json');
const PID = path.join(DIR, 'descendant.pid');
const ACTOR_STAGES = path.join(DIR, 'actor-stages.jsonl');
const DESCENDANT_STAGES = path.join(DIR, 'descendant-stages.jsonl');
const DIR_IDENTITY = lstatSync(DIR, { bigint: true });
const RESOLVED_DIR = realpathSync(DIR);
// Publish readiness only after the grandchild has initialized and flushed both
// inherited pipes; a PID returned by spawn alone does not establish that state.
writeFileSync(DESCENDANT, `
import { writeFileSync } from 'node:fs';
import { appendFileSync, writeFileSync } from 'node:fs';
const record = stage => {
if (process.env.FAKE_MODE === 'timeout') {
try { appendFileSync(${JSON.stringify(DESCENDANT_STAGES)}, JSON.stringify({ stage, atUnixMs: Date.now(), processNs: process.hrtime.bigint().toString(), pid: process.pid }) + '\\n'); } catch {}
}
};
record('descendant_start');
await Bun.sleep(Number(process.env.DESCENDANT_DELAY_MS || 0));
setInterval(() => {}, 1000);
await new Promise(resolve => process.stdout.write(' ', resolve));
record('stdout_flushed');
await new Promise(resolve => process.stderr.write(' ', resolve));
record('stderr_flushed');
writeFileSync(process.env.PID_FILE!, String(process.pid));
record('pid_published');
`);
writeFileSync(FAKE, `
import { spawn } from 'node:child_process';
import { existsSync, rmSync, writeFileSync } from 'node:fs';
import { appendFileSync, existsSync, rmSync, writeFileSync } from 'node:fs';
const record = (stage, pid = process.pid) => {
if (process.env.FAKE_MODE === 'timeout') {
try { appendFileSync(${JSON.stringify(ACTOR_STAGES)}, JSON.stringify({ stage, atUnixMs: Date.now(), processNs: process.hrtime.bigint().toString(), pid }) + '\\n'); } catch {}
}
};
record('actor_start');
const prompt = await Bun.stdin.text();
record('stdin_read');
writeFileSync(process.env.CAPTURE!, JSON.stringify({args:process.argv.slice(2),prompt,cwd:process.cwd(),model:process.env.ANTHROPIC_MODEL,auth:process.env.ANTHROPIC_API_KEY}));
const mode = process.env.FAKE_MODE;
if (mode === 'startup-timeout') {
@@ -39,11 +59,18 @@ if (mode === 'timeout' || mode === 'descendant' || mode === 'escaped') {
// drain fixture must survive that exit so its inherited pipes remain open.
const child = spawn(process.execPath, [process.env.DESCENDANT!], {stdio:['ignore','inherit','inherit'],
detached:mode === 'escaped' || (process.platform === 'win32' && mode === 'descendant')});
record('descendant_spawn_requested');
child.once('spawn', () => record('descendant_spawned', child.pid));
child.once('exit', () => record('descendant_exited', child.pid));
const readyBy = Date.now() + 2000;
while (!existsSync(process.env.PID_FILE!)) {
if (child.exitCode !== null || Date.now() >= readyBy) throw new Error('Descendant did not initialize its inherited pipes');
if (child.exitCode !== null || Date.now() >= readyBy) {
record('readiness_failed');
throw new Error('Descendant did not initialize its inherited pipes');
}
await Bun.sleep(5);
}
record('readiness_observed');
if (mode === 'timeout') await new Promise(() => {});
}
if (mode === 'auth') { process.stderr.write('Not logged in. Please run claude /login.'); process.exit(1); }
@@ -79,6 +106,27 @@ function run(mode = 'success', extra: Partial<Parameters<typeof runClaudeCode>[0
function capture() { return JSON.parse(readFileSync(CAPTURE, 'utf8')); }
function timeoutStageReceipt(file: string): object {
try {
const directory = lstatSync(DIR, { bigint: true });
if (directory.isSymbolicLink() || directory.dev !== DIR_IDENTITY.dev || directory.ino !== DIR_IDENTITY.ino || realpathSync(DIR) !== RESOLVED_DIR || ![ACTOR_STAGES, DESCENDANT_STAGES].includes(file)) return { available: false, reason: 'identity_mismatch' };
const state = lstatSync(file);
if (!state.isFile() || state.isSymbolicLink() || state.size > 4096) return { available: false, reason: 'invalid_receipt' };
const lines = readFileSync(file, 'utf8').trim().split('\n');
if (lines.length > 16) return { available: false, reason: 'invalid_receipt' };
const stages = [];
for (const line of lines) {
let event;
try { event = JSON.parse(line); } catch { return { available: false, reason: 'incomplete_receipt', stages }; }
if (!event || !['actor_start', 'stdin_read', 'descendant_spawn_requested', 'descendant_spawned', 'descendant_exited', 'readiness_failed', 'readiness_observed', 'descendant_start', 'stdout_flushed', 'stderr_flushed', 'pid_published'].includes(event.stage) || !Number.isSafeInteger(event.atUnixMs) || event.atUnixMs <= 0 || typeof event.processNs !== 'string' || !/^\d{1,24}$/.test(event.processNs) || !Number.isSafeInteger(event.pid) || event.pid <= 0) return { available: false, reason: 'invalid_receipt', stages };
stages.push({ stage: event.stage, atUnixMs: event.atUnixMs, processNs: event.processNs, pid: event.pid });
}
return { available: true, stages };
} catch (error) {
return { available: false, reason: (error as NodeJS.ErrnoException).code === 'ENOENT' ? 'not_published' : 'inspection_failed' };
}
}
function running(pid: number): boolean {
try {
process.kill(pid, 0);
@@ -107,6 +155,22 @@ function cleanupDescendant() {
}
describe('Claude Code restricted execution', () => {
test('timeout stage diagnostics retain only bounded safe fixture events', () => {
const event = { stage: 'actor_start', atUnixMs: 123, processNs: '123', pid: process.pid };
try {
expect(timeoutStageReceipt(ACTOR_STAGES)).toEqual({ available: false, reason: 'not_published' });
writeFileSync(ACTOR_STAGES, JSON.stringify({ ...event, secret: 'sensitive-sentinel' }) + '\n');
expect(timeoutStageReceipt(ACTOR_STAGES)).toEqual({ available: true, stages: [event] });
writeFileSync(ACTOR_STAGES, JSON.stringify(event) + '\n{"stage":');
expect(timeoutStageReceipt(ACTOR_STAGES)).toEqual({ available: false, reason: 'incomplete_receipt', stages: [event] });
writeFileSync(ACTOR_STAGES, JSON.stringify({ ...event, stage: 'sensitive-sentinel' }));
expect(JSON.stringify(timeoutStageReceipt(ACTOR_STAGES))).not.toContain('sensitive-sentinel');
writeFileSync(ACTOR_STAGES, 'x'.repeat(4097));
expect(timeoutStageReceipt(ACTOR_STAGES)).toEqual({ available: false, reason: 'invalid_receipt' });
expect(timeoutStageReceipt(CAPTURE)).toEqual({ available: false, reason: 'identity_mismatch' });
} finally { rmSync(ACTOR_STAGES, { force: true }); }
});
test('explicit model override stays one literal argument across access modes and resume', async () => {
const model = 'custom-model "quoted" $(touch /never)';
for (const access of ['none', 'read-only'] as const) {
@@ -211,7 +275,12 @@ describe('Claude Code restricted execution', () => {
test('timeout kills its descendants and clears process signal listeners', async () => {
rmSync(PID, { force: true });
rmSync(ACTOR_STAGES, { force: true });
rmSync(DESCENDANT_STAGES, { force: true });
const before = ['SIGINT','SIGTERM','exit'].map(name => process.listenerCount(name));
const startedAtUnixMs = Date.now();
const startedProcessNs = process.hrtime.bigint().toString();
let returned: { atUnixMs: number; processNs: string; timedOut: boolean } | undefined;
const schedule = globalThis.setTimeout;
let fireTimeout: (() => void) | undefined;
const timer = spyOn(globalThis, 'setTimeout').mockImplementation((callback, delay, ...args) => {
@@ -233,12 +302,14 @@ describe('Claude Code restricted execution', () => {
fireTimeout = undefined;
expire();
const result = await invocation;
returned = { atUnixMs: Date.now(), processNs: process.hrtime.bigint().toString(), timedOut: result.error?.code === 'timeout' };
expect(result.status).toBe('unavailable');
expect(result.error?.code).toBe('timeout');
expect(Date.now() - start).toBeLessThan(2000);
expect(['SIGINT','SIGTERM','exit'].map(name => process.listenerCount(name))).toEqual(before);
await expectDescendantDead();
} finally {
console.error(JSON.stringify({ claudeTimeoutStages: { startedAtUnixMs, startedProcessNs, timeoutMs: 500, readinessMs: 2000, returned, pidPublished: existsSync(PID), actor: timeoutStageReceipt(ACTOR_STAGES), descendant: timeoutStageReceipt(DESCENDANT_STAGES) } }));
fireTimeout?.();
await invocation;
cleanupDescendant();
+138
View File
@@ -0,0 +1,138 @@
import { afterAll, expect, test } from 'bun:test';
import { spawnSync } from 'node:child_process';
import { mkdtempSync, readFileSync, realpathSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import path from 'node:path';
import { parseRunManifest } from '../scripts/test-paid-shards';
const root = path.resolve(import.meta.dir, '..');
const scratch = realpathSync(mkdtempSync(path.join(tmpdir(), 'cookie-phase-')));
const workflow = Bun.YAML.parse(readFileSync(path.join(root, '.github/workflows/evals.yml'), 'utf8')) as any;
const command = workflow.jobs['plan-slices'].steps.find((step: any) => step.name === 'Emit validation-phase manifest').run;
const body = command.match(/^bun --no-install -e '\n([\s\S]*)\n'\s*$/)?.[1];
if (!body) throw new Error('Validation planner script was not found');
afterAll(() => rmSync(scratch, { recursive: true, force: true }));
function manifestFor(phase: string) {
const output = path.join(scratch, phase);
const result = spawnSync(process.execPath, ['--no-env-file', '--no-install', '--no-macros', `--config=${process.platform === 'win32' ? 'NUL' : '/dev/null'}`, '-e', body!.replaceAll('/tmp/paid-plan', output.replaceAll('\\', '/'))], {
cwd: root, env: { ...process.env, VALIDATION_PHASE: phase, EVALS_TIER: 'gate', EVALS_ALL: '1' }, encoding: 'utf8', timeout: 30_000,
});
expect(result.error).toBeUndefined();
expect(result.status).toBe(0);
return parseRunManifest(readFileSync(path.join(output, 'manifest.json'), 'utf8'));
}
test('the actual CI cookie repair planner executes only eight dependent cases without judges or run-all', () => {
const manifest = manifestFor('cookie-behavior');
expect(manifest.profile).toBe('full');
expect(manifest.evalsAll).toBe(false);
expect(manifest.selection).toEqual({ e2e: ['browse-basic', 'browse-snapshot', 'qa-quick', 'qa-only-no-fix', 'design-review-detector-shim-dom', 'diagram-triplet', 'canary-workflow', 'benchmark-workflow'], judges: [] });
expect(manifest.entries.filter(entry => entry.status === 'planned').map(entry => entry.file).sort()).toEqual([
'test/skill-e2e-bws.test.ts', 'test/skill-e2e-deploy.test.ts', 'test/skill-e2e-design.test.ts', 'test/skill-e2e-diagram.test.ts', 'test/skill-e2e-qa-workflow.test.ts',
]);
});
test('the existing quality and behavior phases retain their complete separate shard census', () => {
const quality = manifestFor('quality');
const behavior = manifestFor('behavior');
const qualityFiles = quality.entries.filter(entry => entry.status === 'planned').map(entry => entry.file);
const behaviorFiles = behavior.entries.filter(entry => entry.status === 'planned').map(entry => entry.file);
expect(quality.evalsAll).toBe(true);
expect(behavior.evalsAll).toBe(true);
expect(qualityFiles).toHaveLength(2);
expect(behaviorFiles).toHaveLength(52);
expect(qualityFiles.every(file => file.startsWith('test/skill-llm-eval'))).toBe(true);
expect(behaviorFiles.every(file => !qualityFiles.includes(file))).toBe(true);
});
test('curated Windows and native qualification use the same pinned Node runtime', () => {
const windows = Bun.YAML.parse(readFileSync(path.join(root, '.github/workflows/windows-free-tests.yml'), 'utf8')) as any;
for (const job of ['windows-free-tests', 'cookie-native-qualification']) {
const setup = windows.jobs[job].steps.find((step: any) => step.uses?.startsWith('actions/setup-node@'));
expect(setup.with['node-version']).toBe('24.18.0');
expect(setup.if).toBeUndefined();
}
});
test('Windows retains complete shard logs on successful and failed runs', () => {
const windows = Bun.YAML.parse(readFileSync(path.join(root, '.github/workflows/windows-free-tests.yml'), 'utf8')) as any;
const upload = windows.jobs['windows-free-tests'].steps.find((step: any) => step.with?.name === 'windows-free-test-shard-logs');
expect(upload.if).toBe('always()');
expect(upload.with.path).toBe('${{ runner.temp }}/gstack-free-test-*.log');
});
test('focused Windows diagnostics include the repaired lock and close cases without default-profile qualification', () => {
const windows = Bun.YAML.parse(readFileSync(path.join(root, '.github/workflows/windows-free-tests.yml'), 'utf8')) as any;
const run = windows.jobs['windows-free-tests'].steps.find((step: any) => step.name === 'Run focused native launch and credential diagnostics').run;
const pattern = run.match(/--test-name-pattern '([^']+)'/)?.[1];
expect(pattern).toBeDefined();
const selected = new RegExp(pattern);
for (const name of ['native Windows launch diagnostics > observer', 'native Windows process qualification > a locked real Edge profile leaves its existing owner alive',
'native Windows process qualification > real Edge synthetic profile: normal-close', 'native Windows process qualification > real Edge synthetic profile: stalled-close']) expect(selected.test(name)).toBe(true);
expect(selected.test('native Windows process qualification > an exclusively created default Edge profile persists v20')).toBe(false);
});
test('Dia comparison uses separate pinned runtime jobs and retains diagnostic failures', () => {
const windows = Bun.YAML.parse(readFileSync(path.join(root, '.github/workflows/windows-free-tests.yml'), 'utf8')) as any;
const job = windows.jobs['dia-native-qualification'];
expect(job['runs-on']).toBe('macos-15');
expect(job.strategy['fail-fast']).toBe(false);
expect(job.strategy.matrix.runtime).toContain('["bun","node"]');
expect(job.strategy.matrix.runtime).toContain('inputs.dia_launch_comparison');
const node = job.steps.find((step: any) => step.uses?.startsWith('actions/setup-node@'));
expect(node.with).toEqual({ 'node-version': '24.18.0', architecture: 'arm64' });
const comparison = job.steps.find((step: any) => step.name === 'Compare protected native Dia launch without qualification credit');
expect(comparison.run).toContain('--launch-comparison "$COMPARISON_RUNTIME"');
expect(comparison['continue-on-error']).toBeUndefined();
const upload = job.steps.find((step: any) => step.uses?.startsWith('actions/upload-artifact@'));
expect(upload.if).toBe('always()');
expect(upload.with.name).toContain("format('dia-launch-comparison-{0}', matrix.runtime)");
expect(windows.jobs['windows-free-tests'].if).toContain('!inputs.dia_launch_comparison');
expect(windows.jobs['cookie-native-qualification'].if).toContain('!inputs.dia_launch_comparison');
});
test('Dia GUI readiness is an exclusive, single-job diagnostic without browser installation or launch', () => {
const windows = Bun.YAML.parse(readFileSync(path.join(root, '.github/workflows/windows-free-tests.yml'), 'utf8')) as any;
const job = windows.jobs['dia-native-qualification'];
const input = windows.on.workflow_dispatch.inputs.dia_gui_readiness;
expect(input).toMatchObject({ type: 'boolean', default: false });
expect(job.if).toContain('inputs.dia_gui_readiness');
expect(job.strategy.matrix.runtime).toContain('inputs.dia_launch_comparison && !inputs.dia_gui_readiness');
for (const name of ['windows-free-tests', 'cookie-native-qualification']) {
expect(windows.jobs[name].if).toContain('!inputs.dia_gui_readiness');
}
const probe = job.steps.find((step: any) => step.name === 'Inspect GUI readiness without browser or Keychain access');
expect(probe.if).toBe('inputs.dia_gui_readiness');
expect(probe.run).toEndWith('.github/scripts/run-dia-native-qualification.ts --gui-readiness-only');
expect(probe.env).toEqual({ GSTACK_DIA_NATIVE_QUALIFY: '1' });
const excluded = job.steps.filter((step: any) => step.uses?.startsWith('actions/setup-node@')
|| ['Install pinned dependencies', 'Install the synthetic destination browser', 'Qualify native Dia discovery, decryption, and import',
'Compare protected native Dia launch without qualification credit'].includes(step.name));
expect(excluded).toHaveLength(5);
for (const step of excluded) expect(step.if).toContain('!inputs.dia_gui_readiness');
const upload = job.steps.find((step: any) => step.uses?.startsWith('actions/upload-artifact@'));
expect(upload.if).toBe('always()');
expect(upload.with.name).toContain("inputs.dia_gui_readiness && 'dia-gui-readiness'");
expect(upload.with.path).toBe('${{ runner.temp }}/dia-native-qualification.json');
});
test('the actual GUI readiness selection guard refuses conflicting or malformed mode inputs', () => {
const windows = Bun.YAML.parse(readFileSync(path.join(root, '.github/workflows/windows-free-tests.yml'), 'utf8')) as any;
const steps = windows.jobs['dia-native-qualification'].steps;
const guard = steps.find((step: any) => step.name === 'Validate GUI readiness selection');
expect(guard.if).toBe('inputs.dia_gui_readiness');
expect(guard.env.OTHER_DIA_MODES).toBe('${{ inputs.dia_native_only || inputs.dia_launch_comparison || inputs.native_diagnostics_only }}');
expect(steps.indexOf(guard)).toBeLessThan(steps.findIndex((step: any) => step.name === 'Install pinned dependencies'));
const script = guard.run.match(/ -e '\n([\s\S]*)\n'\s*$/)?.[1];
expect(script).toBeDefined();
for (const input of ['false', 'true', '', 'unknown']) {
const result = spawnSync(process.execPath, ['--no-env-file', '--no-install', '--no-macros', `--config=${process.platform === 'win32' ? 'NUL' : '/dev/null'}`, '-e', script!], {
cwd: root, env: { ...process.env, OTHER_DIA_MODES: input }, encoding: 'utf8', timeout: 5000,
});
expect(result.error).toBeUndefined();
expect(result.status).toBe(input === 'false' ? 0 : 1);
expect(result.stderr.includes('must be selected alone')).toBe(input !== 'false');
}
});
+333
View File
@@ -0,0 +1,333 @@
import { afterEach, describe, expect, test } from 'bun:test';
import { createHash } from 'node:crypto';
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join, resolve } from 'node:path';
import { buildCookieWorkflowJudgeInput, COOKIE_WORKFLOW_JUDGE } from './helpers/cookie-workflow-judge-input';
import { buildWorkflowJudgePrompt, readWorkflowJudgeInput } from './helpers/workflow-judge-input';
import { prepareWorkflowJudgeCache, type WorkflowCacheOptions } from './helpers/workflow-judge-cache';
import type { EvalTestEntry } from './helpers/eval-store';
import { selectTests } from './helpers/test-selection';
import { E2E_TOUCHFILES, LLM_JUDGE_TOUCHFILES } from './helpers/touchfiles-data';
import { selectPrProfile } from '../scripts/test-pr-profile';
import { JUDGE_MS } from './helpers/eval-budgets';
import { JudgeRefusalError, DEFAULT_JUDGE_MAX_TOKENS } from './helpers/llm-judge';
import { COOKIE_MANUAL_REVIEW_FILE, getCookieWorkflowManualReview, isManualReviewEntry } from './helpers/cookie-workflow-manual-review';
const ROOT = resolve(import.meta.dir, '..');
const NAME = 'setup-browser-cookies/SKILL.md workflow';
const scratch: string[] = [];
const skill = 'excluded preamble\n# Setup Browser Cookies\nRead the gstack root\'s `BROWSER.md`, **Choosing a source and checking sign-in**.\nExact body.\n';
const browser = 'excluded browser introduction\n#### Choosing a source and checking sign-in\nExact reference.\n\n### Tabs + frames\nexcluded tab commands\n';
function fixture(entry: string | null = skill, reference: string | null = browser): string {
const root = mkdtempSync(join(tmpdir(), 'gstack-cookie-judge-'));
scratch.push(root);
mkdirSync(join(root, 'setup-browser-cookies'));
if (entry !== null) writeFileSync(join(root, 'setup-browser-cookies/SKILL.md'), entry);
if (reference !== null) writeFileSync(join(root, 'BROWSER.md'), reference);
return root;
}
function approveFixture(root: string) {
const input = buildCookieWorkflowJudgeInput(root);
const approval = { ...JSON.parse(readFileSync(join(ROOT, COOKIE_MANUAL_REVIEW_FILE), 'utf8')),
prompt_sha256: input.sha256, prompt_bytes: Buffer.byteLength(input.prompt), model: 'fixture-model',
reason: 'Synthetic approval fixture, not live review evidence' };
mkdirSync(join(root, '.github'), { recursive: true });
writeFileSync(join(root, COOKIE_MANUAL_REVIEW_FILE), JSON.stringify(approval));
return approval;
}
const refusal = () => new JudgeRefusalError({ id: 'msg_synthetic', _request_id: 'req_synthetic',
model: 'fixture-model', usage: { input_tokens: 1, output_tokens: 0 }, content: [] });
afterEach(() => {
for (const root of scratch.splice(0)) rmSync(root, { recursive: true, force: true });
});
type FixtureScore = { clarity: number; completeness: number; actionability: number; reasoning: string };
const passingScore: FixtureScore = { ...COOKIE_WORKFLOW_JUDGE.thresholds, reasoning: 'Synthetic fixture score' };
function actualCookieCallback(root: string, overrides: {
judge?: () => Promise<FixtureScore>;
clock?: () => number;
budget?: number;
setTimer?: typeof setTimeout;
clearTimer?: typeof clearTimeout;
} = {}) {
const source = readFileSync(join(ROOT, 'test/skill-llm-eval.test.ts'), 'utf8');
const managedStart = source.indexOf('async function runWorkflowJudge');
const managedEnd = source.indexOf('// Block 1:', managedStart);
const start = source.indexOf("describeIfSelected('Cookie setup workflow quality'");
const end = source.indexOf('// Module-level afterAll', start);
expect(managedStart).toBeGreaterThan(-1);
expect(managedEnd).toBeGreaterThan(managedStart);
expect(start).toBeGreaterThan(-1);
expect(end).toBeGreaterThan(start);
const registration = new Bun.Transpiler({ loader: 'ts' }).transformSync(source.slice(managedStart, managedEnd) + source.slice(start, end));
const requests: Array<{ prompt: string; model: undefined; signal: AbortSignal }> = [];
const records: EvalTestEntry[] = [];
const attempts = new Map<string, { attempt: number }>();
let callback: () => Promise<void> = async () => { throw new Error('Judge callback was not registered'); };
new Function('describeIfSelected', 'testIfSelected', 'ROOT', 'buildCookieWorkflowJudgeInput', 'resolveEvalModel', 'callJudge', 'COOKIE_WORKFLOW_JUDGE', 'JUDGE_MS', 'WORKFLOW_JUDGE_TEST_MS', 'WORKFLOW_JUDGE_RECORD_MS', 'evalCollector', 'expect', 'console', 'readWorkflowJudgeInput', 'buildWorkflowJudgePrompt', 'prepareWorkflowJudgeCache', 'workflowJudgeAttempts', 'performance', 'setTimeout', 'clearTimeout', 'JudgeRefusalError', 'getCookieWorkflowManualReview', 'DEFAULT_JUDGE_MAX_TOKENS', registration)(
(_suite: string, names: string[], run: () => void) => { expect(names).toEqual([NAME]); run(); },
(name: string, run: () => Promise<void>, budget: number) => { expect(name).toBe(NAME); expect(budget).toBe(JUDGE_MS + 10_000); callback = run; },
root, buildCookieWorkflowJudgeInput, () => 'fixture-model',
async (prompt: string, model: undefined, options: { signal: AbortSignal }) => { requests.push({ prompt, model, signal: options.signal }); return overrides.judge ? overrides.judge() : passingScore; },
COOKIE_WORKFLOW_JUDGE, overrides.budget ?? JUDGE_MS, JUDGE_MS + 10_000, 5_000,
{ addTest: (record: EvalTestEntry) => records.push(record) }, expect, { log() {} },
readWorkflowJudgeInput, buildWorkflowJudgePrompt,
(options: WorkflowCacheOptions) => prepareWorkflowJudgeCache({ ...options, env: {
EVALS_CACHE_DIR: join(root, 'cache'), EVALS_CACHE_REPOSITORY: 'fixture/cookie', EVALS_CACHE_PR: '1',
EVALS_CACHE_RUNTIME_ID: 'a'.repeat(64), EVALS_TIER: 'gate', EVALS_RUN_ID: 'cookie-fixture',
} }),
attempts, overrides.clock ? { now: overrides.clock } : performance,
overrides.setTimer ?? setTimeout, overrides.clearTimer ?? clearTimeout,
JudgeRefusalError, getCookieWorkflowManualReview, DEFAULT_JUDGE_MAX_TOKENS,
);
return { run: () => callback(), requests, records, attempts };
}
describe('cookie workflow judge input', () => {
test('the registered callback accepts only an approved empty provider refusal, with no score or cache credit', async () => {
const root = fixture();
approveFixture(root);
const h = actualCookieCallback(root, { judge: async () => { throw refusal(); } });
await h.run();
expect(h.requests).toHaveLength(1);
expect(h.records).toHaveLength(1);
expect(h.records[0]).toMatchObject({ passed: false, execution: 'executed', exit_reason: 'provider_refusal' });
expect(isManualReviewEntry(h.records[0])).toBe(true);
expect(h.records[0]).not.toHaveProperty('judge_scores');
expect(h.records[0]).not.toHaveProperty('judge_reasoning');
expect(h.records[0]).not.toHaveProperty('reused_from');
expect(existsSync(join(root, 'cache'))).toBe(false);
});
test('an approval cannot turn low scores, malformed output, or a refusal-named ordinary error into acceptance', async () => {
const root = fixture(); approveFixture(root);
for (const result of [
{ ...passingScore, clarity: 1 },
new SyntaxError('Malformed provider JSON'),
Object.assign(new Error('Synthetic refusal text'), { name: 'JudgeRefusalError' }),
]) {
const h = actualCookieCallback(root, { judge: async () => { if (result instanceof Error) throw result; return result; } });
await expect(h.run()).rejects.toThrow();
expect(h.records).toHaveLength(1);
expect(h.records[0].passed).toBe(false);
expect(h.records[0]).not.toHaveProperty('manual_review');
expect(isManualReviewEntry(h.records[0])).toBe(false);
}
const scored = actualCookieCallback(root);
await scored.run();
expect(scored.records[0]).toMatchObject({ passed: true, judge_scores: COOKIE_WORKFLOW_JUDGE.thresholds });
expect(scored.records[0]).not.toHaveProperty('manual_review');
});
test.each(['prompt', 'model', 'budget', 'thresholds', 'malformed', 'missing', 'evidence', 'response-evidence'])(
'the actual callback rejects an unapproved refusal: %s', async mismatch => {
const root = fixture(); const approval = approveFixture(root);
const error = refusal();
if (mismatch === 'prompt') writeFileSync(join(root, 'BROWSER.md'), browser.replace('Exact reference.', 'Changed reference.'));
if (mismatch === 'model') approval.model = 'different-model';
if (mismatch === 'budget') approval.max_tokens++;
if (mismatch === 'thresholds') approval.thresholds.clarity++;
if (mismatch === 'evidence') error.refusal.request_id = null;
if (mismatch === 'response-evidence') error.refusal.response_id = null;
writeFileSync(join(root, COOKIE_MANUAL_REVIEW_FILE), mismatch === 'malformed' ? '{}' : JSON.stringify(approval));
if (mismatch === 'missing') rmSync(join(root, COOKIE_MANUAL_REVIEW_FILE));
const h = actualCookieCallback(root, { judge: async () => { throw error; } });
await expect(h.run()).rejects.toThrow();
expect(h.records).toHaveLength(1);
expect(h.records[0].passed).toBe(false);
expect(h.records[0]).not.toHaveProperty('manual_review');
expect(existsSync(join(root, 'cache'))).toBe(false);
});
test('a late refusal stays a timeout despite exact approval', async () => {
const root = fixture(); approveFixture(root);
let now = 0;
const h = actualCookieCallback(root, { budget: 20, clock: () => now,
judge: async () => { now = 20; throw refusal(); } });
await expect(h.run()).rejects.toThrow('deadline');
expect(h.records).toHaveLength(1);
expect(h.records[0]).toMatchObject({ passed: false, exit_reason: 'timeout' });
expect(h.records[0]).not.toHaveProperty('manual_review');
});
test('a retry refusal cannot erase an earlier scored failure with manual acceptance', async () => {
const root = fixture(); approveFixture(root);
let calls = 0;
const h = actualCookieCallback(root, { judge: async () => {
if (++calls === 1) return { ...passingScore, clarity: 1 };
throw refusal();
} });
await expect(h.run()).rejects.toThrow();
await expect(h.run()).rejects.toThrow('provider refused');
expect(h.records.map(record => [record.attempt, record.passed, record.exit_reason]))
.toEqual([[1, false, 'validation_failed'], [2, false, 'provider_refusal']]);
for (const record of h.records) expect(record).not.toHaveProperty('manual_review');
});
test('preserves exact source bytes, line ranges, and the immutable rubric', () => {
const input = buildCookieWorkflowJudgeInput(fixture());
expect(input.files).toEqual([
{ path: 'setup-browser-cookies/SKILL.md', kind: 'entrypoint', startLine: 2, endLine: 4, content: skill.slice(skill.indexOf('# Setup')) },
{ path: 'BROWSER.md', kind: 'section', startLine: 2, endLine: 4, content: browser.slice(browser.indexOf('#### Choosing'), browser.indexOf('### Tabs')) },
]);
for (const file of input.files) {
expect(input.text).toContain(`--- BEGIN FILE ${JSON.stringify(file.path)} (lines ${file.startLine}-${file.endLine}; ${file.kind}) ---\n${file.content}\n--- END FILE ${JSON.stringify(file.path)} ---`);
expect(input.text.split(file.content)).toHaveLength(2);
}
expect(input.prompt).toBe(buildWorkflowJudgePrompt(COOKIE_WORKFLOW_JUDGE, input));
expect(input.sha256).toBe(createHash('sha256').update(input.prompt).digest('hex'));
expect(input.prompt).not.toContain('excluded');
expect(COOKIE_WORKFLOW_JUDGE.thresholds).toEqual({ clarity: 4, completeness: 3, actionability: 4 });
});
test('preserves CRLF instead of rewriting excerpts', () => {
const input = buildCookieWorkflowJudgeInput(fixture(skill.replaceAll('\n', '\r\n'), browser.replaceAll('\n', '\r\n')));
expect(input.files[0].content).toBe(skill.slice(skill.indexOf('# Setup')).replaceAll('\n', '\r\n'));
expect(input.files[1].content).toBe(browser.slice(browser.indexOf('#### Choosing'), browser.indexOf('### Tabs')).replaceAll('\n', '\r\n'));
expect(input.files.map(file => [file.startLine, file.endLine])).toEqual([[2, 4], [2, 4]]);
});
test('fingerprints consumed bytes, not excluded source text', () => {
const original = buildCookieWorkflowJudgeInput(fixture());
expect(buildCookieWorkflowJudgeInput(fixture(skill.replace('Exact body.', 'Changed body.'))).sha256).not.toBe(original.sha256);
expect(buildCookieWorkflowJudgeInput(fixture(skill, browser.replace('Exact reference.', 'Changed reference.'))).sha256).not.toBe(original.sha256);
expect(buildCookieWorkflowJudgeInput(fixture(skill.replace('excluded preamble', 'different preamble'), browser.replace('excluded tab commands', 'different tab commands'))).sha256).toBe(original.sha256);
});
test('reads the actual generated workflow and only its referenced cookie section', () => {
const input = buildCookieWorkflowJudgeInput(ROOT);
const entry = readFileSync(join(ROOT, 'setup-browser-cookies/SKILL.md'), 'utf8');
const reference = readFileSync(join(ROOT, 'BROWSER.md'), 'utf8');
expect(input.files[0].content).toBe(entry.slice(entry.indexOf('# Setup Browser Cookies')));
expect(input.files[1].content).toBe(reference.slice(reference.indexOf('#### Choosing a source and checking sign-in'), reference.indexOf('### Tabs + frames')));
expect(input.files[0].content.length).toBeGreaterThan(500);
expect(input.files[1].content.length).toBeGreaterThan(500);
expect(input.prompt).toContain('GSTACK_COOKIE_AUTH_EXPECTED_IDENTITY');
expect(input.prompt).toContain('isolated world');
expect(input.prompt).not.toContain('### Tabs + frames');
});
test('fails closed when either source file is absent', () => {
expect(() => buildCookieWorkflowJudgeInput(fixture(null))).toThrow();
expect(() => buildCookieWorkflowJudgeInput(fixture(skill, null))).toThrow();
});
test('rejects missing, duplicate, or non-heading markers', () => {
for (const broken of [skill.replace('# Setup Browser Cookies', '# Missing'), skill + '# Setup Browser Cookies\nDuplicate\n', skill.replace('# Setup Browser Cookies', 'Quoted # Setup Browser Cookies')]) {
expect(() => buildCookieWorkflowJudgeInput(fixture(broken))).toThrow('expected exactly one marker');
}
for (const marker of ['#### Choosing a source and checking sign-in', '### Tabs + frames']) {
expect(() => buildCookieWorkflowJudgeInput(fixture(skill, browser.replace(marker, 'missing')))).toThrow('expected exactly one marker');
expect(() => buildCookieWorkflowJudgeInput(fixture(skill, browser + marker + '\n'))).toThrow('expected exactly one marker');
}
});
test('rejects empty, reversed, or unreferenced excerpts', () => {
expect(() => buildCookieWorkflowJudgeInput(fixture('# Setup Browser Cookies\n'))).toThrow('empty or reversed');
expect(() => buildCookieWorkflowJudgeInput(fixture(skill, '#### Choosing a source and checking sign-in\n\n### Tabs + frames\n'))).toThrow('empty or reversed');
expect(() => buildCookieWorkflowJudgeInput(fixture(skill, '### Tabs + frames\n#### Choosing a source and checking sign-in\nContent\n'))).toThrow('empty or reversed');
expect(() => buildCookieWorkflowJudgeInput(fixture(skill.replace('`BROWSER.md`', '`other.md`')))).toThrow('missing cookie reference link');
expect(() => buildCookieWorkflowJudgeInput(fixture(skill.replace('**Choosing a source and checking sign-in**', '**Other section**')))).toThrow('missing cookie reference link');
});
test('generated host workflows resolve shared references from the installation root and check Aside first', () => {
for (const file of ['setup-browser-cookies/SKILL.md', '.agents/skills/gstack-setup-browser-cookies/SKILL.md']) {
const source = readFileSync(join(ROOT, file), 'utf8');
const body = source.slice(source.indexOf('# Setup Browser Cookies'));
expect(body).toContain('Use this checkout as the gstack root if it contains `BROWSER.md` and `browse/SKILL.md`');
expect(body).toContain('the installed root containing `bin/gstack-skill-start`, never a generated host stub');
expect(body).toContain("Read that root's `browse/SKILL.md`");
expect(body).toContain('On `READY`, stop importing');
expect(body.indexOf('**BROWSER SETUP**')).toBeLessThan(body.indexOf('## SETUP'));
expect(body).toContain("Read that same root's `BROWSER.md`, **Choosing a source and checking sign-in**");
expect(body).not.toContain('../BROWSER.md');
}
expect(readFileSync(join(ROOT, 'browse/SKILL.md'), 'utf8')).toContain('## BROWSER SETUP (Aside');
expect(readFileSync(join(ROOT, 'BROWSER.md'), 'utf8')).toContain('#### Choosing a source and checking sign-in');
});
test('each owned dependency selects this judge in the fast PR profile', () => {
for (const file of ['setup-browser-cookies/SKILL.md.tmpl', 'setup-browser-cookies/SKILL.md', 'BROWSER.md', 'test/helpers/cookie-workflow-judge-input.ts', 'test/cookie-workflow-judge-input.test.ts', 'test/helpers/cookie-workflow-manual-review.ts', 'test/cookie-workflow-manual-review.test.ts', 'test/helpers/manual-judge-review-fixture.ts', '.github/cookie-workflow-manual-review.json']) {
const selectedJudges = selectTests([file], LLM_JUDGE_TOUCHFILES).selected;
expect(selectedJudges).toEqual([NAME]);
const selectedE2E = selectTests([file], E2E_TOUCHFILES).selected;
const profile = selectPrProfile({ changedFiles: [file], selectedJudges, selectedE2E });
expect(profile.judges).toEqual([NAME]);
expect(profile.deferredPromptFiles).toEqual([]);
expect(profile.missingCoverage).toEqual([]);
expect(profile.needsFullValidation).toBe(false);
}
expect(selectTests(['README.md'], LLM_JUDGE_TOUCHFILES).selected).not.toContain(NAME);
});
test('the managed callback sends and records exact custom input without borrowing a normal cache identity', async () => {
const root = fixture();
const input = buildCookieWorkflowJudgeInput(root);
let scores = passingScore;
const h = actualCookieCallback(root, { judge: async () => scores });
await h.run();
expect(h.requests).toHaveLength(1);
expect(h.requests[0].prompt).toBe(input.prompt);
expect(h.requests[0].model).toBeUndefined();
expect(h.requests[0].signal).toBeInstanceOf(AbortSignal);
expect(h.records[0]).toMatchObject({ name: NAME, prompt: input.prompt, model: 'fixture-model', execution: 'executed', passed: true });
expect(existsSync(join(root, 'cache'))).toBe(false);
const fresh = actualCookieCallback(root);
await fresh.run();
expect(fresh.requests).toHaveLength(1);
for (const dimension of ['clarity', 'completeness', 'actionability'] as const) {
scores = { ...COOKIE_WORKFLOW_JUDGE.thresholds, [dimension]: COOKIE_WORKFLOW_JUDGE.thresholds[dimension] - 1, reasoning: 'Synthetic failing fixture score' };
await expect(h.run()).rejects.toThrow();
expect(h.records.at(-1)).toMatchObject({ passed: false, exit_reason: 'validation_failed', prompt: input.prompt });
}
expect(existsSync(join(root, 'cache'))).toBe(false);
});
test('custom input timeout aborts once and a late response cannot overwrite its successful retry', async () => {
const root = fixture();
let completeLate!: (value: FixtureScore) => void;
let calls = 0, now = 0, timerId = 0;
const timers = new Map<number, () => void>();
const h = actualCookieCallback(root, {
budget: 20, clock: () => now,
setTimer: ((callback: () => void) => { timers.set(++timerId, callback); return timerId; }) as any,
clearTimer: ((id: number) => { timers.delete(id); }) as any,
judge: async () => ++calls === 1 ? new Promise(resolve => { completeLate = resolve; }) : passingScore,
});
const expired = h.run().catch((error: Error) => error);
now = 20;
for (const callback of [...timers.values()]) callback();
expect((await expired as Error).message).toContain('deadline');
expect(h.records).toHaveLength(1);
expect(h.records[0]).toMatchObject({ passed: false, exit_reason: 'timeout', prompt: buildCookieWorkflowJudgeInput(root).prompt });
expect(h.requests[0].signal.aborted).toBe(true);
await h.run();
expect(h.records.map(record => record.passed)).toEqual([false, true]);
expect(h.attempts.get(NAME)?.attempt).toBe(2);
completeLate(passingScore);
await new Promise(resolve => setImmediate(resolve));
expect(h.records).toHaveLength(2);
expect(existsSync(join(root, 'cache'))).toBe(false);
});
test('a superseding custom-input attempt cancels its predecessor without stale recording', async () => {
let completeLate!: (value: FixtureScore) => void;
let calls = 0;
const h = actualCookieCallback(fixture(), {
judge: async () => ++calls === 1 ? new Promise(resolve => { completeLate = resolve; }) : passingScore,
});
const old = h.run().catch((error: Error) => error);
await h.run();
expect((await old as Error).name).toBe('WorkflowJudgeSuperseded');
expect(h.requests[0].signal.aborted).toBe(true);
completeLate(passingScore);
await new Promise(resolve => setImmediate(resolve));
expect(h.records.map(record => [record.passed, record.exit_reason])).toEqual([[false, 'cancelled'], [true, undefined]]);
});
});
+103
View File
@@ -0,0 +1,103 @@
import { afterEach, expect, test } from 'bun:test';
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join, resolve } from 'node:path';
import { buildCookieWorkflowJudgeInput, COOKIE_WORKFLOW_JUDGE } from './helpers/cookie-workflow-judge-input';
import { COOKIE_MANUAL_REVIEW_FILE, getCookieWorkflowManualReview, isManualReviewEntry, manualReviewProblem } from './helpers/cookie-workflow-manual-review';
import { manualReviewFixture } from './helpers/manual-judge-review-fixture';
import { validWorkflowJudgeScore } from './helpers/workflow-judge-cache';
const ROOT = resolve(import.meta.dir, '..');
const roots: string[] = [];
afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); });
function fixture() {
const root = mkdtempSync(join(tmpdir(), 'cookie-policy-')); roots.push(root);
for (const file of [COOKIE_MANUAL_REVIEW_FILE, 'setup-browser-cookies/SKILL.md', 'BROWSER.md']) {
const target = join(root, file); mkdirSync(resolve(target, '..'), { recursive: true });
writeFileSync(target, readFileSync(join(ROOT, file)));
}
const entry = manualReviewFixture(root);
const approval = entry.manual_review!.approval;
const request = { testName: entry.name, prompt: entry.prompt!, model: entry.model!,
maxTokens: approval.max_tokens, thresholds: { ...approval.thresholds }, attempt: 1 };
return { root, entry, approval, request, refusal: entry.manual_review!.refusal };
}
test('the committed approval names precisely the complete reviewed request, not a numerical score', () => {
const f = fixture();
expect(buildCookieWorkflowJudgeInput(ROOT).sha256).toBe(f.approval.prompt_sha256);
expect(f.approval.thresholds).toEqual(COOKIE_WORKFLOW_JUDGE.thresholds);
expect(isManualReviewEntry(f.entry)).toBe(true);
expect(manualReviewProblem(f.entry, ROOT)).toBeNull();
expect(getCookieWorkflowManualReview(f.root, f.request, f.refusal)).toEqual(f.entry.manual_review);
expect(validWorkflowJudgeScore(f.entry as any, f.request.thresholds)).toBe(false);
expect(validWorkflowJudgeScore(f.entry.manual_review as any, f.request.thresholds)).toBe(false);
});
test.each(['case', 'prompt', 'model', 'budget', 'thresholds', 'source', 'approval', 'retry'])(
'admission rejects mismatched %s without changing the approval', mismatch => {
const f = fixture(); const original = readFileSync(join(f.root, COOKIE_MANUAL_REVIEW_FILE), 'utf8');
if (mismatch === 'case') f.request.testName = 'ship/SKILL.md workflow';
if (mismatch === 'prompt') f.request.prompt += '\n';
if (mismatch === 'model') f.request.model += '-other';
if (mismatch === 'budget') f.request.maxTokens++;
if (mismatch === 'thresholds') f.request.thresholds.clarity++;
if (mismatch === 'retry') f.request.attempt++;
if (mismatch === 'source') writeFileSync(join(f.root, 'BROWSER.md'), readFileSync(join(f.root, 'BROWSER.md'), 'utf8').replace('Storage stays intact', 'Changed storage stays intact'));
if (mismatch === 'approval') writeFileSync(join(f.root, COOKIE_MANUAL_REVIEW_FILE), JSON.stringify({ ...f.approval, prompt_sha256: 'a'.repeat(64) }));
expect(getCookieWorkflowManualReview(f.root, f.request, f.refusal)).toBeNull();
if (mismatch !== 'approval') expect(readFileSync(join(f.root, COOKIE_MANUAL_REVIEW_FILE), 'utf8')).toBe(original);
});
test.each(['passed', 'scores', 'reused', 'timeout', 'case', 'suite', 'prompt', 'model', 'refusal', 'missing-id', 'missing-response-id', 'output', 'text', 'tokens', 'schema', 'retry'])(
'malformed manual receipt %s cannot count as acceptance', invalid => {
const f = fixture(); const entry: any = f.entry;
if (invalid === 'passed') entry.passed = true;
if (invalid === 'retry') entry.attempt++;
if (invalid === 'scores') entry.judge_scores = { clarity: 1 };
if (invalid === 'reused') entry.execution = 'reused';
if (invalid === 'timeout') entry.exit_reason = 'timeout';
if (invalid === 'case') entry.name = 'ship/SKILL.md workflow';
if (invalid === 'suite') entry.suite = 'Unrelated suite';
if (invalid === 'prompt') entry.prompt += '\n';
if (invalid === 'model') entry.model = 'another-model';
if (invalid === 'refusal') entry.manual_review.refusal.stop_reason = 'end_turn';
if (invalid === 'missing-id') entry.manual_review.refusal.request_id = null;
if (invalid === 'missing-response-id') entry.manual_review.refusal.response_id = null;
if (invalid === 'output') entry.manual_review.refusal.output_tokens = 1;
if (invalid === 'text') entry.manual_review.refusal.text_blocks = 1;
if (invalid === 'tokens') entry.manual_review.refusal.input_tokens = -1;
if (invalid === 'schema') entry.manual_review.approval.schema_version = 2;
expect(isManualReviewEntry(entry)).toBe(false);
expect(manualReviewProblem(entry, f.root)).not.toBeNull();
});
test('reconciliation rejects changed source, approval provenance, missing approval and corrupt JSON', () => {
const f = fixture();
const modified = structuredClone(f.entry);
modified.manual_review!.approval.approved_by = 'unapproved-reviewer';
expect(manualReviewProblem(modified, f.root)).not.toBeNull();
const file = join(f.root, COOKIE_MANUAL_REVIEW_FILE);
writeFileSync(file, '{');
expect(manualReviewProblem(f.entry, f.root)).not.toBeNull();
rmSync(file);
expect(manualReviewProblem(f.entry, f.root)).not.toBeNull();
writeFileSync(file, JSON.stringify(f.approval));
writeFileSync(join(f.root, 'BROWSER.md'), readFileSync(join(f.root, 'BROWSER.md'), 'utf8').replace('Storage stays intact', 'Changed storage stays intact'));
expect(manualReviewProblem(f.entry, f.root)).not.toBeNull();
expect(isManualReviewEntry(f.entry)).toBe(true);
});
test('a current judge-model override cannot reuse the different approved model', () => {
const f = fixture();
const original = process.env.GSTACK_EVAL_MODEL_JUDGE;
try {
process.env.GSTACK_EVAL_MODEL_JUDGE = 'synthetic-unapproved-model';
expect(manualReviewProblem(f.entry, f.root)).toBe('Manual review model is not the current judge model');
expect(isManualReviewEntry(f.entry)).toBe(true);
} finally {
if (original === undefined) delete process.env.GSTACK_EVAL_MODEL_JUDGE;
else process.env.GSTACK_EVAL_MODEL_JUDGE = original;
}
});
+78
View File
@@ -0,0 +1,78 @@
import { afterEach, expect, test } from 'bun:test';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { createPrecisionLossCandidate } from './helpers/cso-ntfs-fixture';
const roots: string[] = [];
afterEach(() => {
for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true });
});
function fixture() {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ntfs-fixture-'));
roots.push(root);
return { root, target: path.join(root, 'candidate.json') };
}
test('the actual candidate builder reaches a precision-losing generation in a fresh allocator model', () => {
const { root, target } = fixture();
const generations = new Map<number, bigint>();
let reads = 0;
let selectedIdentity: bigint | undefined;
const inode = createPrecisionLossCandidate(target, 'owned candidate', file => {
reads++;
const slot = Number(path.basename(file).split('-').at(-1));
const generation = (generations.get(slot) ?? 0n) + 1n;
generations.set(slot, generation);
const result = (generation << 48n) + BigInt(101 + slot);
if (result > BigInt(Number.MAX_SAFE_INTEGER) && String(Number(result)) !== String(result)) {
selectedIdentity = fs.lstatSync(file, { bigint: true }).ino;
}
return result;
});
expect(inode).toBeGreaterThan(BigInt(Number.MAX_SAFE_INTEGER));
expect(String(Number(inode))).not.toBe(String(inode));
expect(reads).toBeLessThanOrEqual(1024);
expect(fs.lstatSync(target, { bigint: true }).ino).toBe(selectedIdentity);
expect(fs.readFileSync(target, 'utf8')).toBe('owned candidate');
expect(fs.readdirSync(root)).toEqual(['candidate.json']);
});
test('an unavailable precision-losing ID fails at the original creation bound and leaves no candidate', () => {
const { root, target } = fixture();
for (const inode of [1n, 2n ** 54n]) {
let reads = 0;
expect(() => createPrecisionLossCandidate(target, 'owned candidate', () => { reads++; return inode; }))
.toThrow('within 1024 file creations');
expect(reads).toBe(1024);
expect(fs.readdirSync(root)).toEqual([]);
}
});
test('existing targets and candidate-inspection failures never overwrite unrelated fixture state', () => {
const { root, target } = fixture();
fs.writeFileSync(target, 'preserved');
expect(() => createPrecisionLossCandidate(target, 'replacement')).toThrow('already exists');
expect(fs.readFileSync(target, 'utf8')).toBe('preserved');
const next = path.join(root, 'next.json');
expect(() => createPrecisionLossCandidate(next, 'owned candidate', () => { throw new Error('inspection failed'); }))
.toThrow('inspection failed');
expect(fs.readdirSync(root)).toEqual(['candidate.json']);
});
test('a concurrent target and a linked parent are preserved rather than written through', () => {
const { root, target } = fixture();
expect(() => createPrecisionLossCandidate(target, 'replacement', () => {
fs.writeFileSync(target, 'concurrent fixture');
return (1n << 54n) + 1n;
})).toThrow();
expect(fs.readFileSync(target, 'utf8')).toBe('concurrent fixture');
const owned = path.join(root, 'owned');
const link = path.join(root, 'linked');
fs.mkdirSync(owned);
fs.symlinkSync(owned, link, process.platform === 'win32' ? 'junction' : 'dir');
expect(() => createPrecisionLossCandidate(path.join(link, 'candidate.json'), 'replacement'))
.toThrow('must not be linked');
expect(fs.readdirSync(owned)).toEqual([]);
});
+5 -12
View File
@@ -4,6 +4,7 @@ import * as os from 'node:os';
import * as path from 'node:path';
import { createHash } from 'node:crypto';
import { spawn, spawnSync } from 'node:child_process';
import { createPrecisionLossCandidate } from './helpers/cso-ntfs-fixture';
const ROOT = path.resolve(import.meta.dir, '..');
const windows = process.platform === 'win32';
@@ -347,19 +348,11 @@ describe('CSO native Windows build contract', () => {
if(i!==2)continue;
const token='d'.repeat(32),candidate=path.join(leases,`${token}.json`),decision=path.join(leases,`${token}.decision`);
for(const rounded of [false,true]){
let selected:string|undefined;
for(let batch=0;batch<16&&!selected;batch++){
const paths:string[]=[];
for(let index=0;index<64;index++){
const file=path.join(leases,`fixture-${batch}-${index}`);
fs.writeFileSync(file,JSON.stringify({pid:2147483647,token,createdAt:0})+'\n');paths.push(file);
const inode=fs.lstatSync(file,{bigint:true}).ino;
if(!selected&&inode>BigInt(Number.MAX_SAFE_INTEGER)&&String(Number(inode))!==String(inode))selected=file;
}
for(const file of paths){if(file===selected)fs.renameSync(file,candidate);else fs.unlinkSync(file);}
}
expect(selected).toBeDefined();
const selected=createPrecisionLossCandidate(candidate,JSON.stringify({pid:2147483647,token,createdAt:0})+'\n');
const stat=fs.lstatSync(candidate,{bigint:true}),record={schemaVersion:1,token,kind:'ticket',ticket:'0000000000000001',candidateDev:String(stat.dev),candidateIno:rounded?String(Number(stat.ino)):String(stat.ino),ownerPid:2147483647,ownerCreatedAt:0,publisherPid:2147483647,createdAt:0};
expect(stat.ino).toBe(selected);
expect(stat.ino).toBeGreaterThan(BigInt(Number.MAX_SAFE_INTEGER));
expect(String(Number(stat.ino))).not.toBe(String(stat.ino));
fs.writeFileSync(decision,JSON.stringify(record)+'\n');
const result=command(['resume',run.runId]);expect(result.status).not.toBe(0);
if(rounded){
+1 -1
View File
@@ -29,7 +29,7 @@ describe('AO completed manual DX handoff preserves report freshness',()=>{
expect(E2E_TOUCHFILES[owner]).toContain('test/fixtures/dx-manual-handoff-ao.json');
}
const arrays=[...Object.values(E2E_TOUCHFILES),...Object.values(LLM_JUDGE_TOUCHFILES),GLOBAL_TOUCHFILES];
expect(arrays).toHaveLength(233); // Upstream owners plus eleven shared-code audit/review evals.
expect(arrays).toHaveLength(234);
for(const values of arrays)for(let i=0;i<values.length;i++)expect(typeof values[i]).toBe('string');
});
test('exact owned report precedes navigation only, with the current Exit gate recognized',()=>{
+2 -2
View File
@@ -176,9 +176,9 @@ test('current detach supervision covers the live-census floor', () => {
const floor = Math.ceil((Math.ceil(files.length / DEFAULT_JOBS) * DEFAULT_SHARD_TIMEOUT_MS + excess) / 1000 * 1.05);
const pkg = JSON.parse(fs.readFileSync(path.join(import.meta.dir, '../package.json'), 'utf8'));
const configured = Number(pkg.scripts['eval:bg:periodic'].match(/--timeout\s+(\d+)/)[1]);
expect(floor).toBe(64995);
expect(floor).toBe(65268);
expect(configured).toBeGreaterThanOrEqual(floor);
expect(pkg.scripts['eval:bg:gate']).toContain('--timeout 33600');
expect(pkg.scripts['eval:bg:gate']).toContain('--timeout 33800');
});
for (const jobs of [1, 2, 3]) test(`FIFO bound covers partial durations with ${jobs} workers`, () => {
+12 -3
View File
@@ -127,7 +127,7 @@ async function mockedObservation(frames: string[], verdict: 'waiting' | 'working
expect(start).toBeGreaterThan(0); expect(end).toBeGreaterThan(start);
const executable = source.slice(start, end).replace('export async function', 'async function') + '\nreturn runPlanSkillObservation;';
const js = new Bun.Transpiler({ loader: 'ts' }).transformSync(executable);
let clock = 0, tick = -1, closed = 0, judged = 0;
let clock = 0, tick = -1, closed = 0, judged = 0, seedSubmittedAt: number | null = null;
const current = () => frames[Math.min(Math.max(tick, 0), frames.length - 1)]!;
const args: Record<string, unknown> = {
path, process: { cwd: () => '/synthetic-owned' }, Date: { now: () => clock }, randomUUID: () => 'owned',
@@ -136,7 +136,7 @@ async function mockedObservation(frames: string[], verdict: 'waiting' | 'working
visibleSince: current, rawOutput: current, currentScreen: async () => current(), hermeticConfigDir: null,
close: async () => { closed++; } }),
createPlanCountSnapshotWriter: () => () => ({}), logPtySnapshot: () => {},
submitPlanSeed: async () => {}, PlanSeedTimeout: class extends Error {},
submitPlanSeed: async () => { seedSubmittedAt = clock; }, PlanSeedTimeout: class extends Error {},
isRejectedSlashCommand: predicates.isRejectedSlashCommand,
isProseAUQVisible: predicates.isProseAUQVisible, isPlanReadyVisible: predicates.isPlanReadyVisible,
isUnknownSlashCommandVisible: predicates.isUnknownSlashCommandVisible,
@@ -149,9 +149,18 @@ async function mockedObservation(frames: string[], verdict: 'waiting' | 'working
const obs = await run({ skillName: 'plan-eng-review', timeoutMs: 70000,
...(seeded ? { initialPlanContent: '# Plan: Required draft' } : {}) });
expect(closed).toBe(1);
return { obs, judged };
return { obs, judged, seedSubmittedAt };
}
test('seeded preflight checks owned readiness without spending eight seconds before submission', async () => {
const seeded = await mockedObservation([gate], 'working');
expect(seeded.seedSubmittedAt).toBe(0);
expect(seeded.obs.outcome).toBe('plan_ready');
const unseeded = await mockedObservation([gate], 'working', false);
expect(unseeded.seedSubmittedAt).toBeNull();
expect(unseeded.obs.outcome).toBe('plan_ready');
});
for (const [name, current] of [
['cursorless approval', gate.replace('❯ ', '')],
['partial approval', gate.split('\n').slice(0, -1).join('\n')],
+61
View File
@@ -29,11 +29,22 @@ import * as os from 'os';
import * as path from 'path';
import { runBin } from './helpers/run-bin';
import { selectTests, E2E_TOUCHFILES, LLM_JUDGE_TOUCHFILES, GLOBAL_TOUCHFILES } from './helpers/touchfiles';
import { manualReviewFixture } from './helpers/manual-judge-review-fixture';
import { renderDashboard } from '../scripts/eval-watch';
const ROOT = path.resolve(import.meta.dir, '..');
const SCRIPT = (name: string) => path.join(ROOT, 'scripts', name);
const SLUG = 'eval-cli-fixture';
test('eval:watch distinguishes unscored manual acceptance from malformed claims', () => {
const manual = manualReviewFixture();
const output = renderDashboard(null, { tests: [manual, { ...manual, passed: true }], total_cost_usd: 0 });
expect(output).toContain('MANUAL/unscored');
expect(output).toContain(manual.manual_review!.approval.approval_url);
expect(output).toContain('Manual accepted: 1');
expect(output).toContain('✗');
});
let tmpHome: string;
let evalDir: string;
@@ -256,6 +267,22 @@ describe('eval:list CLI (scripts/eval-list.ts)', () => {
// ── eval-compare ─────────────────────────────────────────────────────────────
describe('eval:compare CLI (scripts/eval-compare.ts)', () => {
test('shows a manual transition without calling it a scored regression', () => {
const manual = manualReviewFixture();
const prior = writeRun(evalDir, { tier: 'llm-judge', timestamp: '2026-01-01T01:00:00Z',
tests: [{ name: manual.name, passed: true }] });
const after = writeRun(evalDir, { tier: 'llm-judge', timestamp: '2026-01-02T01:00:00Z',
tests: [{ name: manual.name, passed: false }] });
const body = JSON.parse(fs.readFileSync(after, 'utf8'));
body.tests = [manual]; body.passed = 0; body.failed = 0; body.manual_accepted_tests = 1;
fs.writeFileSync(after, JSON.stringify(body));
const result = runEvalCli('eval-compare.ts', prior, after);
expect(result.status).toBe(0);
expect(result.stdout).toContain('PASS → MANUAL');
expect(result.stdout).toContain('unscored manual review');
expect(result.stdout).not.toContain('REGRESSION:');
});
test('empty eval dir prints the getting-started hint and exits 0', () => {
const result = runEvalCli('eval-compare.ts');
expect(result.status).toBe(0);
@@ -337,6 +364,40 @@ describe('eval:compare CLI (scripts/eval-compare.ts)', () => {
// ── eval-summary ─────────────────────────────────────────────────────────────
describe('eval:summary CLI (scripts/eval-summary.ts)', () => {
test('reports manual provenance without inventing a scored flake', () => {
const manual = manualReviewFixture();
writeRun(evalDir, { tier: 'llm-judge', timestamp: '2026-01-01T01:00:00Z',
tests: [{ name: manual.name, passed: true }] });
const accepted = writeRun(evalDir, { tier: 'llm-judge', timestamp: '2026-01-02T01:00:00Z',
tests: [{ name: manual.name, passed: false }] });
const body = JSON.parse(fs.readFileSync(accepted, 'utf8'));
body.tests = [manual]; body.passed = 0; body.failed = 0; body.manual_accepted_tests = 1;
fs.writeFileSync(accepted, JSON.stringify(body));
const result = runEvalCli('eval-summary.ts');
expect(result.status).toBe(0);
expect(result.stdout).toContain('Manual accepted: 1 unscored provider refusal');
expect(result.stdout).toContain(manual.manual_review!.approval.approval_url);
expect(result.stdout).not.toContain('Flaky tests');
});
test('retried manual claims remain failed in summary history', () => {
const manual = manualReviewFixture();
writeRun(evalDir, { tier: 'llm-judge', timestamp: '2026-01-01T01:00:00Z',
tests: [{ name: manual.name, passed: true }] });
const attempted = writeRun(evalDir, { tier: 'llm-judge', timestamp: '2026-01-02T01:00:00Z',
tests: [{ name: manual.name, passed: false }] });
const body = JSON.parse(fs.readFileSync(attempted, 'utf8'));
const { manual_review: _receipt, ...ordinary } = manual;
body.tests = [{ ...ordinary, attempt: 1 }, { ...manual, attempt: 2 }];
body.total_tests = 2; body.passed = 0; body.failed = 2;
fs.writeFileSync(attempted, JSON.stringify(body));
const result = runEvalCli('eval-summary.ts');
expect(result.status).toBe(0);
expect(result.stdout).not.toContain('Manual accepted:');
expect(result.stdout).toContain('Flaky tests (1):');
expect(result.stdout).toContain(`llm-judge:${manual.name}`);
});
test('empty eval dir prints the getting-started hint and exits 0', () => {
const result = runEvalCli('eval-summary.ts');
expect(result.status).toBe(0);
+29
View File
@@ -10,7 +10,9 @@ import { describe, expect, test } from 'bun:test';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { spawnSync } from 'node:child_process';
import { aggregate, collectEvalFiles } from '../scripts/eval-flake-rank';
import { manualReviewFixture } from './helpers/manual-judge-review-fixture';
const entry = (name: string, passed: boolean, attempt: number) => ({
name, suite: 's', tier: 'e2e', passed, attempt, duration_ms: 1000, cost_usd: 0.1,
@@ -24,6 +26,33 @@ const run = (tests: object[], extra: object = {}) => JSON.stringify({
});
describe('eval-flake-rank aggregate', () => {
test('manual acceptance is visible but not a scored failure or retried pass', () => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'flakerank-manual-'));
const manual = manualReviewFixture();
const { manual_review: _receipt, ...ordinary } = manual;
fs.writeFileSync(path.join(dir, 'prior.json'), run([{ ...ordinary, passed: true, exit_reason: 'success',
judge_scores: { clarity: 4, completeness: 3, actionability: 4 } }]));
fs.writeFileSync(path.join(dir, 'accepted.json'), run([manual]));
const series = aggregate(collectEvalFiles(dir)).get(manual.name);
expect(series).toMatchObject({ runs: 1, passes: 1, fails: 0, manualAccepted: 1,
retriedPasses: 0, totalAttempts: 2 });
const display = spawnSync(process.execPath, [path.resolve(import.meta.dir, '../scripts/eval-flake-rank.ts'), '--dir', dir],
{ encoding: 'utf8', timeout: 10_000 });
expect(display.status, display.stderr).toBe(0);
expect(display.stdout).toContain('fails/runs manual');
expect(display.stdout).toContain('0/1');
expect(display.stdout).toContain(manual.name);
fs.writeFileSync(path.join(dir, 'invalid-retry.json'), run([
{ ...ordinary, attempt: 1 }, { ...manual, attempt: 2 },
]));
expect(aggregate(collectEvalFiles(dir)).get(manual.name)).toMatchObject({ runs: 2, passes: 1,
fails: 1, manualAccepted: 1, retriedPasses: 0 });
fs.writeFileSync(path.join(dir, 'invalid-pass.json'), run([{ ...manual, passed: true }]));
expect(aggregate(collectEvalFiles(dir)).get(manual.name)).toMatchObject({ runs: 3, passes: 1,
fails: 2, manualAccepted: 1, retriedPasses: 0 });
fs.rmSync(dir, { recursive: true, force: true });
});
test('final attempt decides; retried pass counts as retriedPass, not a fail', () => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'flakerank-'));
fs.writeFileSync(path.join(dir, 'run1.json'), run([
+20
View File
@@ -3,6 +3,7 @@ import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
import { spawnSync } from 'child_process';
import { manualReviewFixture } from './helpers/manual-judge-review-fixture';
const ROOT = path.resolve(import.meta.dir, '..');
@@ -75,6 +76,25 @@ function runEvalList(...args: string[]): { stdout: string; stderr: string; statu
}
describe('eval:list CLI', () => {
test('labels validated manual acceptance as unscored and rejects malformed pass claims', () => {
const manual = manualReviewFixture();
const dir = path.join(tmpHome, '.gstack-dev', 'evals');
const body = (entry: typeof manual, timestamp: string) => ({
schema_version: 2, version: '1.90.0', branch: 'main', git_sha: 'fixture', timestamp,
tier: 'llm-judge', total_tests: 1, passed: 0, failed: 0, total_cost_usd: 0,
total_duration_ms: 1, tests: [entry],
});
fs.writeFileSync(path.join(dir, 'manual.json'), JSON.stringify(body(manual, '2026-05-24T03:00:00Z')));
fs.writeFileSync(path.join(dir, 'invalid.json'), JSON.stringify(body({ ...manual, passed: true }, '2026-05-24T04:00:00Z')));
const result = runEvalList('--limit', '2');
expect(result.status).toBe(0);
const lines = result.stdout.split('\n');
expect(lines.find(line => line.includes('03:00'))).toContain('MANUAL/unscored');
expect(lines.find(line => line.includes('03:00'))).toContain(manual.manual_review!.approval.approval_url);
expect(lines.find(line => line.includes('04:00'))).toContain('0/1');
expect(lines.find(line => line.includes('04:00'))).not.toContain('MANUAL');
});
test('limits displayed eval runs with a valid positive integer', () => {
const result = runEvalList('--limit', '1');
@@ -0,0 +1,133 @@
{
"sourceRun": 36036582724,
"sourceRevision": "0d455d385272fee9fcd873951605e2cf46d8757c",
"cases": [
{
"attempt": 1,
"input": {
"questions": [
{
"question": "Core checklist pass: no defects. 1 optional advisory needs your input (prior Skip could NOT be reused — see note).\n\n1. [ADVISORY] src/retry-worker.ts:2 — retrySeconds is re-implemented byte-for-byte identical to lib/retry-after.ts:2–15 (a tested helper already used by src/scheduler.ts:1). src/retry-route.ts:2–15 carries the same copy. Three copies of the 3600s-ceiling / date-parse / fallback contract will drift.\n Fix: worker → `export { retrySeconds } from '../lib/retry-after';` (this is exactly what HEAD had). Same one-liner for the route. Existing test/retry-after.test.ts covers the contract; no new tests needed. Est. impl: −28 / +2 = −26 lines; total −30 / +2 = −28 (comment lines included). Blast radius: a helper bug now hits worker + route + scheduler together — but they already run identical logic.\n\nNOTE: src/retry-route.ts is flagged skip-worktree in the index and its on-disk bytes differ from the tracked blob (an extra trailing comment). git diff/status hide this. Any edit to that file will also be invisible to Git until the flag is cleared.\n\nRECOMMENDATION: A — reuse the helper in both files; the worker change is inside this diff and the route copy is the same duplication.",
"header": "Advisory #1",
"options": [
{ "label": "A) Fix both (Recommended)", "description": "Replace the duplicated function in src/retry-worker.ts AND src/retry-route.ts with a re-export from lib/retry-after.ts." },
{ "label": "A') Fix worker only", "description": "Re-export in src/retry-worker.ts only (the changed file in this diff); leave src/retry-route.ts as-is." },
{ "label": "B) Skip", "description": "Keep the duplicated implementation. Recorded as an explicit skipped advisory (route excluded from reusable coverage)." }
],
"multiSelect": false
},
{
"question": "Separately: should I clear the skip-worktree flag on src/retry-route.ts (`git update-index --no-skip-worktree`) so its hidden local change becomes visible to git status/diff? This changes only the local index flag, not file contents.",
"header": "skip-worktree",
"options": [
{ "label": "Clear the flag (Recommended)", "description": "Make the hidden modification to src/retry-route.ts visible so it can be reviewed/committed normally." },
{ "label": "Leave it", "description": "Keep the flag; the route change stays hidden from Git and is reported only in this review summary." }
],
"multiSelect": false
}
]
}
},
{
"attempt": 2,
"input": {
"questions": [
{
"question": "1. [ADVISORY] src/retry-worker.ts:2 — This diff replaces the worker's `export { retrySeconds } from '../lib/retry-after'` (HEAD) with a verbatim 14-line copy of the helper. Verified callers: src/retry-worker.ts:2-15 (changed anchor) and src/retry-route.ts:2-15 (authored, first-party) both duplicate lib/retry-after.ts:2-15 `retrySeconds` byte-for-byte — same signature, fallback=5 default, 3600s cap, negative→0, unsafe-int→fallback. The helper is already tested (test/retry-after.test.ts 'retry parser contract') and in established use (src/scheduler.ts:1-2). No preserved differences exist; same language/repo/relative-import layout. Smallest adoption: re-export from lib in both files. Estimated implementation: removed 30, added 2, saved 28 (worker −15/+1, route −15/+1). Total incl. tests: same — existing test already covers the contract; grows by ~5-8 lines only if you want an export-identity integration test. Shared-failure blast radius: unchanged (all three already run identical logic; a bug would just become fixable in one place). Note: src/retry-route.ts is flagged assume-unchanged in your index, so any edit to it is invisible to `git diff`/`git commit -a` until the flag is cleared. RECOMMENDATION: A — reuse the proven helper in both callers. Which do you want?",
"header": "Shared-libs",
"multiSelect": false,
"options": [
{ "label": "A) Fix both callers (Recommended)", "description": "Restore the one-line re-export in src/retry-worker.ts and replace the duplicate body in src/retry-route.ts with the same re-export. Net −28 lines." },
{ "label": "B) Fix worker only", "description": "Restore the re-export in src/retry-worker.ts (the file this diff actually changes). Leave src/retry-route.ts's copy alone. Net −14 lines." },
{ "label": "C) Skip", "description": "Keep the duplicated implementation in the worker. Recorded as an explicit skip for this advisory." }
]
},
{
"question": "2. Git state note (not a code defect): src/retry-route.ts has the assume-unchanged bit set (`git ls-files -v` shows `h`), and its working-tree bytes differ from the index — a trailing comment was appended that git diff/status hide. Should I clear the flag with `git update-index --no-assume-unchanged src/retry-route.ts` so the file's real state is visible to Git (and to any edit from question 1)?",
"header": "Index flag",
"multiSelect": false,
"options": [
{ "label": "Yes, clear the flag (Recommended)", "description": "Run `git update-index --no-assume-unchanged src/retry-route.ts`. The hidden trailing-comment change will then appear as a normal modification." },
{ "label": "No, leave it", "description": "Keep the local index flag as is. I will still report the hidden change and exclude the route from reusable snapshot coverage." }
]
}
]
}
}
],
"regressions": [
{
"sourceRun": 36044212977,
"sourceRevision": "70b023b3424453207c95e5b05298955f3094b50f",
"attempt": 1,
"toolUseId": "toolu_01DLWrTd85Uf3PA4gagJCFrW",
"input": {
"questions": [
{
"question": "Pre-Landing Review: 0 issues (0 critical, 0 informational). 1 [ADVISORY] needs your input — the prior 'skipped' decision could not be reused because src/retry-route.ts is skip-worktree with hidden raw changes.\n\n1. [ADVISORY] src/retry-worker.ts:2 — The diff replaces `export { retrySeconds } from '../lib/retry-after'` with a verbatim 15-line copy of the tested helper; src/retry-route.ts:2-15 carries the same copy. Helper: lib/retry-after.ts `retrySeconds` (already used by src/scheduler.ts, covered by test/retry-after.test.ts). Migration: worker → one-line re-export (revert to HEAD form); route → same one-liner. Estimated implementation: −30 / +2 = 28 lines saved; total change same (existing test already covers the contract, no new tests needed). Shared-failure blast radius: scheduler, worker, route — all three copies are identical today, so consolidation removes drift risk without new behavior. Caveat: src/retry-route.ts has the skip-worktree index flag, so an edit there stays invisible to git until the flag is cleared (`git update-index --no-skip-worktree`).\n\nRECOMMENDATION: A — reuse the existing tested helper in both files.\n\nHow should I handle this advisory?",
"header": "Advisory",
"multiSelect": false,
"options": [
{
"label": "A) Fix both (Recommended)",
"description": "Replace the copies in src/retry-worker.ts and src/retry-route.ts with `export { retrySeconds } from '../lib/retry-after';`. I will NOT change the skip-worktree flag; you decide how to stage the route file."
},
{
"label": "A2) Fix worker only",
"description": "Only revert the changed file in this diff (src/retry-worker.ts) to the one-line re-export. Leave src/retry-route.ts untouched."
},
{
"label": "B) Skip",
"description": "Keep the duplicated copies. Recorded as an explicit skip; reuse of this skip will still require revalidation while src/retry-route.ts is skip-worktree."
}
]
}
]
}
},
{
"sourceRun": 36080890009,
"sourceRevision": "5460ce0847574aadfa6cbdd0b9545d2caf9935da",
"attempt": 1,
"toolUseId": "toolu_01Up8B1FR4bhkmRxyhbcAqhY",
"input": {
"questions": [
{
"question": "Pre-Landing Review: 0 issues (0 critical, 0 informational) + 1 advisory.\n\n1. [ADVISORY] (confidence 10/10, core + maintainability, MULTI-SPECIALIST CONFIRMED) src/retry-worker.ts:2 — This diff replaces the re-export of the tested helper with a verbatim copy of retrySeconds; src/retry-route.ts:2-15 carries the same copy. Both bodies are byte-identical to lib/retry-after.ts:2-15 (already used by src/scheduler.ts, covered by test/retry-after.test.ts, 5 assertions passing).\n Fix: `export { retrySeconds } from '../lib/retry-after';` in each caller. Implementation: worker -15/+1, route -15/+1 → ~28 lines saved; no new tests needed (existing contract test covers the shared helper); shared-failure blast radius = scheduler, worker, route all depend on one parser (they already share the contract by copy).\n\nNote: your prior review skipped this same advisory, but src/retry-route.ts has changed on disk since then behind an assume-unchanged index flag, so that decision could not be reused.\n\nRECOMMENDATION: Fix both callers — the copies are identical, the helper is proven, and the diff itself undoes prior reuse.\n\nHow do you want to handle the advisory?",
"header": "Advisory",
"options": [
{
"label": "Fix both callers (Recommended)",
"description": "Restore the re-export in src/retry-worker.ts and replace the copy in src/retry-route.ts with the same re-export (keeping its trailing comment)."
},
{
"label": "Fix worker only",
"description": "Restore the re-export in src/retry-worker.ts (the file changed in this diff); leave src/retry-route.ts as-is."
},
{
"label": "Skip",
"description": "Keep the duplicated implementation; record the advisory as skipped."
}
],
"multiSelect": false
},
{
"question": "src/retry-route.ts is marked assume-unchanged in the index (`git ls-files -v` → `h`), so its uncommitted on-disk edit (trailing comment, line 17) is invisible to `git status`/`git diff` and to the review snapshot fingerprint. Should I clear that flag so the change becomes visible and reviewable?",
"header": "Git index",
"options": [
{
"label": "Clear the flag (Recommended)",
"description": "Run `git update-index --no-assume-unchanged src/retry-route.ts` so Git tracks the working-tree change normally."
},
{
"label": "Leave it",
"description": "Keep the assume-unchanged bit; the path will be excluded from snapshot coverage in the persisted record."
}
],
"multiSelect": false
}
]
}
}
]
}
+1 -1
View File
@@ -4605,7 +4605,7 @@ export async function runPlanSkillObservation(opts: {
};
// Entry deadline → boot → owned paste/receipt/ack → slash → observation.
// Setup consumes the existing case budget; cleanup has its separate grace.
await Bun.sleep(Math.min(8000, Math.max(0, deadlineAt - Date.now())));
if (!opts.initialPlanContent) await Bun.sleep(Math.min(8000, Math.max(0, deadlineAt - Date.now())));
if (opts.initialPlanContent) {
const seed = `Keep this draft plan as context. Briefly acknowledge receipt, then wait for my next message containing a slash command. Do not start the review or call tools yet.\n\n${opts.initialPlanContent}`;
try {
@@ -0,0 +1,50 @@
import { createHash } from 'node:crypto';
import { readFileSync } from 'node:fs';
import { join } from 'node:path';
import { buildWorkflowJudgePrompt, type WorkflowJudgeFile, type WorkflowJudgeInput } from './workflow-judge-input';
export const COOKIE_WORKFLOW_JUDGE = {
judgeContext: 'a fallback-browser cookie import workflow',
judgeGoal: 'how to select an authorized source browser, profile, and domain without guessing an account; configure optional authentication verification before mutation; obtain explicit consent for precisely scoped storage reset; distinguish copied cookies from positive sign-in evidence; and recover within the documented platform and privacy boundaries',
thresholds: { clarity: 4, completeness: 3, actionability: 4 },
} as const;
export function buildCookieWorkflowJudgeInput(root: string): WorkflowJudgeInput & { prompt: string; sha256: string } {
const files: WorkflowJudgeFile[] = [
{ path: 'setup-browser-cookies/SKILL.md', kind: 'entrypoint', start: '# Setup Browser Cookies', end: null },
{ path: 'BROWSER.md', kind: 'section', start: '#### Choosing a source and checking sign-in', end: '### Tabs + frames' },
].map(spec => {
const source = readFileSync(join(root, spec.path), 'utf8');
const locate = (marker: string): number => {
const matches = [...source.matchAll(new RegExp(`^${marker.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\r?$`, 'gm'))];
if (matches.length !== 1) throw new Error(`${spec.path}: expected exactly one marker ${JSON.stringify(marker)}, found ${matches.length}`);
return matches[0].index!;
};
const start = locate(spec.start);
const end = spec.end === null ? source.length : locate(spec.end);
if (end <= start || !source.slice(start + spec.start.length, end).trim()) {
throw new Error(`${spec.path}: empty or reversed cookie workflow excerpt`);
}
return {
path: spec.path,
kind: spec.kind as WorkflowJudgeFile['kind'],
content: source.slice(start, end),
startLine: source.slice(0, start).split('\n').length,
endLine: source.slice(0, end - 1).split('\n').length,
};
});
if (!files[0].content.includes('`BROWSER.md`') || !files[0].content.includes('**Choosing a source and checking sign-in**')) {
throw new Error('setup-browser-cookies/SKILL.md: missing cookie reference link');
}
const text = [
'SKILL.md is the entry point. BROWSER.md supplies the exact referenced cookie section, not an additional execution step.',
...files.map(file => [
`--- BEGIN FILE ${JSON.stringify(file.path)} (lines ${file.startLine}-${file.endLine}; ${file.kind}) ---`,
file.content,
`--- END FILE ${JSON.stringify(file.path)} ---`,
].join('\n')),
].join('\n\n');
const input = { files, text };
const prompt = buildWorkflowJudgePrompt(COOKIE_WORKFLOW_JUDGE, input);
return { ...input, prompt, sha256: createHash('sha256').update(prompt).digest('hex') };
}
@@ -0,0 +1,101 @@
import { createHash } from 'node:crypto';
import { readFileSync } from 'node:fs';
import { join } from 'node:path';
import { buildCookieWorkflowJudgeInput, COOKIE_WORKFLOW_JUDGE } from './cookie-workflow-judge-input';
import type { JudgeRefusalEvidence } from './llm-judge';
import { DEFAULT_JUDGE_MAX_TOKENS, resolveEvalModel } from '../../lib/eval-model';
export const COOKIE_MANUAL_REVIEW_FILE = '.github/cookie-workflow-manual-review.json';
const CASE = 'setup-browser-cookies/SKILL.md workflow';
type Thresholds = { clarity: number; completeness: number; actionability: number };
export interface CookieManualApproval {
schema_version: 1;
test_name: typeof CASE;
prompt_sha256: string;
prompt_bytes: number;
model: string;
max_tokens: number;
thresholds: Thresholds;
approved_by: string;
approved_at: string;
approval_url: string;
reason: string;
}
export interface ManualJudgeReview {
approval: CookieManualApproval;
refusal: JudgeRefusalEvidence;
}
const object = (value: unknown): value is Record<string, any> => value !== null && typeof value === 'object' && !Array.isArray(value);
const nonempty = (value: unknown): value is string => typeof value === 'string' && value.trim().length > 0;
const dimensions = ['clarity', 'completeness', 'actionability'] as const;
const sameThresholds = (a: Thresholds, b: Thresholds) => dimensions.every(key => a[key] === b[key]);
function validApproval(value: unknown): value is CookieManualApproval {
return object(value) && value.schema_version === 1 && value.test_name === CASE
&& typeof value.prompt_sha256 === 'string' && /^[a-f0-9]{64}$/.test(value.prompt_sha256)
&& Number.isSafeInteger(value.prompt_bytes) && value.prompt_bytes > 0
&& nonempty(value.model) && Number.isSafeInteger(value.max_tokens) && value.max_tokens > 0
&& object(value.thresholds) && dimensions.every(key => Number.isInteger(value.thresholds[key]) && value.thresholds[key] >= 1 && value.thresholds[key] <= 5)
&& nonempty(value.approved_by) && typeof value.approved_at === 'string' && /^\d{4}-\d{2}-\d{2}$/.test(value.approved_at)
&& typeof value.approval_url === 'string' && /^https:\/\/github\.com\/garrytan\/gstack\/pull\/2964#issuecomment-\d+$/.test(value.approval_url)
&& nonempty(value.reason);
}
function validRefusal(value: unknown, model: string): value is JudgeRefusalEvidence {
return object(value) && value.stop_reason === 'refusal' && value.model === model
&& nonempty(value.response_id) && nonempty(value.request_id)
&& Number.isSafeInteger(value.input_tokens) && value.input_tokens >= 0
&& value.output_tokens === 0 && value.text_blocks === 0;
}
export function isManualReviewEntry(entry: unknown): boolean {
if (!object(entry) || entry.name !== CASE || entry.suite !== 'Cookie setup workflow quality'
|| entry.tier !== 'llm-judge' || entry.passed !== false
|| entry.attempt !== 1
|| entry.execution !== 'executed' || entry.exit_reason !== 'provider_refusal'
|| entry.judge_scores !== undefined || entry.judge_reasoning !== undefined || entry.reused_from !== undefined
|| typeof entry.prompt !== 'string' || !object(entry.manual_review)) return false;
const { approval, refusal } = entry.manual_review;
return validApproval(approval) && entry.model === approval.model && validRefusal(refusal, approval.model)
&& Buffer.byteLength(entry.prompt) === approval.prompt_bytes
&& createHash('sha256').update(entry.prompt).digest('hex') === approval.prompt_sha256;
}
export function getCookieWorkflowManualReview(root: string, request: {
testName: string; prompt: string; model: string; maxTokens: number; thresholds: Thresholds; attempt: number;
}, refusal: JudgeRefusalEvidence): ManualJudgeReview | null {
if (request.testName !== CASE || request.attempt !== 1 || !validRefusal(refusal, request.model)) return null;
let approval: unknown;
try { approval = JSON.parse(readFileSync(join(root, COOKIE_MANUAL_REVIEW_FILE), 'utf8')); }
catch (error) {
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null;
throw error;
}
if (!validApproval(approval)) throw new Error('Invalid cookie workflow manual-review approval');
const current = buildCookieWorkflowJudgeInput(root);
if (request.prompt !== current.prompt || current.sha256 !== approval.prompt_sha256
|| Buffer.byteLength(request.prompt) !== approval.prompt_bytes || request.model !== approval.model
|| request.maxTokens !== approval.max_tokens || request.maxTokens !== DEFAULT_JUDGE_MAX_TOKENS
|| !sameThresholds(request.thresholds, approval.thresholds)
|| !sameThresholds(request.thresholds, COOKIE_WORKFLOW_JUDGE.thresholds)) return null;
return { approval, refusal };
}
export function manualReviewProblem(entry: unknown, root: string): string | null {
if (!object(entry) || !Object.hasOwn(entry, 'manual_review')) return null;
if (!isManualReviewEntry(entry)) return 'Malformed manual-review claim';
if (entry.model !== resolveEvalModel('judge')) return 'Manual review model is not the current judge model';
try {
const claimed = entry.manual_review as ManualJudgeReview;
const verified = getCookieWorkflowManualReview(root, { testName: entry.name, prompt: entry.prompt,
model: entry.model, maxTokens: claimed.approval.max_tokens, thresholds: claimed.approval.thresholds,
attempt: entry.attempt }, claimed.refusal);
if (!verified || Object.entries(verified.approval).some(([key, value]) => key === 'thresholds'
? !sameThresholds(value as Thresholds, claimed.approval.thresholds)
: value !== claimed.approval[key as keyof CookieManualApproval])) return 'Manual review does not match current source and approval';
return null;
} catch { return 'Manual review approval or current input is unavailable or invalid'; }
}
+34
View File
@@ -0,0 +1,34 @@
import * as fs from 'node:fs';
import * as path from 'node:path';
export function createPrecisionLossCandidate(
target: string,
contents: string,
readFileId: (file: string) => bigint = file => fs.lstatSync(file, { bigint: true }).ino,
): bigint {
if (fs.existsSync(target)) throw new Error('Precision fixture target already exists');
const directory = path.dirname(target);
if (fs.lstatSync(directory).isSymbolicLink()) throw new Error('Precision fixture directory must not be linked');
const pool = fs.mkdtempSync(path.join(directory, 'ntfs-id-'));
let aboveSafe = 0;
try {
for (let batch = 0; batch < 512; batch++) {
const files: string[] = [];
for (let slot = 0; slot < 2; slot++) {
const file = path.join(pool, `candidate-${slot}`);
fs.writeFileSync(file, contents, { flag: 'wx' });
files.push(file);
const inode = readFileId(file);
if (inode > BigInt(Number.MAX_SAFE_INTEGER)) aboveSafe++;
if (inode > BigInt(Number.MAX_SAFE_INTEGER) && String(Number(inode)) !== String(inode)) {
fs.linkSync(file, target);
return inode;
}
}
for (const file of files) fs.unlinkSync(file);
}
throw new Error(`No precision-losing NTFS file ID within 1024 file creations (${aboveSafe} above-safe observations)`);
} finally {
fs.rmSync(pool, { recursive: true, force: true });
}
}
+1 -1
View File
@@ -107,7 +107,7 @@ export const FILE_RETRY_BUDGETS = [
...[
// Fourteen workflow judges include their 10s recording grace; the other
// eleven judges retain 120s. Supervise all 25 and the existing one retry.
{ file: 'test/skill-llm-eval.test.ts', attemptMs: 14 * (JUDGE_MS + 10_000) + 11 * JUDGE_MS, retries: 1 },
{ file: 'test/skill-llm-eval.test.ts', attemptMs: 15 * (JUDGE_MS + 10_000) + 11 * JUDGE_MS, retries: 1 },
{ file: 'test/codex-e2e-plan-format.test.ts', attemptMs: 4 * (CAPTURE_LONG_MS + 10_000), retries: 1 },
{ file: 'test/skill-e2e-auq-matrix.test.ts', attemptMs: 6 * CAPTURE_MS, retries: 1 },
{ file: 'test/skill-e2e-plan-format.test.ts', attemptMs: 4 * (CAPTURE_MS + 10_000), retries: 1 },
+71
View File
@@ -10,11 +10,13 @@ import {
isPartialEval,
listEvalJsonFiles,
compareEvalResults,
evalEntryOutcome,
formatComparison,
generateCommentary,
judgePassed,
} from './eval-store';
import type { EvalResult, EvalTestEntry, ComparisonResult } from './eval-store';
import { manualReviewFixture } from './manual-judge-review-fixture';
let tmpDir: string;
@@ -77,6 +79,36 @@ async function captureStderr(fn: () => Promise<void>): Promise<string> {
// --- EvalCollector tests ---
describe('EvalCollector', () => {
test('manual provider refusal stays unscored and executed; malformed claims stay failed', async () => {
const manual = manualReviewFixture();
const invalidPass = { ...manual, passed: true };
const invalidScore = { ...manual, judge_scores: { clarity: 5 } };
expect(evalEntryOutcome(manual)).toBe('manual-review');
expect(evalEntryOutcome(invalidPass)).toBe('failed');
expect(evalEntryOutcome(invalidScore)).toBe('failed');
expect(evalEntryOutcome({ ...manual, manual_review: { ...manual.manual_review, approval: { ...manual.manual_review!.approval,
prompt_sha256: '0'.repeat(64) } } })).toBe('failed');
const collector = new EvalCollector('llm-judge', tmpDir);
collector.addTest(makeEntry({ name: 'ordinary', tier: 'llm-judge' }));
collector.addTest(manual);
collector.addTest(invalidPass);
collector.addTest(invalidScore);
const partial: EvalResult = JSON.parse(fs.readFileSync(path.join(tmpDir, '_partial-e2e.json'), 'utf8'));
expect(partial).toMatchObject({ passed: 1, failed: 2, manual_accepted_tests: 1,
executed_tests: 4, reused_tests: 0 });
const output = await captureStderr(async () => { await collector.finalize(); });
const result: EvalResult = JSON.parse(fs.readFileSync(fs.readdirSync(tmpDir).map(name => path.join(tmpDir, name))
.find(name => name.endsWith('.json') && !path.basename(name).startsWith('_'))!, 'utf8'));
expect(result).toMatchObject({ passed: 1, failed: 2, manual_accepted_tests: 1, executed_tests: 4 });
expect(result.tests[1]).toMatchObject({ passed: false, execution: 'executed', exit_reason: 'provider_refusal' });
expect(result.tests[1].judge_scores).toBeUndefined();
expect(output).toContain('MANUAL');
expect(output).toContain('1 unscored provider refusal');
expect(output).toContain(manual.manual_review!.approval.approval_url);
expect(result.tests[2].passed).toBe(true);
expect(output).toContain(' FAIL ');
});
test('reused passing evidence preserves origin and remains separate from newly executed attempts', async () => {
const collector = new EvalCollector('llm-judge', tmpDir);
const reused_from = { input_key: 'a'.repeat(64), run_id: '1234/1', revision: 'b'.repeat(40),
@@ -590,6 +622,45 @@ describe('findLatestFinalizedRun', () => {
// --- compareEvalResults tests ---
describe('compareEvalResults', () => {
test('manual acceptance is neither a score regression nor a recovery', () => {
const manual = manualReviewFixture();
const prior = makeResult({ tests: [makeEntry({ name: manual.name, passed: true })] });
const accepted = makeResult({ tests: [manual], passed: 0, failed: 0, manual_accepted_tests: 1 });
const toManual = compareEvalResults(prior, accepted, 'prior.json', 'accepted.json');
expect(toManual).toMatchObject({ improved: 0, regressed: 0, manual_reviewed: 1 });
expect(toManual.deltas[0]).toMatchObject({ status_change: 'manual-review', before: { passed: true },
after: { passed: false, manual_review: true } });
expect(formatComparison(toManual)).toContain('PASS → MANUAL');
expect(formatComparison(toManual)).not.toContain('REGRESSION:');
const fromManual = compareEvalResults(accepted, prior, 'accepted.json', 'prior.json');
expect(fromManual).toMatchObject({ improved: 0, regressed: 0, manual_reviewed: 1 });
expect(formatComparison(fromManual)).toContain('MANUAL → PASS');
const invalid = makeResult({ tests: [{ ...manual, passed: true }] });
expect(compareEvalResults(prior, invalid, 'prior.json', 'invalid.json').regressed).toBe(1);
});
test('manual acceptance followed by a real or malformed failure is a blocking regression', () => {
const manual = manualReviewFixture();
const accepted = makeResult({ tests: [manual], passed: 0, failed: 0, manual_accepted_tests: 1 });
const { manual_review: _receipt, ...ordinary } = manual;
for (const after of [
{ ...ordinary, exit_reason: 'timeout' },
{ ...manual, passed: true },
{ ...manual, judge_scores: { clarity: 5 } },
]) {
const failed = makeResult({ tests: [after] });
const comparison = compareEvalResults(accepted, failed, 'accepted.json', 'failed.json');
expect(comparison).toMatchObject({ improved: 0, regressed: 1 });
expect(comparison.manual_reviewed).toBeUndefined();
expect(comparison.deltas[0]).toMatchObject({ status_change: 'regressed', before: { manual_review: true },
after: { passed: false } });
const output = formatComparison(comparison);
expect(output).toContain('MANUAL → FAIL');
expect(output).toContain('REGRESSION:');
expect(output).toContain('blocking failure');
}
});
test('detects improved/regressed/unchanged per test', () => {
const before = makeResult({
tests: [
+57 -18
View File
@@ -12,6 +12,8 @@ import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
import { spawnSync } from 'child_process';
import { isManualReviewEntry } from './cookie-workflow-manual-review';
import type { ManualJudgeReview } from './cookie-workflow-manual-review';
// v2: EvalTestEntry.harvest gains optional {insertions, deletions, net} and
// may be explicitly null (arm-benchmark harvest-failure taxonomy). Readers
@@ -66,6 +68,7 @@ export interface EvalTestEntry {
/** Absent in older records means executed; reuse is never a new model run. */
execution?: 'executed' | 'reused';
reused_from?: { input_key: string; run_id: string; revision: string; completed_at: string };
manual_review?: ManualJudgeReview;
/** 1-based record attempt for this name in this run. bun's --retry leaves
* retried passes INVISIBLE in its text output (a fail→pass prints no
* (fail) line and recaps as a clean pass — probed on 1.3.10), so the ONLY
@@ -120,6 +123,14 @@ export interface EvalTestEntry {
} | null;
}
export function evalEntryOutcome(entry: unknown): 'passed' | 'failed' | 'manual-review' {
if (!entry || typeof entry !== 'object') return 'failed';
if ('manual_review' in entry) return Object.hasOwn(entry, 'manual_review') && isManualReviewEntry(entry) ? 'manual-review' : 'failed';
const result = entry as EvalTestEntry;
if (result.execution !== undefined && result.execution !== 'executed' && result.execution !== 'reused') return 'failed';
return result.passed === true ? 'passed' : 'failed';
}
export interface EvalResult {
schema_version: number;
version: string;
@@ -135,6 +146,7 @@ export interface EvalResult {
total_tests: number;
executed_tests?: number;
reused_tests?: number;
manual_accepted_tests?: number;
passed: number;
failed: number;
total_cost_usd: number;
@@ -154,10 +166,10 @@ export interface EvalResult {
export interface TestDelta {
name: string;
before: { passed: boolean; cost_usd: number; turns_used?: number; duration_ms?: number;
detection_rate?: number; tool_summary?: Record<string, number> };
detection_rate?: number; tool_summary?: Record<string, number>; manual_review?: boolean };
after: { passed: boolean; cost_usd: number; turns_used?: number; duration_ms?: number;
detection_rate?: number; tool_summary?: Record<string, number> };
status_change: 'improved' | 'regressed' | 'unchanged';
detection_rate?: number; tool_summary?: Record<string, number>; manual_review?: boolean };
status_change: 'improved' | 'regressed' | 'unchanged' | 'manual-review';
}
export interface ComparisonResult {
@@ -173,6 +185,7 @@ export interface ComparisonResult {
improved: number;
regressed: number;
unchanged: number;
manual_reviewed?: number;
tool_count_before: number;
tool_count_after: number;
/** After-tests that had a same-named entry in the before run. 0 = nothing was
@@ -388,6 +401,7 @@ export function compareEvalResults(
): ComparisonResult {
const deltas: TestDelta[] = [];
let improved = 0, regressed = 0, unchanged = 0;
let manualReviewed = 0;
let toolCountBefore = 0, toolCountAfter = 0;
let matched = 0;
@@ -409,33 +423,40 @@ export function compareEvalResults(
toolCountAfter += afterToolCount;
let statusChange: TestDelta['status_change'] = 'unchanged';
const beforeManual = beforeTest !== undefined && evalEntryOutcome(beforeTest) === 'manual-review';
const afterOutcome = evalEntryOutcome(afterTest);
const afterManual = afterOutcome === 'manual-review';
if (beforeTest) {
matched++;
if (!beforeTest.passed && afterTest.passed) { statusChange = 'improved'; improved++; }
else if (beforeTest.passed && !afterTest.passed) { statusChange = 'regressed'; regressed++; }
if (beforeManual && afterOutcome === 'failed') { statusChange = 'regressed'; regressed++; }
else if (beforeManual || afterManual) { statusChange = 'manual-review'; manualReviewed++; }
else if (evalEntryOutcome(beforeTest) === 'failed' && evalEntryOutcome(afterTest) === 'passed') { statusChange = 'improved'; improved++; }
else if (evalEntryOutcome(beforeTest) === 'passed' && evalEntryOutcome(afterTest) === 'failed') { statusChange = 'regressed'; regressed++; }
else { unchanged++; }
} else {
// New test — treat as unchanged (no prior data)
unchanged++;
if (afterManual) { statusChange = 'manual-review'; manualReviewed++; }
else unchanged++;
}
deltas.push({
name: afterTest.name,
before: {
passed: beforeTest?.passed ?? false,
passed: beforeTest !== undefined && evalEntryOutcome(beforeTest) === 'passed',
cost_usd: beforeTest?.cost_usd ?? 0,
turns_used: beforeTest?.turns_used,
duration_ms: beforeTest?.duration_ms,
detection_rate: beforeTest?.detection_rate,
tool_summary: beforeToolSummary,
...(beforeManual ? { manual_review: true } : {}),
},
after: {
passed: afterTest.passed,
passed: afterOutcome === 'passed',
cost_usd: afterTest.cost_usd,
turns_used: afterTest.turns_used,
duration_ms: afterTest.duration_ms,
detection_rate: afterTest.detection_rate,
tool_summary: afterToolSummary,
...(afterManual ? { manual_review: true } : {}),
},
status_change: statusChange,
});
@@ -452,12 +473,13 @@ export function compareEvalResults(
deltas.push({
name: `${name} (removed)`,
before: {
passed: beforeTest.passed,
passed: evalEntryOutcome(beforeTest) === 'passed',
cost_usd: beforeTest.cost_usd,
turns_used: beforeTest.turns_used,
duration_ms: beforeTest.duration_ms,
detection_rate: beforeTest.detection_rate,
tool_summary: beforeToolSummary,
...(evalEntryOutcome(beforeTest) === 'manual-review' ? { manual_review: true } : {}),
},
after: { passed: false, cost_usd: 0, tool_summary: {} },
status_change: 'unchanged',
@@ -477,6 +499,7 @@ export function compareEvalResults(
improved,
regressed,
unchanged,
...(manualReviewed ? { manual_reviewed: manualReviewed } : {}),
tool_count_before: toolCountBefore,
tool_count_after: toolCountAfter,
matched,
@@ -495,8 +518,8 @@ export function formatComparison(c: ComparisonResult): string {
// Per-test deltas
for (const d of c.deltas) {
const arrow = d.status_change === 'improved' ? '↑' : d.status_change === 'regressed' ? '↓' : '=';
const beforeStatus = d.before.passed ? 'PASS' : 'FAIL';
const afterStatus = d.after.passed ? 'PASS' : 'FAIL';
const beforeStatus = d.before.manual_review ? 'MANUAL' : d.before.passed ? 'PASS' : 'FAIL';
const afterStatus = d.after.manual_review ? 'MANUAL' : d.after.passed ? 'PASS' : 'FAIL';
// Turns delta
let turnsDelta = '';
@@ -540,6 +563,7 @@ export function formatComparison(c: ComparisonResult): string {
if (c.improved > 0) parts.push(`${c.improved} improved`);
if (c.regressed > 0) parts.push(`${c.regressed} regressed`);
if (c.unchanged > 0) parts.push(`${c.unchanged} unchanged`);
if (c.manual_reviewed) parts.push(`${c.manual_reviewed} unscored manual review`);
lines.push(` Status: ${parts.join(', ')}`);
const costSign = c.total_cost_delta >= 0 ? '+' : '';
@@ -607,7 +631,9 @@ export function generateCommentary(c: ComparisonResult): string[] {
const regressions = c.deltas.filter(d => d.status_change === 'regressed');
if (regressions.length > 0) {
for (const d of regressions) {
notes.push(`REGRESSION: "${d.name}" was passing, now fails. Investigate immediately.`);
notes.push(d.before.manual_review
? `REGRESSION: "${d.name}" lost its unscored manual acceptance and now has a blocking failure. Investigate immediately.`
: `REGRESSION: "${d.name}" was passing, now fails. Investigate immediately.`);
}
}
@@ -617,6 +643,10 @@ export function generateCommentary(c: ComparisonResult): string[] {
notes.push(`Fixed: "${d.name}" now passes.`);
}
for (const d of c.deltas.filter(delta => delta.status_change === 'manual-review')) {
notes.push(`"${d.name}" includes an unscored manual acceptance; no model-score improvement or regression is inferred.`);
}
// 3. Per-test efficiency changes (only for unchanged-status tests — regressions/improvements are already noted)
const stable = c.deltas.filter(d => d.status_change === 'unchanged' && d.after.passed);
for (const d of stable) {
@@ -893,7 +923,8 @@ export class EvalCollector {
const version = getVersion();
const totalCost = this.tests.reduce((s, t) => s + t.cost_usd, 0);
const totalDuration = this.tests.reduce((s, t) => s + t.duration_ms, 0);
const passed = this.tests.filter(t => t.passed).length;
const passed = this.tests.filter(t => evalEntryOutcome(t) === 'passed').length;
const manual = this.tests.filter(t => evalEntryOutcome(t) === 'manual-review').length;
const partial: EvalResult = {
schema_version: SCHEMA_VERSION,
@@ -907,8 +938,9 @@ export class EvalCollector {
total_tests: this.tests.length,
executed_tests: this.tests.filter(t => t.execution !== 'reused').length,
reused_tests: this.tests.filter(t => t.execution === 'reused').length,
...(manual ? { manual_accepted_tests: manual } : {}),
passed,
failed: this.tests.length - passed,
failed: this.tests.length - passed - manual,
total_cost_usd: Math.round(totalCost * 100) / 100,
total_duration_ms: totalDuration,
tests: this.tests,
@@ -933,7 +965,8 @@ export class EvalCollector {
const timestamp = new Date().toISOString();
const totalCost = this.tests.reduce((s, t) => s + t.cost_usd, 0);
const totalDuration = this.tests.reduce((s, t) => s + t.duration_ms, 0);
const passed = this.tests.filter(t => t.passed).length;
const passed = this.tests.filter(t => evalEntryOutcome(t) === 'passed').length;
const manual = this.tests.filter(t => evalEntryOutcome(t) === 'manual-review').length;
const flaky = this.flakyRetries();
const result: EvalResult = {
@@ -948,8 +981,9 @@ export class EvalCollector {
total_tests: this.tests.length,
executed_tests: this.tests.filter(t => t.execution !== 'reused').length,
reused_tests: this.tests.filter(t => t.execution === 'reused').length,
...(manual ? { manual_accepted_tests: manual } : {}),
passed,
failed: this.tests.length - passed,
failed: this.tests.length - passed - manual,
total_cost_usd: Math.round(totalCost * 100) / 100,
total_duration_ms: totalDuration,
wall_clock_ms: Date.now() - this.createdAt,
@@ -999,7 +1033,9 @@ export class EvalCollector {
lines.push('═'.repeat(70));
for (const t of this.tests) {
const status = !t.passed ? ' FAIL ' : t.execution === 'reused' ? ' REUSE' : ' PASS ';
const outcome = evalEntryOutcome(t);
const status = outcome === 'manual-review' ? 'MANUAL' : outcome === 'failed' ? ' FAIL '
: t.execution === 'reused' ? ' REUSE' : ' PASS ';
const cost = `$${t.cost_usd.toFixed(2)}`;
const dur = t.duration_ms ? `${Math.round(t.duration_ms / 1000)}s` : '';
const turns = t.turns_used !== undefined ? `${t.turns_used}t` : '';
@@ -1010,6 +1046,8 @@ export class EvalCollector {
} else if (t.judge_scores) {
const scores = Object.entries(t.judge_scores).map(([k, v]) => `${k[0]}:${v}`).join(' ');
detail = scores;
} else if (outcome === 'manual-review') {
detail = `unscored; approved by ${t.manual_review!.approval.approved_by} (${t.manual_review!.approval.approval_url})`;
}
const name = t.name.length > 35 ? t.name.slice(0, 32) + '...' : t.name.padEnd(35);
@@ -1020,6 +1058,7 @@ export class EvalCollector {
const totalCost = `$${result.total_cost_usd.toFixed(2)}`;
const totalDur = `${Math.round(result.total_duration_ms / 1000)}s`;
lines.push(` Total: ${result.passed}/${result.total_tests} passed${' '.repeat(20)}${totalCost.padStart(6)} ${totalDur}`);
if (result.manual_accepted_tests) lines.push(` Manual accepted: ${result.manual_accepted_tests} unscored provider refusal(s)`);
lines.push(` Evidence: ${result.executed_tests ?? result.total_tests} executed, ${result.reused_tests ?? 0} reused`);
if (result.flaky_retries && result.flaky_retries.length > 0) {
// Loud, never fatal: a flaky pass must not block anyone, but it must
+33 -2
View File
@@ -13,7 +13,8 @@ import Anthropic from '@anthropic-ai/sdk';
import type { JSONOutputFormat } from '@anthropic-ai/sdk/resources/messages';
import { setTimeout as delay } from 'node:timers/promises';
import { CLAUDE_FRONTIER_EVAL_MODEL, resolveEvalModel } from '../../lib/eval-model';
import { CLAUDE_FRONTIER_EVAL_MODEL, DEFAULT_JUDGE_MAX_TOKENS, resolveEvalModel } from '../../lib/eval-model';
export { DEFAULT_JUDGE_MAX_TOKENS } from '../../lib/eval-model';
export interface JudgeScore {
clarity: number; // 1-5
@@ -22,6 +23,35 @@ export interface JudgeScore {
reasoning: string;
}
export interface JudgeRefusalEvidence {
stop_reason: 'refusal';
response_id: string | null;
request_id: string | null;
model: string | null;
input_tokens: number | null;
output_tokens: number | null;
text_blocks: number;
}
export class JudgeRefusalError extends Error {
readonly refusal: JudgeRefusalEvidence;
constructor(response: { id?: unknown; _request_id?: unknown; model?: unknown;
usage?: { input_tokens?: unknown; output_tokens?: unknown }; content: Array<{ type: string }> }) {
super('Judge provider refused the evaluation; no automated score');
this.name = 'JudgeRefusalError';
this.refusal = {
stop_reason: 'refusal',
response_id: typeof response.id === 'string' ? response.id : null,
request_id: typeof response._request_id === 'string' ? response._request_id : null,
model: typeof response.model === 'string' ? response.model : null,
input_tokens: typeof response.usage?.input_tokens === 'number' ? response.usage.input_tokens : null,
output_tokens: typeof response.usage?.output_tokens === 'number' ? response.usage.output_tokens : null,
text_blocks: response.content.filter(block => block.type === 'text').length,
};
}
}
export interface OutcomeJudgeResult {
detected: string[];
missed: string[];
@@ -89,7 +119,7 @@ export async function callJudge<T>(
// Thinking and answer text share max_tokens. The old 1024-token budget
// could be exhausted before a frontier judge emitted any JSON.
const resolvedModel = resolveEvalModel('judge', model);
const maxTokens = opts?.max_tokens ?? 8192;
const maxTokens = opts?.max_tokens ?? DEFAULT_JUDGE_MAX_TOKENS;
const client = new Anthropic();
const makeRequest = () => client.messages.create({
@@ -134,6 +164,7 @@ export async function callJudge<T>(
.map(block => block.text)
.join('\n');
try {
if (response.stop_reason === 'refusal') throw new JudgeRefusalError(response);
if (opts?.jsonSchema !== undefined) {
if (response.stop_reason !== 'end_turn') throw new Error(`Structured judge did not complete: stop_reason=${response.stop_reason}`);
return JSON.parse(text) as T;
@@ -0,0 +1,17 @@
import { readFileSync } from 'node:fs';
import { resolve } from 'node:path';
import type { EvalTestEntry } from './eval-store';
import { buildCookieWorkflowJudgeInput } from './cookie-workflow-judge-input';
import { COOKIE_MANUAL_REVIEW_FILE } from './cookie-workflow-manual-review';
export function manualReviewFixture(root = resolve(import.meta.dir, '../..')): EvalTestEntry {
const approval = JSON.parse(readFileSync(resolve(root, COOKIE_MANUAL_REVIEW_FILE), 'utf8'));
return {
name: 'setup-browser-cookies/SKILL.md workflow', suite: 'Cookie setup workflow quality', tier: 'llm-judge',
passed: false, execution: 'executed', exit_reason: 'provider_refusal', attempt: 1, duration_ms: 1, cost_usd: 0,
model: approval.model, prompt: buildCookieWorkflowJudgeInput(root).prompt,
error: 'Synthetic provider refusal fixture, not live model evidence',
manual_review: { approval, refusal: { stop_reason: 'refusal', response_id: 'msg_synthetic_fixture',
request_id: 'req_synthetic_fixture', model: approval.model, input_tokens: 1, output_tokens: 0, text_blocks: 0 } },
};
}
+38 -11
View File
@@ -830,17 +830,31 @@ export type SharedQuestionSelector = (input: Record<string, unknown>) => Record<
/** The skip actor may decline work, never approve a mixed fix/preservation choice. */
function skippedReviewOption(question: any): any {
const options = Array.isArray(question?.options) ? question.options : [];
const qualifiedIndexState = /^(?:(?:git\s+)?index|skip-worktree|assume-unchanged)\s+(?:bits?|flags?|attributes?|settings?)$/i;
const candidates = options.flatMap((option: any) => {
if (typeof option?.label !== 'string' || ['description', 'preview'].some(field =>
option[field] !== undefined && typeof option[field] !== 'string')) return [];
const label = option.label.replace(/[‘’]/g, "'").replace(/^\s*(?:[A-Z]|\d+)[.)]\s*/i, '')
.replace(/\s*\(recommended\)\s*$/i, '').trim();
const label = option.label.replace(/[‘’]/g, "'").replace(/`/g, '').replace(/^\s*(?:[A-Z]|\d+)[.)]\s*/i, '')
.replace(/\s*\(recommended\)\s*$/i, '').trim()
.replace(/^no\s*[,.:!?]\s*(?=(?:skip|decline|keep|leave|do not|don't)\b)/i, '');
const referentialRetention = /^(?:keep|leave)\s+(?:it|this|that|them|these)$/i.test(label);
const preservation = option.description?.trim().replace(/`/g, '').match(/^(?:keep|leave|retain|preserve)\s+([^,;.!?]+)/i);
const preservedObject = preservation?.[1].split(/\b(?:and|but|while)\b/i)[0]
.replace(/(?<![-\w])(?:the|this|that|current|existing|local|as[- ]is|unchanged|untouched|set|hidden)\b/gi, '').trim();
const describedRetention = !!preservedObject
&& !/^\w+ing\b/i.test(preservedObject)
&& (/^(?:(?:duplicated|original|prior|tracked|untracked)\s+)*(?:(?:index|skip-worktree|assume-unchanged)\s+)?(?:flags?|code|source|implementations?|copies|copy|files?|routes?|workers?|helpers?|parsers?|changes?|contents?|state|branches|branch|worktrees?)$/i.test(preservedObject)
|| qualifiedIndexState.test(preservedObject));
const description = (option.description ?? '').replace(/[‘’]/g, "'").trim();
const declinesChange = /^(?:do not|don't)\s+(?:apply|change|edit|fix|refactor|extract|modify|touch|clear|remove|update|replace|add|migrate|implement|reuse|import)\b/i;
const labelObject = label.match(/^(?:keep|leave)\s+(?:the\s+)?(.+)$/i)?.[1]
.replace(/\s+(?:as[- ]is|unchanged|untouched|set)$/i, '');
const preservationRank = referentialRetention ? describedRetention || declinesChange.test(description)
: /^(?:keep|leave)\b.*\b(?:current|existing|unchanged|untouched|as[- ]is|alone|set|copies|copy|implementation|code|source)\b/i.test(label)
|| !!labelObject && qualifiedIndexState.test(labelObject);
const rank = /^(?:skip|decline)(?=$|\s|[,.!])/i.test(label) ? 3
: declinesChange.test(label) ? 2
: /^(?:keep|leave)\b.*\b(?:current|existing|unchanged|untouched|as[- ]is|alone|set|copies|copy|implementation|code|source)\b/i.test(label)
|| (/^(?:keep|leave)\b/i.test(label) && declinesChange.test(description)) ? 1 : 0;
: preservationRank ? 1 : 0;
if (!rank) return [];
// A leading decline names rejected work. Classify later commitments rather
// than action words inside recorded metadata or hypothetical consequences.
@@ -848,21 +862,34 @@ function skippedReviewOption(question: any): any {
option.description ?? '', option.preview ?? ''].join('\n').replace(/[‘’]/g, "'");
const actions = new Set(['approve', 'fix', 'apply', 'refactor', 'extract', 'replace', 'rewrite', 'edit', 'modify',
'change', 'clear', 'remove', 'delete', 'add', 'update', 'implement', 'migrate', 'touch', 're-export',
'import', 'reuse', 'share', 'wire', 'convert']);
'import', 'reuse', 'share', 'wire', 'convert', 'set', 'unset', 'toggle', 'flip', 'reset', 'enable', 'disable']);
const isAction = (word = '') => [word, word.replace(/s$/, ''), word.replace(/(?:es|ed|ing)$/, ''),
word.replace(/(?:ed|ing)$/, 'e'), word.replace(/(?:ies|ied)$/, 'y')].some(form => actions.has(form));
word.replace(/(?:ed|ing)$/, 'e'), word.replace(/(?:ies|ied)$/, 'y'),
word.replace(/([a-z])\1(?:ed|ing)$/, '$1')].some(form => actions.has(form));
const changes = commitment.toLowerCase().split(/[,;\n]|[.!?](?:\s|$)|\b(?:and|but|then|while)\b/).some(part => {
const clause = part.replace(/^[^a-z]+/, '')
.replace(/^(?:(?:this|that|the|selected|chosen)\s+(?:option|choice|selection)|i|we|you|it|(?:the\s+)?(?:source|code|route|worker|helper|parser|index(?:\s+flag)?))\s+/, '')
.replace(/^(?:will|would|should|must|can|may|does|do)\s+/, '')
.replace(/^(?:(?:please|also|still|just|now|be)\s+)+/, '');
if (/^(?:not|does not|don't|doesn't|won't|without|no)\b/.test(clause)) return false;
// The no-change choice may persist/reuse its review decision. That is not
// permission to modify source or clear an index flag.
if (/^(?:updates?|updated|updating|reuses?|reused|reusing)\s+(?:the\s+)?(?:(?:prior|recorded|existing)\s+)?(?:review\s+(?:log|record)|decision|advisory|snapshot|ledger)\b/.test(clause)) return false;
const first = clause.match(/^[a-z]+(?:-[a-z]+)*/)?.[0];
const future = clause.match(/\bwill\s+(?:be\s+)?([a-z]+(?:-[a-z]+)*)/)?.[1];
return isAction(first) || isAction(future);
const futureMatch = clause.match(/\b(?:will|would|should|must|can|may)\s+(?:(?:still|also|now|just|[a-z]+ly)\s+)*(?:be\s+)?(?:(?:still|also|now|just|[a-z]+ly)\s+)*([a-z]+(?:-[a-z]+)*)/);
const future = futureMatch?.[1];
const method = /^(?:keep|leave|retain|preserve)\b/.test(clause)
&& [...clause.matchAll(/\b(?:by|through|via)\s+([a-z]+(?:-[a-z]+)*)/g)].some(match => isAction(match[1]));
const state = /^(?:change|modification|file|flag|state|content)\s+(?:stays?|remains?)\b/.test(clause);
const recordedDecision = /^(?:updates?|updated|updating|reuses?|reused|reusing)\s+(?:the\s+)?(?:(?:prior|recorded|existing)\s+)?(?:review\s+(?:log|record)|decision|advisory|snapshot|ledger)\b/.test(clause);
const nominalReuse = /^(?:the\s+)?reuse\s+of\b/.test(clause);
const describedReuse = /\b(?:is|are|was|were|remains?|stays?|requires?|needs?|will|would|should|must|can|may)\s+(?:(?:still|also|now|just|not|never|[a-z]+ly)\s+)*[a-z]+(?:-[a-z]+)*/.test(clause);
const futureSubject = clause.slice(0, futureMatch?.index ?? 0).trim();
const passiveDecision = /\b(?:review\s+(?:log|record)|decision|advisory|snapshot|ledger)$/.test(futureSubject)
|| /\b(?:review\s+(?:log|record)|decision|advisory|snapshot|ledger)\b(?:(?!\b(?:source|code|route|worker|helper|parser|file|flag)\b).)*\bit$/.test(futureSubject);
const futureDecision = /\b(?:can|will|would|should|must|may)\s+(?:(?:still|also|now|just|[a-z]+ly)\s+)*reuse\s+(?:(?:this|the|prior|recorded|existing)\s+)*(?:review\s+(?:log|record)|decision|advisory|snapshot|ledger)\b/.test(clause);
const purpose = [...clause.matchAll(/\b(?:to|by|through|via)\s+(?:[a-z]+ly\s+)*([a-z]+(?:-[a-z]+)*)/g)]
.some(match => isAction(match[1]));
return (isAction(future) && !(future === 'reused' && passiveDecision) && !(future === 'reuse' && futureDecision)) || method || purpose
|| (nominalReuse && !describedReuse)
|| (!state && !recordedDecision && !nominalReuse && isAction(first));
});
return changes ? [] : [{ option, rank }];
});
+1
View File
@@ -1772,6 +1772,7 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic'> = {
* LLM-judge test touchfiles — keyed by test description string.
*/
export const LLM_JUDGE_TOUCHFILES: Record<string, string[]> = {
'setup-browser-cookies/SKILL.md workflow': ['setup-browser-cookies/SKILL.md.tmpl', 'setup-browser-cookies/SKILL.md', 'BROWSER.md', 'test/helpers/cookie-workflow-judge-input.ts', 'test/cookie-workflow-judge-input.test.ts', 'test/helpers/cookie-workflow-manual-review.ts', 'test/cookie-workflow-manual-review.test.ts', 'test/helpers/manual-judge-review-fixture.ts', '.github/cookie-workflow-manual-review.json', 'test/helpers/workflow-judge-input.ts', 'test/skill-llm-eval.test.ts'],
'command reference table': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'browse/src/commands.ts', 'test/skill-llm-eval.test.ts'],
'snapshot flags reference': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'browse/src/snapshot.ts', 'test/skill-llm-eval.test.ts'],
'browse/SKILL.md reference': ['browse/sections/**', 'browse/SKILL.md', 'browse/SKILL.md.tmpl', 'browse/src/**', 'test/skill-llm-eval.test.ts'],
+2 -2
View File
@@ -3,7 +3,7 @@ import * as fs from 'node:fs';
import * as path from 'node:path';
import { isBuiltin } from 'node:module';
import { spawnSync } from 'node:child_process';
import { resolveEvalModel } from '../../lib/eval-model';
import { DEFAULT_JUDGE_MAX_TOKENS, resolveEvalModel } from '../../lib/eval-model';
import { JUDGE_MS } from './eval-budgets';
import type { JudgeScore } from './llm-judge';
import { readWorkflowJudgeInput, buildWorkflowJudgePrompt } from './workflow-judge-input';
@@ -98,7 +98,7 @@ export function prepareWorkflowJudgeCache(opts: WorkflowCacheOptions): {
coverage: { dependencies: 'complete', prompts: 'complete', environment: 'complete' }, unknownDependencies: [],
files: workflowJudgeDependencies(opts.root, input.files.map(file => file.path)),
prompts: { [opts.testName]: prompt },
parameters: { rootPackage, thresholds: opts.thresholds, max_tokens: 8192, temperature: null, budget_ms: JUDGE_MS,
parameters: { rootPackage, thresholds: opts.thresholds, max_tokens: DEFAULT_JUDGE_MAX_TOKENS, temperature: null, budget_ms: JUDGE_MS,
request: 'messages.create/user', retries: 1 },
runtime: { image: env.EVALS_CACHE_RUNTIME_ID!, bun: Bun.version, node: process.versions.node,
platform: process.platform, arch: process.arch, judge: resolveEvalModel('judge', undefined, env),
+15 -2
View File
@@ -1,6 +1,6 @@
import { afterEach, beforeEach, describe, expect, spyOn, test } from 'bun:test';
import Anthropic from '@anthropic-ai/sdk';
import { armJudge, callJudge } from './helpers/llm-judge';
import { armJudge, callJudge, JudgeRefusalError } from './helpers/llm-judge';
describe('frontier Claude judge compatibility', () => {
let originalKey: string | undefined;
@@ -31,6 +31,18 @@ describe('frontier Claude judge compatibility', () => {
expect(diagnostics).not.toHaveBeenCalled();
});
test('an explicit provider refusal retains typed public evidence without parsing a score or retrying', async () => {
create.mockResolvedValue({ id: 'msg_synthetic', _request_id: 'req_synthetic', model: 'claude-fable-5-1',
stop_reason: 'refusal', content: [], usage: { input_tokens: 3886, output_tokens: 0 } } as never);
let failure: unknown;
try { await callJudge('Synthetic test prompt', 'claude-fable-5-1'); } catch (error) { failure = error; }
expect(failure).toBeInstanceOf(JudgeRefusalError);
expect((failure as JudgeRefusalError).refusal).toEqual({ stop_reason: 'refusal', response_id: 'msg_synthetic',
request_id: 'req_synthetic', model: 'claude-fable-5-1', input_tokens: 3886, output_tokens: 0, text_blocks: 0 });
expect(create).toHaveBeenCalledTimes(1);
expect(JSON.parse(diagnostics.mock.calls[0][0]).stopReason).toBe('refusal');
});
test('retains complete public response on malformed JSON and fails without another request', async () => {
// Synthetic malformed response: the actual failed provider text was not retained.
const textBlocks = ['{"questions":[{"toolUseId":"c1","reason":"' + 'evidence '.repeat(500) + '"', '}]'];
@@ -150,7 +162,8 @@ describe('frontier Claude judge compatibility', () => {
create.mockClear(); diagnostics.mockClear();
create.mockResolvedValue({ stop_reason, content: [{ type: 'text', text: '{"score":4}' }] } as never);
await expect(callJudge('score this', 'claude-fable-5-1', { max_tokens: 16_384, jsonSchema }))
.rejects.toThrow(stop_reason === 'max_tokens' ? 'truncated at max_tokens=16384' : 'Structured judge did not complete');
.rejects.toThrow(stop_reason === 'max_tokens' ? 'truncated at max_tokens=16384'
: stop_reason === 'refusal' ? 'Judge provider refused the evaluation; no automated score' : 'Structured judge did not complete');
expect(create).toHaveBeenCalledTimes(1);
}
});
+1 -1
View File
@@ -216,6 +216,6 @@ describe('PR profile paid-runner integration', () => {
{ name: 'retry', suite: 'judge', passed: true, execution: 'executed' },
{ name: 'cached', suite: 'judge', passed: true, execution: 'reused' },
{ name: 'failed', suite: 'native', passed: false },
] }])).toEqual({ executed: 2, reused: 1, passed: 2, failed: 1, attempts: 4 });
] }])).toEqual({ executed: 2, reused: 1, passed: 2, failed: 1, manual_accepted: 0, attempts: 4 });
});
});
+5 -5
View File
@@ -13,7 +13,7 @@ import {
const read = (file: string) => readFileSync(join(import.meta.dir, '..', file), 'utf8');
const newBudgets = FILE_RETRY_BUDGETS.filter(row => !FINDING_RETRY_BUDGETS.some(old => old.file === row.file));
const expectedWalls = {
'test/skill-llm-eval.test.ts': 6_400_000,
'test/skill-llm-eval.test.ts': 6_660_000,
'test/skill-e2e-auq-consistency.test.ts': 2_040_000,
'test/codex-e2e-plan-format.test.ts': 5_000_000,
'test/skill-e2e-auq-matrix.test.ts': 3_720_000,
@@ -139,15 +139,15 @@ test('quality model work, ordinary tiers and the six existing finding registrati
expect(ALL_TIERS).toEqual({ JUDGE_MS: 120000, CAPTURE_MS: 300000, CAPTURE_LONG_MS: 600000, PTY_MS: 900000, PTY_LONG_MS: 1200000 });
const quality = 'test/skill-llm-eval.test.ts';
const qualityBudget = FILE_RETRY_BUDGETS.find(row => row.file === quality)!;
expect(resolvePaidShardBudget([quality])).toEqual({ timeoutMs: 6_400_000, source: 'registered', policyId: qualityBudget.id });
expect(resolvePaidShardBudget([quality])).toEqual({ timeoutMs: 6_660_000, source: 'registered', policyId: qualityBudget.id });
expect(retriesForFiles([quality])).toBe(1);
const qualitySource = read(quality);
const judgeTimeouts = [...qualitySource.matchAll(/}\s*,\s*(JUDGE_MS|WORKFLOW_JUDGE_TEST_MS)\s*\);/g)].map(match => match[1]);
expect(judgeTimeouts.filter(timeout => timeout === 'JUDGE_MS')).toHaveLength(11);
expect(judgeTimeouts.filter(timeout => timeout === 'WORKFLOW_JUDGE_TEST_MS')).toHaveLength(14);
expect(judgeTimeouts.filter(timeout => timeout === 'WORKFLOW_JUDGE_TEST_MS')).toHaveLength(15);
expect(qualitySource).toContain('WORKFLOW_JUDGE_TEST_MS = JUDGE_MS + 10_000');
expect(qualitySource).toContain('const workDeadline = started + JUDGE_MS');
expect(qualityBudget.shardMs).toBe((11 * ALL_TIERS.JUDGE_MS + 14 * (ALL_TIERS.JUDGE_MS + 10_000)) * 2 + 120_000);
expect(qualityBudget.shardMs).toBe((11 * ALL_TIERS.JUDGE_MS + 15 * (ALL_TIERS.JUDGE_MS + 10_000)) * 2 + 120_000);
expect(FINDING_RETRY_BUDGETS.map(row => [row.cases, row.testMs, row.retries, row.shardMs])).toEqual([
[2, 1500000, 1, 6120000], ...Array(5).fill([1, 1500000, 1, 3120000]),
]);
@@ -247,7 +247,7 @@ test('the periodic executor supervises every actual case and retry within its CI
dedicatedAutoplanSlice: planned.dedicatedAutoplanSlice, evalsAll: true, env: { EVALS_ALL: '1' } });
const census = manifest.entries.filter(row => row.status === 'planned');
expect(census).toHaveLength(99);
expect(census.find(row => row.file === 'test/skill-llm-eval.test.ts')?.budget?.timeoutMs).toBe(6_400_000);
expect(census.find(row => row.file === 'test/skill-llm-eval.test.ts')?.budget?.timeoutMs).toBe(6_660_000);
expect(manifest.autoplanSlice).toBe(8);
const walls = executor.strategy.matrix.slice.map((slice: number) => paidShardWallUpperBoundMs(
census.filter(row => row.slice === slice).map(row => row.file), active.jobs,
+1 -1
View File
@@ -124,7 +124,7 @@ for (const mode of ['unseeded-deadline', 'seeded-deadline', 'protocol-error']) t
try {
const run = runPlanSkillObservation({ skillName: 'plan-eng-review', cwd: dir,
...(seeded ? { initialPlanContent: '# Exact plan\nNo new work allowance.' } : {}), timeoutMs: mode === 'protocol-error' ? 10000 : 600, model: 'fixture',
env: { CLAUDE_CONFIG_DIR: config, SEED_CASE: mode === 'protocol-error' ? 'wrong-pid' : 'success' } });
env: { CLAUDE_CONFIG_DIR: config, SEED_CASE: mode === 'protocol-error' ? 'wrong-pid' : mode === 'seeded-deadline' ? 'no-ack' : 'success' } });
if (mode === 'protocol-error') {
await expect(run).rejects.toThrow('Plan seed PID status does not match this launch');
expect(fs.existsSync(path.join(config, 'events.jsonl'))).toBe(false);
+178 -4
View File
@@ -83,6 +83,172 @@ describe('shared-code legacy interactive actor', () => {
expect(input).toEqual(before);
});
test('the registered callback answers both complete native index-flag questions without approving changes', async () => {
const native = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/shared-libs-index-flags-native-questions.json'), 'utf8'));
expect(native.sourceRun).toBe(36036582724);
for (const { attempt, input } of native.cases) {
const original = structuredClone(input), questions: unknown[] = [], answers: unknown[] = [], refusals: Error[] = [];
const callback = createSharedInteractiveToolHandler('skip', {
nonQuestion: () => { throw new Error('unexpected tool'); },
onQuestion: question => { questions.push(question); },
onAnswer: (question, answer) => { answers.push({ question, answer }); },
onRefusal: error => { refusals.push(error); },
});
const expectedLabels = attempt === 1 ? ['B) Skip', 'Leave it'] : ['C) Skip', 'No, leave it'];
const expected = Object.fromEntries(input.questions.map((question: any, index: number) =>
[question.question, expectedLabels[index]]));
expect(await callback('AskUserQuestion', input)).toEqual({ behavior: 'allow', updatedInput: { ...input, answers: expected } });
expect(questions).toEqual([input]);
expect(answers).toEqual([{ question: input, answer: expected }]);
expect(refusals).toEqual([]);
expect(input).toEqual(original);
}
});
test('the captured native no-change questions select every owning interactive lifecycle case', () => {
expect(selectTests(['test/fixtures/shared-libs-index-flags-native-questions.json'], E2E_TOUCHFILES, GLOBAL_TOUCHFILES).selected.sort())
.toEqual(['shared-libs-review-index-flags', 'shared-libs-review-lifecycle', 'shared-libs-review-path-eligibility',
'shared-libs-review-prior-coverage', 'shared-libs-review-revalidation']);
});
test('the registered callback acknowledges the complete first-attempt native skip despite descriptive reuse', async () => {
const native = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/shared-libs-index-flags-native-questions.json'), 'utf8'));
const { sourceRun, attempt, input } = native.regressions[0];
expect({ sourceRun, attempt }).toEqual({ sourceRun: 36044212977, attempt: 1 });
const before = structuredClone(input), questions: unknown[] = [], answers: unknown[] = [], refusals: Error[] = [];
const expected = { [input.questions[0].question]: 'B) Skip' };
const callback = createSharedInteractiveToolHandler('skip', {
nonQuestion: () => { throw new Error('unexpected tool'); },
onQuestion: question => { questions.push(question); },
onAnswer: (question, answer) => { answers.push({ question, answer }); },
onRefusal: error => { refusals.push(error); },
});
expect(await callback('AskUserQuestion', input)).toEqual({ behavior: 'allow', updatedInput: { ...input, answers: expected } });
expect(questions).toEqual([input]);
expect(answers).toEqual([{ question: input, answer: expected }]);
expect(refusals).toEqual([]);
expect(input).toEqual(before);
});
test('the registered callback answers the complete 5460 first-attempt native packet', async () => {
const native = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/shared-libs-index-flags-native-questions.json'), 'utf8'));
const { sourceRun, sourceRevision, attempt, toolUseId, input } = native.regressions[1];
expect({ sourceRun, sourceRevision, attempt, toolUseId }).toEqual({ sourceRun: 36080890009,
sourceRevision: '5460ce0847574aadfa6cbdd0b9545d2caf9935da', attempt: 1,
toolUseId: 'toolu_01Up8B1FR4bhkmRxyhbcAqhY' });
const before = structuredClone(input), questions: unknown[] = [], answers: unknown[] = [], refusals: Error[] = [];
const expected = { [input.questions[0].question]: 'Skip', [input.questions[1].question]: 'Leave it' };
const callback = createSharedInteractiveToolHandler('skip', {
nonQuestion: () => { throw new Error('unexpected tool'); },
onQuestion: question => { questions.push(question); },
onAnswer: (question, answer) => { answers.push({ question, answer }); },
onRefusal: error => { refusals.push(error); },
});
expect(await callback('AskUserQuestion', input)).toEqual({ behavior: 'allow', updatedInput: { ...input, answers: expected } });
expect(questions).toEqual([input]);
expect(answers).toEqual([{ question: input, answer: expected }]);
expect(refusals).toEqual([]);
expect(input).toEqual(before);
});
test.each([
{ label: 'Leave it', description: 'Keep the index flag as-is and record the decision in the review log.' },
{ label: 'No, leave it', description: 'Preserve the current index flag. Report its hidden source without modifying it.' },
{ label: 'Keep it', description: 'Leave the index flag set; update the review log with the skipped advisory.' },
])('a referential retention choice requires an explicit no-change description: $label', async option => {
const input = { questions: [{ question: 'Should I clear the index flag?', options: [
{ label: 'Clear the index flag', description: 'Make its hidden changes visible.' }, option,
] }] };
const answer = await createSharedInteractiveToolHandler('skip', {
nonQuestion: () => {}, onQuestion: () => {}, onAnswer: () => {},
})('AskUserQuestion', input);
expect(answer.updatedInput.answers).toEqual({ [input.questions[0].question]: option.label });
});
test.each([
{ label: 'Leave it', description: 'Keep the assume-unchanged bit; exclude its path from snapshot coverage.' },
{ label: 'Leave it', description: 'Preserve the `skip-worktree` bit; report its hidden source.' },
{ label: 'Leave it', description: 'Keep the Git index bits unchanged; record the advisory.' },
{ label: 'Leave it', description: 'Keep the index attributes; report the hidden route.' },
{ label: 'Leave it', description: 'Retain the index settings; report without changing them.' },
{ label: 'Keep the assume-unchanged bit' },
{ label: 'Leave the `skip-worktree` flag' },
{ label: 'Keep Git index attributes' },
])('a qualified Git-index state may be preserved without authorizing edits: $label', async option => {
const input = { questions: [{ question: 'Should I clear the index flag?', options: [
{ label: 'Clear the index flag', description: 'Make its hidden changes visible.' }, option,
] }] };
const answer = await createSharedInteractiveToolHandler('skip', {
nonQuestion: () => {}, onQuestion: () => {}, onAnswer: () => {},
})('AskUserQuestion', input);
expect(answer.updatedInput.answers).toEqual({ [input.questions[0].question]: option.label });
});
test.each([
[{ label: 'Leave it' }],
[{ label: 'Leave it', description: 'Keep it.' }],
[{ label: 'Leave it', description: 'Keep going.' }],
[{ label: 'Leave it', description: 'Keep working on the fix.' }],
[{ label: 'Leave it', description: 'Keep trying.' }],
[{ label: 'Leave it', description: 'Keep investigating the source state.' }],
[{ label: 'Leave it', description: 'Keep pursuing the source fix.' }],
[{ label: 'Leave it', description: 'Keep progress going.' }],
[{ label: 'Leave it', description: 'Keep source changing.' }],
[{ label: 'Leave it', description: 'Keep the changes going.' }],
[{ label: 'Leave it', description: 'Preserve the implementation by rewriting the helper.' }],
[{ label: 'Skip', description: 'Keep the duplicated copies. Reuse of this skip.' }],
[{ label: 'Skip', description: 'Keep the duplicated copies. The reuse of this decision.' }],
[{ label: 'Skip', description: 'Keep the duplicated copies. Reuse of the review decision will still clear the flag.' }],
[{ label: 'Skip', description: 'Keep the duplicated copies. Reuse of the review decision will eventually modify source.' }],
[{ label: 'Skip', description: 'Keep the duplicated copies. Reuse of the review decision can quietly replace the helper.' }],
[{ label: 'Skip', description: 'Keep the duplicated copies. Reuse of this decision will still be applied to source.' }],
[{ label: 'Skip', description: 'Keep the duplicated copies. Reuse of this decision will need to clear the flag.' }],
[{ label: 'Skip', description: 'Keep the duplicated copies. Reuse the recorded decision will still clear the flag.' }],
[{ label: 'Skip', description: 'Keep the code unchanged; the recorded decision says the route will be reused.' }],
[{ label: 'Skip', description: 'Keep the code unchanged; the decision names the source so it will be reused.' }],
[{ label: 'Skip', description: 'Keep both duplicated copies; a later review can reuse this decision to clear the flag.' }],
[{ label: 'No, leave it', description: 'No changes.' }],
[{ label: 'Leave it', description: 'Clear the index flag and report it.' }],
[{ label: 'No, leave it', description: 'Keep the index flag as-is; apply the worker fix.' }],
[{ label: 'Leave it', description: 'Keep the flag set; the route change remains hidden yet will clear the flag tomorrow.' }],
[{ label: 'No, leave it and clear the flag', description: 'Keep the index flag set.' }],
[{ label: 'Leave it', description: 'Update the review log with this decision.' }],
[{ label: 'Leave it', description: 'Keep the flag set.' }, { label: 'Keep it', description: 'Preserve the index flag as-is.' }],
])('referential retention refuses vague, mixed, or duplicate choices: %j', async options => {
const refusals: Error[] = [], answers: unknown[] = [];
const callback = createSharedInteractiveToolHandler('skip', {
nonQuestion: () => {}, onQuestion: () => {}, onAnswer: answer => { answers.push(answer); },
onRefusal: error => { refusals.push(error); },
});
await expect(callback('AskUserQuestion', { questions: [{ question: 'Should I clear the index flag?', options }] }))
.rejects.toThrow('No unambiguous no-change option');
expect(refusals).toHaveLength(1);
expect(answers).toEqual([]);
});
test.each([
[{ label: 'Leave it', description: 'Keep the bit unchanged.' }],
[{ label: 'Keep the bit' }],
[{ label: 'Keep the index bit and set the other flag' }],
[{ label: 'Leave it', description: 'Keep the index bit; set the skip-worktree flag.' }],
[{ label: 'Leave it', description: 'Keep the assume-unchanged bit while unsetting the skip-worktree flag.' }],
[{ label: 'Leave it', description: 'Keep the index setting by toggling the other flag.' }],
[{ label: 'Leave it', description: 'Keep the `assume-unchanged` bit; flip the skip-worktree flag.' }],
[{ label: 'Leave it', description: 'Keep the index flag, then reset the other index bit.' }],
[{ label: 'Leave it', description: 'Keep the index flag; enable the skip-worktree bit.' }],
[{ label: 'Leave it', description: 'Keep the index flag; disable the skip-worktree bit.' }],
])('qualified index retention rejects ambiguous bits and state-changing commitments: %j', async options => {
const refusals: Error[] = [], answers: unknown[] = [];
const callback = createSharedInteractiveToolHandler('skip', {
nonQuestion: () => {}, onQuestion: () => {}, onAnswer: answer => { answers.push(answer); },
onRefusal: error => { refusals.push(error); },
});
await expect(callback('AskUserQuestion', { questions: [{ question: 'Should I clear the index flag?', options }] }))
.rejects.toThrow('No unambiguous no-change option');
expect(refusals).toHaveLength(1);
expect(answers).toEqual([]);
});
const preservationCaptures = JSON.parse(fs.readFileSync(path.join(import.meta.dir,
'fixtures/shared-libs-index-flags-no-change-description.json'), 'utf8')).cases;
@@ -141,11 +307,16 @@ describe('shared-code legacy interactive actor', () => {
{ label: 'Decline', description: 'Don’t refactor either caller.' },
{ label: 'Don’t refactor', description: 'Keep the current implementation.' },
{ label: 'Do not change', description: 'Leave source untouched. No code edits or new tests.' },
{ label: 'Skip', description: 'Do not refactor by extracting a helper. Keep the source unchanged.' },
{ label: 'Leave it set', description: 'Do not touch the index flag; report missing snapshot coverage.' },
{ label: 'Keep current', description: 'Keep both implementations unchanged.', preview: '// no edits; record skipped advisory' },
{ label: 'Leave it set', description: 'Do not touch the index flag. Any edit to retry-route.ts stays local-only until you clear it yourself; it stays excluded from snapshot coverage.' },
{ label: 'Skip', description: 'Keep the duplicated implementation as-is. Recorded as an explicit skipped advisory with full snapshot coverage so it can be reused next review.' },
{ label: 'Skip', description: 'Keep both inline copies. Recorded as an explicit skip with verified snapshot coverage for future reuse.' },
{ label: 'Skip', description: 'Keep the duplicated copies. Reuse of this skip will still require revalidation while the route remains hidden.' },
{ label: 'Skip', description: 'Keep the duplicated copies. The reuse of this decision is recorded as requiring revalidation.' },
{ label: 'Skip', description: 'Keep both duplicated copies; a later review can reuse this decision.' },
{ label: 'Skip', description: 'Keep both copies. The recorded review decision can be reused in a later review.' },
{ label: 'Skip', description: 'Update the review log with the skipped advisory; reuse the recorded decision next review.' },
{ label: 'Skip', description: 'This option does not refactor the route. You should not fix the worker.' },
{ label: 'Skip', description: 'This option updates the review log. We will reuse the recorded decision.' },
@@ -178,6 +349,9 @@ describe('shared-code legacy interactive actor', () => {
[{ label: 'Skip', description: 'You should fix the worker' }],
[{ label: 'Skip', description: 'The worker imports the helper' }],
[{ label: 'Skip', description: 'We will clear the index flag' }],
[{ label: 'Skip', description: 'Preserve the implementation by rewriting the helper.' }],
[{ label: 'Skip', description: 'Keep the source through applying the fix.' }],
[{ label: 'Skip', description: 'Retain the implementation via extracting a helper.' }],
[{ label: 'Keep going' }],
[{ label: 'Do not warn' }],
[{ label: 'Leave logging disabled and fix parser' }],
@@ -631,7 +805,7 @@ describe('shared-code capture attempt accounting', () => {
expect(result.tests.map((row: EvalTestEntry) => row.transcript!.filter(event => event.scenario_name).length))
.toEqual([3, 3, 2, 2, 4, 4]);
expect(result.total_cost_usd).toBe(0.18);
expect(collectorOutcomeCounts([result])).toEqual({ executed: 3, reused: 0, passed: 3, failed: 0, attempts: 6 });
expect(collectorOutcomeCounts([result])).toEqual({ executed: 3, reused: 0, passed: 3, failed: 0, manual_accepted: 0, attempts: 6 });
});
test('missing scenarios in a later attempt cannot inherit an earlier pass', async () => {
@@ -645,7 +819,7 @@ describe('shared-code capture attempt accounting', () => {
})).rejects.toThrow('missing scenarios: second');
const result = await finalized(captures);
expect(result.tests[1]).toMatchObject({ attempt: 2, passed: false, exit_reason: 'attempt_incomplete' });
expect(collectorOutcomeCounts([result])).toEqual({ executed: 1, reused: 0, passed: 0, failed: 1, attempts: 2 });
expect(collectorOutcomeCounts([result])).toEqual({ executed: 1, reused: 0, passed: 0, failed: 1, manual_accepted: 0, attempts: 2 });
});
test('setup, verification and cleanup failures survive even when all recorded captures passed', async () => {
@@ -764,7 +938,7 @@ test('actual-retry', () => captures.runAttempt('actual-retry', ['audit'], 5_000,
const result = JSON.parse(fs.readFileSync(path.join(resultDir, file), 'utf8'));
expect(result.tests.map((row: EvalTestEntry) => [row.attempt, row.passed, row.exit_reason]))
.toEqual([[1, false, 'timeout'], [2, true, 'success']]);
expect(collectorOutcomeCounts([result])).toEqual({ executed: 1, reused: 0, passed: 1, failed: 0, attempts: 2 });
expect(collectorOutcomeCounts([result])).toEqual({ executed: 1, reused: 0, passed: 1, failed: 0, manual_accepted: 0, attempts: 2 });
});
test('Bun outer timeouts stay failed after late completion, with and without a retry', () => {
@@ -806,7 +980,7 @@ test('outer-timeout', () => captures.runAttempt('outer-timeout', ['audit'], 50,
.toEqual(mode === 'retry' ? [[1, false], [2, true]] : [[1, false]]);
expect(['timeout', 'attempt_incomplete']).toContain(result.tests[0].exit_reason);
expect(result.tests[0].error).toContain('Test attempt stopped:');
expect(collectorOutcomeCounts([result])).toEqual({ executed: 1, reused: 0,
expect(collectorOutcomeCounts([result])).toEqual({ executed: 1, reused: 0, manual_accepted: 0,
passed: mode === 'retry' ? 1 : 0, failed: mode === 'retry' ? 0 : 1, attempts: mode === 'retry' ? 2 : 1 });
}
});
+51 -7
View File
@@ -15,11 +15,14 @@ import { JUDGE_MS } from './helpers/eval-budgets';
import Anthropic from '@anthropic-ai/sdk';
import * as fs from 'fs';
import * as path from 'path';
import { callJudge, judge } from './helpers/llm-judge';
import { callJudge, judge, JudgeRefusalError, DEFAULT_JUDGE_MAX_TOKENS } from './helpers/llm-judge';
import { ENG_REVIEW_EXCERPT } from './helpers/workflow-excerpt';
import type { JudgeScore } from './helpers/llm-judge';
import { readWorkflowJudgeInput, buildWorkflowJudgePrompt } from './helpers/workflow-judge-input';
import { readWorkflowJudgeInput, buildWorkflowJudgePrompt, type WorkflowJudgeInput } from './helpers/workflow-judge-input';
import { prepareWorkflowJudgeCache } from './helpers/workflow-judge-cache';
import { buildCookieWorkflowJudgeInput, COOKIE_WORKFLOW_JUDGE } from './helpers/cookie-workflow-judge-input';
import { getCookieWorkflowManualReview, type ManualJudgeReview } from './helpers/cookie-workflow-manual-review';
import { resolveEvalModel } from '../lib/eval-model';
import { LLM_JUDGE_TOUCHFILES } from './helpers/touchfiles';
// Runs when EVALS=1 is set (requires ANTHROPIC_API_KEY in env) — the EVALS
// gate lives in the shared describeIfSelected. Selection machinery is shared
@@ -593,6 +596,7 @@ async function runWorkflowJudge(opts: {
judgeContext: string;
judgeGoal: string;
thresholds?: { clarity: number; completeness: number; actionability: number };
readInput?: () => WorkflowJudgeInput;
}) {
const started = performance.now();
const previous = workflowJudgeAttempts.get(opts.testName);
@@ -603,6 +607,8 @@ async function runWorkflowJudge(opts: {
let stage: 'input' | 'judge' | 'validation' | 'recording' = 'input';
let finalized = false;
let scores: JudgeScore | undefined;
let manualReview: ManualJudgeReview | undefined;
let customInputMetadata: { prompt: string; model: string } | undefined;
let reused: ReturnType<ReturnType<typeof prepareWorkflowJudgeCache>['lookup']> = null;
let timer: ReturnType<typeof setTimeout>;
let rejectStopped: (error: Error) => void;
@@ -617,18 +623,23 @@ async function runWorkflowJudge(opts: {
clearTimeout(timer);
if (!passed) controller.abort(error);
evalCollector?.addTest({
name: opts.testName, suite: opts.suite, tier: 'llm-judge', passed,
name: opts.testName, suite: opts.suite, tier: 'llm-judge', passed, attempt,
duration_ms: Math.max(0, performance.now() - started),
cost_usd: reused || !scores ? 0 : 0.02,
execution: reused ? 'reused' : 'executed',
...customInputMetadata,
...(manualReview ? { manual_review: manualReview } : {}),
...(reused ? { reused_from: { input_key: reused.reuse.key, run_id: reused.reuse.source.runId,
revision: reused.reuse.source.revision, completed_at: new Date(reused.reuse.source.completedAt).toISOString() } } : {}),
...(scores ? { judge_scores: { clarity: scores.clarity, completeness: scores.completeness, actionability: scores.actionability },
judge_reasoning: scores.reasoning } : {}),
...(passed ? {} : { exit_reason: error instanceof Error && error.name === 'WorkflowJudgeDeadline' ? 'timeout'
...(passed ? {} : { exit_reason: error instanceof JudgeRefusalError ? 'provider_refusal'
: error instanceof Error && error.name === 'WorkflowJudgeDeadline' ? 'timeout'
: error instanceof Error && error.name === 'WorkflowJudgeSuperseded' ? 'cancelled'
: stage === 'validation' ? 'validation_failed' : 'harness_error',
error: `${error instanceof Error ? error.message : String(error)}${scores ? '' : '\nNo completed model response; cost and usage unavailable.'}` }),
error: `${error instanceof Error ? error.message : String(error)}${scores ? '' : error instanceof JudgeRefusalError
? '\nNo automated score; provider refusal usage retained when manually accepted; cost unavailable.'
: '\nNo completed model response; cost and usage unavailable.'}` }),
});
};
const stop = (error: Error) => {
@@ -653,16 +664,35 @@ async function runWorkflowJudge(opts: {
const work = async () => {
checkActive();
const thresholds = { clarity: 4, completeness: 3, actionability: 4, ...opts.thresholds };
const input = readWorkflowJudgeInput({ root: ROOT, skillPath: opts.skillPath,
const input = opts.readInput ? opts.readInput() : readWorkflowJudgeInput({ root: ROOT, skillPath: opts.skillPath,
startMarker: opts.startMarker, endMarker: opts.endMarker });
checkActive();
const prompt = buildWorkflowJudgePrompt(opts, input);
if (opts.readInput) customInputMetadata = { prompt, model: resolveEvalModel('judge') };
const cache = prepareWorkflowJudgeCache({ ...opts, root: ROOT, thresholds, prompt, attempt });
checkActive();
reused = cache.lookup();
checkActive();
stage = 'judge';
const result = reused?.scores ?? await callJudge<JudgeScore>(prompt, undefined, { signal: controller.signal });
const maxTokens = DEFAULT_JUDGE_MAX_TOKENS;
let result: JudgeScore;
try {
result = reused?.scores ?? await callJudge<JudgeScore>(prompt, undefined, { signal: controller.signal, max_tokens: maxTokens });
} catch (error) {
checkActive();
if (error instanceof JudgeRefusalError && customInputMetadata) {
const approved = getCookieWorkflowManualReview(ROOT, { testName: opts.testName, prompt,
model: customInputMetadata.model, maxTokens, thresholds, attempt }, error.refusal);
checkActive();
if (approved) {
manualReview = approved;
console.log(`[workflow-judge] ${opts.testName}: MANUAL ACCEPTANCE, no automated score; ${approved.approval.approval_url}`);
finish(false, error);
return;
}
}
throw error;
}
checkActive();
scores = result;
console.log(`[workflow-judge] ${opts.testName}: ${reused ? `reused ${reused.reuse.source.runId} @ ${reused.reuse.source.revision} (${new Date(reused.reuse.source.completedAt).toISOString()})` : 'executed'}`);
@@ -941,5 +971,19 @@ ${voiceSection}`);
}, JUDGE_MS);
});
describeIfSelected('Cookie setup workflow quality', ['setup-browser-cookies/SKILL.md workflow'], () => {
testIfSelected('setup-browser-cookies/SKILL.md workflow', async () => {
await runWorkflowJudge({
testName: 'setup-browser-cookies/SKILL.md workflow',
suite: 'Cookie setup workflow quality',
skillPath: 'setup-browser-cookies/SKILL.md',
startMarker: '# Setup Browser Cookies',
endMarker: null,
...COOKIE_WORKFLOW_JUDGE,
readInput: () => buildCookieWorkflowJudgeInput(ROOT),
});
}, WORKFLOW_JUDGE_TEST_MS);
});
// Module-level afterAll — finalize eval collector after all tests complete
afterAll(() => finalizeEvalCollector(evalCollector));
+2 -2
View File
@@ -594,7 +594,7 @@ describe('TOUCHFILES completeness', () => {
);
const unique = registeredJudgeTestNames(llmContent);
expect(unique).toHaveLength(25);
expect(unique).toHaveLength(26);
const missing = unique.filter(name => !(name in LLM_JUDGE_TOUCHFILES));
if (missing.length > 0) {
@@ -613,7 +613,7 @@ describe('TOUCHFILES completeness', () => {
testIfSelected('unmapped judge case', async () => {}, 120_000);
`;
const names = registeredJudgeTestNames(withUnmappedCase);
expect(names).toHaveLength(26);
expect(names).toHaveLength(27);
expect(names.filter(name => !(name in LLM_JUDGE_TOUCHFILES))).toEqual(['unmapped judge case']);
});
+9 -5
View File
@@ -1,6 +1,7 @@
import { afterEach, expect, spyOn, test } from 'bun:test';
import { Messages } from '@anthropic-ai/sdk/resources/messages';
import { callJudge } from './helpers/llm-judge';
import { callJudge, JudgeRefusalError, DEFAULT_JUDGE_MAX_TOKENS } from './helpers/llm-judge';
import { getCookieWorkflowManualReview } from './helpers/cookie-workflow-manual-review';
import { resolveEvalModel } from '../lib/eval-model';
import * as fs from 'node:fs';
import * as os from 'node:os';
@@ -141,7 +142,7 @@ test('workflow registration preserves model work and reserves only terminal-reco
const source = fs.readFileSync(path.join(import.meta.dir, 'skill-llm-eval.test.ts'), 'utf8');
const body = source.split('async function runWorkflowJudge')[1]!.split('// Block 1:')[0]!;
const stages = ['workflowJudgeAttempts.set', 'readWorkflowJudgeInput(', 'cache.lookup()',
'callJudge<JudgeScore>(prompt, undefined, { signal: controller.signal })',
'callJudge<JudgeScore>(prompt, undefined, { signal: controller.signal, max_tokens: maxTokens })',
'expect(scores.clarity)', 'expect(scores.completeness)', 'expect(scores.actionability)', 'cache.publish(scores, active)']
.map(stage => body.indexOf(stage));
expect(stages.every(position => position >= 0)).toBe(true);
@@ -150,7 +151,7 @@ test('workflow registration preserves model work and reserves only terminal-reco
expect(body).toContain('const workDeadline = started + JUDGE_MS;');
expect(source).toContain('const WORKFLOW_JUDGE_RECORD_MS = 5_000;');
expect(source).toContain('const WORKFLOW_JUDGE_TEST_MS = JUDGE_MS + 10_000;');
expect(source.match(/\}, WORKFLOW_JUDGE_TEST_MS\);/g)).toHaveLength(14);
expect(source.match(/\}, WORKFLOW_JUDGE_TEST_MS\);/g)).toHaveLength(15);
expect(source.match(/\}, JUDGE_MS\);/g)).toHaveLength(11);
});
@@ -174,7 +175,7 @@ function actualCallback(f: ReturnType<typeof fixture>, overrides: {
const run = new Function('ROOT', 'readWorkflowJudgeInput',
'buildWorkflowJudgePrompt', 'prepareWorkflowJudgeCache', 'workflowJudgeAttempts', 'callJudge',
'evalCollector', 'expect', 'console', 'performance', 'JUDGE_MS', 'WORKFLOW_JUDGE_RECORD_MS',
'setTimeout', 'clearTimeout',
'setTimeout', 'clearTimeout', 'JudgeRefusalError', 'getCookieWorkflowManualReview', 'DEFAULT_JUDGE_MAX_TOKENS',
`${javascript}\nreturn runWorkflowJudge;`)(
f.root, overrides.read ?? readWorkflowJudgeInput, buildWorkflowJudgePrompt,
(options: WorkflowCacheOptions) => (overrides.prepare ?? prepareWorkflowJudgeCache)({ ...options, env: f.env }),
@@ -183,7 +184,8 @@ function actualCallback(f: ReturnType<typeof fixture>, overrides: {
return overrides.judge ? overrides.judge(prompt, model, options) : scores;
}, { addTest: (entry: any) => records.push(entry) }, expect, { log() {} },
overrides.clock ? { now: overrides.clock } : performance, overrides.budget ?? 120_000, overrides.allowance ?? 5_000,
overrides.setTimer ?? setTimeout, overrides.clearTimer ?? clearTimeout);
overrides.setTimer ?? setTimeout, overrides.clearTimer ?? clearTimeout,
JudgeRefusalError, getCookieWorkflowManualReview, DEFAULT_JUDGE_MAX_TOKENS);
return { run, records, signals, prompts, attempts, options: { ...f.opts, suite: 'Cache regression' } };
}
@@ -193,6 +195,8 @@ test('the actual workflow callback executes once, reuses with provenance, and pr
await first.run(options);
expect(first.prompts).toEqual([f.opts.prompt]); expect(f.entries()).toHaveLength(1);
expect(first.records[0]).toMatchObject({ passed: true, execution: 'executed', cost_usd: 0.02 });
expect(first.records[0]).not.toHaveProperty('prompt');
expect(first.records[0]).not.toHaveProperty('model');
const reused = actualCallback(f, { judge: async () => ({ ...scores, clarity: 1 }) });
await reused.run(options);
expect(reused.prompts).toHaveLength(0);