Merge remote-tracking branch 'origin/main' into garrytan/fix-wave-oct6

This commit is contained in:
garrytan committed 2026-10-06 18:09:49 +00:00
commit 9207fabcbe
322 files changed
+22550 -1051

No files matched your search

+47 -18
View File
@@ -44,6 +44,8 @@
* bun run eval:pass-rates --dir <path> # local eval dirs / downloaded artifacts (repeatable)
* bun run eval:pass-rates --backfill # also import legacy slice artifacts, labeled pre-policy
* bun run eval:pass-rates --json | --gate
* bun run eval:pass-rates --headroom | --reds | --run <id> # report views (scripts/lib/eval-history.ts)
* bun run eval:pass-rates --help # every flag, no network
*/
import * as fs from 'node:fs';
@@ -55,7 +57,9 @@ import { E2E_KINDS, E2E_TIERS, E2E_TOUCHFILES, GLOBAL_TOUCHFILES, LLM_JUDGE_TOUC
import { CASE_QUARANTINE, EVAL_POLICY } from '../test/helpers/periodic-exclude-data';
import { CASE_TEST_NAMES } from './test-paid-shards';
import { resolveStateRoot } from '../lib/state-root';
import { downloadRunArtifacts, isPooledTrialRun, isWeeklyHistoryRun, listWeeklyRuns, parseFlakeLedger, repoSlug, TRIAL_OUTCOMES_MAX_BYTES } from './lib/ci-history';
import { GH_JOBS, PASS_RATES_USAGE, criticalPath, formatCriticalPath, formatHeadroom, formatRedLedger, headroom, headroomAlarms, parsePassRatesArgs,
redLedger, triageRun, CENSUS_RED_GUIDE, type CriticalPath } from './lib/eval-history';
import { downloadRunArtifacts, GH_HISTORY, isPooledTrialRun, isWeeklyHistoryRun, listWeeklyRuns, parseFlakeLedger, repoSlug, TRIAL_OUTCOMES_MAX_BYTES, type WeeklyRun } from './lib/ci-history';
interface TestSeries {
name: string;
@@ -237,7 +241,7 @@ export function backfillEvalFiles(files: string[], run?: { run_id: string; sha?:
tier: caseTier(id, registry), kind: registry.kinds[id]!, trial: 1, panel: { n: 1, k: 1 }, attempt: 1,
outcome, ...(outcome === 'failed' ? { failure_class: failureClassOf(entry) } : {}),
exit_reason: entry.exit_reason, error: sanitizeTrialError(entry.error),
duration_ms: Math.max(0, entry.duration_ms || 0), cost_usd: Math.max(0, entry.cost_usd || 0),
duration_ms: Math.max(0, entry.duration_ms || 0), cost_usd: Math.max(0, entry.cost_usd || 0), ...(entry.cost_known === false ? { cost_known: false } : {}),
model: entry.model, cli_version: result.claude_cli_version, policy_version: 0, quarantined: false,
execution: entry.execution === 'reused' ? 'reused' : 'executed', source: 'backfill',
run_id: run?.run_id ?? `local:${file}`, sha: run?.sha ?? result.git_sha, recorded_at: run?.timestamp ?? result.timestamp,
@@ -292,7 +296,7 @@ export function holmRejections(pValues: number[], alpha: number): Set<number> {
export type PassRateLabel = 'INCONCLUSIVE' | 'BROKEN' | 'FLAKY' | 'FAILING' | 'PASSING';
export type AlarmKind = 'drift' | 'rule-as-behavior' | 'regression' | 'quarantine-exit' | 'quarantine-expired'
| 'quarantine-cap' | 'quarantine-invalid';
| 'quarantine-cap' | 'quarantine-invalid' | 'headroom';
/** The EVAL_POLICY fields pass-rates reads (structural, so tests can vary them). */
export interface PassRatePolicy {
@@ -567,24 +571,22 @@ export { downloadRunArtifacts, GH_HISTORY, listWeeklyRuns, TRIAL_OUTCOMES_MAX_BY
type HistoryFetcher, type RunArtifact, type WeeklyRun } from './lib/ci-history';
if (import.meta.main) {
const argv = process.argv.slice(2);
const flag = (name: string) => { const index = argv.indexOf(name); return index === -1 ? undefined : argv[index + 1]; };
const dirs = argv.flatMap((arg, index) => arg === '--dir' && argv[index + 1] ? [argv[index + 1]!] : []);
const asJson = argv.includes('--json');
const gate = argv.includes('--gate');
const backfill = argv.includes('--backfill');
const caseFilter = flag('--case');
const runsLimit = Number(flag('--runs')) || 10;
const sinceDays = Number(flag('--since-days')) || 60;
const repo = flag('--repo') ?? repoSlug();
const workflow = flag('--workflow') ?? 'evals-periodic.yml';
const branch = flag('--branch') ?? 'main';
const parsed = parsePassRatesArgs(process.argv.slice(2), id => Object.hasOwn(E2E_TIERS, id) || Object.hasOwn(LLM_JUDGE_TOUCHFILES, id)
|| (/^test\/[\w./-]+\.test\.ts$/.test(id) && fs.existsSync(id)));
if ('help' in parsed) { console.log(PASS_RATES_USAGE); process.exit(0); }
if ('error' in parsed) { console.error(`eval:pass-rates: ${parsed.error}\n${PASS_RATES_USAGE}`); process.exit(2); }
const { dirs, json: asJson, gate, backfill, caseFilter, runs: runsLimit, sinceDays, view } = parsed;
const repo = parsed.repo ?? repoSlug();
const workflow = parsed.workflow ?? 'evals-periodic.yml';
const branch = parsed.branch ?? 'main';
const records: TrialRecord[] = [];
const unattributed = new Set<string>();
const errors: string[] = [];
let historyError: string | null = null;
let weeklyRuns: string[] | undefined;
let fetched: WeeklyRun[] = [];
const cacheDir = path.join(path.resolve(resolveStateRoot()), 'eval-pass-rates-cache', repo.replace('/', '-'));
let pooledRunIds: Set<string> | undefined;
const manualReviews: string[] = [];
@@ -611,7 +613,8 @@ if (import.meta.main) {
: [];
pooledRunIds = new Set(pooledRuns.map(run => `${run.id}`));
const runs = [...weekly, ...pooledRuns];
const cacheDir = path.join(path.resolve(resolveStateRoot()), 'eval-pass-rates-cache', repo.replace('/', '-'));
if (parsed.runId !== undefined && !runs.some(run => run.id === parsed.runId)) runs.push(GH_JOBS.getRun(repo, parsed.runId));
fetched = runs;
const match = backfill
? (name: string) => name.startsWith('trial-outcomes') || /^(paid-slice-\d+|gate-census-\d+)(-a\d+)?$/.test(name)
: (name: string) => name.startsWith('trial-outcomes');
@@ -624,11 +627,37 @@ if (import.meta.main) {
}
}
const report = analyzePassRates(records, { weeklyRuns, pooledRunIds, unattributed: [...unattributed].sort(), errors, manualReviews });
const scope = dirs.length ? `local dirs ${dirs.join(', ')}` : `${repo} ${workflow} on ${branch}${pooledRunIds?.size ? ` + ${pooledRunIds.size} pooled branch census run(s)` : ''}, `
+ `last ${runsLimit} completed run(s): ${fetched.map(run => run.id).join(', ') || 'none'}`;
const scoped = caseFilter ? records.filter(record => record.case === caseFilter) : records;
const caseHeadroom = headroom(scoped);
if (view !== 'rates') {
const critical: CriticalPath[] = [];
if (view !== 'reds' && !dirs.length) {
for (const run of fetched.filter(run => view === 'headroom' || run.id === parsed.runId)) {
try { critical.push(criticalPath(String(run.id), GH_JOBS.listJobs(repo, run.id))); } catch { critical.push({ run: String(run.id), jobs: 0, slowest: null, wallMs: null }); }
}
}
const lines = view === 'headroom' ? formatHeadroom(caseHeadroom, critical) : view === 'reds' ? formatRedLedger(redLedger(scoped))
: historyError ? [] : triageRun({ repo, run: fetched.find(run => run.id === parsed.runId)!, records, caseFilter, cacheDir, fetcher: GH_HISTORY,
download: (match, maxBytes) => downloadRunArtifacts({ repo, run: fetched.find(run => run.id === parsed.runId)!, match, cacheDir, maxBytes }) })
.concat(formatCriticalPath(critical), historyError ? [] : [`guide: ${CENSUS_RED_GUIDE}`]);
if (asJson) console.log(JSON.stringify({ scope, historyError, view, lines }, null, 2));
else {
console.log(`scope: ${scope}`);
if (historyError) console.log(`history unavailable (${historyError})`);
for (const line of lines) console.log(line);
}
process.exit(historyError ? 1 : 0);
}
const full = analyzePassRates(records, { weeklyRuns, pooledRunIds, unattributed: [...unattributed].sort(), errors, manualReviews });
const alarms = [...full.alarms, ...(gate ? headroomAlarms(caseHeadroom) : [])].filter(alarm => !caseFilter || alarm.case === caseFilter);
const report = { ...full, cases: full.cases.filter(entry => !caseFilter || entry.case === caseFilter), alarms };
const ledger = readFreeLedger();
if (asJson) {
console.log(JSON.stringify({ repo, workflow, branch, dirs, historyError, ...report, freeLedger: ledger }, null, 2));
console.log(JSON.stringify({ repo, workflow, branch, scope, dirs, historyError, ...report, freeLedger: ledger }, null, 2));
} else {
console.log(`scope: ${scope}`);
if (historyError) console.log(`pass-rates: history unavailable (${historyError}); every label below is INCONCLUSIVE`);
console.log(formatPassRates(report, { caseFilter }));
if (ledger.length > 0) {
+40 -37
View File
@@ -1,5 +1,5 @@
{
"version": 1,
"version": 2,
"patterns": [
"test/helpers/agent-sdk-runner.ts",
"test/helpers/codex-session-runner.ts",
@@ -10,41 +10,44 @@
"test/helpers/llm-judge.ts"
],
"files": {
"scripts/lib/paid-cases.ts": {"blob":"cfb25a673c8f9c95aacfdbcd91b5dc471488da72","since":1,"nonBehavioral":"audit-wave integration before the first v2 census (EVAL_POLICY v2 already starts fresh series)"},
"scripts/lib/paid-plan.ts": {"blob":"45721ce128a112666a0caed6c13c66637b74ca54","since":1,"nonBehavioral":"audit-wave integration before the first v2 census (EVAL_POLICY v2 already starts fresh series)"},
"scripts/lib/paid-report.ts": {"blob":"72e2a3647f7e56f77ed27ced79ca9c4d239bb62c","since":1,"nonBehavioral":"audit-wave integration before the first v2 census (EVAL_POLICY v2 already starts fresh series)"},
"scripts/lib/shard-engine.ts": {"blob":"3cd0b1aaa84018b00bf9ea0ce48093f3c8492f66","since":1,"nonBehavioral":"audit-wave integration before the first v2 census (EVAL_POLICY v2 already starts fresh series)"},
"scripts/test-paid-shards.ts": {"blob":"93717f22a85d842152f49c4275367a1d943cf13e","since":1,"nonBehavioral":"audit-wave integration before the first v2 census (EVAL_POLICY v2 already starts fresh series)"},
"scripts/test-pr-profile.ts": {"blob":"d7860897397de8fd605623570d1e7ebbbd5ec743","since":1,"nonBehavioral":"audit-wave integration before the first v2 census (EVAL_POLICY v2 already starts fresh series)"},
"scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":1},
"test/helpers/agent-sdk-runner.ts": {"blob":"7edff19228222e3582f3efc0dbe12c35649e16b7","since":1},
"test/helpers/claude-pty-runner.ts": {"blob":"20cc5894532873aa6145fc6617fdeec8cef94a2d","since":1},
"test/helpers/codex-session-runner.ts": {"blob":"4eea35931a35a0afb006363402e3d4191e379737","since":1},
"test/helpers/e2e-gate.ts": {"blob":"efb7e197dd5f66c3ad0e34074b14e640754f7a45","since":1,"nonBehavioral":"audit-wave integration before the first v2 census (EVAL_POLICY v2 already starts fresh series)"},
"test/helpers/e2e-helpers.ts": {"blob":"2d4a2a53063cfc1f003db366d2c861ab1e379e6e","since":1},
"test/helpers/eval-budgets.ts": {"blob":"824be22c1ddad036bdac9ad89367eb061c07475f","since":1},
"test/helpers/eval-store.ts": {"blob":"23ec90ffdc235ecff27e734e4652e9ddda649cb1","since":1},
"test/helpers/hermetic-env.ts": {"blob":"6473f78285384f4420f48cbe5928658919078e0b","since":1},
"test/helpers/llm-judge.ts": {"blob":"453a4c1d355f3811deca53f953767109a9ec9292","since":1},
"test/helpers/paid-test-set.ts": {"blob":"ff9bfe43c43ec2bdd11114f7b8ec7c5736116f75","since":1,"nonBehavioral":"audit-wave integration before the first v2 census (EVAL_POLICY v2 already starts fresh series)"},
"test/helpers/pty-screen.ts": {"blob":"4f77569f9ce11f9cee9f347216e468cd7c956820","since":1},
"test/helpers/pty/auq.ts": {"blob":"16e5ea2bb5b9f28f44808a1a09024dbd3e6f28c4","since":1},
"test/helpers/pty/binary.ts": {"blob":"404d7515076d5c00804f89e23ea69296312c00e1","since":1},
"test/helpers/pty/boundaries.ts": {"blob":"53eb76e57c0558c00254366d6334a1c833126575","since":1},
"test/helpers/pty/classify.ts": {"blob":"3f2daa12ff6484f96b8d28d910a50231c4bb3ae8","since":1},
"test/helpers/pty/fake-session.ts": {"blob":"554e064ef259b8444280eddfe9b0db93c66a8202","since":1},
"test/helpers/pty/judge.ts": {"blob":"81c5bb4c48ab928aca2f186a1ade9df7169d7fbe","since":1},
"test/helpers/pty/launch.ts": {"blob":"86136d044f496c11aaaaeeee9572b5831bd13208","since":1},
"test/helpers/pty/plan-native.ts": {"blob":"ddba9cb68c7c0da935cff7c1cea258dee2dff740","since":1},
"test/helpers/pty/runners/counting.ts": {"blob":"a4ac5961c136e3c18540d5b4075ef44c2b87a425","since":1},
"test/helpers/pty/runners/floor.ts": {"blob":"a8d6d180280da349a9609af9141ccfd4205313d6","since":1},
"test/helpers/pty/runners/observation.ts": {"blob":"032298c30ebeff0a9314749670895406169bde1d","since":1},
"test/helpers/pty/screen.ts": {"blob":"ba86fa311c94401ecb7c1376a7a2bc7c1c516cf6","since":1},
"test/helpers/pty/session.ts": {"blob":"68f56d907edc6ceaa731f06dbd99c99ee77bcfe6","since":1},
"test/helpers/session-drain-policy.ts": {"blob":"6211239dd0485e6495a700b824a7e0e61d1bcd97","since":1},
"test/helpers/session-runner.ts": {"blob":"702444c3b3f7e129d101ea3864752e09951adb77","since":1},
"test/helpers/skill-fixture.ts": {"blob":"b188064594eb4201a54a80ba1bc6f28e31d7f49e","since":1},
"test/helpers/test-selection.ts": {"blob":"51655f89f66be7a5ec487e2abfe4744300ebd5f6","since":1},
"test/helpers/touchfiles.ts": {"blob":"15f369056d41b9dc42dcd86f81f60c878074704c","since":1}
"scripts/lib/paid-cases.ts": {"blob":"9bacd52446192a7ecf3dfaae901ea443cf9c9ae2","since":2},
"scripts/lib/paid-plan.ts": {"blob":"45721ce128a112666a0caed6c13c66637b74ca54","since":2},
"scripts/lib/paid-report.ts": {"blob":"3fda96a8b1f7f3d55797934847353f254cc4d054","since":2},
"scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":2},
"scripts/lib/shard-engine.ts": {"blob":"3cd0b1aaa84018b00bf9ea0ce48093f3c8492f66","since":2},
"scripts/test-paid-shards.ts": {"blob":"8ddee5f11ed7856876135520dd8025ab989e2db9","since":2},
"scripts/test-pr-profile.ts": {"blob":"d7860897397de8fd605623570d1e7ebbbd5ec743","since":2},
"scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":2},
"test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":2},
"test/helpers/auq-substance-panel.ts": {"blob":"4881260ef793668e4963a062db6d06a5e1f7a40f","since":2},
"test/helpers/claude-pty-runner.ts": {"blob":"20cc5894532873aa6145fc6617fdeec8cef94a2d","since":2},
"test/helpers/codex-session-runner.ts": {"blob":"4eea35931a35a0afb006363402e3d4191e379737","since":2},
"test/helpers/e2e-gate.ts": {"blob":"efb7e197dd5f66c3ad0e34074b14e640754f7a45","since":2},
"test/helpers/e2e-helpers.ts": {"blob":"2d4a2a53063cfc1f003db366d2c861ab1e379e6e","since":2},
"test/helpers/eval-budgets.ts": {"blob":"667d0e09b20fe5d7366f57b77b5fb9b04baa37ef","since":2},
"test/helpers/eval-store.ts": {"blob":"609c6d099108ff2696eb014b47aa318a618dd2af","since":2},
"test/helpers/hermetic-env.ts": {"blob":"6473f78285384f4420f48cbe5928658919078e0b","since":2},
"test/helpers/llm-judge.ts": {"blob":"54e3306b494fb5d7f132ccdaaae5c680465028f9","since":2},
"test/helpers/paid-test-set.ts": {"blob":"ff9bfe43c43ec2bdd11114f7b8ec7c5736116f75","since":2},
"test/helpers/pty-screen.ts": {"blob":"4f77569f9ce11f9cee9f347216e468cd7c956820","since":2},
"test/helpers/pty/auq.ts": {"blob":"16e5ea2bb5b9f28f44808a1a09024dbd3e6f28c4","since":2},
"test/helpers/pty/binary.ts": {"blob":"404d7515076d5c00804f89e23ea69296312c00e1","since":2},
"test/helpers/pty/boundaries.ts": {"blob":"53eb76e57c0558c00254366d6334a1c833126575","since":2},
"test/helpers/pty/classify.ts": {"blob":"3f2daa12ff6484f96b8d28d910a50231c4bb3ae8","since":2},
"test/helpers/pty/fake-session.ts": {"blob":"554e064ef259b8444280eddfe9b0db93c66a8202","since":2},
"test/helpers/pty/judge.ts": {"blob":"81c5bb4c48ab928aca2f186a1ade9df7169d7fbe","since":2},
"test/helpers/pty/launch.ts": {"blob":"4ef0bcb7b5e7c35d2aac0cbffd7f8a303440b6fe","since":2},
"test/helpers/pty/plan-native.ts": {"blob":"ddba9cb68c7c0da935cff7c1cea258dee2dff740","since":2},
"test/helpers/pty/runners/counting.ts": {"blob":"8c5c05e54abf961259d34d9c3d77db67138d7794","since":2},
"test/helpers/pty/runners/floor.ts": {"blob":"465f01641fb3a18ae1c9d2613a9c2b26f8fb503e","since":2},
"test/helpers/pty/runners/observation.ts": {"blob":"1c2b75fbb984708725abaa8939125f7cab969f9f","since":2},
"test/helpers/pty/screen.ts": {"blob":"ee005f36ffa48617e6832ccbeb35e8e96873df9b","since":2},
"test/helpers/pty/session.ts": {"blob":"58533cea9d81cc87bb3b499bdfe0db8e3a8709c5","since":2},
"test/helpers/session-drain-policy.ts": {"blob":"6211239dd0485e6495a700b824a7e0e61d1bcd97","since":2},
"test/helpers/session-ledger.ts": {"blob":"6e3c9e837e46f98addba2d12eaebff802da320bb","since":2},
"test/helpers/session-runner.ts": {"blob":"d40eb29e8b8fdc0675b511693e4a90823b5963fc","since":2},
"test/helpers/skill-fixture.ts": {"blob":"b188064594eb4201a54a80ba1bc6f28e31d7f49e","since":2},
"test/helpers/test-selection.ts": {"blob":"51655f89f66be7a5ec487e2abfe4744300ebd5f6","since":2},
"test/helpers/touchfiles.ts": {"blob":"15f369056d41b9dc42dcd86f81f60c878074704c","since":2}
}
}
+8 -8
View File
@@ -10,7 +10,7 @@ import {
buildCrossSkillConsistencyJudgePrompt, buildVoiceDirectiveJudgePrompt,
JUDGE_SCORE_SCHEMA, QA_ANTI_REFUSAL_JUDGE_SCHEMA, CROSS_SKILL_CONSISTENCY_JUDGE_SCHEMA,
VOICE_DIRECTIVE_JUDGE_SCHEMA, VOICE_DIRECTIVE_DIMENSIONS, JUDGE_SCORE_DIMENSIONS, DEFAULT_JUDGE_MAX_TOKENS,
judgePanelMean, judgePanelMajority,
judgePanelMedian, judgePanelMajority,
} from '../test/helpers/llm-judge';
import { buildWorkflowJudgePrompt, WORKFLOW_JUDGE_RESPONSE_SCHEMA } from '../test/helpers/workflow-judge-input';
import { CLAUDE_FRONTIER_EVAL_MODEL } from '../lib/eval-model';
@@ -18,7 +18,7 @@ import type { CalibrationConfig, Verdict } from './judge-calibration';
type Sample = Record<string, unknown>;
/** What production rejects: judgePanelMean needs finite numbers, judgePanelMajority needs booleans. */
/** What production rejects: judgePanelMedian needs finite numbers, judgePanelMajority needs booleans. */
function requireFields(value: unknown, numbers: readonly string[], booleans: readonly string[] = []): void {
const sample = (value ?? {}) as Sample;
for (const key of numbers) {
@@ -29,13 +29,13 @@ function requireFields(value: unknown, numbers: readonly string[], booleans: rea
}
}
const meets = (mean: Record<string, number>, floors: Record<string, number>): Verdict =>
Object.entries(floors).every(([key, floor]) => mean[key]! >= floor) ? 'pass' : 'fail';
const meets = (median: Record<string, number>, floors: Record<string, number>): Verdict =>
Object.entries(floors).every(([key, floor]) => median[key]! >= floor) ? 'pass' : 'fail';
const QA_FLOORS = { clarity: 3, completeness: 3, actionability: 4 };
const scoreJudge = (floors: Record<string, number>) => ({
validate: (value: unknown) => requireFields(value, JUDGE_SCORE_DIMENSIONS),
verdict: (samples: unknown[]) => meets(judgePanelMean<string>(samples as Sample[], JUDGE_SCORE_DIMENSIONS), floors),
verdict: (samples: unknown[]) => meets(judgePanelMedian<string>(samples as Sample[], JUDGE_SCORE_DIMENSIONS), floors),
});
const workflow = (id: string, priority: number, cases: string[], extra: { agentCapability?: 'frontier'; model?: string; floors?: Record<string, number> } = {}): CalibrationConfig => ({
@@ -80,7 +80,7 @@ export const CALIBRATION_CONFIGS: CalibrationConfig[] = [
oldOptions: {}, newOptions: { jsonSchema: QA_ANTI_REFUSAL_JUDGE_SCHEMA },
validate: value => requireFields(value, ['confidence'], ['would_browse']),
verdict: samples => judgePanelMajority<string>(samples as Sample[], 'would_browse')
&& judgePanelMean<string>(samples as Sample[], ['confidence']).confidence >= 4 ? 'pass' : 'fail',
&& judgePanelMedian<string>(samples as Sample[], ['confidence']).confidence >= 4 ? 'pass' : 'fail',
estimatedOutputTokens: 1200,
},
{
@@ -89,7 +89,7 @@ export const CALIBRATION_CONFIGS: CalibrationConfig[] = [
oldOptions: {}, newOptions: { jsonSchema: CROSS_SKILL_CONSISTENCY_JUDGE_SCHEMA },
validate: value => requireFields(value, ['score'], ['consistent']),
verdict: samples => judgePanelMajority<string>(samples as Sample[], 'consistent')
&& judgePanelMean<string>(samples as Sample[], ['score']).score >= 4 ? 'pass' : 'fail',
&& judgePanelMedian<string>(samples as Sample[], ['score']).score >= 4 ? 'pass' : 'fail',
estimatedOutputTokens: 1200,
},
{
@@ -97,7 +97,7 @@ export const CALIBRATION_CONFIGS: CalibrationConfig[] = [
build: inputs => buildVoiceDirectiveJudgePrompt(inputs.voiceSection!),
oldOptions: {}, newOptions: { jsonSchema: VOICE_DIRECTIVE_JUDGE_SCHEMA },
validate: value => requireFields(value, VOICE_DIRECTIVE_DIMENSIONS),
verdict: samples => meets(judgePanelMean<string>(samples as Sample[], VOICE_DIRECTIVE_DIMENSIONS),
verdict: samples => meets(judgePanelMedian<string>(samples as Sample[], VOICE_DIRECTIVE_DIMENSIONS),
Object.fromEntries(VOICE_DIRECTIVE_DIMENSIONS.map(key => [key, 4]))),
estimatedOutputTokens: 1200,
},
+123
View File
@@ -0,0 +1,123 @@
#!/usr/bin/env bun
/**
* C1 workflow-judge control bundles (pre-registered in docs/evals/judge-controls-2026-10.md).
*
* For each judge that produced low samples, builds the production bundle with the
* eval's own reader and prompt builder, derives a negative and a targeted-defect
* bundle, and runs one 3-sample panel per bundle with the eval's request options.
*
* Usage:
* bun run scripts/judge-controls.ts --list (builds every bundle, no API calls)
* bun run scripts/judge-controls.ts --run (36 judge calls; writes docs/evals/judge-controls/results.json)
*/
import * as fs from 'node:fs';
import * as path from 'node:path';
import { buildQaWorkflowJudgePrompt, callJudge, judgePanelMean, judgePanelMedian, DEFAULT_JUDGE_MAX_TOKENS, JUDGE_SCORE_DIMENSIONS, type CallJudgeOptions } from '../test/helpers/llm-judge';
import { buildWorkflowJudgePrompt, readWorkflowJudgeInput, QA_DISCOVERY_REFERENCES, WORKFLOW_JUDGE_RESPONSE_SCHEMA } from '../test/helpers/workflow-judge-input';
const ROOT = path.resolve(import.meta.dir, '..');
const FLOORS = { clarity: 3, completeness: 3, actionability: 4 } as const;
const SAMPLES = 3;
type Judge = {
name: string;
read: () => string;
prompt: (text: string) => string;
options: CallJudgeOptions;
defect: (text: string) => string;
};
function cut(text: string, start: string, end: string): string {
const from = text.indexOf(start);
const to = text.indexOf(end, from + start.length);
if (from < 0 || to < 0) throw new Error(`defect anchor missing: ${start} … ${end}`);
return text.slice(0, from) + text.slice(to);
}
function insertAfter(text: string, anchor: string, line: string): string {
const at = text.indexOf(anchor);
if (at < 0) throw new Error(`defect anchor missing: ${anchor}`);
return text.slice(0, at + anchor.length) + '\n\n' + line + text.slice(at + anchor.length);
}
const workflow = (skillPath: string, startMarker: string, endMarker: string | null, references: readonly string[] | undefined) =>
() => readWorkflowJudgeInput({ root: ROOT, skillPath, startMarker, endMarker, references }).text;
const JUDGES: Judge[] = [
{
name: 'qa/SKILL.md workflow',
read: workflow('qa/SKILL.md', '# /qa: Test', null,
['qa/templates/functional-report-template.md', 'qa/templates/qa-report-template.md', 'qa/references/issue-taxonomy.md']),
prompt: buildQaWorkflowJudgePrompt,
options: {},
defect: text => cut(text, '## Phase 10: Report', '## Phase 11:'),
},
{
name: 'qa-only/SKILL.md workflow',
read: workflow('qa-only/SKILL.md', '# /qa-only:', null, QA_DISCOVERY_REFERENCES.filter(file => file !== 'qa/sections/exploratory.md')),
prompt: text => buildWorkflowJudgePrompt({ judgeContext: 'a report-only QA testing workflow',
judgeGoal: 'how to select browser/native functional/mixed targets, explore safely with repository tools, report exact contract evidence and coverage limits, conditionally load browser/DX instructions and never mutate product/tests/Git through any tool' }, { files: [], text }),
options: { max_tokens: DEFAULT_JUDGE_MAX_TOKENS, jsonSchema: WORKFLOW_JUDGE_RESPONSE_SCHEMA },
defect: text => insertAfter(text, '# /qa-only: Report-Only QA Testing', 'After reporting each bug, fix it in the source and commit the fix.'),
},
{
name: 'review/SKILL.md workflow',
read: workflow('review/SKILL.md', '## Step 0: Detect platform and base branch', null,
[...QA_DISCOVERY_REFERENCES, 'review/checklist.md', 'review/specialists/testing.md']),
prompt: text => buildWorkflowJudgePrompt({ judgeContext: 'a pre-landing review with bounded exploratory QA',
judgeGoal: 'how to review and explore changed behavior even for small diffs without a plan or server, preserve report-only discovery and the test_stub ASK gate, handle incomplete probes honestly, and rerun affected evidence after approved repairs',
agentCapability: 'frontier' }, { files: [], text }),
options: { max_tokens: DEFAULT_JUDGE_MAX_TOKENS },
defect: text => cut(text, '## Step 3: Get the diff', '## Step 3.4'),
},
{
name: 'setup-deploy/SKILL.md platform setup',
read: workflow('setup-deploy/SKILL.md', '### Step 2: Detect platform', '## Important Rules', undefined),
prompt: text => buildWorkflowJudgePrompt({ judgeContext: 'a deployment configuration setup workflow that detects deploy platforms and writes config to CLAUDE.md',
judgeGoal: 'how to detect deploy platforms (Fly.io, Render, Vercel, Netlify, Heroku, GitHub Actions, custom), gather platform-specific configuration (URLs, status commands, health checks, custom hooks), and persist everything to CLAUDE.md for future automated use' }, { files: [], text }),
options: { max_tokens: DEFAULT_JUDGE_MAX_TOKENS, jsonSchema: WORKFLOW_JUDGE_RESPONSE_SCHEMA },
defect: text => {
const step = text.indexOf('### Step 2: Detect platform');
const open = text.indexOf('```bash', step);
const close = text.indexOf('```', open + 7);
if (step < 0 || open < 0 || close < 0) throw new Error('defect anchor missing: Step 2 bash block');
return text.slice(0, open) + text.slice(close + 3);
},
},
];
/** Keep only file boundaries and headings. */
const negative = (text: string) => text.split('\n')
.filter(line => /^--- (BEGIN|END) FILE /.test(line) || /^#{1,6} /.test(line)).join('\n');
function bundles(judge: Judge): Record<'positive' | 'negative' | 'defect', string> {
const positive = judge.read();
return { positive, negative: negative(positive), defect: judge.defect(positive) };
}
const mode = process.argv[2];
if (mode !== '--list' && mode !== '--run') {
console.error('usage: bun run scripts/judge-controls.ts --list | --run');
process.exit(2);
}
if (mode === '--list') {
for (const judge of JUDGES) {
const built = bundles(judge);
console.log(judge.name, Object.fromEntries(Object.entries(built).map(([k, v]) => [k, v.length])));
}
process.exit(0);
}
type Sample = { clarity: number; completeness: number; actionability: number; reasoning: string };
const results = await Promise.all(JUDGES.flatMap(judge => Object.entries(bundles(judge)).map(async ([bundle, text]) => {
const prompt = judge.prompt(text);
const samples = await Promise.all(Array.from({ length: SAMPLES }, () => callJudge<Sample>(prompt, undefined, judge.options)));
const mean = judgePanelMean(samples, JUDGE_SCORE_DIMENSIONS);
const median = judgePanelMedian(samples, JUDGE_SCORE_DIMENSIONS);
const verdict = Object.entries(FLOORS).every(([key, floor]) => median[key as keyof typeof FLOORS] >= floor) ? 'pass' : 'fail';
return { judge: judge.name, bundle, verdict, median, mean, samples };
})));
const out = path.join(ROOT, 'docs/evals/judge-controls/results.json');
fs.mkdirSync(path.dirname(out), { recursive: true });
fs.writeFileSync(out, JSON.stringify({ recorded_at: new Date().toISOString(), results }, null, 2) + '\n');
for (const r of results) console.log(`${r.judge} | ${r.bundle} | ${r.verdict} | median ${JSON.stringify(r.median)} mean ${JSON.stringify(r.mean)}`);
+354
View File
@@ -0,0 +1,354 @@
/**
* eval:pass-rates report views over stored trial records (plan 0.1):
* session headroom (--headroom), the verdict red ledger with the all-green
* probability (--reds), and one census's triage view (--run <id>). Pure
* functions plus the run-scoped fetches they need; scripts/eval-flake-rank.ts
* parses flags, loads history and dispatches here.
*
* Headroom is per armed session (session ledger, `sessions[]` on a record):
* elapsed over the timeout it armed. A case's headroom is its slowest
* session; a timed-out sample is censored (its true duration is unknown).
* Records without sessions are unknown, never scored; their case wall
* (JUnit or shard clock) is shown separately as an upper bound.
*
* The all-green probability is an approximation that assumes independent
* verdicts: P(green) = Π (1 - p_i) over the cases a census verdicts, where
* p_i = (reds_i + K * p̄) / (verdicts_i + K) shrinks each case's red rate
* toward the pooled rate p̄. Its interval recomputes the same product with p̄
* replaced by the pooled rate's 95% Wilson bounds.
*/
import * as fs from 'node:fs';
import * as path from 'node:path';
import { spawnSync } from 'node:child_process';
import { panelVerdict, type TrialOutcomeRecord } from '../../test/helpers/eval-store';
import { HEADROOM_FAIL, HEADROOM_WARN } from '../../test/helpers/eval-budgets';
import { sanitizeFixedFenceLine } from './published-text';
import type { HistoryFetcher, WeeklyRun } from './ci-history';
import { CASE_SHARDED_FILES, trialShardKey } from './paid-cases';
import { shardSlug } from '../test-paid-shards';
import { CENSUS_RED_GUIDE } from './paid-report';
export { CENSUS_RED_GUIDE };
/** Below this many session samples a case's headroom is insufficient, never a pass. */
export const HEADROOM_MIN_SAMPLES = 3;
/** Pseudo-verdicts of pooled rate added to each case's red rate. */
export const RED_RATE_SHRINK = 4;
/** Largest slice artifact --run downloads. */
export const EVIDENCE_MAX_BYTES = 64 * 1024 * 1024;
type Rec = TrialOutcomeRecord;
const pct = (x: number) => `${Math.round(x * 1000) / 10}%`;
const secs = (ms: number) => `${Math.round(ms / 1000)}s`;
// --- Headroom ---
export interface SessionHeadroom { key: string; samples: number; censored: number; maxRatio: number; maxElapsedMs: number; budgetMs: number }
export interface CaseHeadroom {
case: string;
status: 'over' | 'warn' | 'ok' | 'insufficient' | 'unknown';
sessions: SessionHeadroom[];
/** Slowest session's worst ratio, when any session was recorded. */
worst?: SessionHeadroom;
/** Largest case wall (JUnit or shard clock) over the window: an upper bound, never a session clock. */
caseWallMs: number;
runs: number;
}
export function headroom(records: readonly Rec[]): CaseHeadroom[] {
const byCase = new Map<string, Rec[]>();
for (const r of records) if (r.outcome !== 'skipped' && r.execution !== 'reused') byCase.set(r.case, [...(byCase.get(r.case) ?? []), r]);
return [...byCase.entries()].map(([id, rs]) => {
const sessions = new Map<string, SessionHeadroom>();
for (const s of rs.flatMap(r => r.sessions ?? [])) {
if (!s.budget_ms) continue;
const key = s.key;
const entry = sessions.get(key) ?? { key, samples: 0, censored: 0, maxRatio: 0, maxElapsedMs: 0, budgetMs: s.budget_ms };
entry.samples++;
if (s.end === 'session_timeout' || s.end === 'observer_timeout') entry.censored++;
const ratio = s.elapsed_ms / s.budget_ms;
if (ratio > entry.maxRatio) Object.assign(entry, { maxRatio: ratio, maxElapsedMs: s.elapsed_ms, budgetMs: s.budget_ms });
sessions.set(key, entry);
}
const list = [...sessions.values()].sort((a, b) => b.maxRatio - a.maxRatio);
const worst = list[0];
const status: CaseHeadroom['status'] = !worst ? 'unknown' : worst.samples < HEADROOM_MIN_SAMPLES ? 'insufficient'
: worst.maxRatio > HEADROOM_FAIL ? 'over' : worst.maxRatio > HEADROOM_WARN ? 'warn' : 'ok';
return { case: id, status, sessions: list, ...(worst ? { worst } : {}), caseWallMs: Math.max(0, ...rs.map(r => r.duration_ms)),
runs: new Set(rs.map(r => r.run_id ?? 'local')).size };
}).sort((a, b) => (b.worst?.maxRatio ?? -1) - (a.worst?.maxRatio ?? -1) || b.caseWallMs - a.caseWallMs);
}
/** --gate alarms: a case whose slowest session ran above HEADROOM_FAIL of its armed budget. */
export function headroomAlarms(cases: readonly CaseHeadroom[]): Array<{ kind: 'headroom'; case: string; message: string }> {
return cases.filter(c => c.status === 'over').map(c => ({ kind: 'headroom' as const, case: c.case,
message: `${c.case} session ${c.worst!.key}: max ${secs(c.worst!.maxElapsedMs)} of ${secs(c.worst!.budgetMs)} (${pct(c.worst!.maxRatio)})`
+ `${c.worst!.censored ? `, ${c.worst!.censored} timed-out sample(s) censored` : ''} over ${c.worst!.samples} sample(s), above the ${pct(HEADROOM_FAIL)} cap. `
+ 'Cut work in the skill or fixture; budgets are never raised (TESTING_INTERNALS "Timeout policy").' }));
}
export function formatHeadroom(cases: readonly CaseHeadroom[], critical: readonly CriticalPath[] = []): string[] {
const lines = [`headroom: slowest armed session per case (warn > ${pct(HEADROOM_WARN)}, alarm > ${pct(HEADROOM_FAIL)}; `
+ `insufficient below ${HEADROOM_MIN_SAMPLES} samples; censored = timed out)`,
' status session max/budget samples case wall (upper bound, not a session clock) case'];
for (const c of cases) {
const w = c.worst;
const session = w ? `${pct(w.maxRatio)} ${secs(w.maxElapsedMs)}/${secs(w.budgetMs)}` : 'unknown (no session ledger)';
lines.push(` ${c.status.padEnd(12)} ${session.padEnd(26)} ${(w ? `${w.samples}${w.censored ? ` (${w.censored} censored)` : ''}` : '-').padEnd(7)} `
+ `${secs(c.caseWallMs).padEnd(45)} ${c.case}${w ? ` [${w.key}]` : ''}`);
}
return [...lines, ...formatCriticalPath(critical)];
}
export function formatCriticalPath(critical: readonly CriticalPath[]): string[] {
return critical.map(cp => `critical path ${cp.run}: ${cp.slowest ? `${cp.slowest.name} ${secs(cp.slowest.ms)}` : 'unknown'}`
+ ` of ${cp.jobs} slice job(s), run wall ${cp.wallMs === null ? 'unknown' : secs(cp.wallMs)}${cp.files?.length ? `; slice files: ${cp.files.join(', ')}` : ''}`);
}
// --- Critical path from runner slice timings ---
export interface JobTiming { name: string; startedAt: string | null; completedAt: string | null }
export interface CriticalPath { run: string; jobs: number; slowest: { name: string; ms: number } | null; wallMs: number | null; files?: string[] }
const SLICE_JOB = /^(eval-slices|eval-codex-slices|gate-census) \((\d+)\)$/;
export function criticalPath(run: string, jobs: readonly JobTiming[], plan?: { entries: Array<{ file: string; slice?: number; status?: string }> }): CriticalPath {
const timed = jobs.filter(j => SLICE_JOB.test(j.name) && j.startedAt && j.completedAt)
.map(j => ({ name: j.name, ms: Date.parse(j.completedAt!) - Date.parse(j.startedAt!) }));
const slowest = timed.sort((a, b) => b.ms - a.ms)[0] ?? null;
const all = jobs.filter(j => j.startedAt && j.completedAt);
const wallMs = all.length ? Math.max(...all.map(j => Date.parse(j.completedAt!))) - Math.min(...all.map(j => Date.parse(j.startedAt!))) : null;
const slice = slowest ? Number(SLICE_JOB.exec(slowest.name)![2]) : null;
const files = plan && slice !== null && slowest!.name.startsWith('gate-census')
? plan.entries.filter(e => e.slice === slice && e.status === 'planned').map(e => e.file) : undefined;
return { run, jobs: timed.length, slowest, wallMs, ...(files ? { files } : {}) };
}
// --- Red ledger ---
export interface VerdictRed { run: string; lane: string; case: string; status: string; trials: Rec[] }
export interface CensusReds { run: string; lane: string; verdicts: number; reds: VerdictRed[]; byClass: Record<string, number>; byCause: Record<string, number> }
export interface RedLedger {
censuses: CensusReds[];
pooled: { verdicts: number; reds: number; rate: number; lo: number; hi: number };
perCase: Array<{ case: string; verdicts: number; reds: number; rate: number }>;
allGreen: { p: number; lo: number; hi: number; verdicts: number } | null;
}
function wilson(k: number, n: number, z = 1.96): { lo: number; hi: number } {
if (n === 0) return { lo: 0, hi: 1 };
const p = k / n, d = 1 + z * z / n, c = p + z * z / (2 * n), m = z * Math.sqrt(p * (1 - p) / n + z * z / (4 * n * n));
return { lo: Math.max(0, (c - m) / d), hi: Math.min(1, (c + m) / d) };
}
/** One panelVerdict per case per census lane (earliest attempt), exactly as the report computes it. */
export function verdictsOf(records: readonly Rec[]): Array<{ run: string; lane: string; case: string; status: string; failsLane: boolean; trials: Rec[] }> {
const groups = new Map<string, Rec[]>();
for (const r of records) {
if (r.policy_version < 1 || r.execution === 'reused') continue;
// A JUnit case is one verdict per shard file, as the report counts it; trial shards form one panel.
const key = `${r.run_id ?? 'local'}\0${r.lane ?? r.tier}\0${r.case}\0${r.source === 'shard' ? '' : r.file}`;
groups.set(key, [...(groups.get(key) ?? []), r]);
}
return [...groups.entries()].map(([key, rs]) => {
const [run, lane, id] = key.split('\0') as [string, string, string, string];
const attempt = Math.min(...rs.map(r => r.attempt));
const trials = rs.filter(r => r.attempt === attempt);
const v = panelVerdict({ case: id, kind: trials[0]!.kind, panel: trials[0]!.panel, quarantined: trials[0]!.quarantined,
trials: trials.map(t => ({ trial: t.trial, outcome: t.outcome, attempt, ...(t.failure_class ? { failure_class: t.failure_class } : {}) })) });
return { run, lane, case: id, status: v.status, failsLane: v.failsLane, trials };
});
}
export function redLedger(records: readonly Rec[]): RedLedger {
const verdicts = verdictsOf(records).filter(v => v.status !== 'SKIPPED');
const censusKey = (v: { run: string; lane: string }) => `${v.run}\0${v.lane}`;
const censuses = new Map<string, CensusReds>();
for (const v of verdicts) {
const c = censuses.get(censusKey(v)) ?? { run: v.run, lane: v.lane, verdicts: 0, reds: [], byClass: {}, byCause: {} };
c.verdicts++;
if (v.failsLane) {
c.reds.push({ run: v.run, lane: v.lane, case: v.case, status: v.status, trials: v.trials });
for (const t of v.trials.filter(t => t.outcome === 'failed')) {
c.byClass[t.failure_class ?? 'assertion'] = (c.byClass[t.failure_class ?? 'assertion'] ?? 0) + 1;
c.byCause[t.failure_cause ?? 'unrecorded'] = (c.byCause[t.failure_cause ?? 'unrecorded'] ?? 0) + 1;
}
}
censuses.set(censusKey(v), c);
}
const reds = verdicts.filter(v => v.failsLane).length;
const rate = verdicts.length ? reds / verdicts.length : 0;
const pooled = { verdicts: verdicts.length, reds, rate, ...wilson(reds, verdicts.length) };
const byCase = new Map<string, { verdicts: number; reds: number }>();
for (const v of verdicts) {
const e = byCase.get(v.case) ?? { verdicts: 0, reds: 0 };
e.verdicts++; if (v.failsLane) e.reds++;
byCase.set(v.case, e);
}
const perCase = [...byCase.entries()].map(([id, e]) => ({ case: id, ...e, rate: (e.reds + RED_RATE_SHRINK * rate) / (e.verdicts + RED_RATE_SHRINK) }))
.sort((a, b) => b.rate - a.rate || (a.case < b.case ? -1 : 1));
const ordered = [...censuses.values()];
const latestRun = ordered.map(c => c.run).sort().at(-1);
const census = verdicts.filter(v => v.run === latestRun);
const green = (prior: number) => census.reduce((p, v) => {
const e = byCase.get(v.case)!;
return p * (1 - (e.reds + RED_RATE_SHRINK * prior) / (e.verdicts + RED_RATE_SHRINK));
}, 1);
const allGreen = census.length ? { p: green(rate), lo: green(pooled.hi), hi: green(pooled.lo), verdicts: census.length } : null;
return { censuses: ordered.sort((a, b) => (a.run < b.run ? -1 : a.run > b.run ? 1 : a.lane < b.lane ? -1 : 1)), pooled, perCase, allGreen };
}
const tally = (counts: Record<string, number>) => Object.entries(counts).sort(([a, x], [b, y]) => y - x || (a < b ? -1 : 1)).map(([k, n]) => `${k} ${n}`).join(', ') || 'none';
export function formatRedLedger(ledger: RedLedger, opts: { top?: number } = {}): string[] {
const lines = ['reds: verdict reds per census lane by failure class and machine cause (cause "unrecorded" = written before failure_cause existed)'];
for (const c of ledger.censuses) {
lines.push(` ${c.run} ${c.lane}: ${c.reds.length} red of ${c.verdicts} verdicts${c.reds.length ? ` — class: ${tally(c.byClass)}; cause: ${tally(c.byCause)}` : ''}`);
for (const red of c.reds) {
const failed = red.trials.filter(t => t.outcome === 'failed');
lines.push(` ✗ ${red.case} ${red.status} ${failed.map(t => `t${t.trial} ${t.failure_class ?? 'assertion'}/${t.failure_cause ?? 'unrecorded'}`
+ `${t.error ? ` — ${sanitizeFixedFenceLine(t.error, 160)}` : ''}`).join('; ')}`);
}
}
const p = ledger.pooled;
lines.push(`pooled: ${p.reds} red of ${p.verdicts} verdicts = ${pct(p.rate)} per verdict [95% ${pct(p.lo)}–${pct(p.hi)}]`);
const top = ledger.perCase.filter(c => c.reds > 0).slice(0, opts.top ?? 15);
if (top.length) lines.push(`per-case red rates (shrunk toward the pooled rate, K=${RED_RATE_SHRINK}):`, ...top.map(c => ` ${pct(c.rate).padStart(6)} ${c.reds}/${c.verdicts} ${c.case}`));
const g = ledger.allGreen;
lines.push(g ? `all-green probability (approximation, assumes independent verdicts): ${pct(g.p)} for the latest census's ${g.verdicts} verdicts [${pct(g.lo)}–${pct(g.hi)} with the pooled rate's 95% interval as prior]; `
+ 'formula Π(1 - p_i) over its cases; rerun: bun run eval:pass-rates --reds' : 'all-green probability: no post-policy verdicts');
lines.push(`guide: ${CENSUS_RED_GUIDE}`);
return lines;
}
// --- One census's triage view ---
/** Fetches the report views make beyond the history loader: a named run and its slice jobs. */
export interface RunJobsFetcher { listJobs(repo: string, runId: number): JobTiming[]; getRun(repo: string, runId: number): WeeklyRun }
function ghJson(args: string[]): string[] {
const result = spawnSync('gh', args, { timeout: 120_000, maxBuffer: 64 * 1024 * 1024 });
if (result.status !== 0) throw new Error(`gh ${args.slice(0, 2).join(' ')} failed: ${String(result.stderr || result.error || '').trim()}`);
return result.stdout.toString('utf8').split('\n').filter(Boolean);
}
export const GH_JOBS: RunJobsFetcher = {
listJobs: (repo, runId) => ghJson(['api', `repos/${repo}/actions/runs/${runId}/jobs?per_page=100`, '--paginate',
'--jq', '.jobs[] | {name, startedAt: .started_at, completedAt: .completed_at}']).map(line => JSON.parse(line) as JobTiming),
getRun: (repo, runId) => JSON.parse(ghJson(['api', `repos/${repo}/actions/runs/${runId}`,
'--jq', '{id, attempt: .run_attempt, sha: .head_sha, branch: .head_branch, createdAt: .created_at}'])[0]!) as WeeklyRun,
};
/** The manifest shard key a record ran under: its trial shard, its case shard, or its file. */
export function recordShardKey(r: Pick<Rec, 'file' | 'case' | 'trial' | 'source'>): string {
return r.source === 'shard' ? trialShardKey(r.file, r.case, r.trial) : CASE_SHARDED_FILES.includes(r.file) ? `${r.file}#${r.case}` : r.file;
}
/**
* The run's reds with their values and history, and the slice artifacts that
* hold their transcripts. Only the slices a red names are downloaded; a slice
* over EVIDENCE_MAX_BYTES is reported, never silently skipped.
*/
export function triageRun(opts: { repo: string; run: WeeklyRun; records: readonly Rec[]; caseFilter?: string; cacheDir: string;
fetcher: HistoryFetcher; download: (match: (name: string) => boolean, maxBytes: number) => string[] }): string[] {
const runId = String(opts.run.id);
const mine = opts.records.filter(r => r.run_id === runId);
const reds = verdictsOf(mine).filter(v => v.failsLane && (!opts.caseFilter || v.case === opts.caseFilter));
const lines = [`run ${runId} (${opts.run.branch} @ ${opts.run.sha.slice(0, 9)}, ${opts.run.createdAt}): ${reds.length} verdict red(s)${opts.caseFilter ? ` for ${opts.caseFilter}` : ''}`];
const history = verdictsOf(opts.records);
const plans = new Map<string, Map<string, number>>();
const planFor = (lane: string) => {
const name = lane.startsWith('gate') && lane.includes('census') ? 'gate-census-plan' : 'paid-plan';
if (!plans.has(name)) {
const files = new Map<string, number>();
for (const dir of opts.download(n => n === name, 8 * 1024 * 1024)) {
try {
const manifest = JSON.parse(fs.readFileSync(path.join(dir, 'manifest.json'), 'utf8'));
for (const e of manifest.entries ?? []) if (e.status === 'planned' && Number.isInteger(e.slice)) files.set(e.file, e.slice);
} catch { /* no manifest: slices stay unknown */ }
}
plans.set(name, files);
}
return { prefix: name === 'gate-census-plan' ? 'gate-census' : 'paid-slice', files: plans.get(name)! };
};
const artifacts = opts.fetcher.listArtifacts(opts.repo, opts.run.id);
for (const red of reds) {
const runs = history.filter(v => v.case === red.case);
lines.push(`✗ ${red.case} ${red.lane} ${red.status} (history: ${runs.filter(v => !v.failsLane).length}/${runs.length} verdicts green)`);
for (const t of red.trials.filter(t => t.outcome === 'failed')) {
const d = t.failure_detail;
const detail = !d ? '' : 'judge' in d ? ` ${d.judge.map(j => `${j.dimension} ${j.mean} < ${j.threshold} (${j.samples} samples)${j.rationale ? ` "${j.rationale}"` : ''}`).join('; ')}`
: ` Expected: ${d.expected} · Received: ${d.received}`;
lines.push(sanitizeFixedFenceLine(` t${t.trial}: ${t.failure_class ?? 'assertion'} / cause ${t.failure_cause ?? 'unrecorded'}`
+ `${t.failure_cause_evidence ? `: ${t.failure_cause_evidence}` : ''}${t.error ? ` — ${t.error}` : ''}${detail}`, 2000));
const plan = planFor(red.lane);
const slice = plan.files.get(recordShardKey(t));
if (slice === undefined) { lines.push(' evidence: slice unknown (no plan artifact for this lane)'); continue; }
const name = `${plan.prefix}-${slice}-a${t.attempt}`;
const artifact = artifacts.find(a => a.name === name);
if (!artifact) { lines.push(` evidence: ${name} (expired or missing)`); continue; }
if (artifact.size > EVIDENCE_MAX_BYTES) { lines.push(` evidence: ${name} too large to fetch (${Math.round(artifact.size / 1024 / 1024)} MB)`); continue; }
const [dir] = opts.download(n => n === name, EVIDENCE_MAX_BYTES);
lines.push(` evidence: ${dir ? path.join(dir, 'shards', shardSlug([recordShardKey(t)])) : `${name} (download failed)`}`);
}
}
return lines;
}
// --- Flags ---
export const PASS_RATES_USAGE = `Usage: bun run eval:pass-rates [options]
(no view flag) per-case trial pass rates with 95% Wilson intervals
--headroom slowest armed session per case vs its budget, case wall upper bound, census critical path
--reds verdict reds per census by failure class and cause, pooled and per-case rates, all-green probability
--run <id> one census: each red with its values, history and fetched transcript evidence
--case <id> restrict every view, the JSON and the alarms to one registered case
--runs <n> completed runs per branch to read (default 10)
--branch <name> branch whose runs to read beside main (default: current branch)
--repo <owner/repo> --workflow <file> --dir <path> (repeatable, local artifacts; no network)
--backfill also import legacy slice artifacts, labeled pre-policy
--since-days <n> legacy eval files window for --dir (default 60)
--json machine-readable output
--gate exit 1 on ACTION REQUIRED (pass-rate alarms and sessions above ${pct(HEADROOM_FAIL)} of budget)
--help this text (no network)`;
export interface PassRatesArgs {
view: 'rates' | 'headroom' | 'reds' | 'run';
runId?: number; caseFilter?: string; runs: number; sinceDays: number; dirs: string[];
json: boolean; gate: boolean; backfill: boolean; repo?: string; workflow?: string; branch?: string;
}
/** Parse eval:pass-rates flags; unknown flags, bad values and unknown case ids are errors (exit 2, before any fetch). */
export function parsePassRatesArgs(argv: readonly string[], isKnownCase: (id: string) => boolean): { help: true } | { error: string } | PassRatesArgs {
const out: PassRatesArgs = { view: 'rates', runs: 10, sinceDays: 60, dirs: [], json: false, gate: false, backfill: false };
const views: string[] = [];
for (let i = 0; i < argv.length; i++) {
const arg = argv[i]!;
const value = () => { const v = argv[++i]; if (v === undefined || v.startsWith('--')) throw new Error(`${arg} needs a value`); return v; };
const count = () => { const v = value(); if (!/^[1-9]\d*$/.test(v)) throw new Error(`${arg} needs a positive integer, got ${v}`); return Number(v); };
try {
switch (arg) {
case '--help': case '-h': return { help: true };
case '--headroom': views.push('headroom'); break;
case '--reds': views.push('reds'); break;
case '--run': views.push('run'); out.runId = count(); break;
case '--case': out.caseFilter = value(); break;
case '--runs': out.runs = count(); break;
case '--since-days': out.sinceDays = count(); break;
case '--dir': out.dirs.push(value()); break;
case '--repo': out.repo = value(); break;
case '--workflow': out.workflow = value(); break;
case '--branch': out.branch = value(); break;
case '--json': out.json = true; break;
case '--gate': out.gate = true; break;
case '--backfill': out.backfill = true; break;
default: return { error: `unknown flag ${arg}` };
}
} catch (error) { return { error: (error as Error).message }; }
}
if (views.length > 1) return { error: `choose one of --headroom, --reds, --run (got ${views.join(', ')})` };
if (views[0]) out.view = views[0] as PassRatesArgs['view'];
if (out.caseFilter !== undefined && !isKnownCase(out.caseFilter)) return { error: `unknown case ${out.caseFilter} (not an E2E, judge or paid test file id)` };
if (out.view === 'run' && out.dirs.length) return { error: '--run reads one GitHub run; it cannot combine with --dir' };
return out;
}
+1
View File
@@ -251,6 +251,7 @@ export const CODEX_CI_FILES: readonly string[] = [
'test/skill-e2e-outside-voice.test.ts',
'test/skill-e2e-outside-plan-disabled.test.ts',
'test/skill-e2e-safety-codex-boundary.test.ts',
'test/codex-e2e-multiblock-live.test.ts',
];
/**
+158 -42
View File
@@ -5,15 +5,19 @@ import * as fs from 'node:fs';
import * as path from 'node:path';
import { normalizeRelativePath } from './shard-engine';
import { EVAL_POLICY } from '../../test/helpers/periodic-exclude-data';
import { STALL_WINDOW_MS } from '../../test/helpers/eval-budgets';
import {
isFinalizedEvalResultFile, failureClassOf, panelVerdict, sanitizeTrialError, formatTrialOutcomes, TRIAL_OUTCOME_SCHEMA, TRIAL_OUTCOMES_FILE,
type EvalCaseKind, type PanelVerdict, type TrialFailureClass, type TrialOutcome, type TrialOutcomeRecord,
trialCostKnown, trialFailureFields, trialSessions, type TrialFailureDetail,
} from '../../test/helpers/eval-store';
import { readSessionLedger, type SessionLedgerRow } from '../../test/helpers/session-ledger';
import { publishedFence, sanitizeFixedFenceLine } from './published-text';
import { E2E_KINDS } from '../../test/helpers/touchfiles-data';
import { manualReviewProblem } from '../../test/helpers/cookie-workflow-manual-review';
import { writeNegativeReceipt, writePanelReceipt } from '../e2e-shard-reuse';
import { E2E_TOUCHFILES, E2E_TIERS, LLM_JUDGE_TOUCHFILES } from '../../test/helpers/touchfiles';
import { CASE_TEST_NAMES, type CaseTrialPlan, caseTrialPlan, fileCaseRegistration, shardCaseId, shardFile, trialShardKey } from './paid-cases';
import { CASE_TEST_NAMES, type CaseTrialPlan, caseTestNamePattern, caseTrialPlan, fileCaseRegistration, shardCaseId, shardFile, trialShardKey } from './paid-cases';
import { type ManifestEntry, PAID_TEST_DURATIONS_FILE, type PaidRunManifest, type SliceResult, collectorOutcomeCounts, formatProfileCoverage, loadPaidTestDurations, mergePaidTestDurations, parseRunManifest, trialPanelKey, verifySliceResults, writePaidTestDurations } from './paid-plan';
import { DEFAULT_JOBS, type PaidCaseSelection, type PaidTier, ROOT, type ShardTrialRecord } from './paid-types';
import { collectPaidTestFiles, isAllSkippedPass, shardSlug, paidSelectionEnv } from './paid-select';
@@ -21,14 +25,39 @@ import type { RunShardsOptions } from '../test-paid-shards';
// ─── Local diagnosis: one case through the CI panel runner (A9) ────────────
/** The one paid file that statically registers `id`, else a thrown reason. */
/** How --case runs exactly one case: trials selected by its Bun test name, or its whole file when that file registers no other case. */
export interface CaseSelection { file: string; mode: 'name' | 'file'; reason: string }
const escapeRe = (text: string) => text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
/**
* The paid file that registers `id` and how a trial selects it. A file whose
* registration is complete wins over one that also lists the id; a test
* registered literally under the id (or its CASE_TEST_NAMES label) is
* selected by name; a loop that registers ids at runtime is selected by the
* id it names its test; a file registering only this case runs whole (its
* test names are not the id). Anything else throws before any process starts.
*/
export function caseSelection(id: string, rootDir = ROOT, discovered = collectPaidTestFiles(rootDir)): CaseSelection {
const files = discovered.map(file => {
const source = fs.readFileSync(path.join(rootDir, file), 'utf8');
return { file, source, ...fileCaseRegistration(file, source) };
}).filter(f => f.registered.includes(id));
const owners = files.filter(f => f.known).length ? files.filter(f => f.known) : files;
if (owners.length !== 1) throw new Error(`--case ${id}: ${owners.length ? `registered by ${owners.map(f => f.file).join(', ')}` : 'no paid file statically registers it'}; it needs exactly one`);
const owner = owners[0]!;
const name = escapeRe(CASE_TEST_NAMES[id] ?? id);
if (owner.registered.length === 1) return { file: owner.file, mode: 'file', reason: 'the whole file (it registers no other case)' };
if (new RegExp(`\\b(?:test|it|test\\.\\w+|test(?:Concurrent)?IfSelected)\\s*\\(\\s*(['"\`])${name}\\1|testName\\s*:\\s*(['"\`])${name}\\2`).test(owner.source)) {
return { file: owner.file, mode: 'name', reason: `its Bun test is named ${CASE_TEST_NAMES[id] ?? id}` };
}
if (owner.computed) return { file: owner.file, mode: 'name', reason: `its loop names the Bun test ${id} at runtime` };
throw new Error(`--case ${id}: ${owner.file} registers several cases and none of its tests is named ${id}; add its test name to CASE_TEST_NAMES`);
}
/** The one paid file that registers `id`, else a thrown reason. */
export function caseFile(id: string, rootDir = ROOT, discovered = collectPaidTestFiles(rootDir)): string {
const owners = discovered.filter(file => {
const { registered, known } = fileCaseRegistration(file, fs.readFileSync(path.join(rootDir, file), 'utf8'));
return known && registered.includes(id);
});
if (owners.length !== 1) throw new Error(`--case ${id}: ${owners.length ? `registered by ${owners.join(', ')}` : 'no paid file statically registers it'}; it needs exactly one`);
return owners[0]!;
return caseSelection(id, rootDir, discovered).file;
}
/**
@@ -41,15 +70,13 @@ export function caseFile(id: string, rootDir = ROOT, discovered = collectPaidTes
export async function runCaseDiagnosis(id: string, options: {
trials?: number; jobs?: number; withinShardConcurrency?: number; timeoutMs?: number; rootDir?: string; env?: NodeJS.ProcessEnv;
evalDirBase?: string; commandFor?: RunShardsOptions['commandFor']; log?: (line: string) => void; file?: string;
/** The CLI passes its own runner: importing the entry module during its top-level await never resolves. */
runShards?: typeof import('../test-paid-shards').runPaidShards;
} = {}): Promise<PanelVerdict> {
const rootDir = options.rootDir ?? ROOT;
const log = options.log ?? ((line: string) => console.log(line));
const file = options.file ?? caseFile(id, rootDir);
const testName = CASE_TEST_NAMES[id] ?? id;
if (!options.commandFor && !fs.readFileSync(path.join(rootDir, file), 'utf8').includes(testName)) {
throw new Error(`--case ${id}: ${file} has no literal Bun test named "${testName}", so a trial could not select it. `
+ 'Run the whole file (bun test <file> with EVALS=1) or add its literal name to CASE_TEST_NAMES.');
}
const selection = options.file ? { file: options.file, mode: 'name' as const } : caseSelection(id, rootDir);
const file = selection.file;
const policy = caseTrialPlan(id);
const n = options.trials ?? policy.panel.n;
const plan: CaseTrialPlan = { ...policy, panel: { n, k: policy.panel.k === policy.panel.n ? n : Math.min(policy.panel.k, n) } };
@@ -57,14 +84,27 @@ export async function runCaseDiagnosis(id: string, options: {
const tier = E2E_TIERS[id] as PaidTier;
log(`[test:paid] --case ${id}: ${n} trial(s) of ${file} (kind ${plan.kind}, PASS at ${plan.panel.k}/${n}${plan.quarantined ? ', quarantined' : ''}), tier=${tier}`);
// The runner is the CLI module; load it lazily so this library never imports it statically (no cycle).
const { runPaidShards } = await import('../test-paid-shards');
const summary = await runPaidShards(keys.map(key => [key]), {
jobs: Math.min(options.jobs ?? DEFAULT_JOBS, n), withinShardConcurrency: options.withinShardConcurrency, timeoutMs: options.timeoutMs,
rootDir, log, commandFor: options.commandFor, evalDirBase: options.evalDirBase,
trials: Object.fromEntries(keys.map(key => [key, plan])),
const runPaidShards = options.runShards ?? (await import('../test-paid-shards')).runPaidShards;
const shardOptions = { withinShardConcurrency: options.withinShardConcurrency, timeoutMs: options.timeoutMs, rootDir, log, commandFor: options.commandFor,
env: { ...(options.env ?? process.env), EVALS: '1', EVALS_TIER: tier, EVALS_PREFLIGHT_OK: '1', EVALS_ALL: '1',
...paidSelectionEnv('full', { e2e: [id], judges: [] }, `--case ${id}`) },
});
...paidSelectionEnv('full', { e2e: [id], judges: [] }, `--case ${id}`) } };
if (selection.mode === 'file') {
// The whole file is the case: one file shard per trial, each with its own eval dir, judged by shard status as CI does.
const fileTrials = [];
for (let trial = 1; trial <= n; trial++) {
const [outcome] = (await runPaidShards([[file]], { ...shardOptions, jobs: 1,
...(options.evalDirBase ? { evalDirBase: path.join(options.evalDirBase, `t${trial}`) } : {}) })).outcomes;
if (!outcome || outcome.runnerError !== undefined || isAllSkippedPass(outcome) || !['passed', 'failed', 'timed-out'].includes(outcome.status)) continue;
fileTrials.push({ trial, outcome: outcome.status === 'passed' ? 'passed' as const : 'failed' as const,
...(outcome.status === 'passed' ? {} : { failure_class: outcome.status === 'timed-out' ? 'timeout' as const : 'assertion' as const }) });
}
const verdict = panelVerdict({ case: id, kind: plan.kind, panel: plan.panel, trials: fileTrials, quarantined: plan.quarantined });
log(formatPanelLine({ ...verdict, file, slices: {} }, tier));
return verdict;
}
const summary = await runPaidShards(keys.map(key => [key]), { ...shardOptions,
jobs: Math.min(options.jobs ?? DEFAULT_JOBS, n), evalDirBase: options.evalDirBase,
trials: Object.fromEntries(keys.map(key => [key, plan])) });
const trials = summary.outcomes.flatMap(outcome => outcome.trial && outcome.trial.outcome !== null ? [{
trial: outcome.trial.trial, outcome: outcome.trial.outcome, ...(outcome.trial.failure_class ? { failure_class: outcome.trial.failure_class } : {}),
...(outcome.trial.exit_reason ? { exit_reason: outcome.trial.exit_reason } : {}), ...(outcome.trial.error ? { error: outcome.trial.error } : {}),
@@ -187,6 +227,20 @@ export interface JUnitCensus {
const bump = (counts: Map<string, number>, key: string) => counts.set(key, (counts.get(key) ?? 0) + 1);
/** Ledger rows of one JUnit case: its own id or test name, else the shard's only case. */
function caseSessions(rows: SessionLedgerRow[], id: string, onlyCase: boolean): SessionLedgerRow[] {
return rows.filter(row => row.case === id || (!row.case && (onlyCase || row.test_name === id || row.test_name === CASE_TEST_NAMES[id])));
}
/** The artifact holding a slice's shards: the downloaded directory's own name when it is one. */
/** The red-census guide the report and every eval:pass-rates view link. */
export const CENSUS_RED_GUIDE = 'docs/evals/census-red.md';
export function sliceArtifactName(root: string, slice: number, attempt: number): string {
const name = path.basename(root);
return /^(?:paid-slice|gate-census)-\d+-a\d+$/.test(name) ? name : `slice ${slice} attempt ${attempt}`;
}
export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact[], recordsByShard: Map<string, any[]>,
base: Record<string, unknown>, cliVersion?: string): JUnitCensus {
const census: JUnitCensus = { ruleCases: [], history: [], failedShards: new Set(), skipped: [], deselected: new Map(), shardSkipReasons: new Map(), unattributed: [] };
@@ -207,6 +261,7 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact
else census.unattributed.push(`${key} :: ${tc.name || '(no name)'} (${tc.outcome})`);
}
const records = recordsByShard.get(key) ?? [];
const ledger = readSessionLedger(path.join(root, 'shards', shardSlug([key])));
for (const [id, cases] of byCase) {
const kind = (E2E_KINDS[id] ?? 'rule') as EvalCaseKind;
const failed = cases.find(tc => tc.outcome === 'failed');
@@ -216,18 +271,24 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact
const failureClass: TrialFailureClass | undefined = !failed ? undefined
: failed.failureType === 'TimeoutError' ? 'timeout' : failedRecord ? failureClassOf(failedRecord) : 'assertion';
const error = sanitizeTrialError(failedRecord?.error ?? failed?.message);
const rerun = Object.hasOwn(E2E_TIERS, id) ? rerunCommand(manifest.tier, id, 1) : `EVALS=1 EVALS_TIER=${manifest.tier} bun test ${shardFile(key)}`;
const sessions = caseSessions(ledger, id, byCase.size === 1);
// A case's JUnit times sum its concurrent tests (plan-mode-no-op: 346 s in a 176 s shard), so its wall is capped by the shard's.
const diagnosis = failed && failureClass ? trialFailureFields({ failure_class: failureClass, exit_reason: failedRecord?.exit_reason,
error: failedRecord?.error ?? failed.message, sessions, record: failedRecord }, STALL_WINDOW_MS) : undefined;
if (failed) census.failedShards.add(key);
if (caseOutcome === 'skipped') {
bump(reasons, RUNTIME_SKIP_REASON);
census.skipped.push(`◌ ${id} ${key} (${cases.map(tc => tc.name).join('; ')}): ${RUNTIME_SKIP_REASON}`);
}
census.ruleCases.push({ id, kind, outcome: caseOutcome,
...(failed ? { line: `✗ ${id} ${kind} FAIL ${failureClass}${failedRecord?.exit_reason === 'timeout' && failedRecord?.timeout_at_turn !== undefined ? ` at turn ${failedRecord.timeout_at_turn}` : ''}${error ? ` — ${error}` : ''} [slice ${result.sliceIndex}, attempt ${result.attempt ?? 1}] rerun: ${rerun}` } : {}) });
...(failed ? { line: `✗ ${id} ${kind} FAIL ${failureClass}${failedRecord?.exit_reason === 'timeout' && failedRecord?.timeout_at_turn !== undefined ? ` at turn ${failedRecord.timeout_at_turn}` : ''}${error ? ` — ${error}` : ''}`
+ `${diagnosisText(diagnosis, error)} [slice ${result.sliceIndex}, attempt ${result.attempt ?? 1}]`
+ `${evidencePointer(sliceArtifactName(root, result.sliceIndex, result.attempt ?? 1), key, typeof base.run_id === 'string' ? base.run_id : undefined, id)}`
+ ` after a repair: ${afterRepairCommand(manifest.tier, id, shardFile(key))}` } : {}) });
census.history.push({ ...base, case: id, file: shardFile(key), kind, trial: 1, panel: { n: 1, k: 1 }, outcome: caseOutcome,
...(failureClass ? { failure_class: failureClass } : {}), ...(failedRecord?.exit_reason ? { exit_reason: String(failedRecord.exit_reason) } : {}),
...(error && failed ? { error } : {}), duration_ms: cases.reduce((sum, tc) => sum + tc.timeMs, 0),
cost_usd: Math.round(mine.reduce((sum: number, r: any) => sum + (Number(r.cost_usd) || 0), 0) * 100) / 100,
...(error && failed ? { error } : {}), ...diagnosis, ...trialSessions(sessions), duration_ms: Math.min(cases.reduce((sum, tc) => sum + tc.timeMs, 0), outcome.elapsedMs),
cost_usd: Math.round(mine.reduce((sum: number, r: any) => sum + (Number(r.cost_usd) || 0), 0) * 100) / 100, ...trialCostKnown(mine, sessions),
...(typeof mine[0]?.model === 'string' ? { model: mine[0].model } : {}), ...(cliVersion ? { cli_version: cliVersion } : {}),
quarantined: false, execution: outcome.reused ? 'reused' : 'executed', source: 'junit' } as TrialOutcomeRecord);
}
@@ -290,7 +351,9 @@ export function panelReports(manifest: PaidRunManifest, results: SliceResult[],
return [{ trial: t.trial, outcome: t.outcome, attempt, ...(t.failure_class ? { failure_class: t.failure_class } : {}),
...(t.exit_reason ? { exit_reason: t.exit_reason } : {}), ...(t.error ? { error: t.error } : {}),
...(got.outcome.reused ? { execution: 'reused' as const } : {}),
...(t.timeout_at_turn !== undefined ? { timeout_at_turn: t.timeout_at_turn } : {}) }];
...(t.timeout_at_turn !== undefined ? { timeout_at_turn: t.timeout_at_turn } : {}),
...(t.failure_cause ? { failure_cause: t.failure_cause } : {}), ...(t.failure_cause_evidence ? { failure_cause_evidence: t.failure_cause_evidence } : {}),
...(t.failure_detail ? { failure_detail: t.failure_detail } : {}) }];
});
const verdict = panelVerdict({ case: shardCaseId(key)!, kind: plan.kind, panel: plan.panel, trials, quarantined: plan.quarantined });
// Reuse is whole-panel only: every trial reused from one run, or none.
@@ -303,29 +366,69 @@ export function panelReports(manifest: PaidRunManifest, results: SliceResult[],
});
}
type TrialDiagnosis = { failure_cause?: string; failure_cause_evidence?: string; failure_detail?: TrialFailureDetail };
/** Cause (when it says more than the class), its evidence and the failure detail, appended to a red line. */
function diagnosisText(d: TrialDiagnosis | undefined, error?: string): string {
if (!d?.failure_cause) return '';
const evidence = d.failure_cause_evidence && d.failure_cause_evidence !== error ? `: ${d.failure_cause_evidence}` : '';
const detail = !d.failure_detail ? ''
: 'judge' in d.failure_detail ? ` ${d.failure_detail.judge.map(j => `${j.dimension} ${j.mean} < ${j.threshold} (${j.samples} samples)${j.rationale ? ` "${j.rationale}"` : ''}`).join('; ')}`
: ` Expected: ${d.failure_detail.expected} · Received: ${d.failure_detail.received}`;
return ` · cause ${d.failure_cause}${evidence}${detail}`;
}
/** Where a red's transcript lives and how to fetch it. */
function evidencePointer(artifact: string, key: string, runId: string | undefined, id: string): string {
const arg = /^[\w./-]+$/.test(id) ? id : `'${id.replace(/'/g, `'\\''`)}'`;
return ` evidence: ${artifact}/shards/${shardSlug([key])}${runId ? ` (fetch: bun run eval:pass-rates --run ${runId} --case ${arg})` : ''}`;
}
/** Why one failed trial failed, in one line: a case timeout names its turn. */
function trialCause(trial: PanelVerdict['trials'][number] & { timeout_at_turn?: number }): string {
function trialCause(trial: PanelVerdict['trials'][number] & TrialDiagnosis & { timeout_at_turn?: number }): string {
if (trial.outcome === 'skipped') return `t${trial.trial}: skipped`;
const cls = trial.failure_class ?? 'assertion';
const head = cls === 'timeout' || trial.exit_reason === 'timeout'
? `timeout${trial.timeout_at_turn !== undefined ? ` at turn ${trial.timeout_at_turn}` : ''}`
: cls;
return `t${trial.trial}: ${head}${trial.exit_reason && trial.exit_reason !== 'timeout' ? ` (${trial.exit_reason})` : ''}${trial.error ? ` — ${trial.error}` : ''}`;
const exit = trial.exit_reason === 'success' ? ' (session completed; check failed)'
: trial.exit_reason && trial.exit_reason !== 'timeout' ? ` (${trial.exit_reason})` : '';
return `t${trial.trial}: ${head}${exit}${trial.error ? ` — ${trial.error}` : ''}${diagnosisText(trial, trial.error)}`;
}
export function rerunCommand(tier: PaidTier, id: string, trials: number): string {
return `bun run scripts/test-paid-shards.ts --tier ${tier} --case ${id}${trials > 1 ? ` --trials ${trials}` : ''}`;
/**
* A paid run of exactly one case, for after a repair (never on unchanged
* inputs): E2E cases through --case, judges through their selection and an
* exact test-name filter, a keyless file (its own case) by file.
*/
export function afterRepairCommand(tier: PaidTier, id: string, file: string, trials = 1): string {
if (Object.hasOwn(E2E_TIERS, id) && resolvesAlone(id)) return `bun run scripts/test-paid-shards.ts --tier ${tier} --case ${id}${trials > 1 ? ` --trials ${trials}` : ''}`;
if (Object.hasOwn(LLM_JUDGE_TOUCHFILES, id)) {
return `EVALS=1 EVALS_TIER=${tier} EVALS_JUDGE_SELECTION_JSON='${JSON.stringify({ version: 1, selected: [id], reason: 'after a repair' })}' bun test ${file} -t '${caseTestNamePattern([id])}'`;
}
return `EVALS=1 EVALS_TIER=${tier} bun test ${file}`;
}
/** One line per non-PASS or split panel verdict. */
export function formatPanelLine(panel: PanelReport, tier: PaidTier): string {
const resolved = new Map<string, boolean>();
/** Whether --case can select this E2E id (a case several files register runs by its shard file instead). */
function resolvesAlone(id: string): boolean {
if (!resolved.has(id)) { try { caseSelection(id); resolved.set(id, true); } catch { resolved.set(id, false); } }
return resolved.get(id)!;
}
/** One line per non-PASS or split panel verdict. `artifacts` names each slice's artifact (slice -> name). */
export function formatPanelLine(panel: PanelReport, tier: PaidTier, ctx: { artifacts?: Map<number, string>; runId?: string } = {}): string {
const mark = panel.status === 'PASS' ? '⚠' : panel.failsLane ? '✗' : '◌';
const label = panel.status === 'PASS' ? `PASS ${panel.passed}/${panel.panel.n}` : `${panel.status} ${panel.passed}/${panel.panel.n}`;
const causes = panel.trials.filter(t => t.outcome !== 'passed').map(t => trialCause(t as any));
const where = Object.entries(panel.slices).map(([trial, slice]) => `t${trial}@slice ${slice}`).join(', ');
const red = panel.trials.find(t => t.outcome === 'failed');
const slice = red ? panel.slices[red.trial] : undefined;
const evidence = red && slice !== undefined
? evidencePointer(ctx.artifacts?.get(slice) ?? `slice ${slice} attempt ${panel.attempt}`, trialShardKey(panel.file, panel.case, red.trial), ctx.runId, panel.case) : '';
return `${mark} ${panel.case} ${panel.kind}${panel.quarantined ? ' (quarantined)' : ''} ${label} (${panel.marks})`
+ `${causes.length ? ` ${causes.join('; ')}` : ''}${panel.status === 'INCOMPLETE' || panel.status === 'SKIPPED' ? ` [${panel.reason}]` : ''}`
+ `${where ? ` [${where}, attempt ${panel.attempt}]` : ''} rerun: ${rerunCommand(tier, panel.case, panel.panel.n)}`;
+ `${where ? ` [${where}, attempt ${panel.attempt}]` : ''}${evidence} after a repair: ${afterRepairCommand(tier, panel.case, panel.file, panel.panel.n)}`;
}
export interface ReportHeadline {
@@ -348,7 +451,13 @@ export interface ReportHeadline {
redispatchEligible: boolean;
}
export function formatHeadline(h: ReportHeadline): string[] {
/** Known cost, never a bare total when any trial's cost is unknown, and no total when none is known. */
export function formatCost(knownUsd: number, unknownTrials: number, trials: number): string {
if (trials > 0 && unknownTrials === trials) return `cost unknown (no billing captured for any of ${trials} trial(s))`;
return `cost $${knownUsd.toFixed(2)}${unknownTrials ? ` known + ${unknownTrials} of ${trials} trial(s) cost unknown` : ''}`;
}
export function formatHeadline(h: ReportHeadline, cost: { unknownTrials: number; trials: number } = { unknownTrials: 0, trials: 0 }): string[] {
const c = h.counts;
const minutes = h.wallMs === null ? 'unknown' : `${Math.floor(h.wallMs / 60_000)}m${String(Math.round((h.wallMs % 60_000) / 1000)).padStart(2, '0')}s`;
return [
@@ -356,7 +465,7 @@ export function formatHeadline(h: ReportHeadline): string[] {
` rule ${c.rule.passed}/${c.rule.total} · behavior ${c.behavior.passed}/${c.behavior.total}${c.behavior.split ? ` (${c.behavior.split} split)` : ''}`
+ ` · judge ${c.judge.passed}/${c.judge.total} · quarantined ${c.quarantined.total} (${c.quarantined.failingLane} failing the lane)`,
` SKIPPED ${c.skipped} · INFRA ${c.infra} · INCOMPLETE ${c.incomplete} · unattributed ${c.unattributed} · ACTION REQUIRED ${h.actionRequired}`,
` wall ${minutes} · cost $${h.costUsd.toFixed(2)}${h.redispatchEligible ? ' · every red is machine-classified INFRA/INCOMPLETE: eligible for ONE re-dispatch as a new run (EVAL_POLICY.infraRedispatch); report both runs' : ''}`,
` wall ${minutes} · ${formatCost(h.costUsd, cost.unknownTrials, cost.trials)}${h.redispatchEligible ? ' · every red is machine-classified INFRA/INCOMPLETE: eligible for ONE re-dispatch as a new run (EVAL_POLICY.infraRedispatch); report both runs' : ''}`,
];
}
@@ -538,11 +647,16 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo
history.push({ ...common(attempt), case: t.case, file: shardFile(outcome.files[0]!), kind: t.kind, trial: t.trial, panel: t.panel,
outcome: t.outcome, ...(t.outcome === 'failed' ? { failure_class: t.failure_class ?? 'assertion' } : {}),
...(t.exit_reason ? { exit_reason: t.exit_reason } : {}), ...(t.error ? { error: t.error } : {}),
duration_ms: t.duration_ms, cost_usd: t.cost_usd, ...(t.model ? { model: t.model } : {}),
...(t.outcome === 'failed' && t.failure_cause ? { failure_cause: t.failure_cause } : {}),
...(t.outcome === 'failed' && t.failure_cause_evidence ? { failure_cause_evidence: t.failure_cause_evidence } : {}),
...(t.outcome === 'failed' && t.failure_detail ? { failure_detail: t.failure_detail } : {}), ...(t.sessions ? { sessions: t.sessions } : {}),
duration_ms: t.duration_ms, cost_usd: t.cost_usd, ...(t.cost_known === false ? { cost_known: false } : {}), ...(t.model ? { model: t.model } : {}),
...(outcome.reused ? { input_identity: outcome.reused.inputKey } : {}),
quarantined: t.quarantined, execution: outcome.reused ? 'reused' : 'executed', source: 'shard' } as TrialOutcomeRecord);
}
}
const sliceArtifacts = new Map(artifacts.filter(a => (a.result.attempt ?? 1) === primary)
.map(a => [a.result.sliceIndex, sliceArtifactName(a.root, a.result.sliceIndex, primary)] as const));
const census = junitCensus(manifest, artifacts.filter(a => (a.result.attempt ?? 1) === primary), recordsByShard,
common(primary), env.GSTACK_CLAUDE_CLI_VERSION);
history.push(...census.history);
@@ -568,7 +682,7 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo
if (got && got.o.status === 'passed') return [];
if (got && census.failedShards.has(normalizeRelativePath(entry.file))) return [];
const id = shardCaseId(entry.file);
return [`✗ ${entry.file} rule shard ${got ? got.o.status : 'NOT REPORTED'}${got?.o.runnerError ? ` — ${sanitizeTrialError(got.o.runnerError)}` : ''} [slice ${entry.slice}, attempt ${primary}]${id ? ` rerun: ${rerunCommand(manifest.tier, id, 1)}` : ''}`];
return [`✗ ${entry.file} rule shard ${got ? got.o.status : 'NOT REPORTED'}${got?.o.runnerError ? ` — ${sanitizeTrialError(got.o.runnerError)}` : ''} [slice ${entry.slice}, attempt ${primary}]${id ? ` after a repair: ${afterRepairCommand(manifest.tier, id, shardFile(entry.file))}` : ''}`];
});
const behaviorPanels = panels.filter(p => !p.quarantined && p.kind === 'behavior');
const lanePanels = panels.filter(p => !p.quarantined);
@@ -582,7 +696,7 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo
const failureLines = [
...ruleShardFailures,
...ruleCases.filter(c => c.line).map(c => c.line!),
...panels.filter(p => p.status !== 'PASS' || p.split).map(p => formatPanelLine(p, manifest.tier)),
...panels.filter(p => p.status !== 'PASS' || p.split).map(p => formatPanelLine(p, manifest.tier, { artifacts: sliceArtifacts, runId })),
];
const red = verdict.problems.length > 0;
const headline: ReportHeadline = {
@@ -603,19 +717,20 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo
costUsd: Math.round(costUsd * 100) / 100,
redispatchEligible: red && infraOnly(verdict.problems),
};
const headlineLines = formatHeadline(headline);
const executed = history.filter(r => r.attempt === primary && r.execution === 'executed' && r.outcome !== 'skipped');
const headlineLines = formatHeadline(headline, { unknownTrials: executed.filter(r => r.cost_known === false).length, trials: executed.length });
for (const line of headlineLines) console.log(line);
if (failureLines.length) {
console.log('[test:paid] failures and split verdicts:');
for (const line of failureLines) console.log(` ${line}`);
}
for (const panel of laterPanels) console.log(` attempt ${panel.attempt} (re-run; reported, never replacing attempt ${primary}): ${formatPanelLine(panel, manifest.tier)}`);
const fence = (lines: string[]) => ['```', ...lines.map(line => line.replace(/```/g, "'''")), '```'];
const fence = (lines: string[]) => publishedFence(lines);
fs.writeFileSync(summaryMdPath, [
...fence(headlineLines),
...(failureLines.length ? ['', '**Failures and split verdicts**', '', ...fence(failureLines)] : []),
...(censusLines.length ? ['', '**Skipped, deselected and unattributed testcases**', '', ...fence(censusLines)] : []),
...(verdict.problems.length ? ['', `**ACTION REQUIRED (${verdict.problems.length})**`, '', ...fence(verdict.problems.map(p => sanitizeTrialError(p) ?? p))] : []),
...(verdict.problems.length ? ['', `**ACTION REQUIRED (${verdict.problems.length})**`, '', ...fence(verdict.problems.map(p => sanitizeTrialError(p) ?? p)), '', `Guide: ${CENSUS_RED_GUIDE}`] : []),
].join('\n') + '\n');
if (!manualProblems.length) fs.writeFileSync(summaryPath, JSON.stringify({ version: 2, files, totals: {
...evidence, total: evidence.passed + evidence.failed + evidence.manual_accepted,
@@ -626,7 +741,8 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo
marks: p.marks, split: p.split, quarantined: p.quarantined, failsLane: p.failsLane, redClass: p.redClass, reason: p.reason,
trials: p.trials.map(t => ({ trial: t.trial, outcome: t.outcome, ...(t.failure_class ? { failure_class: t.failure_class } : {}),
...(t.exit_reason ? { exit_reason: t.exit_reason } : {}), ...(t.error ? { error: t.error } : {}) })) })),
failures: failureLines.map(line => sanitizeTrialError(line) ?? line) }, null, 2) + '\n');
// evals.yml wraps these in a fixed fence for the PR comment.
failures: failureLines.map(line => sanitizeFixedFenceLine(line)) }, null, 2) + '\n');
if (verdict.problems.length) {
console.error(`[test:paid] report: ${verdict.problems.length} problem(s):`);
for (const problem of verdict.problems) console.error(` ✗ ${problem}`);
+6 -1
View File
@@ -4,7 +4,7 @@
*/
import * as path from 'node:path';
import { normalizeRelativePath, type LanePolicy } from './shard-engine';
import type { EvalCaseKind, PanelShape, TrialFailureClass, TrialOutcome } from '../../test/helpers/eval-store';
import type { EvalCaseKind, PanelShape, TrialFailureCause, TrialFailureClass, TrialFailureDetail, TrialOutcome, TrialSessionSummary } from '../../test/helpers/eval-store';
export const ROOT = path.resolve(import.meta.dir, '..', '..');
@@ -119,6 +119,11 @@ export interface ShardTrialRecord {
cost_usd: number;
duration_ms: number;
model?: string;
failure_cause?: TrialFailureCause;
failure_cause_evidence?: string;
failure_detail?: TrialFailureDetail;
sessions?: TrialSessionSummary[];
cost_known?: false;
}
/** Upload reserve between a slice's in-process deadline and its CI job ceiling (ENG-2). */
+41
View File
@@ -0,0 +1,41 @@
/**
* The one sanitizer for model- or fixture-written text that reaches a public
* issue, PR comment or committed file (census report, tracking issue,
* eval:pass-rates output). Control and bidi characters are stripped, the
* repo's credential-scan patterns (lib/redact-engine, the engine the quality
* gate's credential scan runs) are redacted, @-mentions are neutralized and
* length is capped. Inline text also has HTML and Markdown link brackets
* escaped; blocks go inside a code fence longer than any backtick run they
* contain, so their content stays literal and cannot close the fence.
*/
import { redactFindingSpans } from '../../lib/redact-engine';
export const PUBLISHED_TEXT_MAX = 1000;
const cap = (text: string, max: number) => (text.length > max ? `${text.slice(0, max - 1)}…` : text);
function clean(text: string): string {
// eslint-disable-next-line no-control-regex
const stripped = text.replace(/[\u0000-\u0008\u000a-\u001f\u007f-\u009f\u200b-\u200f\u202a-\u202e\u2066-\u2069]/g, ' ').slice(0, 64 * 1024);
const redacted = redactFindingSpans(stripped) ?? '[redacted: credential-shaped text that could not be isolated]';
return redacted.replace(/@(?=[A-Za-z0-9_-])/g, '@\u200b');
}
/** One inline line: cleaned, HTML, Markdown link brackets and backticks neutralized, capped. */
export function sanitizePublishedText(text: string, max = PUBLISHED_TEXT_MAX): string {
return cap(clean(text).replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;')
.replace(/\[/g, '&#91;').replace(/\]/g, '&#93;').replace(/`/g, "'"), max);
}
/** One line for a consumer that wraps it in a fixed three-backtick fence (the PR comment): backticks become quotes. */
export function sanitizeFixedFenceLine(text: string, max = PUBLISHED_TEXT_MAX): string {
return cap(clean(text).replace(/`/g, "'"), max);
}
/** Cleaned, capped lines inside a fence longer than the longest backtick run they contain. */
export function publishedFence(lines: readonly string[], max = PUBLISHED_TEXT_MAX): string[] {
const body = lines.map(line => cap(clean(line), max));
const longest = Math.max(0, ...body.flatMap(line => (line.match(/`+/g) ?? []).map(run => run.length)));
const fence = '`'.repeat(Math.max(3, longest + 1));
return [fence, ...body, fence];
}
+140 -138
View File
@@ -1,109 +1,109 @@
{
"version": 2,
"recordedAt": "2026-09-29T19:29:33.835Z",
"recordedAt": "2026-10-04T18:45:27.037Z",
"tiers": {
"gate": {
"test/llm-judge-recommendation.test.ts": 1000,
"test/skill-e2e-ask-user-question-format-compliance.test.ts": 63904,
"test/skill-e2e-ask-user-question-format-compliance.test.ts": 54768,
"test/skill-e2e-autoplan-dual-voice.test.ts": 1000,
"test/skill-e2e-bws.test.ts": 101092,
"test/skill-e2e-bws.test.ts": 74261,
"test/skill-e2e-context-skills.test.ts": 1000,
"test/skill-e2e-coverage-audit.test.ts": 50849,
"test/skill-e2e-cso.test.ts": 227394,
"test/skill-e2e-deploy.test.ts#benchmark-workflow": 62156,
"test/skill-e2e-deploy.test.ts#canary-workflow": 50673,
"test/skill-e2e-deploy.test.ts#land-and-deploy-first-run": 125350,
"test/skill-e2e-deploy.test.ts#land-and-deploy-review-gate": 177755,
"test/skill-e2e-deploy.test.ts#land-and-deploy-workflow": 200984,
"test/skill-e2e-deploy.test.ts#setup-deploy-workflow": 51171,
"test/skill-e2e-coverage-audit.test.ts": 65669,
"test/skill-e2e-cso.test.ts": 229620,
"test/skill-e2e-deploy.test.ts#benchmark-workflow": 120646,
"test/skill-e2e-deploy.test.ts#canary-workflow": 42804,
"test/skill-e2e-deploy.test.ts#land-and-deploy-first-run": 103709,
"test/skill-e2e-deploy.test.ts#land-and-deploy-review-gate": 147499,
"test/skill-e2e-deploy.test.ts#land-and-deploy-workflow": 148339,
"test/skill-e2e-deploy.test.ts#setup-deploy-workflow": 36798,
"test/skill-e2e-design.test.ts": 261000,
"test/skill-e2e-design.test.ts#design-review-detector-shim": 42811,
"test/skill-e2e-design.test.ts#design-review-detector-shim-dom": 81914,
"test/skill-e2e-design.test.ts#design-review-plugin-handoff": 126642,
"test/skill-e2e-design.test.ts#plan-design-review-no-ui-scope": 24431,
"test/skill-e2e-diagram.test.ts": 29998,
"test/skill-e2e-docsync-spawned.test.ts": 137929,
"test/skill-e2e-design.test.ts#design-review-detector-shim": 45224,
"test/skill-e2e-design.test.ts#design-review-detector-shim-dom": 87823,
"test/skill-e2e-design.test.ts#design-review-plugin-handoff": 74874,
"test/skill-e2e-design.test.ts#plan-design-review-no-ui-scope": 39970,
"test/skill-e2e-diagram.test.ts": 30124,
"test/skill-e2e-docsync-spawned.test.ts": 108078,
"test/skill-e2e-gbrain-roundtrip-local.test.ts": 1000,
"test/skill-e2e-hermetic-canary.test.ts": 8813,
"test/skill-e2e-investigate-owned-completion.test.ts": 45178,
"test/skill-e2e-investigate-owned-termination.test.ts": 48608,
"test/skill-e2e-hermetic-canary.test.ts": 4951,
"test/skill-e2e-investigate-owned-completion.test.ts": 36179,
"test/skill-e2e-investigate-owned-termination.test.ts": 48428,
"test/skill-e2e-ios-device.test.ts": 1000,
"test/skill-e2e-learnings.test.ts": 28899,
"test/skill-e2e-office-hours-auto-mode.test.ts": 83101,
"test/skill-e2e-learnings.test.ts": 33590,
"test/skill-e2e-office-hours-auto-mode.test.ts": 78399,
"test/skill-e2e-office-hours-brain-writeback.test.ts": 1000,
"test/skill-e2e-office-hours-phase4.test.ts": 1000,
"test/skill-e2e-office-hours.test.ts": 1000,
"test/skill-e2e-plan-ceo-finding-floor.test.ts": 297738,
"test/skill-e2e-plan-ceo-plan-mode.test.ts": 36812,
"test/skill-e2e-plan-ceo-finding-floor.test.ts": 248087,
"test/skill-e2e-plan-ceo-plan-mode.test.ts": 112067,
"test/skill-e2e-plan-decision-classification.test.ts": 1000,
"test/skill-e2e-plan-design-with-ui.test.ts": 240000,
"test/skill-e2e-plan-devex-finding-floor.test.ts": 207635,
"test/skill-e2e-plan-design-with-ui.test.ts": 285788,
"test/skill-e2e-plan-devex-finding-floor.test.ts": 181446,
"test/skill-e2e-plan-devex-peer-comparison-classification.test.ts": 1000,
"test/skill-e2e-plan-devex-plan-mode.test.ts": 115679,
"test/skill-e2e-plan-devex-plan-mode.test.ts": 105953,
"test/skill-e2e-plan-format.test.ts": 1000,
"test/skill-e2e-plan-mode-no-op.test.ts": 217184,
"test/skill-e2e-plan-mode-no-op.test.ts": 271596,
"test/skill-e2e-plan-prosons.test.ts": 1000,
"test/skill-e2e-plan-tune.test.ts": 63925,
"test/skill-e2e-plan-tune.test.ts": 70702,
"test/skill-e2e-plan.test.ts": 251000,
"test/skill-e2e-plan.test.ts#plan-review-report": 76406,
"test/skill-e2e-plan.test.ts#plan-review-report": 197666,
"test/skill-e2e-qa-bugs.test.ts": 1000,
"test/skill-e2e-qa-callers.test.ts": 737164,
"test/skill-e2e-qa-callers.test.ts#review-exploratory-small-cli": 294865,
"test/skill-e2e-qa-callers.test.ts#ship-exploratory-late-input": 294865,
"test/skill-e2e-qa-callers.test.ts#ship-exploratory-plan-checks": 294865,
"test/skill-e2e-qa-callers.test.ts#ship-exploratory-small-cli": 294865,
"test/skill-e2e-qa-callers.test.ts#ship-exploratory-unavailable": 294865,
"test/skill-e2e-qa-functional-fix.test.ts": 235133,
"test/skill-e2e-qa-functional.test.ts": 356641,
"test/skill-e2e-qa-callers.test.ts#review-exploratory-small-cli": 224426,
"test/skill-e2e-qa-callers.test.ts#ship-exploratory-late-input": 234381,
"test/skill-e2e-qa-callers.test.ts#ship-exploratory-plan-checks": 192841,
"test/skill-e2e-qa-callers.test.ts#ship-exploratory-small-cli": 147399,
"test/skill-e2e-qa-callers.test.ts#ship-exploratory-unavailable": 191231,
"test/skill-e2e-qa-functional-fix.test.ts": 200616,
"test/skill-e2e-qa-functional.test.ts": 188766,
"test/skill-e2e-qa-workflow.test.ts#qa-bootstrap": 35863,
"test/skill-e2e-qa-workflow.test.ts#qa-only-no-fix": 201333,
"test/skill-e2e-qa-workflow.test.ts#qa-quick": 169852,
"test/skill-e2e-retro.test.ts": 141804,
"test/skill-e2e-retro.test.ts": 200434,
"test/skill-e2e-review-army.test.ts": 520000,
"test/skill-e2e-review-army.test.ts#review-army-delivery-audit": 50219,
"test/skill-e2e-review-army.test.ts#review-army-json-findings": 21332,
"test/skill-e2e-review-army.test.ts#review-army-migration-safety": 145112,
"test/skill-e2e-review-army.test.ts#review-army-perf-n-plus-one": 241299,
"test/skill-e2e-review-army.test.ts#review-army-quality-score": 86836,
"test/skill-e2e-review-attribution.test.ts": 77592,
"test/skill-e2e-review.test.ts": 136132,
"test/skill-e2e-review-army.test.ts#review-army-delivery-audit": 39313,
"test/skill-e2e-review-army.test.ts#review-army-json-findings": 22950,
"test/skill-e2e-review-army.test.ts#review-army-migration-safety": 162617,
"test/skill-e2e-review-army.test.ts#review-army-perf-n-plus-one": 114005,
"test/skill-e2e-review-army.test.ts#review-army-quality-score": 55496,
"test/skill-e2e-review-attribution.test.ts": 80562,
"test/skill-e2e-review.test.ts": 132863,
"test/skill-e2e-safety-codex-boundary.test.ts": 1000,
"test/skill-e2e-safety-codex-consult.test.ts": 82940,
"test/skill-e2e-safety-codex-consult.test.ts": 150155,
"test/skill-e2e-safety-design-risk.test.ts": 1000,
"test/skill-e2e-safety-ios-demo.test.ts": 1000,
"test/skill-e2e-safety-pair-agent.test.ts": 39860,
"test/skill-e2e-safety-ship-evidence.test.ts": 106130,
"test/skill-e2e-session-intelligence.test.ts": 55332,
"test/skill-e2e-safety-pair-agent.test.ts": 36028,
"test/skill-e2e-safety-ship-evidence.test.ts": 100833,
"test/skill-e2e-session-intelligence.test.ts": 53080,
"test/skill-e2e-shared-libs-paths.test.ts": 680000,
"test/skill-e2e-shared-libs-paths.test.ts#shared-libs-review-index-flags": 190202,
"test/skill-e2e-shared-libs-paths.test.ts#shared-libs-review-path-eligibility": 205615,
"test/skill-e2e-shared-libs-paths.test.ts#shared-libs-review-prior-coverage": 214728,
"test/skill-e2e-shared-libs-paths.test.ts#shared-libs-review-index-flags": 145972,
"test/skill-e2e-shared-libs-paths.test.ts#shared-libs-review-path-eligibility": 170775,
"test/skill-e2e-shared-libs-paths.test.ts#shared-libs-review-prior-coverage": 172422,
"test/skill-e2e-shared-libs.test.ts": 658000,
"test/skill-e2e-shared-libs.test.ts#shared-libs-read-only": 197454,
"test/skill-e2e-shared-libs.test.ts#shared-libs-review-lifecycle": 267080,
"test/skill-e2e-shared-libs.test.ts#shared-libs-review-revalidation": 316752,
"test/skill-e2e-shared-libs.test.ts#shared-libs-unsupported-git": 170776,
"test/skill-e2e-shared-libs.test.ts#shared-libs-read-only": 160556,
"test/skill-e2e-shared-libs.test.ts#shared-libs-review-lifecycle": 164017,
"test/skill-e2e-shared-libs.test.ts#shared-libs-review-revalidation": 281097,
"test/skill-e2e-shared-libs.test.ts#shared-libs-unsupported-git": 87581,
"test/skill-e2e-ship-docsync.test.ts": 129000,
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-completion": 366116,
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-current": 434135,
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-failure": 164667,
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-late-result": 214218,
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-launch-failure": 140987,
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-missing-asset": 142564,
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-missing-marker": 117648,
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-recovery": 225432,
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-stale-after": 269572,
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-stale-before": 197633,
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-store": 396588,
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-timeout-unsettled": 152278,
"test/skill-e2e-ship-hook-consent.test.ts": 69995,
"test/skill-e2e-ship-hook-refresh.test.ts": 70007,
"test/skill-e2e-ship-skip.test.ts": 80318,
"test/skill-e2e-skillify.test.ts": 191127,
"test/skill-e2e-test-value.test.ts": 332000,
"test/skill-e2e-third-party-actions.test.ts": 82102,
"test/skill-e2e-triage.test.ts": 54088,
"test/skill-e2e-workflow.test.ts": 184925,
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-completion": 398349,
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-current": 382630,
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-failure": 238690,
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-late-result": 132034,
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-launch-failure": 53611,
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-missing-asset": 106550,
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-missing-marker": 59963,
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-recovery": 132137,
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-stale-after": 155419,
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-stale-before": 164443,
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-store": 392835,
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-timeout-unsettled": 63645,
"test/skill-e2e-ship-hook-consent.test.ts": 49572,
"test/skill-e2e-ship-hook-refresh.test.ts": 40853,
"test/skill-e2e-ship-skip.test.ts": 81807,
"test/skill-e2e-skillify.test.ts": 177262,
"test/skill-e2e-test-value.test.ts": 216843,
"test/skill-e2e-third-party-actions.test.ts": 71961,
"test/skill-e2e-triage.test.ts": 64816,
"test/skill-e2e-workflow.test.ts": 265175,
"test/skill-llm-eval.test.ts": 354000,
"test/skill-routing-e2e.test.ts": 1000
},
@@ -123,98 +123,100 @@
"test/carve-section-loading.test.ts#carve-section-loading-review": 247902,
"test/carve-section-loading.test.ts#carve-section-loading-setup-gbrain": 80994,
"test/carve-section-loading.test.ts#carve-section-loading-spec": 176639,
"test/codex-e2e-recommendation-substance.test.ts": 17984,
"test/codex-e2e-shared-libs.test.ts": 53442,
"test/codex-e2e-sol-scope.test.ts": 37063,
"test/codex-e2e.test.ts": 120390,
"test/llm-judge-recommendation.test.ts": 15334,
"test/skill-e2e-arm-benchmark.test.ts": 75588,
"test/codex-e2e-multiblock-live.test.ts": 180000,
"test/codex-e2e-recommendation-substance.test.ts": 14617,
"test/codex-e2e-shared-libs.test.ts": 49563,
"test/codex-e2e-sol-scope.test.ts": 32228,
"test/codex-e2e.test.ts": 110677,
"test/llm-judge-recommendation.test.ts": 21562,
"test/skill-e2e-arm-benchmark.test.ts": 72714,
"test/skill-e2e-aside.test.ts": 1000,
"test/skill-e2e-auq-consistency.test.ts": 70429,
"test/skill-e2e-auq-matrix.test.ts": 165796,
"test/skill-e2e-auq-verbose-vs-carved-ab.test.ts": 58547,
"test/skill-e2e-auto-decide-preserved.test.ts": 154931,
"test/skill-e2e-autoplan-dual-voice.test.ts": 167918,
"test/skill-e2e-benchmark-providers.test.ts": 9495,
"test/skill-e2e-auq-consistency.test.ts": 63226,
"test/skill-e2e-auq-matrix.test.ts": 145925,
"test/skill-e2e-auq-verbose-vs-carved-ab.test.ts": 65480,
"test/skill-e2e-auto-decide-preserved.test.ts": 234151,
"test/skill-e2e-autoplan-dual-voice.test.ts": 154207,
"test/skill-e2e-benchmark-providers.test.ts": 8847,
"test/skill-e2e-bws.test.ts": 1000,
"test/skill-e2e-context-skills.test.ts": 169497,
"test/skill-e2e-context-skills.test.ts": 144874,
"test/skill-e2e-coverage-audit.test.ts": 1000,
"test/skill-e2e-cso.test.ts": 253498,
"test/skill-e2e-cso.test.ts": 303949,
"test/skill-e2e-design-model-smoke.test.ts": 90000,
"test/skill-e2e-design.test.ts": 817000,
"test/skill-e2e-design.test.ts#design-consultation-core": 159257,
"test/skill-e2e-design.test.ts#design-consultation-existing": 175607,
"test/skill-e2e-design.test.ts#design-consultation-preview": 114638,
"test/skill-e2e-design.test.ts#design-consultation-research": 109573,
"test/skill-e2e-design.test.ts#design-html-slop-gate": 158970,
"test/skill-e2e-design.test.ts#plan-design-review-plan-mode": 300166,
"test/skill-e2e-diagram.test.ts": 55187,
"test/skill-e2e-design.test.ts#design-consultation-core": 173162,
"test/skill-e2e-design.test.ts#design-consultation-existing": 225306,
"test/skill-e2e-design.test.ts#design-consultation-preview": 107037,
"test/skill-e2e-design.test.ts#design-consultation-research": 99890,
"test/skill-e2e-design.test.ts#design-html-slop-gate": 139668,
"test/skill-e2e-design.test.ts#plan-design-review-plan-mode": 169183,
"test/skill-e2e-diagram.test.ts": 52819,
"test/skill-e2e-gbrain-roundtrip-local.test.ts": 1000,
"test/skill-e2e-health.test.ts": 188584,
"test/skill-e2e-health.test.ts": 164674,
"test/skill-e2e-hermetic-canary.test.ts": 1000,
"test/skill-e2e-ios-device.test.ts": 1000,
"test/skill-e2e-learnings.test.ts": 1000,
"test/skill-e2e-office-hours-brain-writeback.test.ts": 203044,
"test/skill-e2e-office-hours-design-draft.test.ts": 285868,
"test/skill-e2e-office-hours-design-draft.test.ts": 210162,
"test/skill-e2e-office-hours-phase4.test.ts": 41421,
"test/skill-e2e-office-hours-section-loading.test.ts": 1200000,
"test/skill-e2e-office-hours.test.ts": 136970,
"test/skill-e2e-outside-plan-disabled.test.ts": 54255,
"test/skill-e2e-outside-voice.test.ts": 186881,
"test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-opus-5-5.test.ts": 56080,
"test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-sonnet.test.ts": 85282,
"test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash.test.ts": 80465,
"test/skill-e2e-overlay-harness-opus-4-7-effort-match-trivial.test.ts": 49282,
"test/skill-e2e-overlay-harness-opus-4-7-literal-interpretation.test.ts": 318532,
"test/skill-e2e-plan-ceo-mode-routing.test.ts": 443504,
"test/skill-e2e-plan-ceo-review-section-loading.test.ts": 342250,
"test/skill-e2e-outside-plan-disabled.test.ts": 72070,
"test/skill-e2e-outside-voice.test.ts": 195221,
"test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-opus-5-5.test.ts": 57374,
"test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-sonnet.test.ts": 77984,
"test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash.test.ts": 55431,
"test/skill-e2e-overlay-harness-opus-4-7-effort-match-trivial.test.ts": 34905,
"test/skill-e2e-overlay-harness-opus-4-7-literal-interpretation.test.ts": 226920,
"test/skill-e2e-plan-ceo-mode-routing.test.ts": 378316,
"test/skill-e2e-plan-ceo-review-section-loading.test.ts": 302668,
"test/skill-e2e-plan-ceo-split-overflow.test.ts": 504266,
"test/skill-e2e-plan-decision-classification.test.ts": 93469,
"test/skill-e2e-plan-design-finding-floor.test.ts": 155538,
"test/skill-e2e-plan-design-plan-mode.test.ts": 106608,
"test/skill-e2e-plan-devex-peer-comparison-classification.test.ts": 79786,
"test/skill-e2e-plan-decision-classification.test.ts": 88190,
"test/skill-e2e-plan-design-finding-floor.test.ts": 131042,
"test/skill-e2e-plan-design-plan-mode.test.ts": 111838,
"test/skill-e2e-plan-devex-peer-comparison-classification.test.ts": 77201,
"test/skill-e2e-plan-eng-artifact-full.test.ts": 595009,
"test/skill-e2e-plan-eng-artifact.test.ts": 498474,
"test/skill-e2e-plan-eng-finding-floor.test.ts": 388666,
"test/skill-e2e-plan-eng-multi-finding-batching.test.ts": 597500,
"test/skill-e2e-plan-eng-plan-mode.test.ts": 229226,
"test/skill-e2e-plan-eng-artifact.test.ts": 406795,
"test/skill-e2e-plan-eng-finding-floor.test.ts": 234397,
"test/skill-e2e-plan-eng-multi-finding-batching.test.ts": 409161,
"test/skill-e2e-plan-eng-plan-mode.test.ts": 170914,
"test/skill-e2e-plan-format.test.ts": 219670,
"test/skill-e2e-plan-prosons.test.ts": 163659,
"test/skill-e2e-plan-tune.test.ts": 1000,
"test/skill-e2e-plan.test.ts": 824000,
"test/skill-e2e-plan.test.ts#plan-ceo-review": 136101,
"test/skill-e2e-plan.test.ts#plan-ceo-review-expansion-energy": 91984,
"test/skill-e2e-plan.test.ts#plan-ceo-review-selective": 267212,
"test/skill-e2e-plan.test.ts#plan-eng-review": 128506,
"test/skill-e2e-qa-bugs.test.ts": 333847,
"test/skill-e2e-plan.test.ts#plan-ceo-review": 274619,
"test/skill-e2e-plan.test.ts#plan-ceo-review-expansion-energy": 100000,
"test/skill-e2e-plan.test.ts#plan-ceo-review-selective": 265163,
"test/skill-e2e-plan.test.ts#plan-eng-review": 169400,
"test/skill-e2e-qa-bugs.test.ts": 355966,
"test/skill-e2e-qa-workflow.test.ts#qa-fix-loop": 411728,
"test/skill-e2e-retro.test.ts": 166711,
"test/skill-e2e-retro.test.ts": 150139,
"test/skill-e2e-review-army.test.ts": 511000,
"test/skill-e2e-review-army.test.ts#review-army-consensus": 277908,
"test/skill-e2e-review-army.test.ts#review-army-red-team": 88882,
"test/skill-e2e-review-army.test.ts#review-army-simplification": 136644,
"test/skill-e2e-review-army.test.ts#review-army-simplification-precision": 23884,
"test/skill-e2e-review-army.test.ts#review-army-consensus": 107375,
"test/skill-e2e-review-army.test.ts#review-army-red-team": 110743,
"test/skill-e2e-review-army.test.ts#review-army-simplification": 140434,
"test/skill-e2e-review-army.test.ts#review-army-simplification-precision": 25020,
"test/skill-e2e-review-attribution.test.ts": 1000,
"test/skill-e2e-review.test.ts": 179362,
"test/skill-e2e-safety-codex-boundary.test.ts": 100310,
"test/skill-e2e-safety-codex-boundary.test.ts": 102851,
"test/skill-e2e-safety-codex-consult.test.ts": 1000,
"test/skill-e2e-safety-design-risk.test.ts": 52700,
"test/skill-e2e-safety-ios-demo.test.ts": 53270,
"test/skill-e2e-safety-design-risk.test.ts": 54889,
"test/skill-e2e-safety-ios-demo.test.ts": 100092,
"test/skill-e2e-safety-pair-agent.test.ts": 1000,
"test/skill-e2e-safety-ship-evidence.test.ts": 1000,
"test/skill-e2e-session-intelligence.test.ts": 1000,
"test/skill-e2e-setup-gbrain-bad-token.test.ts": 42912,
"test/skill-e2e-setup-gbrain-path4-local-pglite.test.ts": 165743,
"test/skill-e2e-setup-gbrain-remote.test.ts": 121127,
"test/skill-e2e-shared-libs-periodic.test.ts": 369660,
"test/skill-e2e-ship-section-loading.test.ts": 255273,
"test/skill-e2e-setup-gbrain-bad-token.test.ts": 39292,
"test/skill-e2e-setup-gbrain-path4-local-pglite.test.ts": 128414,
"test/skill-e2e-setup-gbrain-remote.test.ts": 89701,
"test/skill-e2e-shared-libs-periodic.test.ts": 217221,
"test/skill-e2e-ship-section-loading.test.ts": 189463,
"test/skill-e2e-skillify.test.ts": 58522,
"test/skill-e2e-sync-gbrain-readiness.test.ts": 60234,
"test/skill-e2e-sync-gbrain-readiness.test.ts": 47502,
"test/skill-e2e-test-value.test.ts": 332000,
"test/skill-e2e-third-party-actions.test.ts": 1000,
"test/skill-e2e-triage.test.ts": 1000,
"test/skill-e2e-workflow.test.ts": 110357,
"test/skill-llm-eval.test.ts": 383454,
"test/skill-routing-e2e.test.ts": 68843
"test/skill-e2e-workflow.test.ts": 121594,
"test/skill-llm-eval.test.ts": 339704,
"test/skill-routing-e2e.test.ts": 81360
}
}
}
+1 -1
View File
@@ -133,7 +133,7 @@ If \`NEEDS_SETUP\`:
3. If \`bun\` is not installed:
\`\`\`bash
if ! command -v bun >/dev/null 2>&1; then
BUN_VERSION="1.3.10"
BUN_VERSION="1.4.2"
BUN_INSTALL_SHA="bab8acfb046aac8c72407bdcce903957665d655d7acaa3e11c7c4616beae68dd"
tmpfile=$(mktemp "\${TMPDIR:-/tmp}/bun-install.XXXXXX")
curl -fsSL "https://bun.sh/install" -o "$tmpfile"
+4
View File
@@ -140,6 +140,10 @@ export const RESOLVERS: Record<string, ResolverFn> = {
DX_FRAMEWORK: generateDxFramework,
TASTE_PROFILE: generateTasteProfile,
BIN_DIR: (ctx) => ctx.paths.binDir,
// Quotable runtime root for paths under gstack's install (freeze/bin, bin, lib):
// $GSTACK_ROOT on env-var hosts (each fence gets the runtime prelude), the
// literal $HOME path on Claude, so a quoted path keeps working with spaces.
RUNTIME_ROOT: (ctx) => getHostConfig(ctx.host).usesEnvVars ? '$GSTACK_ROOT' : `$HOME/${getHostConfig(ctx.host).globalRoot}`,
// Literal bin dir for skills that skip the shared preamble, where
// $GSTACK_BIN is never set (#2906): the per-install root when rendering for
// one, else the host's default global root.
+18 -21
View File
@@ -1,29 +1,26 @@
import { type TemplateContext, toShellPath } from './types';
import { type TemplateContext } from './types';
import { binaryAssignment } from './runtime-root';
/**
* {{MAKE_PDF_SETUP}} — emits the shell preamble that resolves $P to the
* make-pdf binary. Mirrors generateDesignSetup.
* {{MAKE_PDF_SETUP}} — emits the readiness check that resolves $P to the
* make-pdf binary (make-pdf/dist/pdf). Mirrors generateDesignSetup: the
* assignment comes from binaryAssignment(ctx, 'make-pdf'), the same resolver
* the runtime prelude puts into every later block that uses $P on env-var
* hosts, so readiness and use always name the same binary.
*
* $P = make-pdf/dist/pdf.
*
* Resolution order:
* 1. Local skill root: $_ROOT/{localSkillRoot}/make-pdf/dist/pdf
* 2. Global: ~/{globalRoot}/make-pdf/dist/pdf
* 3. Env override (MAKE_PDF_BIN) — for contributor dev builds
* Resolution order: MAKE_PDF_BIN (contributor dev builds), then the
* repo-local install, then the global install (env-var hosts: $GSTACK_ROOT,
* which the prelude resolves repo-local first).
*/
export function generateMakePdfSetup(ctx: TemplateContext): string {
return `## MAKE-PDF SETUP (run this check BEFORE any make-pdf command)
\`\`\`bash
_ROOT=$(git rev-parse --show-toplevel 2>/dev/null)
P=""
[ -n "$MAKE_PDF_BIN" ] && [ -x "$MAKE_PDF_BIN" ] && P="$MAKE_PDF_BIN"
[ -z "$P" ] && [ -n "$_ROOT" ] && [ -x "$_ROOT/${ctx.paths.localSkillRoot}/make-pdf/dist/pdf" ] && P="$_ROOT/${ctx.paths.localSkillRoot}/make-pdf/dist/pdf"
[ -z "$P" ] && P="${toShellPath(ctx.paths.skillRoot)}/make-pdf/dist/pdf"
${binaryAssignment(ctx, 'make-pdf')}
if [ -x "$P" ]; then
echo "MAKE_PDF_READY: $P"
alias _p_="$P" # shellcheck alias helper (not exported)
export P # available as $P in subsequent blocks within the same skill invocation
export P # env-var hosts re-derive $P in every later block (runtime prelude)
else
echo "MAKE_PDF_NOT_AVAILABLE (run './setup' in the gstack repo to build it)"
fi
@@ -36,12 +33,12 @@ If \`MAKE_PDF_READY\` is printed: \`$P\` is the binary path for the rest of
the skill. Use \`$P\` (not an explicit path) so the skill body stays portable.
Core commands:
- \`$P generate <input.md> [output.pdf]\` — render markdown to PDF (80% use case)
- \`$P generate --cover --toc essay.md out.pdf\` — full publication layout
- \`$P generate --watermark DRAFT memo.md draft.pdf\` — diagonal DRAFT watermark
- \`$P preview <input.md>\` — render HTML and open in browser (fast iteration)
- \`$P setup\` — verify the browser (Aside, or gstack's own headless fallback) + pdftotext and run a smoke test
- \`$P --help\` — full flag reference
- \`"$P" generate <input.md> [output.pdf]\` — render markdown to PDF (80% use case)
- \`"$P" generate --cover --toc essay.md out.pdf\` — full publication layout
- \`"$P" generate --watermark DRAFT memo.md draft.pdf\` — diagonal DRAFT watermark
- \`"$P" preview <input.md>\` — render HTML and open in browser (fast iteration)
- \`"$P" setup\` — verify the browser (Aside, or gstack's own headless fallback) + pdftotext and run a smoke test
- \`"$P" --help\` — full flag reference
Output contract:
- \`stdout\`: ONLY the output path on success. One line.
+2 -2
View File
@@ -176,7 +176,7 @@ ${outsideVoiceInvocation(ctx, { timeoutMs: 540000, nativeAlreadyRequired: true,
${outsideVoiceFor(ctx).id === 'codex' ? 'The Codex backend uses `codex review --base` without a positional prompt: those arguments are mutually exclusive. Never drop --base to resolve an argv error; prompt-only review changes the diff scope.' : 'The Claude Code backend receives the parent-captured base diff, including committed and working-tree changes, because review mode cannot execute git.'}
Present output under \`${outsideVoiceFor(ctx).label.toUpperCase()} SAYS (code review):\` inside a \`tool-output\` fence.
Only a completed response with severity tags or an explicit no-findings conclusion establishes the gate. P0/P1 findings (\`[P0]\`/\`[P1]\` or native \`P0:\`/\`P1:\` labels; \`VERDICT: findings\`) → GATE: FAIL. Completed without P0/P1 → GATE: PASS. Refusal, failure, missing markers or \`OUTSIDE_STATUS: unverified\` → GATE: MISSING COVERAGE; preserve the existing user decision flow.
Only a completed response with severity tags or an explicit no-findings conclusion establishes the gate. P0/P1 findings (\`[P0]\`/\`[P1]\` or native \`P0:\`/\`P1:\` labels; \`VERDICT: findings\`) → GATE: FAIL. Completed without P0/P1 → GATE: PASS. Refusal, failure, missing markers or \`OUTSIDE_STATUS: unverified\` → GATE: MISSING COVERAGE; no fix question.
If GATE is FAIL, use AskUserQuestion:
\`\`\`
@@ -409,7 +409,7 @@ THE PLAN:
function codexPlanReviewRun(ctx: TemplateContext, ceo: boolean, needsApprovalReadiness: boolean): string {
return `**If \`CODEX_MODE: ready\` (or \`unverified\`) — run ${outsideVoiceFor(ctx).label}:**
${['plan-ceo-review', 'plan-eng-review'].includes(ctx.skillName) ? `Run this block only for \`ready\`, in the one foreground Bash call described below.
${['plan-ceo-review', 'plan-eng-review'].includes(ctx.skillName) ? `Run this block for \`ready\` or \`unverified\`, in the one foreground Bash call described below.
Its opening harness guard rechecks the fresh shell: exit 78 uses the same Native
fallback below, never a replacement provider. Finish termination before fallback and consume only
completed output. Use private temporary paths, with no background jobs.` : `Run the selected backend in the one foreground Bash invocation described below.
+3 -3
View File
@@ -467,13 +467,13 @@ The plan completion results augment the existing Scope Drift Detection. If a pla
- **NOT DONE items** become additional evidence for **MISSING REQUIREMENTS** in the scope drift report.
- **Items in the diff that don't match any plan item** become evidence for **SCOPE CREEP** detection.
- **HIGH-impact plan-file discrepancies** trigger AskUserQuestion:
- Show the investigation findings
- **HIGH-impact plan-file discrepancies** trigger AskUserQuestion showing the investigation findings:
- Options: A) Stop this review for implementation, B) Continue this review with P1 TODOs, C) Record the items as intentionally dropped
- A ends this invocation before code review or implementation. List the missing work; after implementation, start a fresh /review.
- B queues the approved TODO changes for Step 5, not this read-only audit. B/C continue to the final Scope Check and Step 2. None of these choices authorizes shipping or waives required verification.
- Spawned or non-interactive: report REQUIREMENTS MISSING, continue; record nothing.
This is **INFORMATIONAL** unless HIGH-impact plan-file discrepancies are found (then it gates via AskUserQuestion).
Otherwise the audit is **INFORMATIONAL**.
Discrepancies derived only from fallback sources (commit messages, TODOS.md, PR description) never trigger
this question, whatever their IMPACT: report them in the Scope Check as lower-confidence missing requirements.
-3
View File
@@ -97,9 +97,6 @@ Never reset DEADLINE_FILE/bypass DEADLINE_TOOL. Expiry or an invalid/missing DEA
Each probe is one native command/interaction plus checks, excluding bookkeeping.
Never batch probes.
How one probe works: (1) if bounded, check the clock and choose the next command;
(2) publish a checkpoint naming it (functional: \`capture --after\` publishes it);
(3) run exactly that command via DEADLINE_TOOL or EVIDENCE_TOOL, read its result, repeat.
1. First demonstrate success: output AND durable effects. Guard if bounded; await completion.
2. **Decide whether another probe is needed.** If bounded, run \`bun DEADLINE_TOOL status DEADLINE_FILE\`.
+28 -9
View File
@@ -2,8 +2,8 @@
* Runtime root for env-var hosts (C1, #1159): the one owner of how generated
* bash finds gstack on Codex and every other `usesEnvVars` host.
*
* Those hosts run each fenced block in a fresh shell, so `GSTACK_ROOT`, `B`
* and `D` set by an earlier block are gone. The preamble's root resolution and
* Those hosts run each fenced block in a fresh shell, so `GSTACK_ROOT`, `B`,
* `D` and `P` set by an earlier block are gone. The preamble's root resolution and
* the per-fence prelude come from `runtimeRootPrelude()`; `insertRuntimePreludes()`
* is the single post-render pass (gen-skill-docs, before rewriteInstallRoot)
* that puts it into every fence that uses these variables without assigning
@@ -19,6 +19,13 @@ import { toShellPath, type TemplateContext } from './types';
/** Per-fence byte budget for the prelude (T-ENG1), asserted by the INV-3 test. */
export const PRELUDE_BYTE_BUDGET = 400;
/**
* make-pdf's dev-build override (E2), appended to `P`'s assignment. It is the
* one fixed segment outside PRELUDE_BYTE_BUDGET: root plus `P=...` fits the
* budget like `B`/`D` do, and the override is needed in every block that runs
* `$P` so readiness and use name the same binary.
*/
export const MAKE_PDF_OVERRIDE = ';[ -x "${MAKE_PDF_BIN:-}" ]&&P=$MAKE_PDF_BIN||:';
const ROOT_VARS = /\$\{?GSTACK_(?:ROOT|BIN|BROWSE|DESIGN|MAKE_PDF)\b/;
const assigns = (name: string) => new RegExp(`(?:^|[\\s;&|(])${name}=`, 'm');
@@ -48,13 +55,23 @@ export function runtimeRootPrelude(ctx: TemplateContext): string {
}
/**
* Lines that assign `B` (browse) or `D` (design). BROWSE SETUP, DESIGN SETUP and
* the prelude all use this, so a later fence runs the binary setup checked.
* Env-var hosts derive from `GSTACK_ROOT`, which the prelude already points at
* the repo-local install when one exists; other hosts keep their
* repo-local-first probe.
* Lines that assign `B` (browse), `D` (design) or `P` (make-pdf). BROWSE SETUP,
* DESIGN SETUP, MAKE-PDF SETUP and the prelude all use this, so a later fence
* runs the binary setup checked. Env-var hosts derive from `GSTACK_ROOT`, which
* the prelude already points at the repo-local install when one exists; other
* hosts keep their repo-local-first probe. make-pdf's binary is
* `make-pdf/dist/pdf`, and `MAKE_PDF_BIN` (a contributor dev build) wins on
* every host.
*/
export function binaryAssignment(ctx: TemplateContext, tool: 'browse' | 'design'): string {
export function binaryAssignment(ctx: TemplateContext, tool: 'browse' | 'design' | 'make-pdf'): string {
if (tool === 'make-pdf') {
if (getHostConfig(ctx.host).usesEnvVars) return `P=$GSTACK_ROOT/make-pdf/dist/pdf${MAKE_PDF_OVERRIDE}`;
return `_ROOT=$(git rev-parse --show-toplevel 2>/dev/null)
P=""
[ -n "$MAKE_PDF_BIN" ] && [ -x "$MAKE_PDF_BIN" ] && P="$MAKE_PDF_BIN"
[ -z "$P" ] && [ -n "$_ROOT" ] && [ -x "$_ROOT/${ctx.paths.localSkillRoot}/make-pdf/dist/pdf" ] && P="$_ROOT/${ctx.paths.localSkillRoot}/make-pdf/dist/pdf"
[ -z "$P" ] && P="${toShellPath(ctx.paths.skillRoot)}/make-pdf/dist/pdf"`;
}
const v = tool === 'browse' ? 'B' : 'D';
const dir = tool === 'browse' ? ctx.paths.browseDir : ctx.paths.designDir;
if (getHostConfig(ctx.host).usesEnvVars) return `${v}=$GSTACK_ROOT/${tool}/dist/${tool}`;
@@ -69,8 +86,9 @@ export function fencePrelude(ctx: TemplateContext, body: string): string {
if (!getHostConfig(ctx.host).usesEnvVars) return '';
const needB = uses('B').test(body) && !assigns('B').test(body);
const needD = uses('D').test(body) && !assigns('D').test(body);
const needP = uses('P').test(body) && !assigns('P').test(body);
if (assigns('GSTACK_ROOT').test(body)) return '';
const needRoot = needB || needD || ROOT_VARS.test(body);
const needRoot = needB || needD || needP || ROOT_VARS.test(body);
const derived = Object.entries(DERIVED)
.filter(([name]) => uses(`GSTACK_${name}`).test(body) && !assigns(`GSTACK_${name}`).test(body))
.map(([name, dir]) => `GSTACK_${name}=$GSTACK_ROOT/${dir}`);
@@ -79,6 +97,7 @@ export function fencePrelude(ctx: TemplateContext, body: string): string {
derived.join(' '),
needB ? binaryAssignment(ctx, 'browse') : '',
needD ? binaryAssignment(ctx, 'design') : '',
needP ? binaryAssignment(ctx, 'make-pdf') : '',
].filter(Boolean).join('\n');
}
+30 -13
View File
@@ -47,37 +47,54 @@ all preceding valid round files in order (omit them for round 1):
~/.claude/skills/gstack/bin/gstack-office-hours-review prepare --design "<design-path>" --out-dir "<review-directory>" "<round-1.json if present>" "<round-2.json if present>"
\`\`\`
Omit absent arguments rather than passing placeholders. The helper chooses the next
Omit absent arguments rather than passing placeholders. Users get 3 rounds; only
when a caller sets a lower review round limit, add \`--max-rounds <N>\` with that
same value to every prepare, check, and finalize command. The helper chooses the next
round and writes \`round-N.prompt.md\`. It includes the full finding schema, all five
review dimensions (Completeness, Consistency, Clarity, Scope, Feasibility), the
office-hours coaching contract, and the COMPLETE preceding JSON verdict.
office-hours coaching contract, and the COMPLETE preceding JSON verdict. It also
saves the design as reviewed (\`round-N.design.md\`). Round 1 is a full review;
rounds 2 and 3 are delta re-reviews of the exact design diff since the last round,
so prepare every round in the same review directory and do not edit the design
between prepare and its review.
Use the Agent tool with \`run_in_background: false\` and its returned \`dispatch\`
string unchanged as the prompt. The reviewer must Read the entire prepared prompt
file before reviewing the design. Do not recreate the prompt, copy selected fields,
or summarize prior findings. A parent Read does not deliver the file to the reviewer.
The reviewer has fresh context and cannot see the brainstorming conversation.
Its prepared contract requires a complete JSON Write and an identical JSON response.
Its prepared contract requires a complete JSON Write, sealed by the dispatch's
\`Seal:\` command, and a one-line \`OFFICE_HOURS_VERDICT\` receipt as its entire response.
It protects the required coaching and Assignment sections, distinguishes unknown
customer facts from committed behavior, and requires evidence for every prior status.
**Step 2: Check stop conditions, then fix and re-dispatch**
After each verdict, BEFORE fixing any findings or dispatching again, validate the
saved files with the helper (list every completed round in order):
saved files with the helper. Pass the reviewer's entire response unchanged as the
receipt and list every completed round in order:
\`\`\`bash
~/.claude/skills/gstack/bin/gstack-office-hours-review check "<round-1.json>" "<round-2.json if present>" "<round-3.json if present>"
~/.claude/skills/gstack/bin/gstack-office-hours-review check --receipt "<reviewer response>" "<round-1.json>" "<round-2.json if present>" "<round-3.json if present>"
\`\`\`
A missing, malformed, or mismatched receipt fails the check: that attempt is a failed review.
Omit absent arguments rather than passing placeholders.
**Convergence guard and stopping rules:** Read its stop reason:
- PASS: no unresolved findings; proceed to Step 3.
- CONVERGENCE: the reviewer explicitly marked a prior obligation persisting with
a concrete prior/current finding pair and document evidence. Stop even if new
findings appear. Shared topic labels or new refinements alone are insufficient.
- MAX_ITERATIONS: round 3 completed; stop.
- CONTINUE: fix the listed findings in the design, then return to Step 1 to prepare and dispatch the next review.
**Convergence guard and stopping rules:** Each finding is blocking or minor. Only
blocking findings require another round. Read its stop reason:
- PASS: no blocking findings remain. Any minor findings are recorded, not fixed,
and never justify another round; proceed to Step 3.
- CONVERGENCE: the reviewer explicitly marked a blocking prior obligation
persisting with a concrete prior/current finding pair and document evidence.
Stop even if new findings appear. Shared topic labels or new refinements alone
are insufficient.
- MAX_ITERATIONS: round 3 completed (or the caller's lower round limit); stop.
- CONTINUE: fix only the blocking findings in the design, then return to Step 1
to prepare and dispatch the next review. Do not edit for minor findings
mid-loop: they stay recorded for the user, and new text only gives the next
diff more to review. Its reviewer checks every prior finding and raises new
blocking findings only for problems your changes introduced or exposed.
On a stop, do not fix again or re-dispatch. Run the finalizer before approval:
@@ -119,7 +136,7 @@ mkdir -p "$GSTACK_STATE_ROOT/analytics"
echo '{"skill":"office-hours","ts":"'$(date -u +%Y-%m-%dT%H:%M:%SZ)'","iterations":ITERATIONS,"issues_found":FOUND,"issues_fixed":FIXED,"remaining":REMAINING,"quality_score":SCORE}' >> "$GSTACK_STATE_ROOT/analytics/spec-review.jsonl" 2>/dev/null || true
\`\`\`
Use iterations, issues_found, issues_fixed, remaining, and quality_score from the
helper. FOUND counts finding observations across rounds; FIXED counts only
helper; its remaining_blocking and remaining_minor split the remaining count. FOUND counts finding observations across rounds; FIXED counts only
reviewer-confirmed resolutions. An unavailable score is null, never invented.`;
}
+3 -3
View File
@@ -60,13 +60,13 @@ backslashes serialize cleanly — never use hand-rolled \`echo\` / \`printf\`.
\`\`\`bash
GSTACK_STATE_ROOT=$(~/.claude/skills/gstack/bin/gstack-paths --get GSTACK_STATE_ROOT); : "\${GSTACK_STATE_ROOT:?gstack-paths failed; reinstall with ./setup or /gstack-upgrade}"
BRANCH=$(~/.claude/skills/gstack/bin/gstack-slug --get BRANCH 2>/dev/null)
SLUG=$(~/.claude/skills/gstack/bin/gstack-slug --get SLUG 2>/dev/null)
TASKS_DIR="$GSTACK_STATE_ROOT/projects/\${SLUG:-unknown}"
mkdir -p "$TASKS_DIR"
TASKS_FILE="$TASKS_DIR/tasks-${phase}-$(date +%Y%m%d-%H%M%S).jsonl"
COMMIT=$(git rev-parse HEAD 2>/dev/null || echo unknown)
BRANCH=$(git branch --show-current 2>/dev/null || echo unknown)
# The task record keeps the raw git branch; artifact filenames keep gstack-slug's sanitized BRANCH.
GIT_BRANCH=$(git branch --show-current 2>/dev/null || echo unknown)
RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)-$$"
# Repeat ONE jq invocation per task identified during this review.
@@ -77,7 +77,7 @@ RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)-$$"
jq -nc \\
--arg phase '${phase}' \\
--arg run_id "$RUN_ID" \\
--arg branch "$BRANCH" \\
--arg branch "$GIT_BRANCH" \\
--arg commit "$COMMIT" \\
--arg id "$TASK_ID" \\
--arg priority "$PRIORITY" \\
+1 -1
View File
@@ -129,7 +129,7 @@ function cardRules(mode: TestValueBarMode, skillName: string): string {
qa: skillName === 'qa-only' ? 'Put it under each proposed test.' : 'Put it in the 8e.5 record.',
audit: 'Read cards from test header comments when present.',
}[mode];
return `Value card: \`Value: protects=<...>; fails_when=<...>; why_new=<...>; seam=none\` (seam: \`none\` or its name); each field at most ${CARD_FIELD_MAX_BYTES} UTF-8 bytes here (clamp to 157 plus \`...\`; JSON keeps full values). ${where} A missing upstream card never blocks: derive it; ignore unknown fields.
return `Value card: \`Value: protects=<...>; fails_when=<...>; why_new=<...>; seam=none\` (seam: \`none\` or its name); each field at most ${CARD_FIELD_MAX_BYTES} UTF-8 bytes here (clamp to 157 plus \`...\`; written JSON keeps full values). ${where} A missing upstream card never blocks: derive it; ignore unknown fields.
Example: ${EXAMPLE_CARD}
${EXAMPLE_REJECTED}`;
+9
View File
@@ -676,6 +676,8 @@ export interface FreeRunReport {
inFlight: string[];
/** Planned files never observed in the stream (silent passers + never-flushed wedges). */
filesWithNoOutput: number;
/** The last nonblank line either stream printed: a native abort's message when bun dies without a summary. */
lastOutputLine: string | null;
}
interface FileProgress {
@@ -711,6 +713,7 @@ export class FreeRunReporter {
private testsRan: number | null = null;
private filesRan: number | null = null;
private sawSummary = false;
private lastOutputLine: string | null = null;
constructor(
private readonly plannedFiles: string[],
@@ -758,6 +761,7 @@ export class FreeRunReporter {
unhandledErrors: [...this.unhandled],
inFlight,
filesWithNoOutput: this.plannedFiles.filter((f) => !this.progress.has(normalizeRelativePath(f))).length,
lastOutputLine: this.lastOutputLine,
};
}
@@ -768,6 +772,7 @@ export class FreeRunReporter {
// (fail) lines land under a second phantom file (observed on the first
// Linux run: 5 real failures reported as 10 across 2 files).
const line = stripAnsiLine(rawLine).replace(/^::group::/, '');
if (line.trim() !== '') this.lastOutputLine = line.trim();
let visible = false;
const failedCount = this.failureSummary.consume(line, origin);
if (failedCount !== null) {
@@ -903,6 +908,10 @@ export function buildRunEpilogue(
for (const u of report.unhandledErrors) {
lines.push(` ⚠ unhandled error between tests (around ${u.file ?? 'unknown file'})`);
}
if (status === 'failed' && !report.sawTerminalSummary) {
lines.push(` ⚠ test process ended before its summary; in flight: ${report.inFlight.length > 0 ? report.inFlight.join(', ') : 'unknown'}`);
if (report.lastOutputLine) lines.push(` ⚠ last output: ${report.lastOutputLine}`);
}
if (status === 'timed-out') {
if (report.inFlight.length > 0) {
lines.push(` ⏱ in flight at kill: ${report.inFlight.join(', ')}`);
+25 -16
View File
@@ -66,13 +66,16 @@ import { EVAL_POLICY, PERIODIC_CI_EXCLUDE } from '../test/helpers/periodic-exclu
import {
getProjectEvalDir, getClaudeCliVersion, isFinalizedEvalResultFile, failureClassOf, panelVerdict,
sanitizeTrialError, CONTRACT_VIOLATIONS_FILE, TRIAL_ENV, type TrialFailureClass,
trialCostKnown, trialFailureFields, trialSessions,
} from '../test/helpers/eval-store';
import { STALL_WINDOW_MS } from '../test/helpers/eval-budgets';
import { readSessionLedger, type SessionLedgerRow } from '../test/helpers/session-ledger';
import { preflightAnthropicApi } from '../test/helpers/anthropic-preflight';
import { e2eReuseLaneProblem, prepareE2EShardReuse, selectPlanReceipts } from './e2e-shard-reuse';
import { E2E_TOUCHFILES, E2E_TIERS } from '../test/helpers/touchfiles';
import { scopeCodexAccess, shardFile, shardCaseId, shardTrial, trialShardKey, type CaseTrialPlan, caseTrialPlan, excludedCasesNamePattern, caseTestNamePattern, expandCaseShards, expandTrialShards, fileCaseRegistration, partitionCaseExclusions } from './lib/paid-cases';
import { retriesForFiles, trialPanelKey, sliceExecutionOrder, buildRunManifest, parseRunManifest, type SliceResult, sliceExitCode, guardTrialRecords, formatSlicePlan, formatCapacityPreflight, sliceDeadlineMs } from './lib/paid-plan';
import { caseFile, runCaseDiagnosis, formatPanelLine, runPaidReport } from './lib/paid-report';
import { caseSelection, runCaseDiagnosis, formatPanelLine, runPaidReport } from './lib/paid-report';
import {
DEFAULT_JOBS, DEFAULT_MAX_FILES_PER_SHARD, DEFAULT_SHARD_TIMEOUT_MS, DEFAULT_TIER, DEFAULT_WITHIN_SHARD_CONCURRENCY, OVERLAY_MAX_ACTIVE_SHARDS,
PAID_LANE_POLICY, PAID_TIERS, ROOT, SLICE_JOB_STARTED_AT_ENV, SLICE_UPLOAD_RESERVE_MS, isOverlayTestFile,
@@ -94,8 +97,10 @@ export * from './lib/paid-cases';
export * from './lib/paid-plan';
export * from './lib/paid-report';
/** Records and contract evidence an isolated shard left in its eval dir. */
export function readTrialEvidence(evalDir: string | undefined): { records: any[]; contract: string | null } {
type TrialEvidence = { records: any[]; contract: string | null; sessions?: SessionLedgerRow[] };
/** Records, contract evidence and session-ledger rows an isolated shard left in its eval dir. */
export function readTrialEvidence(evalDir: string | undefined): TrialEvidence {
if (!evalDir || !fs.existsSync(evalDir)) return { records: [], contract: null };
const names = fs.readdirSync(evalDir);
const parse = (name: string) => { try { return JSON.parse(fs.readFileSync(path.join(evalDir, name), 'utf8')); } catch { return null; } };
@@ -108,28 +113,32 @@ export function readTrialEvidence(evalDir: string | undefined): { records: any[]
const line = fs.readFileSync(path.join(evalDir, CONTRACT_VIOLATIONS_FILE), 'utf8').split('\n').find(l => l.trim());
if (line) contract = String(JSON.parse(line).message ?? 'contract violation');
} catch { /* no sidecar */ }
return { records, contract };
return { records, contract, sessions: readSessionLedger(evalDir) };
}
/** Classify one isolated trial shard. Contract evidence always fails the trial. */
export function classifyTrialShard(
outcome: Pick<ShardOutcome, 'status' | 'executedTests' | 'skippedTests' | 'elapsedMs' | 'runnerError'>,
caseId: string, trial: number, plan: CaseTrialPlan,
evidence: { records: any[]; contract: string | null },
evidence: TrialEvidence,
): ShardTrialRecord {
const failedRecord = evidence.records.find(record => record.passed === false) ?? evidence.records[0];
const base: ShardTrialRecord = {
case: caseId, trial, kind: plan.kind, panel: plan.panel, quarantined: plan.quarantined, outcome: null,
cost_usd: Math.round(evidence.records.reduce((sum, record) => sum + (Number(record.cost_usd) || 0), 0) * 100) / 100,
duration_ms: outcome.elapsedMs,
...(typeof failedRecord?.model === 'string' ? { model: failedRecord.model } : {}),
...(typeof failedRecord?.model === 'string' ? { model: failedRecord.model } : {}), ...trialSessions(evidence.sessions ?? []),
...trialCostKnown(evidence.records, evidence.sessions),
};
const failed = (failureClass: TrialFailureClass, error?: string): ShardTrialRecord => {
const cls = evidence.contract !== null ? 'contract' : failureClass;
const raw = evidence.contract ?? failedRecord?.error ?? error;
return { ...base, outcome: 'failed', failure_class: cls,
...(failedRecord?.exit_reason ? { exit_reason: String(failedRecord.exit_reason) } : {}),
...(Number.isInteger(failedRecord?.timeout_at_turn) ? { timeout_at_turn: failedRecord.timeout_at_turn } : {}),
...(sanitizeTrialError(raw) ? { error: sanitizeTrialError(raw) } : {}),
...trialFailureFields({ failure_class: cls, exit_reason: failedRecord?.exit_reason, error: raw, sessions: evidence.sessions, record: failedRecord }, STALL_WINDOW_MS) };
};
const failed = (failureClass: TrialFailureClass, error?: string): ShardTrialRecord => ({
...base, outcome: 'failed', failure_class: evidence.contract !== null ? 'contract' : failureClass,
...(failedRecord?.exit_reason ? { exit_reason: String(failedRecord.exit_reason) } : {}),
...(Number.isInteger(failedRecord?.timeout_at_turn) ? { timeout_at_turn: failedRecord.timeout_at_turn } : {}),
...(sanitizeTrialError(evidence.contract ?? failedRecord?.error ?? error) ? { error: sanitizeTrialError(evidence.contract ?? failedRecord?.error ?? error) } : {}),
});
if (outcome.runnerError !== undefined) return { ...base, harness: `runner error: ${sanitizeTrialError(outcome.runnerError) ?? 'unknown'}` };
if (outcome.status === 'never-started') return { ...base, harness: 'never started' };
if (outcome.status === 'passed-empty') return { ...base, harness: 'hollow: executed no case' };
@@ -873,16 +882,16 @@ async function main(): Promise<number> {
if (options.reportDir) return runPaidReport(options.reportDir, { writeDurations: options.writeDurations });
if (options.caseId && options.listOnly) {
const file = caseFile(options.caseId);
const { file, mode, reason } = caseSelection(options.caseId);
const plan = caseTrialPlan(options.caseId);
const n = options.trials ?? plan.panel.n;
console.log(`[test:paid] --case ${options.caseId}: ${n} trial(s) of ${file} (kind ${plan.kind}), list only`);
for (let trial = 1; trial <= n; trial++) console.log(` ${trialShardKey(file, options.caseId, trial)}`);
console.log(`[test:paid] --case ${options.caseId}: ${n} trial(s) of ${file} (kind ${plan.kind}), selects ${reason}, list only`);
for (let trial = 1; trial <= n; trial++) console.log(` ${mode === 'file' ? `${file} (trial ${trial})` : trialShardKey(file, options.caseId, trial)}`);
return 0;
}
if (options.caseId) {
preflightAnthropicApi(process.env);
const verdict = await runCaseDiagnosis(options.caseId, { trials: options.trials ?? undefined, jobs: options.jobs,
const verdict = await runCaseDiagnosis(options.caseId, { trials: options.trials ?? undefined, jobs: options.jobs, runShards: runPaidShards,
withinShardConcurrency: options.withinShardConcurrency, timeoutMs: timeoutOverride,
evalDirBase: process.env.GSTACK_EVAL_DIR || getProjectEvalDir() });
return verdict.status === 'PASS' ? 0 : 1;
-1
View File
@@ -265,7 +265,6 @@
"test/model-overlays.test.ts\tTS2322\tType 'string' is not assignable to type '\"claude\" | \"fable-5\" | \"gemini\" | \"gpt\" | \"gpt-5.4\" | \"gpt-5.6-sol\" | \"gpt-6-astra\" | \"o-series\" | \"opus-4-7\" | \"opus-4-8\" | \"sonnet-5\" | undefined'.": 4,
"test/native-auto-decide-pty.test.ts\tTS2345\tArgument of type 'string | undefined' is not assignable to parameter of type 'string'. Type 'undefined' is not assignable to type 'string'.": 1,
"test/native-auto-decide.test.ts\tTS2345\tArgument of type '{ status: string; calls: never[]; assistantMessages: { sessionId: string; text: string; timestamp: string; }[]; }' is not assignable to parameter of type 'PlanCountTranscript'. Types of property 'status' are incompatible. Type 'string' is not assignable to type '\"error\" | \"missing\" | \"ready\"'.": 1,
"test/office-hours-completion.test.ts\tTS18046\t'reordered.findings' is of type 'unknown'.": 1,
"test/office-hours-review.test.ts\tTS18048\t'check.stderr' is possibly 'undefined'.": 1,
"test/office-hours-review.test.ts\tTS18048\t'check.stdout' is possibly 'undefined'.": 1,
"test/office-hours-review.test.ts\tTS18048\t'direct.stderr' is possibly 'undefined'.": 1,
+1 -1
View File
@@ -4,7 +4,7 @@
# Runs as user `ubi` (passwordless sudo) from the synced checkout.
set -euo pipefail
BUN_VERSION=1.4.0
BUN_VERSION=1.4.2
NODE_VERSION=22.20.0
export DEBIAN_FRONTEND=noninteractive