mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-19 22:54:36 +01:00
Merge remote-tracking branch 'origin/main' into garrytan/fix-wave-oct6
This commit is contained in:
commit
9207fabcbe
322 files changed
+22550
-1051
No files matched your search
+47
-18
@@ -44,6 +44,8 @@
|
||||
* bun run eval:pass-rates --dir <path> # local eval dirs / downloaded artifacts (repeatable)
|
||||
* bun run eval:pass-rates --backfill # also import legacy slice artifacts, labeled pre-policy
|
||||
* bun run eval:pass-rates --json | --gate
|
||||
* bun run eval:pass-rates --headroom | --reds | --run <id> # report views (scripts/lib/eval-history.ts)
|
||||
* bun run eval:pass-rates --help # every flag, no network
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
@@ -55,7 +57,9 @@ import { E2E_KINDS, E2E_TIERS, E2E_TOUCHFILES, GLOBAL_TOUCHFILES, LLM_JUDGE_TOUC
|
||||
import { CASE_QUARANTINE, EVAL_POLICY } from '../test/helpers/periodic-exclude-data';
|
||||
import { CASE_TEST_NAMES } from './test-paid-shards';
|
||||
import { resolveStateRoot } from '../lib/state-root';
|
||||
import { downloadRunArtifacts, isPooledTrialRun, isWeeklyHistoryRun, listWeeklyRuns, parseFlakeLedger, repoSlug, TRIAL_OUTCOMES_MAX_BYTES } from './lib/ci-history';
|
||||
import { GH_JOBS, PASS_RATES_USAGE, criticalPath, formatCriticalPath, formatHeadroom, formatRedLedger, headroom, headroomAlarms, parsePassRatesArgs,
|
||||
redLedger, triageRun, CENSUS_RED_GUIDE, type CriticalPath } from './lib/eval-history';
|
||||
import { downloadRunArtifacts, GH_HISTORY, isPooledTrialRun, isWeeklyHistoryRun, listWeeklyRuns, parseFlakeLedger, repoSlug, TRIAL_OUTCOMES_MAX_BYTES, type WeeklyRun } from './lib/ci-history';
|
||||
|
||||
interface TestSeries {
|
||||
name: string;
|
||||
@@ -237,7 +241,7 @@ export function backfillEvalFiles(files: string[], run?: { run_id: string; sha?:
|
||||
tier: caseTier(id, registry), kind: registry.kinds[id]!, trial: 1, panel: { n: 1, k: 1 }, attempt: 1,
|
||||
outcome, ...(outcome === 'failed' ? { failure_class: failureClassOf(entry) } : {}),
|
||||
exit_reason: entry.exit_reason, error: sanitizeTrialError(entry.error),
|
||||
duration_ms: Math.max(0, entry.duration_ms || 0), cost_usd: Math.max(0, entry.cost_usd || 0),
|
||||
duration_ms: Math.max(0, entry.duration_ms || 0), cost_usd: Math.max(0, entry.cost_usd || 0), ...(entry.cost_known === false ? { cost_known: false } : {}),
|
||||
model: entry.model, cli_version: result.claude_cli_version, policy_version: 0, quarantined: false,
|
||||
execution: entry.execution === 'reused' ? 'reused' : 'executed', source: 'backfill',
|
||||
run_id: run?.run_id ?? `local:${file}`, sha: run?.sha ?? result.git_sha, recorded_at: run?.timestamp ?? result.timestamp,
|
||||
@@ -292,7 +296,7 @@ export function holmRejections(pValues: number[], alpha: number): Set<number> {
|
||||
|
||||
export type PassRateLabel = 'INCONCLUSIVE' | 'BROKEN' | 'FLAKY' | 'FAILING' | 'PASSING';
|
||||
export type AlarmKind = 'drift' | 'rule-as-behavior' | 'regression' | 'quarantine-exit' | 'quarantine-expired'
|
||||
| 'quarantine-cap' | 'quarantine-invalid';
|
||||
| 'quarantine-cap' | 'quarantine-invalid' | 'headroom';
|
||||
|
||||
/** The EVAL_POLICY fields pass-rates reads (structural, so tests can vary them). */
|
||||
export interface PassRatePolicy {
|
||||
@@ -567,24 +571,22 @@ export { downloadRunArtifacts, GH_HISTORY, listWeeklyRuns, TRIAL_OUTCOMES_MAX_BY
|
||||
type HistoryFetcher, type RunArtifact, type WeeklyRun } from './lib/ci-history';
|
||||
|
||||
if (import.meta.main) {
|
||||
const argv = process.argv.slice(2);
|
||||
const flag = (name: string) => { const index = argv.indexOf(name); return index === -1 ? undefined : argv[index + 1]; };
|
||||
const dirs = argv.flatMap((arg, index) => arg === '--dir' && argv[index + 1] ? [argv[index + 1]!] : []);
|
||||
const asJson = argv.includes('--json');
|
||||
const gate = argv.includes('--gate');
|
||||
const backfill = argv.includes('--backfill');
|
||||
const caseFilter = flag('--case');
|
||||
const runsLimit = Number(flag('--runs')) || 10;
|
||||
const sinceDays = Number(flag('--since-days')) || 60;
|
||||
const repo = flag('--repo') ?? repoSlug();
|
||||
const workflow = flag('--workflow') ?? 'evals-periodic.yml';
|
||||
const branch = flag('--branch') ?? 'main';
|
||||
const parsed = parsePassRatesArgs(process.argv.slice(2), id => Object.hasOwn(E2E_TIERS, id) || Object.hasOwn(LLM_JUDGE_TOUCHFILES, id)
|
||||
|| (/^test\/[\w./-]+\.test\.ts$/.test(id) && fs.existsSync(id)));
|
||||
if ('help' in parsed) { console.log(PASS_RATES_USAGE); process.exit(0); }
|
||||
if ('error' in parsed) { console.error(`eval:pass-rates: ${parsed.error}\n${PASS_RATES_USAGE}`); process.exit(2); }
|
||||
const { dirs, json: asJson, gate, backfill, caseFilter, runs: runsLimit, sinceDays, view } = parsed;
|
||||
const repo = parsed.repo ?? repoSlug();
|
||||
const workflow = parsed.workflow ?? 'evals-periodic.yml';
|
||||
const branch = parsed.branch ?? 'main';
|
||||
|
||||
const records: TrialRecord[] = [];
|
||||
const unattributed = new Set<string>();
|
||||
const errors: string[] = [];
|
||||
let historyError: string | null = null;
|
||||
let weeklyRuns: string[] | undefined;
|
||||
let fetched: WeeklyRun[] = [];
|
||||
const cacheDir = path.join(path.resolve(resolveStateRoot()), 'eval-pass-rates-cache', repo.replace('/', '-'));
|
||||
let pooledRunIds: Set<string> | undefined;
|
||||
|
||||
const manualReviews: string[] = [];
|
||||
@@ -611,7 +613,8 @@ if (import.meta.main) {
|
||||
: [];
|
||||
pooledRunIds = new Set(pooledRuns.map(run => `${run.id}`));
|
||||
const runs = [...weekly, ...pooledRuns];
|
||||
const cacheDir = path.join(path.resolve(resolveStateRoot()), 'eval-pass-rates-cache', repo.replace('/', '-'));
|
||||
if (parsed.runId !== undefined && !runs.some(run => run.id === parsed.runId)) runs.push(GH_JOBS.getRun(repo, parsed.runId));
|
||||
fetched = runs;
|
||||
const match = backfill
|
||||
? (name: string) => name.startsWith('trial-outcomes') || /^(paid-slice-\d+|gate-census-\d+)(-a\d+)?$/.test(name)
|
||||
: (name: string) => name.startsWith('trial-outcomes');
|
||||
@@ -624,11 +627,37 @@ if (import.meta.main) {
|
||||
}
|
||||
}
|
||||
|
||||
const report = analyzePassRates(records, { weeklyRuns, pooledRunIds, unattributed: [...unattributed].sort(), errors, manualReviews });
|
||||
const scope = dirs.length ? `local dirs ${dirs.join(', ')}` : `${repo} ${workflow} on ${branch}${pooledRunIds?.size ? ` + ${pooledRunIds.size} pooled branch census run(s)` : ''}, `
|
||||
+ `last ${runsLimit} completed run(s): ${fetched.map(run => run.id).join(', ') || 'none'}`;
|
||||
const scoped = caseFilter ? records.filter(record => record.case === caseFilter) : records;
|
||||
const caseHeadroom = headroom(scoped);
|
||||
if (view !== 'rates') {
|
||||
const critical: CriticalPath[] = [];
|
||||
if (view !== 'reds' && !dirs.length) {
|
||||
for (const run of fetched.filter(run => view === 'headroom' || run.id === parsed.runId)) {
|
||||
try { critical.push(criticalPath(String(run.id), GH_JOBS.listJobs(repo, run.id))); } catch { critical.push({ run: String(run.id), jobs: 0, slowest: null, wallMs: null }); }
|
||||
}
|
||||
}
|
||||
const lines = view === 'headroom' ? formatHeadroom(caseHeadroom, critical) : view === 'reds' ? formatRedLedger(redLedger(scoped))
|
||||
: historyError ? [] : triageRun({ repo, run: fetched.find(run => run.id === parsed.runId)!, records, caseFilter, cacheDir, fetcher: GH_HISTORY,
|
||||
download: (match, maxBytes) => downloadRunArtifacts({ repo, run: fetched.find(run => run.id === parsed.runId)!, match, cacheDir, maxBytes }) })
|
||||
.concat(formatCriticalPath(critical), historyError ? [] : [`guide: ${CENSUS_RED_GUIDE}`]);
|
||||
if (asJson) console.log(JSON.stringify({ scope, historyError, view, lines }, null, 2));
|
||||
else {
|
||||
console.log(`scope: ${scope}`);
|
||||
if (historyError) console.log(`history unavailable (${historyError})`);
|
||||
for (const line of lines) console.log(line);
|
||||
}
|
||||
process.exit(historyError ? 1 : 0);
|
||||
}
|
||||
const full = analyzePassRates(records, { weeklyRuns, pooledRunIds, unattributed: [...unattributed].sort(), errors, manualReviews });
|
||||
const alarms = [...full.alarms, ...(gate ? headroomAlarms(caseHeadroom) : [])].filter(alarm => !caseFilter || alarm.case === caseFilter);
|
||||
const report = { ...full, cases: full.cases.filter(entry => !caseFilter || entry.case === caseFilter), alarms };
|
||||
const ledger = readFreeLedger();
|
||||
if (asJson) {
|
||||
console.log(JSON.stringify({ repo, workflow, branch, dirs, historyError, ...report, freeLedger: ledger }, null, 2));
|
||||
console.log(JSON.stringify({ repo, workflow, branch, scope, dirs, historyError, ...report, freeLedger: ledger }, null, 2));
|
||||
} else {
|
||||
console.log(`scope: ${scope}`);
|
||||
if (historyError) console.log(`pass-rates: history unavailable (${historyError}); every label below is INCONCLUSIVE`);
|
||||
console.log(formatPassRates(report, { caseFilter }));
|
||||
if (ledger.length > 0) {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"version": 1,
|
||||
"version": 2,
|
||||
"patterns": [
|
||||
"test/helpers/agent-sdk-runner.ts",
|
||||
"test/helpers/codex-session-runner.ts",
|
||||
@@ -10,41 +10,44 @@
|
||||
"test/helpers/llm-judge.ts"
|
||||
],
|
||||
"files": {
|
||||
"scripts/lib/paid-cases.ts": {"blob":"cfb25a673c8f9c95aacfdbcd91b5dc471488da72","since":1,"nonBehavioral":"audit-wave integration before the first v2 census (EVAL_POLICY v2 already starts fresh series)"},
|
||||
"scripts/lib/paid-plan.ts": {"blob":"45721ce128a112666a0caed6c13c66637b74ca54","since":1,"nonBehavioral":"audit-wave integration before the first v2 census (EVAL_POLICY v2 already starts fresh series)"},
|
||||
"scripts/lib/paid-report.ts": {"blob":"72e2a3647f7e56f77ed27ced79ca9c4d239bb62c","since":1,"nonBehavioral":"audit-wave integration before the first v2 census (EVAL_POLICY v2 already starts fresh series)"},
|
||||
"scripts/lib/shard-engine.ts": {"blob":"3cd0b1aaa84018b00bf9ea0ce48093f3c8492f66","since":1,"nonBehavioral":"audit-wave integration before the first v2 census (EVAL_POLICY v2 already starts fresh series)"},
|
||||
"scripts/test-paid-shards.ts": {"blob":"93717f22a85d842152f49c4275367a1d943cf13e","since":1,"nonBehavioral":"audit-wave integration before the first v2 census (EVAL_POLICY v2 already starts fresh series)"},
|
||||
"scripts/test-pr-profile.ts": {"blob":"d7860897397de8fd605623570d1e7ebbbd5ec743","since":1,"nonBehavioral":"audit-wave integration before the first v2 census (EVAL_POLICY v2 already starts fresh series)"},
|
||||
"scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":1},
|
||||
"test/helpers/agent-sdk-runner.ts": {"blob":"7edff19228222e3582f3efc0dbe12c35649e16b7","since":1},
|
||||
"test/helpers/claude-pty-runner.ts": {"blob":"20cc5894532873aa6145fc6617fdeec8cef94a2d","since":1},
|
||||
"test/helpers/codex-session-runner.ts": {"blob":"4eea35931a35a0afb006363402e3d4191e379737","since":1},
|
||||
"test/helpers/e2e-gate.ts": {"blob":"efb7e197dd5f66c3ad0e34074b14e640754f7a45","since":1,"nonBehavioral":"audit-wave integration before the first v2 census (EVAL_POLICY v2 already starts fresh series)"},
|
||||
"test/helpers/e2e-helpers.ts": {"blob":"2d4a2a53063cfc1f003db366d2c861ab1e379e6e","since":1},
|
||||
"test/helpers/eval-budgets.ts": {"blob":"824be22c1ddad036bdac9ad89367eb061c07475f","since":1},
|
||||
"test/helpers/eval-store.ts": {"blob":"23ec90ffdc235ecff27e734e4652e9ddda649cb1","since":1},
|
||||
"test/helpers/hermetic-env.ts": {"blob":"6473f78285384f4420f48cbe5928658919078e0b","since":1},
|
||||
"test/helpers/llm-judge.ts": {"blob":"453a4c1d355f3811deca53f953767109a9ec9292","since":1},
|
||||
"test/helpers/paid-test-set.ts": {"blob":"ff9bfe43c43ec2bdd11114f7b8ec7c5736116f75","since":1,"nonBehavioral":"audit-wave integration before the first v2 census (EVAL_POLICY v2 already starts fresh series)"},
|
||||
"test/helpers/pty-screen.ts": {"blob":"4f77569f9ce11f9cee9f347216e468cd7c956820","since":1},
|
||||
"test/helpers/pty/auq.ts": {"blob":"16e5ea2bb5b9f28f44808a1a09024dbd3e6f28c4","since":1},
|
||||
"test/helpers/pty/binary.ts": {"blob":"404d7515076d5c00804f89e23ea69296312c00e1","since":1},
|
||||
"test/helpers/pty/boundaries.ts": {"blob":"53eb76e57c0558c00254366d6334a1c833126575","since":1},
|
||||
"test/helpers/pty/classify.ts": {"blob":"3f2daa12ff6484f96b8d28d910a50231c4bb3ae8","since":1},
|
||||
"test/helpers/pty/fake-session.ts": {"blob":"554e064ef259b8444280eddfe9b0db93c66a8202","since":1},
|
||||
"test/helpers/pty/judge.ts": {"blob":"81c5bb4c48ab928aca2f186a1ade9df7169d7fbe","since":1},
|
||||
"test/helpers/pty/launch.ts": {"blob":"86136d044f496c11aaaaeeee9572b5831bd13208","since":1},
|
||||
"test/helpers/pty/plan-native.ts": {"blob":"ddba9cb68c7c0da935cff7c1cea258dee2dff740","since":1},
|
||||
"test/helpers/pty/runners/counting.ts": {"blob":"a4ac5961c136e3c18540d5b4075ef44c2b87a425","since":1},
|
||||
"test/helpers/pty/runners/floor.ts": {"blob":"a8d6d180280da349a9609af9141ccfd4205313d6","since":1},
|
||||
"test/helpers/pty/runners/observation.ts": {"blob":"032298c30ebeff0a9314749670895406169bde1d","since":1},
|
||||
"test/helpers/pty/screen.ts": {"blob":"ba86fa311c94401ecb7c1376a7a2bc7c1c516cf6","since":1},
|
||||
"test/helpers/pty/session.ts": {"blob":"68f56d907edc6ceaa731f06dbd99c99ee77bcfe6","since":1},
|
||||
"test/helpers/session-drain-policy.ts": {"blob":"6211239dd0485e6495a700b824a7e0e61d1bcd97","since":1},
|
||||
"test/helpers/session-runner.ts": {"blob":"702444c3b3f7e129d101ea3864752e09951adb77","since":1},
|
||||
"test/helpers/skill-fixture.ts": {"blob":"b188064594eb4201a54a80ba1bc6f28e31d7f49e","since":1},
|
||||
"test/helpers/test-selection.ts": {"blob":"51655f89f66be7a5ec487e2abfe4744300ebd5f6","since":1},
|
||||
"test/helpers/touchfiles.ts": {"blob":"15f369056d41b9dc42dcd86f81f60c878074704c","since":1}
|
||||
"scripts/lib/paid-cases.ts": {"blob":"9bacd52446192a7ecf3dfaae901ea443cf9c9ae2","since":2},
|
||||
"scripts/lib/paid-plan.ts": {"blob":"45721ce128a112666a0caed6c13c66637b74ca54","since":2},
|
||||
"scripts/lib/paid-report.ts": {"blob":"3fda96a8b1f7f3d55797934847353f254cc4d054","since":2},
|
||||
"scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":2},
|
||||
"scripts/lib/shard-engine.ts": {"blob":"3cd0b1aaa84018b00bf9ea0ce48093f3c8492f66","since":2},
|
||||
"scripts/test-paid-shards.ts": {"blob":"8ddee5f11ed7856876135520dd8025ab989e2db9","since":2},
|
||||
"scripts/test-pr-profile.ts": {"blob":"d7860897397de8fd605623570d1e7ebbbd5ec743","since":2},
|
||||
"scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":2},
|
||||
"test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":2},
|
||||
"test/helpers/auq-substance-panel.ts": {"blob":"4881260ef793668e4963a062db6d06a5e1f7a40f","since":2},
|
||||
"test/helpers/claude-pty-runner.ts": {"blob":"20cc5894532873aa6145fc6617fdeec8cef94a2d","since":2},
|
||||
"test/helpers/codex-session-runner.ts": {"blob":"4eea35931a35a0afb006363402e3d4191e379737","since":2},
|
||||
"test/helpers/e2e-gate.ts": {"blob":"efb7e197dd5f66c3ad0e34074b14e640754f7a45","since":2},
|
||||
"test/helpers/e2e-helpers.ts": {"blob":"2d4a2a53063cfc1f003db366d2c861ab1e379e6e","since":2},
|
||||
"test/helpers/eval-budgets.ts": {"blob":"667d0e09b20fe5d7366f57b77b5fb9b04baa37ef","since":2},
|
||||
"test/helpers/eval-store.ts": {"blob":"609c6d099108ff2696eb014b47aa318a618dd2af","since":2},
|
||||
"test/helpers/hermetic-env.ts": {"blob":"6473f78285384f4420f48cbe5928658919078e0b","since":2},
|
||||
"test/helpers/llm-judge.ts": {"blob":"54e3306b494fb5d7f132ccdaaae5c680465028f9","since":2},
|
||||
"test/helpers/paid-test-set.ts": {"blob":"ff9bfe43c43ec2bdd11114f7b8ec7c5736116f75","since":2},
|
||||
"test/helpers/pty-screen.ts": {"blob":"4f77569f9ce11f9cee9f347216e468cd7c956820","since":2},
|
||||
"test/helpers/pty/auq.ts": {"blob":"16e5ea2bb5b9f28f44808a1a09024dbd3e6f28c4","since":2},
|
||||
"test/helpers/pty/binary.ts": {"blob":"404d7515076d5c00804f89e23ea69296312c00e1","since":2},
|
||||
"test/helpers/pty/boundaries.ts": {"blob":"53eb76e57c0558c00254366d6334a1c833126575","since":2},
|
||||
"test/helpers/pty/classify.ts": {"blob":"3f2daa12ff6484f96b8d28d910a50231c4bb3ae8","since":2},
|
||||
"test/helpers/pty/fake-session.ts": {"blob":"554e064ef259b8444280eddfe9b0db93c66a8202","since":2},
|
||||
"test/helpers/pty/judge.ts": {"blob":"81c5bb4c48ab928aca2f186a1ade9df7169d7fbe","since":2},
|
||||
"test/helpers/pty/launch.ts": {"blob":"4ef0bcb7b5e7c35d2aac0cbffd7f8a303440b6fe","since":2},
|
||||
"test/helpers/pty/plan-native.ts": {"blob":"ddba9cb68c7c0da935cff7c1cea258dee2dff740","since":2},
|
||||
"test/helpers/pty/runners/counting.ts": {"blob":"8c5c05e54abf961259d34d9c3d77db67138d7794","since":2},
|
||||
"test/helpers/pty/runners/floor.ts": {"blob":"465f01641fb3a18ae1c9d2613a9c2b26f8fb503e","since":2},
|
||||
"test/helpers/pty/runners/observation.ts": {"blob":"1c2b75fbb984708725abaa8939125f7cab969f9f","since":2},
|
||||
"test/helpers/pty/screen.ts": {"blob":"ee005f36ffa48617e6832ccbeb35e8e96873df9b","since":2},
|
||||
"test/helpers/pty/session.ts": {"blob":"58533cea9d81cc87bb3b499bdfe0db8e3a8709c5","since":2},
|
||||
"test/helpers/session-drain-policy.ts": {"blob":"6211239dd0485e6495a700b824a7e0e61d1bcd97","since":2},
|
||||
"test/helpers/session-ledger.ts": {"blob":"6e3c9e837e46f98addba2d12eaebff802da320bb","since":2},
|
||||
"test/helpers/session-runner.ts": {"blob":"d40eb29e8b8fdc0675b511693e4a90823b5963fc","since":2},
|
||||
"test/helpers/skill-fixture.ts": {"blob":"b188064594eb4201a54a80ba1bc6f28e31d7f49e","since":2},
|
||||
"test/helpers/test-selection.ts": {"blob":"51655f89f66be7a5ec487e2abfe4744300ebd5f6","since":2},
|
||||
"test/helpers/touchfiles.ts": {"blob":"15f369056d41b9dc42dcd86f81f60c878074704c","since":2}
|
||||
}
|
||||
}
|
||||
@@ -10,7 +10,7 @@ import {
|
||||
buildCrossSkillConsistencyJudgePrompt, buildVoiceDirectiveJudgePrompt,
|
||||
JUDGE_SCORE_SCHEMA, QA_ANTI_REFUSAL_JUDGE_SCHEMA, CROSS_SKILL_CONSISTENCY_JUDGE_SCHEMA,
|
||||
VOICE_DIRECTIVE_JUDGE_SCHEMA, VOICE_DIRECTIVE_DIMENSIONS, JUDGE_SCORE_DIMENSIONS, DEFAULT_JUDGE_MAX_TOKENS,
|
||||
judgePanelMean, judgePanelMajority,
|
||||
judgePanelMedian, judgePanelMajority,
|
||||
} from '../test/helpers/llm-judge';
|
||||
import { buildWorkflowJudgePrompt, WORKFLOW_JUDGE_RESPONSE_SCHEMA } from '../test/helpers/workflow-judge-input';
|
||||
import { CLAUDE_FRONTIER_EVAL_MODEL } from '../lib/eval-model';
|
||||
@@ -18,7 +18,7 @@ import type { CalibrationConfig, Verdict } from './judge-calibration';
|
||||
|
||||
type Sample = Record<string, unknown>;
|
||||
|
||||
/** What production rejects: judgePanelMean needs finite numbers, judgePanelMajority needs booleans. */
|
||||
/** What production rejects: judgePanelMedian needs finite numbers, judgePanelMajority needs booleans. */
|
||||
function requireFields(value: unknown, numbers: readonly string[], booleans: readonly string[] = []): void {
|
||||
const sample = (value ?? {}) as Sample;
|
||||
for (const key of numbers) {
|
||||
@@ -29,13 +29,13 @@ function requireFields(value: unknown, numbers: readonly string[], booleans: rea
|
||||
}
|
||||
}
|
||||
|
||||
const meets = (mean: Record<string, number>, floors: Record<string, number>): Verdict =>
|
||||
Object.entries(floors).every(([key, floor]) => mean[key]! >= floor) ? 'pass' : 'fail';
|
||||
const meets = (median: Record<string, number>, floors: Record<string, number>): Verdict =>
|
||||
Object.entries(floors).every(([key, floor]) => median[key]! >= floor) ? 'pass' : 'fail';
|
||||
|
||||
const QA_FLOORS = { clarity: 3, completeness: 3, actionability: 4 };
|
||||
const scoreJudge = (floors: Record<string, number>) => ({
|
||||
validate: (value: unknown) => requireFields(value, JUDGE_SCORE_DIMENSIONS),
|
||||
verdict: (samples: unknown[]) => meets(judgePanelMean<string>(samples as Sample[], JUDGE_SCORE_DIMENSIONS), floors),
|
||||
verdict: (samples: unknown[]) => meets(judgePanelMedian<string>(samples as Sample[], JUDGE_SCORE_DIMENSIONS), floors),
|
||||
});
|
||||
|
||||
const workflow = (id: string, priority: number, cases: string[], extra: { agentCapability?: 'frontier'; model?: string; floors?: Record<string, number> } = {}): CalibrationConfig => ({
|
||||
@@ -80,7 +80,7 @@ export const CALIBRATION_CONFIGS: CalibrationConfig[] = [
|
||||
oldOptions: {}, newOptions: { jsonSchema: QA_ANTI_REFUSAL_JUDGE_SCHEMA },
|
||||
validate: value => requireFields(value, ['confidence'], ['would_browse']),
|
||||
verdict: samples => judgePanelMajority<string>(samples as Sample[], 'would_browse')
|
||||
&& judgePanelMean<string>(samples as Sample[], ['confidence']).confidence >= 4 ? 'pass' : 'fail',
|
||||
&& judgePanelMedian<string>(samples as Sample[], ['confidence']).confidence >= 4 ? 'pass' : 'fail',
|
||||
estimatedOutputTokens: 1200,
|
||||
},
|
||||
{
|
||||
@@ -89,7 +89,7 @@ export const CALIBRATION_CONFIGS: CalibrationConfig[] = [
|
||||
oldOptions: {}, newOptions: { jsonSchema: CROSS_SKILL_CONSISTENCY_JUDGE_SCHEMA },
|
||||
validate: value => requireFields(value, ['score'], ['consistent']),
|
||||
verdict: samples => judgePanelMajority<string>(samples as Sample[], 'consistent')
|
||||
&& judgePanelMean<string>(samples as Sample[], ['score']).score >= 4 ? 'pass' : 'fail',
|
||||
&& judgePanelMedian<string>(samples as Sample[], ['score']).score >= 4 ? 'pass' : 'fail',
|
||||
estimatedOutputTokens: 1200,
|
||||
},
|
||||
{
|
||||
@@ -97,7 +97,7 @@ export const CALIBRATION_CONFIGS: CalibrationConfig[] = [
|
||||
build: inputs => buildVoiceDirectiveJudgePrompt(inputs.voiceSection!),
|
||||
oldOptions: {}, newOptions: { jsonSchema: VOICE_DIRECTIVE_JUDGE_SCHEMA },
|
||||
validate: value => requireFields(value, VOICE_DIRECTIVE_DIMENSIONS),
|
||||
verdict: samples => meets(judgePanelMean<string>(samples as Sample[], VOICE_DIRECTIVE_DIMENSIONS),
|
||||
verdict: samples => meets(judgePanelMedian<string>(samples as Sample[], VOICE_DIRECTIVE_DIMENSIONS),
|
||||
Object.fromEntries(VOICE_DIRECTIVE_DIMENSIONS.map(key => [key, 4]))),
|
||||
estimatedOutputTokens: 1200,
|
||||
},
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
#!/usr/bin/env bun
|
||||
/**
|
||||
* C1 workflow-judge control bundles (pre-registered in docs/evals/judge-controls-2026-10.md).
|
||||
*
|
||||
* For each judge that produced low samples, builds the production bundle with the
|
||||
* eval's own reader and prompt builder, derives a negative and a targeted-defect
|
||||
* bundle, and runs one 3-sample panel per bundle with the eval's request options.
|
||||
*
|
||||
* Usage:
|
||||
* bun run scripts/judge-controls.ts --list (builds every bundle, no API calls)
|
||||
* bun run scripts/judge-controls.ts --run (36 judge calls; writes docs/evals/judge-controls/results.json)
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { buildQaWorkflowJudgePrompt, callJudge, judgePanelMean, judgePanelMedian, DEFAULT_JUDGE_MAX_TOKENS, JUDGE_SCORE_DIMENSIONS, type CallJudgeOptions } from '../test/helpers/llm-judge';
|
||||
import { buildWorkflowJudgePrompt, readWorkflowJudgeInput, QA_DISCOVERY_REFERENCES, WORKFLOW_JUDGE_RESPONSE_SCHEMA } from '../test/helpers/workflow-judge-input';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const FLOORS = { clarity: 3, completeness: 3, actionability: 4 } as const;
|
||||
const SAMPLES = 3;
|
||||
|
||||
type Judge = {
|
||||
name: string;
|
||||
read: () => string;
|
||||
prompt: (text: string) => string;
|
||||
options: CallJudgeOptions;
|
||||
defect: (text: string) => string;
|
||||
};
|
||||
|
||||
function cut(text: string, start: string, end: string): string {
|
||||
const from = text.indexOf(start);
|
||||
const to = text.indexOf(end, from + start.length);
|
||||
if (from < 0 || to < 0) throw new Error(`defect anchor missing: ${start} … ${end}`);
|
||||
return text.slice(0, from) + text.slice(to);
|
||||
}
|
||||
|
||||
function insertAfter(text: string, anchor: string, line: string): string {
|
||||
const at = text.indexOf(anchor);
|
||||
if (at < 0) throw new Error(`defect anchor missing: ${anchor}`);
|
||||
return text.slice(0, at + anchor.length) + '\n\n' + line + text.slice(at + anchor.length);
|
||||
}
|
||||
|
||||
const workflow = (skillPath: string, startMarker: string, endMarker: string | null, references: readonly string[] | undefined) =>
|
||||
() => readWorkflowJudgeInput({ root: ROOT, skillPath, startMarker, endMarker, references }).text;
|
||||
|
||||
const JUDGES: Judge[] = [
|
||||
{
|
||||
name: 'qa/SKILL.md workflow',
|
||||
read: workflow('qa/SKILL.md', '# /qa: Test', null,
|
||||
['qa/templates/functional-report-template.md', 'qa/templates/qa-report-template.md', 'qa/references/issue-taxonomy.md']),
|
||||
prompt: buildQaWorkflowJudgePrompt,
|
||||
options: {},
|
||||
defect: text => cut(text, '## Phase 10: Report', '## Phase 11:'),
|
||||
},
|
||||
{
|
||||
name: 'qa-only/SKILL.md workflow',
|
||||
read: workflow('qa-only/SKILL.md', '# /qa-only:', null, QA_DISCOVERY_REFERENCES.filter(file => file !== 'qa/sections/exploratory.md')),
|
||||
prompt: text => buildWorkflowJudgePrompt({ judgeContext: 'a report-only QA testing workflow',
|
||||
judgeGoal: 'how to select browser/native functional/mixed targets, explore safely with repository tools, report exact contract evidence and coverage limits, conditionally load browser/DX instructions and never mutate product/tests/Git through any tool' }, { files: [], text }),
|
||||
options: { max_tokens: DEFAULT_JUDGE_MAX_TOKENS, jsonSchema: WORKFLOW_JUDGE_RESPONSE_SCHEMA },
|
||||
defect: text => insertAfter(text, '# /qa-only: Report-Only QA Testing', 'After reporting each bug, fix it in the source and commit the fix.'),
|
||||
},
|
||||
{
|
||||
name: 'review/SKILL.md workflow',
|
||||
read: workflow('review/SKILL.md', '## Step 0: Detect platform and base branch', null,
|
||||
[...QA_DISCOVERY_REFERENCES, 'review/checklist.md', 'review/specialists/testing.md']),
|
||||
prompt: text => buildWorkflowJudgePrompt({ judgeContext: 'a pre-landing review with bounded exploratory QA',
|
||||
judgeGoal: 'how to review and explore changed behavior even for small diffs without a plan or server, preserve report-only discovery and the test_stub ASK gate, handle incomplete probes honestly, and rerun affected evidence after approved repairs',
|
||||
agentCapability: 'frontier' }, { files: [], text }),
|
||||
options: { max_tokens: DEFAULT_JUDGE_MAX_TOKENS },
|
||||
defect: text => cut(text, '## Step 3: Get the diff', '## Step 3.4'),
|
||||
},
|
||||
{
|
||||
name: 'setup-deploy/SKILL.md platform setup',
|
||||
read: workflow('setup-deploy/SKILL.md', '### Step 2: Detect platform', '## Important Rules', undefined),
|
||||
prompt: text => buildWorkflowJudgePrompt({ judgeContext: 'a deployment configuration setup workflow that detects deploy platforms and writes config to CLAUDE.md',
|
||||
judgeGoal: 'how to detect deploy platforms (Fly.io, Render, Vercel, Netlify, Heroku, GitHub Actions, custom), gather platform-specific configuration (URLs, status commands, health checks, custom hooks), and persist everything to CLAUDE.md for future automated use' }, { files: [], text }),
|
||||
options: { max_tokens: DEFAULT_JUDGE_MAX_TOKENS, jsonSchema: WORKFLOW_JUDGE_RESPONSE_SCHEMA },
|
||||
defect: text => {
|
||||
const step = text.indexOf('### Step 2: Detect platform');
|
||||
const open = text.indexOf('```bash', step);
|
||||
const close = text.indexOf('```', open + 7);
|
||||
if (step < 0 || open < 0 || close < 0) throw new Error('defect anchor missing: Step 2 bash block');
|
||||
return text.slice(0, open) + text.slice(close + 3);
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
/** Keep only file boundaries and headings. */
|
||||
const negative = (text: string) => text.split('\n')
|
||||
.filter(line => /^--- (BEGIN|END) FILE /.test(line) || /^#{1,6} /.test(line)).join('\n');
|
||||
|
||||
function bundles(judge: Judge): Record<'positive' | 'negative' | 'defect', string> {
|
||||
const positive = judge.read();
|
||||
return { positive, negative: negative(positive), defect: judge.defect(positive) };
|
||||
}
|
||||
|
||||
const mode = process.argv[2];
|
||||
if (mode !== '--list' && mode !== '--run') {
|
||||
console.error('usage: bun run scripts/judge-controls.ts --list | --run');
|
||||
process.exit(2);
|
||||
}
|
||||
if (mode === '--list') {
|
||||
for (const judge of JUDGES) {
|
||||
const built = bundles(judge);
|
||||
console.log(judge.name, Object.fromEntries(Object.entries(built).map(([k, v]) => [k, v.length])));
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
type Sample = { clarity: number; completeness: number; actionability: number; reasoning: string };
|
||||
const results = await Promise.all(JUDGES.flatMap(judge => Object.entries(bundles(judge)).map(async ([bundle, text]) => {
|
||||
const prompt = judge.prompt(text);
|
||||
const samples = await Promise.all(Array.from({ length: SAMPLES }, () => callJudge<Sample>(prompt, undefined, judge.options)));
|
||||
const mean = judgePanelMean(samples, JUDGE_SCORE_DIMENSIONS);
|
||||
const median = judgePanelMedian(samples, JUDGE_SCORE_DIMENSIONS);
|
||||
const verdict = Object.entries(FLOORS).every(([key, floor]) => median[key as keyof typeof FLOORS] >= floor) ? 'pass' : 'fail';
|
||||
return { judge: judge.name, bundle, verdict, median, mean, samples };
|
||||
})));
|
||||
const out = path.join(ROOT, 'docs/evals/judge-controls/results.json');
|
||||
fs.mkdirSync(path.dirname(out), { recursive: true });
|
||||
fs.writeFileSync(out, JSON.stringify({ recorded_at: new Date().toISOString(), results }, null, 2) + '\n');
|
||||
for (const r of results) console.log(`${r.judge} | ${r.bundle} | ${r.verdict} | median ${JSON.stringify(r.median)} mean ${JSON.stringify(r.mean)}`);
|
||||
@@ -0,0 +1,354 @@
|
||||
/**
|
||||
* eval:pass-rates report views over stored trial records (plan 0.1):
|
||||
* session headroom (--headroom), the verdict red ledger with the all-green
|
||||
* probability (--reds), and one census's triage view (--run <id>). Pure
|
||||
* functions plus the run-scoped fetches they need; scripts/eval-flake-rank.ts
|
||||
* parses flags, loads history and dispatches here.
|
||||
*
|
||||
* Headroom is per armed session (session ledger, `sessions[]` on a record):
|
||||
* elapsed over the timeout it armed. A case's headroom is its slowest
|
||||
* session; a timed-out sample is censored (its true duration is unknown).
|
||||
* Records without sessions are unknown, never scored; their case wall
|
||||
* (JUnit or shard clock) is shown separately as an upper bound.
|
||||
*
|
||||
* The all-green probability is an approximation that assumes independent
|
||||
* verdicts: P(green) = Π (1 - p_i) over the cases a census verdicts, where
|
||||
* p_i = (reds_i + K * p̄) / (verdicts_i + K) shrinks each case's red rate
|
||||
* toward the pooled rate p̄. Its interval recomputes the same product with p̄
|
||||
* replaced by the pooled rate's 95% Wilson bounds.
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { panelVerdict, type TrialOutcomeRecord } from '../../test/helpers/eval-store';
|
||||
import { HEADROOM_FAIL, HEADROOM_WARN } from '../../test/helpers/eval-budgets';
|
||||
import { sanitizeFixedFenceLine } from './published-text';
|
||||
import type { HistoryFetcher, WeeklyRun } from './ci-history';
|
||||
import { CASE_SHARDED_FILES, trialShardKey } from './paid-cases';
|
||||
import { shardSlug } from '../test-paid-shards';
|
||||
import { CENSUS_RED_GUIDE } from './paid-report';
|
||||
|
||||
export { CENSUS_RED_GUIDE };
|
||||
|
||||
/** Below this many session samples a case's headroom is insufficient, never a pass. */
|
||||
export const HEADROOM_MIN_SAMPLES = 3;
|
||||
/** Pseudo-verdicts of pooled rate added to each case's red rate. */
|
||||
export const RED_RATE_SHRINK = 4;
|
||||
/** Largest slice artifact --run downloads. */
|
||||
export const EVIDENCE_MAX_BYTES = 64 * 1024 * 1024;
|
||||
|
||||
type Rec = TrialOutcomeRecord;
|
||||
const pct = (x: number) => `${Math.round(x * 1000) / 10}%`;
|
||||
const secs = (ms: number) => `${Math.round(ms / 1000)}s`;
|
||||
|
||||
// --- Headroom ---
|
||||
|
||||
export interface SessionHeadroom { key: string; samples: number; censored: number; maxRatio: number; maxElapsedMs: number; budgetMs: number }
|
||||
export interface CaseHeadroom {
|
||||
case: string;
|
||||
status: 'over' | 'warn' | 'ok' | 'insufficient' | 'unknown';
|
||||
sessions: SessionHeadroom[];
|
||||
/** Slowest session's worst ratio, when any session was recorded. */
|
||||
worst?: SessionHeadroom;
|
||||
/** Largest case wall (JUnit or shard clock) over the window: an upper bound, never a session clock. */
|
||||
caseWallMs: number;
|
||||
runs: number;
|
||||
}
|
||||
|
||||
export function headroom(records: readonly Rec[]): CaseHeadroom[] {
|
||||
const byCase = new Map<string, Rec[]>();
|
||||
for (const r of records) if (r.outcome !== 'skipped' && r.execution !== 'reused') byCase.set(r.case, [...(byCase.get(r.case) ?? []), r]);
|
||||
return [...byCase.entries()].map(([id, rs]) => {
|
||||
const sessions = new Map<string, SessionHeadroom>();
|
||||
for (const s of rs.flatMap(r => r.sessions ?? [])) {
|
||||
if (!s.budget_ms) continue;
|
||||
const key = s.key;
|
||||
const entry = sessions.get(key) ?? { key, samples: 0, censored: 0, maxRatio: 0, maxElapsedMs: 0, budgetMs: s.budget_ms };
|
||||
entry.samples++;
|
||||
if (s.end === 'session_timeout' || s.end === 'observer_timeout') entry.censored++;
|
||||
const ratio = s.elapsed_ms / s.budget_ms;
|
||||
if (ratio > entry.maxRatio) Object.assign(entry, { maxRatio: ratio, maxElapsedMs: s.elapsed_ms, budgetMs: s.budget_ms });
|
||||
sessions.set(key, entry);
|
||||
}
|
||||
const list = [...sessions.values()].sort((a, b) => b.maxRatio - a.maxRatio);
|
||||
const worst = list[0];
|
||||
const status: CaseHeadroom['status'] = !worst ? 'unknown' : worst.samples < HEADROOM_MIN_SAMPLES ? 'insufficient'
|
||||
: worst.maxRatio > HEADROOM_FAIL ? 'over' : worst.maxRatio > HEADROOM_WARN ? 'warn' : 'ok';
|
||||
return { case: id, status, sessions: list, ...(worst ? { worst } : {}), caseWallMs: Math.max(0, ...rs.map(r => r.duration_ms)),
|
||||
runs: new Set(rs.map(r => r.run_id ?? 'local')).size };
|
||||
}).sort((a, b) => (b.worst?.maxRatio ?? -1) - (a.worst?.maxRatio ?? -1) || b.caseWallMs - a.caseWallMs);
|
||||
}
|
||||
|
||||
/** --gate alarms: a case whose slowest session ran above HEADROOM_FAIL of its armed budget. */
|
||||
export function headroomAlarms(cases: readonly CaseHeadroom[]): Array<{ kind: 'headroom'; case: string; message: string }> {
|
||||
return cases.filter(c => c.status === 'over').map(c => ({ kind: 'headroom' as const, case: c.case,
|
||||
message: `${c.case} session ${c.worst!.key}: max ${secs(c.worst!.maxElapsedMs)} of ${secs(c.worst!.budgetMs)} (${pct(c.worst!.maxRatio)})`
|
||||
+ `${c.worst!.censored ? `, ${c.worst!.censored} timed-out sample(s) censored` : ''} over ${c.worst!.samples} sample(s), above the ${pct(HEADROOM_FAIL)} cap. `
|
||||
+ 'Cut work in the skill or fixture; budgets are never raised (TESTING_INTERNALS "Timeout policy").' }));
|
||||
}
|
||||
|
||||
export function formatHeadroom(cases: readonly CaseHeadroom[], critical: readonly CriticalPath[] = []): string[] {
|
||||
const lines = [`headroom: slowest armed session per case (warn > ${pct(HEADROOM_WARN)}, alarm > ${pct(HEADROOM_FAIL)}; `
|
||||
+ `insufficient below ${HEADROOM_MIN_SAMPLES} samples; censored = timed out)`,
|
||||
' status session max/budget samples case wall (upper bound, not a session clock) case'];
|
||||
for (const c of cases) {
|
||||
const w = c.worst;
|
||||
const session = w ? `${pct(w.maxRatio)} ${secs(w.maxElapsedMs)}/${secs(w.budgetMs)}` : 'unknown (no session ledger)';
|
||||
lines.push(` ${c.status.padEnd(12)} ${session.padEnd(26)} ${(w ? `${w.samples}${w.censored ? ` (${w.censored} censored)` : ''}` : '-').padEnd(7)} `
|
||||
+ `${secs(c.caseWallMs).padEnd(45)} ${c.case}${w ? ` [${w.key}]` : ''}`);
|
||||
}
|
||||
return [...lines, ...formatCriticalPath(critical)];
|
||||
}
|
||||
|
||||
export function formatCriticalPath(critical: readonly CriticalPath[]): string[] {
|
||||
return critical.map(cp => `critical path ${cp.run}: ${cp.slowest ? `${cp.slowest.name} ${secs(cp.slowest.ms)}` : 'unknown'}`
|
||||
+ ` of ${cp.jobs} slice job(s), run wall ${cp.wallMs === null ? 'unknown' : secs(cp.wallMs)}${cp.files?.length ? `; slice files: ${cp.files.join(', ')}` : ''}`);
|
||||
}
|
||||
|
||||
// --- Critical path from runner slice timings ---
|
||||
|
||||
export interface JobTiming { name: string; startedAt: string | null; completedAt: string | null }
|
||||
export interface CriticalPath { run: string; jobs: number; slowest: { name: string; ms: number } | null; wallMs: number | null; files?: string[] }
|
||||
|
||||
const SLICE_JOB = /^(eval-slices|eval-codex-slices|gate-census) \((\d+)\)$/;
|
||||
|
||||
export function criticalPath(run: string, jobs: readonly JobTiming[], plan?: { entries: Array<{ file: string; slice?: number; status?: string }> }): CriticalPath {
|
||||
const timed = jobs.filter(j => SLICE_JOB.test(j.name) && j.startedAt && j.completedAt)
|
||||
.map(j => ({ name: j.name, ms: Date.parse(j.completedAt!) - Date.parse(j.startedAt!) }));
|
||||
const slowest = timed.sort((a, b) => b.ms - a.ms)[0] ?? null;
|
||||
const all = jobs.filter(j => j.startedAt && j.completedAt);
|
||||
const wallMs = all.length ? Math.max(...all.map(j => Date.parse(j.completedAt!))) - Math.min(...all.map(j => Date.parse(j.startedAt!))) : null;
|
||||
const slice = slowest ? Number(SLICE_JOB.exec(slowest.name)![2]) : null;
|
||||
const files = plan && slice !== null && slowest!.name.startsWith('gate-census')
|
||||
? plan.entries.filter(e => e.slice === slice && e.status === 'planned').map(e => e.file) : undefined;
|
||||
return { run, jobs: timed.length, slowest, wallMs, ...(files ? { files } : {}) };
|
||||
}
|
||||
|
||||
// --- Red ledger ---
|
||||
|
||||
export interface VerdictRed { run: string; lane: string; case: string; status: string; trials: Rec[] }
|
||||
export interface CensusReds { run: string; lane: string; verdicts: number; reds: VerdictRed[]; byClass: Record<string, number>; byCause: Record<string, number> }
|
||||
export interface RedLedger {
|
||||
censuses: CensusReds[];
|
||||
pooled: { verdicts: number; reds: number; rate: number; lo: number; hi: number };
|
||||
perCase: Array<{ case: string; verdicts: number; reds: number; rate: number }>;
|
||||
allGreen: { p: number; lo: number; hi: number; verdicts: number } | null;
|
||||
}
|
||||
|
||||
function wilson(k: number, n: number, z = 1.96): { lo: number; hi: number } {
|
||||
if (n === 0) return { lo: 0, hi: 1 };
|
||||
const p = k / n, d = 1 + z * z / n, c = p + z * z / (2 * n), m = z * Math.sqrt(p * (1 - p) / n + z * z / (4 * n * n));
|
||||
return { lo: Math.max(0, (c - m) / d), hi: Math.min(1, (c + m) / d) };
|
||||
}
|
||||
|
||||
/** One panelVerdict per case per census lane (earliest attempt), exactly as the report computes it. */
|
||||
export function verdictsOf(records: readonly Rec[]): Array<{ run: string; lane: string; case: string; status: string; failsLane: boolean; trials: Rec[] }> {
|
||||
const groups = new Map<string, Rec[]>();
|
||||
for (const r of records) {
|
||||
if (r.policy_version < 1 || r.execution === 'reused') continue;
|
||||
// A JUnit case is one verdict per shard file, as the report counts it; trial shards form one panel.
|
||||
const key = `${r.run_id ?? 'local'}\0${r.lane ?? r.tier}\0${r.case}\0${r.source === 'shard' ? '' : r.file}`;
|
||||
groups.set(key, [...(groups.get(key) ?? []), r]);
|
||||
}
|
||||
return [...groups.entries()].map(([key, rs]) => {
|
||||
const [run, lane, id] = key.split('\0') as [string, string, string, string];
|
||||
const attempt = Math.min(...rs.map(r => r.attempt));
|
||||
const trials = rs.filter(r => r.attempt === attempt);
|
||||
const v = panelVerdict({ case: id, kind: trials[0]!.kind, panel: trials[0]!.panel, quarantined: trials[0]!.quarantined,
|
||||
trials: trials.map(t => ({ trial: t.trial, outcome: t.outcome, attempt, ...(t.failure_class ? { failure_class: t.failure_class } : {}) })) });
|
||||
return { run, lane, case: id, status: v.status, failsLane: v.failsLane, trials };
|
||||
});
|
||||
}
|
||||
|
||||
export function redLedger(records: readonly Rec[]): RedLedger {
|
||||
const verdicts = verdictsOf(records).filter(v => v.status !== 'SKIPPED');
|
||||
const censusKey = (v: { run: string; lane: string }) => `${v.run}\0${v.lane}`;
|
||||
const censuses = new Map<string, CensusReds>();
|
||||
for (const v of verdicts) {
|
||||
const c = censuses.get(censusKey(v)) ?? { run: v.run, lane: v.lane, verdicts: 0, reds: [], byClass: {}, byCause: {} };
|
||||
c.verdicts++;
|
||||
if (v.failsLane) {
|
||||
c.reds.push({ run: v.run, lane: v.lane, case: v.case, status: v.status, trials: v.trials });
|
||||
for (const t of v.trials.filter(t => t.outcome === 'failed')) {
|
||||
c.byClass[t.failure_class ?? 'assertion'] = (c.byClass[t.failure_class ?? 'assertion'] ?? 0) + 1;
|
||||
c.byCause[t.failure_cause ?? 'unrecorded'] = (c.byCause[t.failure_cause ?? 'unrecorded'] ?? 0) + 1;
|
||||
}
|
||||
}
|
||||
censuses.set(censusKey(v), c);
|
||||
}
|
||||
const reds = verdicts.filter(v => v.failsLane).length;
|
||||
const rate = verdicts.length ? reds / verdicts.length : 0;
|
||||
const pooled = { verdicts: verdicts.length, reds, rate, ...wilson(reds, verdicts.length) };
|
||||
const byCase = new Map<string, { verdicts: number; reds: number }>();
|
||||
for (const v of verdicts) {
|
||||
const e = byCase.get(v.case) ?? { verdicts: 0, reds: 0 };
|
||||
e.verdicts++; if (v.failsLane) e.reds++;
|
||||
byCase.set(v.case, e);
|
||||
}
|
||||
const perCase = [...byCase.entries()].map(([id, e]) => ({ case: id, ...e, rate: (e.reds + RED_RATE_SHRINK * rate) / (e.verdicts + RED_RATE_SHRINK) }))
|
||||
.sort((a, b) => b.rate - a.rate || (a.case < b.case ? -1 : 1));
|
||||
const ordered = [...censuses.values()];
|
||||
const latestRun = ordered.map(c => c.run).sort().at(-1);
|
||||
const census = verdicts.filter(v => v.run === latestRun);
|
||||
const green = (prior: number) => census.reduce((p, v) => {
|
||||
const e = byCase.get(v.case)!;
|
||||
return p * (1 - (e.reds + RED_RATE_SHRINK * prior) / (e.verdicts + RED_RATE_SHRINK));
|
||||
}, 1);
|
||||
const allGreen = census.length ? { p: green(rate), lo: green(pooled.hi), hi: green(pooled.lo), verdicts: census.length } : null;
|
||||
return { censuses: ordered.sort((a, b) => (a.run < b.run ? -1 : a.run > b.run ? 1 : a.lane < b.lane ? -1 : 1)), pooled, perCase, allGreen };
|
||||
}
|
||||
|
||||
const tally = (counts: Record<string, number>) => Object.entries(counts).sort(([a, x], [b, y]) => y - x || (a < b ? -1 : 1)).map(([k, n]) => `${k} ${n}`).join(', ') || 'none';
|
||||
|
||||
export function formatRedLedger(ledger: RedLedger, opts: { top?: number } = {}): string[] {
|
||||
const lines = ['reds: verdict reds per census lane by failure class and machine cause (cause "unrecorded" = written before failure_cause existed)'];
|
||||
for (const c of ledger.censuses) {
|
||||
lines.push(` ${c.run} ${c.lane}: ${c.reds.length} red of ${c.verdicts} verdicts${c.reds.length ? ` — class: ${tally(c.byClass)}; cause: ${tally(c.byCause)}` : ''}`);
|
||||
for (const red of c.reds) {
|
||||
const failed = red.trials.filter(t => t.outcome === 'failed');
|
||||
lines.push(` ✗ ${red.case} ${red.status} ${failed.map(t => `t${t.trial} ${t.failure_class ?? 'assertion'}/${t.failure_cause ?? 'unrecorded'}`
|
||||
+ `${t.error ? ` — ${sanitizeFixedFenceLine(t.error, 160)}` : ''}`).join('; ')}`);
|
||||
}
|
||||
}
|
||||
const p = ledger.pooled;
|
||||
lines.push(`pooled: ${p.reds} red of ${p.verdicts} verdicts = ${pct(p.rate)} per verdict [95% ${pct(p.lo)}–${pct(p.hi)}]`);
|
||||
const top = ledger.perCase.filter(c => c.reds > 0).slice(0, opts.top ?? 15);
|
||||
if (top.length) lines.push(`per-case red rates (shrunk toward the pooled rate, K=${RED_RATE_SHRINK}):`, ...top.map(c => ` ${pct(c.rate).padStart(6)} ${c.reds}/${c.verdicts} ${c.case}`));
|
||||
const g = ledger.allGreen;
|
||||
lines.push(g ? `all-green probability (approximation, assumes independent verdicts): ${pct(g.p)} for the latest census's ${g.verdicts} verdicts [${pct(g.lo)}–${pct(g.hi)} with the pooled rate's 95% interval as prior]; `
|
||||
+ 'formula Π(1 - p_i) over its cases; rerun: bun run eval:pass-rates --reds' : 'all-green probability: no post-policy verdicts');
|
||||
lines.push(`guide: ${CENSUS_RED_GUIDE}`);
|
||||
return lines;
|
||||
}
|
||||
|
||||
// --- One census's triage view ---
|
||||
|
||||
/** Fetches the report views make beyond the history loader: a named run and its slice jobs. */
|
||||
export interface RunJobsFetcher { listJobs(repo: string, runId: number): JobTiming[]; getRun(repo: string, runId: number): WeeklyRun }
|
||||
|
||||
function ghJson(args: string[]): string[] {
|
||||
const result = spawnSync('gh', args, { timeout: 120_000, maxBuffer: 64 * 1024 * 1024 });
|
||||
if (result.status !== 0) throw new Error(`gh ${args.slice(0, 2).join(' ')} failed: ${String(result.stderr || result.error || '').trim()}`);
|
||||
return result.stdout.toString('utf8').split('\n').filter(Boolean);
|
||||
}
|
||||
|
||||
export const GH_JOBS: RunJobsFetcher = {
|
||||
listJobs: (repo, runId) => ghJson(['api', `repos/${repo}/actions/runs/${runId}/jobs?per_page=100`, '--paginate',
|
||||
'--jq', '.jobs[] | {name, startedAt: .started_at, completedAt: .completed_at}']).map(line => JSON.parse(line) as JobTiming),
|
||||
getRun: (repo, runId) => JSON.parse(ghJson(['api', `repos/${repo}/actions/runs/${runId}`,
|
||||
'--jq', '{id, attempt: .run_attempt, sha: .head_sha, branch: .head_branch, createdAt: .created_at}'])[0]!) as WeeklyRun,
|
||||
};
|
||||
|
||||
/** The manifest shard key a record ran under: its trial shard, its case shard, or its file. */
|
||||
export function recordShardKey(r: Pick<Rec, 'file' | 'case' | 'trial' | 'source'>): string {
|
||||
return r.source === 'shard' ? trialShardKey(r.file, r.case, r.trial) : CASE_SHARDED_FILES.includes(r.file) ? `${r.file}#${r.case}` : r.file;
|
||||
}
|
||||
|
||||
/**
|
||||
* The run's reds with their values and history, and the slice artifacts that
|
||||
* hold their transcripts. Only the slices a red names are downloaded; a slice
|
||||
* over EVIDENCE_MAX_BYTES is reported, never silently skipped.
|
||||
*/
|
||||
export function triageRun(opts: { repo: string; run: WeeklyRun; records: readonly Rec[]; caseFilter?: string; cacheDir: string;
|
||||
fetcher: HistoryFetcher; download: (match: (name: string) => boolean, maxBytes: number) => string[] }): string[] {
|
||||
const runId = String(opts.run.id);
|
||||
const mine = opts.records.filter(r => r.run_id === runId);
|
||||
const reds = verdictsOf(mine).filter(v => v.failsLane && (!opts.caseFilter || v.case === opts.caseFilter));
|
||||
const lines = [`run ${runId} (${opts.run.branch} @ ${opts.run.sha.slice(0, 9)}, ${opts.run.createdAt}): ${reds.length} verdict red(s)${opts.caseFilter ? ` for ${opts.caseFilter}` : ''}`];
|
||||
const history = verdictsOf(opts.records);
|
||||
const plans = new Map<string, Map<string, number>>();
|
||||
const planFor = (lane: string) => {
|
||||
const name = lane.startsWith('gate') && lane.includes('census') ? 'gate-census-plan' : 'paid-plan';
|
||||
if (!plans.has(name)) {
|
||||
const files = new Map<string, number>();
|
||||
for (const dir of opts.download(n => n === name, 8 * 1024 * 1024)) {
|
||||
try {
|
||||
const manifest = JSON.parse(fs.readFileSync(path.join(dir, 'manifest.json'), 'utf8'));
|
||||
for (const e of manifest.entries ?? []) if (e.status === 'planned' && Number.isInteger(e.slice)) files.set(e.file, e.slice);
|
||||
} catch { /* no manifest: slices stay unknown */ }
|
||||
}
|
||||
plans.set(name, files);
|
||||
}
|
||||
return { prefix: name === 'gate-census-plan' ? 'gate-census' : 'paid-slice', files: plans.get(name)! };
|
||||
};
|
||||
const artifacts = opts.fetcher.listArtifacts(opts.repo, opts.run.id);
|
||||
for (const red of reds) {
|
||||
const runs = history.filter(v => v.case === red.case);
|
||||
lines.push(`✗ ${red.case} ${red.lane} ${red.status} (history: ${runs.filter(v => !v.failsLane).length}/${runs.length} verdicts green)`);
|
||||
for (const t of red.trials.filter(t => t.outcome === 'failed')) {
|
||||
const d = t.failure_detail;
|
||||
const detail = !d ? '' : 'judge' in d ? ` ${d.judge.map(j => `${j.dimension} ${j.mean} < ${j.threshold} (${j.samples} samples)${j.rationale ? ` "${j.rationale}"` : ''}`).join('; ')}`
|
||||
: ` Expected: ${d.expected} · Received: ${d.received}`;
|
||||
lines.push(sanitizeFixedFenceLine(` t${t.trial}: ${t.failure_class ?? 'assertion'} / cause ${t.failure_cause ?? 'unrecorded'}`
|
||||
+ `${t.failure_cause_evidence ? `: ${t.failure_cause_evidence}` : ''}${t.error ? ` — ${t.error}` : ''}${detail}`, 2000));
|
||||
const plan = planFor(red.lane);
|
||||
const slice = plan.files.get(recordShardKey(t));
|
||||
if (slice === undefined) { lines.push(' evidence: slice unknown (no plan artifact for this lane)'); continue; }
|
||||
const name = `${plan.prefix}-${slice}-a${t.attempt}`;
|
||||
const artifact = artifacts.find(a => a.name === name);
|
||||
if (!artifact) { lines.push(` evidence: ${name} (expired or missing)`); continue; }
|
||||
if (artifact.size > EVIDENCE_MAX_BYTES) { lines.push(` evidence: ${name} too large to fetch (${Math.round(artifact.size / 1024 / 1024)} MB)`); continue; }
|
||||
const [dir] = opts.download(n => n === name, EVIDENCE_MAX_BYTES);
|
||||
lines.push(` evidence: ${dir ? path.join(dir, 'shards', shardSlug([recordShardKey(t)])) : `${name} (download failed)`}`);
|
||||
}
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
// --- Flags ---
|
||||
|
||||
export const PASS_RATES_USAGE = `Usage: bun run eval:pass-rates [options]
|
||||
(no view flag) per-case trial pass rates with 95% Wilson intervals
|
||||
--headroom slowest armed session per case vs its budget, case wall upper bound, census critical path
|
||||
--reds verdict reds per census by failure class and cause, pooled and per-case rates, all-green probability
|
||||
--run <id> one census: each red with its values, history and fetched transcript evidence
|
||||
--case <id> restrict every view, the JSON and the alarms to one registered case
|
||||
--runs <n> completed runs per branch to read (default 10)
|
||||
--branch <name> branch whose runs to read beside main (default: current branch)
|
||||
--repo <owner/repo> --workflow <file> --dir <path> (repeatable, local artifacts; no network)
|
||||
--backfill also import legacy slice artifacts, labeled pre-policy
|
||||
--since-days <n> legacy eval files window for --dir (default 60)
|
||||
--json machine-readable output
|
||||
--gate exit 1 on ACTION REQUIRED (pass-rate alarms and sessions above ${pct(HEADROOM_FAIL)} of budget)
|
||||
--help this text (no network)`;
|
||||
|
||||
export interface PassRatesArgs {
|
||||
view: 'rates' | 'headroom' | 'reds' | 'run';
|
||||
runId?: number; caseFilter?: string; runs: number; sinceDays: number; dirs: string[];
|
||||
json: boolean; gate: boolean; backfill: boolean; repo?: string; workflow?: string; branch?: string;
|
||||
}
|
||||
|
||||
/** Parse eval:pass-rates flags; unknown flags, bad values and unknown case ids are errors (exit 2, before any fetch). */
|
||||
export function parsePassRatesArgs(argv: readonly string[], isKnownCase: (id: string) => boolean): { help: true } | { error: string } | PassRatesArgs {
|
||||
const out: PassRatesArgs = { view: 'rates', runs: 10, sinceDays: 60, dirs: [], json: false, gate: false, backfill: false };
|
||||
const views: string[] = [];
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const arg = argv[i]!;
|
||||
const value = () => { const v = argv[++i]; if (v === undefined || v.startsWith('--')) throw new Error(`${arg} needs a value`); return v; };
|
||||
const count = () => { const v = value(); if (!/^[1-9]\d*$/.test(v)) throw new Error(`${arg} needs a positive integer, got ${v}`); return Number(v); };
|
||||
try {
|
||||
switch (arg) {
|
||||
case '--help': case '-h': return { help: true };
|
||||
case '--headroom': views.push('headroom'); break;
|
||||
case '--reds': views.push('reds'); break;
|
||||
case '--run': views.push('run'); out.runId = count(); break;
|
||||
case '--case': out.caseFilter = value(); break;
|
||||
case '--runs': out.runs = count(); break;
|
||||
case '--since-days': out.sinceDays = count(); break;
|
||||
case '--dir': out.dirs.push(value()); break;
|
||||
case '--repo': out.repo = value(); break;
|
||||
case '--workflow': out.workflow = value(); break;
|
||||
case '--branch': out.branch = value(); break;
|
||||
case '--json': out.json = true; break;
|
||||
case '--gate': out.gate = true; break;
|
||||
case '--backfill': out.backfill = true; break;
|
||||
default: return { error: `unknown flag ${arg}` };
|
||||
}
|
||||
} catch (error) { return { error: (error as Error).message }; }
|
||||
}
|
||||
if (views.length > 1) return { error: `choose one of --headroom, --reds, --run (got ${views.join(', ')})` };
|
||||
if (views[0]) out.view = views[0] as PassRatesArgs['view'];
|
||||
if (out.caseFilter !== undefined && !isKnownCase(out.caseFilter)) return { error: `unknown case ${out.caseFilter} (not an E2E, judge or paid test file id)` };
|
||||
if (out.view === 'run' && out.dirs.length) return { error: '--run reads one GitHub run; it cannot combine with --dir' };
|
||||
return out;
|
||||
}
|
||||
@@ -251,6 +251,7 @@ export const CODEX_CI_FILES: readonly string[] = [
|
||||
'test/skill-e2e-outside-voice.test.ts',
|
||||
'test/skill-e2e-outside-plan-disabled.test.ts',
|
||||
'test/skill-e2e-safety-codex-boundary.test.ts',
|
||||
'test/codex-e2e-multiblock-live.test.ts',
|
||||
];
|
||||
|
||||
/**
|
||||
|
||||
+158
-42
@@ -5,15 +5,19 @@ import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { normalizeRelativePath } from './shard-engine';
|
||||
import { EVAL_POLICY } from '../../test/helpers/periodic-exclude-data';
|
||||
import { STALL_WINDOW_MS } from '../../test/helpers/eval-budgets';
|
||||
import {
|
||||
isFinalizedEvalResultFile, failureClassOf, panelVerdict, sanitizeTrialError, formatTrialOutcomes, TRIAL_OUTCOME_SCHEMA, TRIAL_OUTCOMES_FILE,
|
||||
type EvalCaseKind, type PanelVerdict, type TrialFailureClass, type TrialOutcome, type TrialOutcomeRecord,
|
||||
trialCostKnown, trialFailureFields, trialSessions, type TrialFailureDetail,
|
||||
} from '../../test/helpers/eval-store';
|
||||
import { readSessionLedger, type SessionLedgerRow } from '../../test/helpers/session-ledger';
|
||||
import { publishedFence, sanitizeFixedFenceLine } from './published-text';
|
||||
import { E2E_KINDS } from '../../test/helpers/touchfiles-data';
|
||||
import { manualReviewProblem } from '../../test/helpers/cookie-workflow-manual-review';
|
||||
import { writeNegativeReceipt, writePanelReceipt } from '../e2e-shard-reuse';
|
||||
import { E2E_TOUCHFILES, E2E_TIERS, LLM_JUDGE_TOUCHFILES } from '../../test/helpers/touchfiles';
|
||||
import { CASE_TEST_NAMES, type CaseTrialPlan, caseTrialPlan, fileCaseRegistration, shardCaseId, shardFile, trialShardKey } from './paid-cases';
|
||||
import { CASE_TEST_NAMES, type CaseTrialPlan, caseTestNamePattern, caseTrialPlan, fileCaseRegistration, shardCaseId, shardFile, trialShardKey } from './paid-cases';
|
||||
import { type ManifestEntry, PAID_TEST_DURATIONS_FILE, type PaidRunManifest, type SliceResult, collectorOutcomeCounts, formatProfileCoverage, loadPaidTestDurations, mergePaidTestDurations, parseRunManifest, trialPanelKey, verifySliceResults, writePaidTestDurations } from './paid-plan';
|
||||
import { DEFAULT_JOBS, type PaidCaseSelection, type PaidTier, ROOT, type ShardTrialRecord } from './paid-types';
|
||||
import { collectPaidTestFiles, isAllSkippedPass, shardSlug, paidSelectionEnv } from './paid-select';
|
||||
@@ -21,14 +25,39 @@ import type { RunShardsOptions } from '../test-paid-shards';
|
||||
|
||||
// ─── Local diagnosis: one case through the CI panel runner (A9) ────────────
|
||||
|
||||
/** The one paid file that statically registers `id`, else a thrown reason. */
|
||||
/** How --case runs exactly one case: trials selected by its Bun test name, or its whole file when that file registers no other case. */
|
||||
export interface CaseSelection { file: string; mode: 'name' | 'file'; reason: string }
|
||||
|
||||
const escapeRe = (text: string) => text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
|
||||
/**
|
||||
* The paid file that registers `id` and how a trial selects it. A file whose
|
||||
* registration is complete wins over one that also lists the id; a test
|
||||
* registered literally under the id (or its CASE_TEST_NAMES label) is
|
||||
* selected by name; a loop that registers ids at runtime is selected by the
|
||||
* id it names its test; a file registering only this case runs whole (its
|
||||
* test names are not the id). Anything else throws before any process starts.
|
||||
*/
|
||||
export function caseSelection(id: string, rootDir = ROOT, discovered = collectPaidTestFiles(rootDir)): CaseSelection {
|
||||
const files = discovered.map(file => {
|
||||
const source = fs.readFileSync(path.join(rootDir, file), 'utf8');
|
||||
return { file, source, ...fileCaseRegistration(file, source) };
|
||||
}).filter(f => f.registered.includes(id));
|
||||
const owners = files.filter(f => f.known).length ? files.filter(f => f.known) : files;
|
||||
if (owners.length !== 1) throw new Error(`--case ${id}: ${owners.length ? `registered by ${owners.map(f => f.file).join(', ')}` : 'no paid file statically registers it'}; it needs exactly one`);
|
||||
const owner = owners[0]!;
|
||||
const name = escapeRe(CASE_TEST_NAMES[id] ?? id);
|
||||
if (owner.registered.length === 1) return { file: owner.file, mode: 'file', reason: 'the whole file (it registers no other case)' };
|
||||
if (new RegExp(`\\b(?:test|it|test\\.\\w+|test(?:Concurrent)?IfSelected)\\s*\\(\\s*(['"\`])${name}\\1|testName\\s*:\\s*(['"\`])${name}\\2`).test(owner.source)) {
|
||||
return { file: owner.file, mode: 'name', reason: `its Bun test is named ${CASE_TEST_NAMES[id] ?? id}` };
|
||||
}
|
||||
if (owner.computed) return { file: owner.file, mode: 'name', reason: `its loop names the Bun test ${id} at runtime` };
|
||||
throw new Error(`--case ${id}: ${owner.file} registers several cases and none of its tests is named ${id}; add its test name to CASE_TEST_NAMES`);
|
||||
}
|
||||
|
||||
/** The one paid file that registers `id`, else a thrown reason. */
|
||||
export function caseFile(id: string, rootDir = ROOT, discovered = collectPaidTestFiles(rootDir)): string {
|
||||
const owners = discovered.filter(file => {
|
||||
const { registered, known } = fileCaseRegistration(file, fs.readFileSync(path.join(rootDir, file), 'utf8'));
|
||||
return known && registered.includes(id);
|
||||
});
|
||||
if (owners.length !== 1) throw new Error(`--case ${id}: ${owners.length ? `registered by ${owners.join(', ')}` : 'no paid file statically registers it'}; it needs exactly one`);
|
||||
return owners[0]!;
|
||||
return caseSelection(id, rootDir, discovered).file;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -41,15 +70,13 @@ export function caseFile(id: string, rootDir = ROOT, discovered = collectPaidTes
|
||||
export async function runCaseDiagnosis(id: string, options: {
|
||||
trials?: number; jobs?: number; withinShardConcurrency?: number; timeoutMs?: number; rootDir?: string; env?: NodeJS.ProcessEnv;
|
||||
evalDirBase?: string; commandFor?: RunShardsOptions['commandFor']; log?: (line: string) => void; file?: string;
|
||||
/** The CLI passes its own runner: importing the entry module during its top-level await never resolves. */
|
||||
runShards?: typeof import('../test-paid-shards').runPaidShards;
|
||||
} = {}): Promise<PanelVerdict> {
|
||||
const rootDir = options.rootDir ?? ROOT;
|
||||
const log = options.log ?? ((line: string) => console.log(line));
|
||||
const file = options.file ?? caseFile(id, rootDir);
|
||||
const testName = CASE_TEST_NAMES[id] ?? id;
|
||||
if (!options.commandFor && !fs.readFileSync(path.join(rootDir, file), 'utf8').includes(testName)) {
|
||||
throw new Error(`--case ${id}: ${file} has no literal Bun test named "${testName}", so a trial could not select it. `
|
||||
+ 'Run the whole file (bun test <file> with EVALS=1) or add its literal name to CASE_TEST_NAMES.');
|
||||
}
|
||||
const selection = options.file ? { file: options.file, mode: 'name' as const } : caseSelection(id, rootDir);
|
||||
const file = selection.file;
|
||||
const policy = caseTrialPlan(id);
|
||||
const n = options.trials ?? policy.panel.n;
|
||||
const plan: CaseTrialPlan = { ...policy, panel: { n, k: policy.panel.k === policy.panel.n ? n : Math.min(policy.panel.k, n) } };
|
||||
@@ -57,14 +84,27 @@ export async function runCaseDiagnosis(id: string, options: {
|
||||
const tier = E2E_TIERS[id] as PaidTier;
|
||||
log(`[test:paid] --case ${id}: ${n} trial(s) of ${file} (kind ${plan.kind}, PASS at ${plan.panel.k}/${n}${plan.quarantined ? ', quarantined' : ''}), tier=${tier}`);
|
||||
// The runner is the CLI module; load it lazily so this library never imports it statically (no cycle).
|
||||
const { runPaidShards } = await import('../test-paid-shards');
|
||||
const summary = await runPaidShards(keys.map(key => [key]), {
|
||||
jobs: Math.min(options.jobs ?? DEFAULT_JOBS, n), withinShardConcurrency: options.withinShardConcurrency, timeoutMs: options.timeoutMs,
|
||||
rootDir, log, commandFor: options.commandFor, evalDirBase: options.evalDirBase,
|
||||
trials: Object.fromEntries(keys.map(key => [key, plan])),
|
||||
const runPaidShards = options.runShards ?? (await import('../test-paid-shards')).runPaidShards;
|
||||
const shardOptions = { withinShardConcurrency: options.withinShardConcurrency, timeoutMs: options.timeoutMs, rootDir, log, commandFor: options.commandFor,
|
||||
env: { ...(options.env ?? process.env), EVALS: '1', EVALS_TIER: tier, EVALS_PREFLIGHT_OK: '1', EVALS_ALL: '1',
|
||||
...paidSelectionEnv('full', { e2e: [id], judges: [] }, `--case ${id}`) },
|
||||
});
|
||||
...paidSelectionEnv('full', { e2e: [id], judges: [] }, `--case ${id}`) } };
|
||||
if (selection.mode === 'file') {
|
||||
// The whole file is the case: one file shard per trial, each with its own eval dir, judged by shard status as CI does.
|
||||
const fileTrials = [];
|
||||
for (let trial = 1; trial <= n; trial++) {
|
||||
const [outcome] = (await runPaidShards([[file]], { ...shardOptions, jobs: 1,
|
||||
...(options.evalDirBase ? { evalDirBase: path.join(options.evalDirBase, `t${trial}`) } : {}) })).outcomes;
|
||||
if (!outcome || outcome.runnerError !== undefined || isAllSkippedPass(outcome) || !['passed', 'failed', 'timed-out'].includes(outcome.status)) continue;
|
||||
fileTrials.push({ trial, outcome: outcome.status === 'passed' ? 'passed' as const : 'failed' as const,
|
||||
...(outcome.status === 'passed' ? {} : { failure_class: outcome.status === 'timed-out' ? 'timeout' as const : 'assertion' as const }) });
|
||||
}
|
||||
const verdict = panelVerdict({ case: id, kind: plan.kind, panel: plan.panel, trials: fileTrials, quarantined: plan.quarantined });
|
||||
log(formatPanelLine({ ...verdict, file, slices: {} }, tier));
|
||||
return verdict;
|
||||
}
|
||||
const summary = await runPaidShards(keys.map(key => [key]), { ...shardOptions,
|
||||
jobs: Math.min(options.jobs ?? DEFAULT_JOBS, n), evalDirBase: options.evalDirBase,
|
||||
trials: Object.fromEntries(keys.map(key => [key, plan])) });
|
||||
const trials = summary.outcomes.flatMap(outcome => outcome.trial && outcome.trial.outcome !== null ? [{
|
||||
trial: outcome.trial.trial, outcome: outcome.trial.outcome, ...(outcome.trial.failure_class ? { failure_class: outcome.trial.failure_class } : {}),
|
||||
...(outcome.trial.exit_reason ? { exit_reason: outcome.trial.exit_reason } : {}), ...(outcome.trial.error ? { error: outcome.trial.error } : {}),
|
||||
@@ -187,6 +227,20 @@ export interface JUnitCensus {
|
||||
|
||||
const bump = (counts: Map<string, number>, key: string) => counts.set(key, (counts.get(key) ?? 0) + 1);
|
||||
|
||||
/** Ledger rows of one JUnit case: its own id or test name, else the shard's only case. */
|
||||
function caseSessions(rows: SessionLedgerRow[], id: string, onlyCase: boolean): SessionLedgerRow[] {
|
||||
return rows.filter(row => row.case === id || (!row.case && (onlyCase || row.test_name === id || row.test_name === CASE_TEST_NAMES[id])));
|
||||
}
|
||||
|
||||
/** The artifact holding a slice's shards: the downloaded directory's own name when it is one. */
|
||||
/** The red-census guide the report and every eval:pass-rates view link. */
|
||||
export const CENSUS_RED_GUIDE = 'docs/evals/census-red.md';
|
||||
|
||||
export function sliceArtifactName(root: string, slice: number, attempt: number): string {
|
||||
const name = path.basename(root);
|
||||
return /^(?:paid-slice|gate-census)-\d+-a\d+$/.test(name) ? name : `slice ${slice} attempt ${attempt}`;
|
||||
}
|
||||
|
||||
export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact[], recordsByShard: Map<string, any[]>,
|
||||
base: Record<string, unknown>, cliVersion?: string): JUnitCensus {
|
||||
const census: JUnitCensus = { ruleCases: [], history: [], failedShards: new Set(), skipped: [], deselected: new Map(), shardSkipReasons: new Map(), unattributed: [] };
|
||||
@@ -207,6 +261,7 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact
|
||||
else census.unattributed.push(`${key} :: ${tc.name || '(no name)'} (${tc.outcome})`);
|
||||
}
|
||||
const records = recordsByShard.get(key) ?? [];
|
||||
const ledger = readSessionLedger(path.join(root, 'shards', shardSlug([key])));
|
||||
for (const [id, cases] of byCase) {
|
||||
const kind = (E2E_KINDS[id] ?? 'rule') as EvalCaseKind;
|
||||
const failed = cases.find(tc => tc.outcome === 'failed');
|
||||
@@ -216,18 +271,24 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact
|
||||
const failureClass: TrialFailureClass | undefined = !failed ? undefined
|
||||
: failed.failureType === 'TimeoutError' ? 'timeout' : failedRecord ? failureClassOf(failedRecord) : 'assertion';
|
||||
const error = sanitizeTrialError(failedRecord?.error ?? failed?.message);
|
||||
const rerun = Object.hasOwn(E2E_TIERS, id) ? rerunCommand(manifest.tier, id, 1) : `EVALS=1 EVALS_TIER=${manifest.tier} bun test ${shardFile(key)}`;
|
||||
const sessions = caseSessions(ledger, id, byCase.size === 1);
|
||||
// A case's JUnit times sum its concurrent tests (plan-mode-no-op: 346 s in a 176 s shard), so its wall is capped by the shard's.
|
||||
const diagnosis = failed && failureClass ? trialFailureFields({ failure_class: failureClass, exit_reason: failedRecord?.exit_reason,
|
||||
error: failedRecord?.error ?? failed.message, sessions, record: failedRecord }, STALL_WINDOW_MS) : undefined;
|
||||
if (failed) census.failedShards.add(key);
|
||||
if (caseOutcome === 'skipped') {
|
||||
bump(reasons, RUNTIME_SKIP_REASON);
|
||||
census.skipped.push(`◌ ${id} ${key} (${cases.map(tc => tc.name).join('; ')}): ${RUNTIME_SKIP_REASON}`);
|
||||
}
|
||||
census.ruleCases.push({ id, kind, outcome: caseOutcome,
|
||||
...(failed ? { line: `✗ ${id} ${kind} FAIL ${failureClass}${failedRecord?.exit_reason === 'timeout' && failedRecord?.timeout_at_turn !== undefined ? ` at turn ${failedRecord.timeout_at_turn}` : ''}${error ? ` — ${error}` : ''} [slice ${result.sliceIndex}, attempt ${result.attempt ?? 1}] rerun: ${rerun}` } : {}) });
|
||||
...(failed ? { line: `✗ ${id} ${kind} FAIL ${failureClass}${failedRecord?.exit_reason === 'timeout' && failedRecord?.timeout_at_turn !== undefined ? ` at turn ${failedRecord.timeout_at_turn}` : ''}${error ? ` — ${error}` : ''}`
|
||||
+ `${diagnosisText(diagnosis, error)} [slice ${result.sliceIndex}, attempt ${result.attempt ?? 1}]`
|
||||
+ `${evidencePointer(sliceArtifactName(root, result.sliceIndex, result.attempt ?? 1), key, typeof base.run_id === 'string' ? base.run_id : undefined, id)}`
|
||||
+ ` after a repair: ${afterRepairCommand(manifest.tier, id, shardFile(key))}` } : {}) });
|
||||
census.history.push({ ...base, case: id, file: shardFile(key), kind, trial: 1, panel: { n: 1, k: 1 }, outcome: caseOutcome,
|
||||
...(failureClass ? { failure_class: failureClass } : {}), ...(failedRecord?.exit_reason ? { exit_reason: String(failedRecord.exit_reason) } : {}),
|
||||
...(error && failed ? { error } : {}), duration_ms: cases.reduce((sum, tc) => sum + tc.timeMs, 0),
|
||||
cost_usd: Math.round(mine.reduce((sum: number, r: any) => sum + (Number(r.cost_usd) || 0), 0) * 100) / 100,
|
||||
...(error && failed ? { error } : {}), ...diagnosis, ...trialSessions(sessions), duration_ms: Math.min(cases.reduce((sum, tc) => sum + tc.timeMs, 0), outcome.elapsedMs),
|
||||
cost_usd: Math.round(mine.reduce((sum: number, r: any) => sum + (Number(r.cost_usd) || 0), 0) * 100) / 100, ...trialCostKnown(mine, sessions),
|
||||
...(typeof mine[0]?.model === 'string' ? { model: mine[0].model } : {}), ...(cliVersion ? { cli_version: cliVersion } : {}),
|
||||
quarantined: false, execution: outcome.reused ? 'reused' : 'executed', source: 'junit' } as TrialOutcomeRecord);
|
||||
}
|
||||
@@ -290,7 +351,9 @@ export function panelReports(manifest: PaidRunManifest, results: SliceResult[],
|
||||
return [{ trial: t.trial, outcome: t.outcome, attempt, ...(t.failure_class ? { failure_class: t.failure_class } : {}),
|
||||
...(t.exit_reason ? { exit_reason: t.exit_reason } : {}), ...(t.error ? { error: t.error } : {}),
|
||||
...(got.outcome.reused ? { execution: 'reused' as const } : {}),
|
||||
...(t.timeout_at_turn !== undefined ? { timeout_at_turn: t.timeout_at_turn } : {}) }];
|
||||
...(t.timeout_at_turn !== undefined ? { timeout_at_turn: t.timeout_at_turn } : {}),
|
||||
...(t.failure_cause ? { failure_cause: t.failure_cause } : {}), ...(t.failure_cause_evidence ? { failure_cause_evidence: t.failure_cause_evidence } : {}),
|
||||
...(t.failure_detail ? { failure_detail: t.failure_detail } : {}) }];
|
||||
});
|
||||
const verdict = panelVerdict({ case: shardCaseId(key)!, kind: plan.kind, panel: plan.panel, trials, quarantined: plan.quarantined });
|
||||
// Reuse is whole-panel only: every trial reused from one run, or none.
|
||||
@@ -303,29 +366,69 @@ export function panelReports(manifest: PaidRunManifest, results: SliceResult[],
|
||||
});
|
||||
}
|
||||
|
||||
type TrialDiagnosis = { failure_cause?: string; failure_cause_evidence?: string; failure_detail?: TrialFailureDetail };
|
||||
|
||||
/** Cause (when it says more than the class), its evidence and the failure detail, appended to a red line. */
|
||||
function diagnosisText(d: TrialDiagnosis | undefined, error?: string): string {
|
||||
if (!d?.failure_cause) return '';
|
||||
const evidence = d.failure_cause_evidence && d.failure_cause_evidence !== error ? `: ${d.failure_cause_evidence}` : '';
|
||||
const detail = !d.failure_detail ? ''
|
||||
: 'judge' in d.failure_detail ? ` ${d.failure_detail.judge.map(j => `${j.dimension} ${j.mean} < ${j.threshold} (${j.samples} samples)${j.rationale ? ` "${j.rationale}"` : ''}`).join('; ')}`
|
||||
: ` Expected: ${d.failure_detail.expected} · Received: ${d.failure_detail.received}`;
|
||||
return ` · cause ${d.failure_cause}${evidence}${detail}`;
|
||||
}
|
||||
|
||||
/** Where a red's transcript lives and how to fetch it. */
|
||||
function evidencePointer(artifact: string, key: string, runId: string | undefined, id: string): string {
|
||||
const arg = /^[\w./-]+$/.test(id) ? id : `'${id.replace(/'/g, `'\\''`)}'`;
|
||||
return ` evidence: ${artifact}/shards/${shardSlug([key])}${runId ? ` (fetch: bun run eval:pass-rates --run ${runId} --case ${arg})` : ''}`;
|
||||
}
|
||||
|
||||
/** Why one failed trial failed, in one line: a case timeout names its turn. */
|
||||
function trialCause(trial: PanelVerdict['trials'][number] & { timeout_at_turn?: number }): string {
|
||||
function trialCause(trial: PanelVerdict['trials'][number] & TrialDiagnosis & { timeout_at_turn?: number }): string {
|
||||
if (trial.outcome === 'skipped') return `t${trial.trial}: skipped`;
|
||||
const cls = trial.failure_class ?? 'assertion';
|
||||
const head = cls === 'timeout' || trial.exit_reason === 'timeout'
|
||||
? `timeout${trial.timeout_at_turn !== undefined ? ` at turn ${trial.timeout_at_turn}` : ''}`
|
||||
: cls;
|
||||
return `t${trial.trial}: ${head}${trial.exit_reason && trial.exit_reason !== 'timeout' ? ` (${trial.exit_reason})` : ''}${trial.error ? ` — ${trial.error}` : ''}`;
|
||||
const exit = trial.exit_reason === 'success' ? ' (session completed; check failed)'
|
||||
: trial.exit_reason && trial.exit_reason !== 'timeout' ? ` (${trial.exit_reason})` : '';
|
||||
return `t${trial.trial}: ${head}${exit}${trial.error ? ` — ${trial.error}` : ''}${diagnosisText(trial, trial.error)}`;
|
||||
}
|
||||
|
||||
export function rerunCommand(tier: PaidTier, id: string, trials: number): string {
|
||||
return `bun run scripts/test-paid-shards.ts --tier ${tier} --case ${id}${trials > 1 ? ` --trials ${trials}` : ''}`;
|
||||
/**
|
||||
* A paid run of exactly one case, for after a repair (never on unchanged
|
||||
* inputs): E2E cases through --case, judges through their selection and an
|
||||
* exact test-name filter, a keyless file (its own case) by file.
|
||||
*/
|
||||
export function afterRepairCommand(tier: PaidTier, id: string, file: string, trials = 1): string {
|
||||
if (Object.hasOwn(E2E_TIERS, id) && resolvesAlone(id)) return `bun run scripts/test-paid-shards.ts --tier ${tier} --case ${id}${trials > 1 ? ` --trials ${trials}` : ''}`;
|
||||
if (Object.hasOwn(LLM_JUDGE_TOUCHFILES, id)) {
|
||||
return `EVALS=1 EVALS_TIER=${tier} EVALS_JUDGE_SELECTION_JSON='${JSON.stringify({ version: 1, selected: [id], reason: 'after a repair' })}' bun test ${file} -t '${caseTestNamePattern([id])}'`;
|
||||
}
|
||||
return `EVALS=1 EVALS_TIER=${tier} bun test ${file}`;
|
||||
}
|
||||
|
||||
/** One line per non-PASS or split panel verdict. */
|
||||
export function formatPanelLine(panel: PanelReport, tier: PaidTier): string {
|
||||
const resolved = new Map<string, boolean>();
|
||||
/** Whether --case can select this E2E id (a case several files register runs by its shard file instead). */
|
||||
function resolvesAlone(id: string): boolean {
|
||||
if (!resolved.has(id)) { try { caseSelection(id); resolved.set(id, true); } catch { resolved.set(id, false); } }
|
||||
return resolved.get(id)!;
|
||||
}
|
||||
|
||||
/** One line per non-PASS or split panel verdict. `artifacts` names each slice's artifact (slice -> name). */
|
||||
export function formatPanelLine(panel: PanelReport, tier: PaidTier, ctx: { artifacts?: Map<number, string>; runId?: string } = {}): string {
|
||||
const mark = panel.status === 'PASS' ? '⚠' : panel.failsLane ? '✗' : '◌';
|
||||
const label = panel.status === 'PASS' ? `PASS ${panel.passed}/${panel.panel.n}` : `${panel.status} ${panel.passed}/${panel.panel.n}`;
|
||||
const causes = panel.trials.filter(t => t.outcome !== 'passed').map(t => trialCause(t as any));
|
||||
const where = Object.entries(panel.slices).map(([trial, slice]) => `t${trial}@slice ${slice}`).join(', ');
|
||||
const red = panel.trials.find(t => t.outcome === 'failed');
|
||||
const slice = red ? panel.slices[red.trial] : undefined;
|
||||
const evidence = red && slice !== undefined
|
||||
? evidencePointer(ctx.artifacts?.get(slice) ?? `slice ${slice} attempt ${panel.attempt}`, trialShardKey(panel.file, panel.case, red.trial), ctx.runId, panel.case) : '';
|
||||
return `${mark} ${panel.case} ${panel.kind}${panel.quarantined ? ' (quarantined)' : ''} ${label} (${panel.marks})`
|
||||
+ `${causes.length ? ` ${causes.join('; ')}` : ''}${panel.status === 'INCOMPLETE' || panel.status === 'SKIPPED' ? ` [${panel.reason}]` : ''}`
|
||||
+ `${where ? ` [${where}, attempt ${panel.attempt}]` : ''} rerun: ${rerunCommand(tier, panel.case, panel.panel.n)}`;
|
||||
+ `${where ? ` [${where}, attempt ${panel.attempt}]` : ''}${evidence} after a repair: ${afterRepairCommand(tier, panel.case, panel.file, panel.panel.n)}`;
|
||||
}
|
||||
|
||||
export interface ReportHeadline {
|
||||
@@ -348,7 +451,13 @@ export interface ReportHeadline {
|
||||
redispatchEligible: boolean;
|
||||
}
|
||||
|
||||
export function formatHeadline(h: ReportHeadline): string[] {
|
||||
/** Known cost, never a bare total when any trial's cost is unknown, and no total when none is known. */
|
||||
export function formatCost(knownUsd: number, unknownTrials: number, trials: number): string {
|
||||
if (trials > 0 && unknownTrials === trials) return `cost unknown (no billing captured for any of ${trials} trial(s))`;
|
||||
return `cost $${knownUsd.toFixed(2)}${unknownTrials ? ` known + ${unknownTrials} of ${trials} trial(s) cost unknown` : ''}`;
|
||||
}
|
||||
|
||||
export function formatHeadline(h: ReportHeadline, cost: { unknownTrials: number; trials: number } = { unknownTrials: 0, trials: 0 }): string[] {
|
||||
const c = h.counts;
|
||||
const minutes = h.wallMs === null ? 'unknown' : `${Math.floor(h.wallMs / 60_000)}m${String(Math.round((h.wallMs % 60_000) / 1000)).padStart(2, '0')}s`;
|
||||
return [
|
||||
@@ -356,7 +465,7 @@ export function formatHeadline(h: ReportHeadline): string[] {
|
||||
` rule ${c.rule.passed}/${c.rule.total} · behavior ${c.behavior.passed}/${c.behavior.total}${c.behavior.split ? ` (${c.behavior.split} split)` : ''}`
|
||||
+ ` · judge ${c.judge.passed}/${c.judge.total} · quarantined ${c.quarantined.total} (${c.quarantined.failingLane} failing the lane)`,
|
||||
` SKIPPED ${c.skipped} · INFRA ${c.infra} · INCOMPLETE ${c.incomplete} · unattributed ${c.unattributed} · ACTION REQUIRED ${h.actionRequired}`,
|
||||
` wall ${minutes} · cost $${h.costUsd.toFixed(2)}${h.redispatchEligible ? ' · every red is machine-classified INFRA/INCOMPLETE: eligible for ONE re-dispatch as a new run (EVAL_POLICY.infraRedispatch); report both runs' : ''}`,
|
||||
` wall ${minutes} · ${formatCost(h.costUsd, cost.unknownTrials, cost.trials)}${h.redispatchEligible ? ' · every red is machine-classified INFRA/INCOMPLETE: eligible for ONE re-dispatch as a new run (EVAL_POLICY.infraRedispatch); report both runs' : ''}`,
|
||||
];
|
||||
}
|
||||
|
||||
@@ -538,11 +647,16 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo
|
||||
history.push({ ...common(attempt), case: t.case, file: shardFile(outcome.files[0]!), kind: t.kind, trial: t.trial, panel: t.panel,
|
||||
outcome: t.outcome, ...(t.outcome === 'failed' ? { failure_class: t.failure_class ?? 'assertion' } : {}),
|
||||
...(t.exit_reason ? { exit_reason: t.exit_reason } : {}), ...(t.error ? { error: t.error } : {}),
|
||||
duration_ms: t.duration_ms, cost_usd: t.cost_usd, ...(t.model ? { model: t.model } : {}),
|
||||
...(t.outcome === 'failed' && t.failure_cause ? { failure_cause: t.failure_cause } : {}),
|
||||
...(t.outcome === 'failed' && t.failure_cause_evidence ? { failure_cause_evidence: t.failure_cause_evidence } : {}),
|
||||
...(t.outcome === 'failed' && t.failure_detail ? { failure_detail: t.failure_detail } : {}), ...(t.sessions ? { sessions: t.sessions } : {}),
|
||||
duration_ms: t.duration_ms, cost_usd: t.cost_usd, ...(t.cost_known === false ? { cost_known: false } : {}), ...(t.model ? { model: t.model } : {}),
|
||||
...(outcome.reused ? { input_identity: outcome.reused.inputKey } : {}),
|
||||
quarantined: t.quarantined, execution: outcome.reused ? 'reused' : 'executed', source: 'shard' } as TrialOutcomeRecord);
|
||||
}
|
||||
}
|
||||
const sliceArtifacts = new Map(artifacts.filter(a => (a.result.attempt ?? 1) === primary)
|
||||
.map(a => [a.result.sliceIndex, sliceArtifactName(a.root, a.result.sliceIndex, primary)] as const));
|
||||
const census = junitCensus(manifest, artifacts.filter(a => (a.result.attempt ?? 1) === primary), recordsByShard,
|
||||
common(primary), env.GSTACK_CLAUDE_CLI_VERSION);
|
||||
history.push(...census.history);
|
||||
@@ -568,7 +682,7 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo
|
||||
if (got && got.o.status === 'passed') return [];
|
||||
if (got && census.failedShards.has(normalizeRelativePath(entry.file))) return [];
|
||||
const id = shardCaseId(entry.file);
|
||||
return [`✗ ${entry.file} rule shard ${got ? got.o.status : 'NOT REPORTED'}${got?.o.runnerError ? ` — ${sanitizeTrialError(got.o.runnerError)}` : ''} [slice ${entry.slice}, attempt ${primary}]${id ? ` rerun: ${rerunCommand(manifest.tier, id, 1)}` : ''}`];
|
||||
return [`✗ ${entry.file} rule shard ${got ? got.o.status : 'NOT REPORTED'}${got?.o.runnerError ? ` — ${sanitizeTrialError(got.o.runnerError)}` : ''} [slice ${entry.slice}, attempt ${primary}]${id ? ` after a repair: ${afterRepairCommand(manifest.tier, id, shardFile(entry.file))}` : ''}`];
|
||||
});
|
||||
const behaviorPanels = panels.filter(p => !p.quarantined && p.kind === 'behavior');
|
||||
const lanePanels = panels.filter(p => !p.quarantined);
|
||||
@@ -582,7 +696,7 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo
|
||||
const failureLines = [
|
||||
...ruleShardFailures,
|
||||
...ruleCases.filter(c => c.line).map(c => c.line!),
|
||||
...panels.filter(p => p.status !== 'PASS' || p.split).map(p => formatPanelLine(p, manifest.tier)),
|
||||
...panels.filter(p => p.status !== 'PASS' || p.split).map(p => formatPanelLine(p, manifest.tier, { artifacts: sliceArtifacts, runId })),
|
||||
];
|
||||
const red = verdict.problems.length > 0;
|
||||
const headline: ReportHeadline = {
|
||||
@@ -603,19 +717,20 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo
|
||||
costUsd: Math.round(costUsd * 100) / 100,
|
||||
redispatchEligible: red && infraOnly(verdict.problems),
|
||||
};
|
||||
const headlineLines = formatHeadline(headline);
|
||||
const executed = history.filter(r => r.attempt === primary && r.execution === 'executed' && r.outcome !== 'skipped');
|
||||
const headlineLines = formatHeadline(headline, { unknownTrials: executed.filter(r => r.cost_known === false).length, trials: executed.length });
|
||||
for (const line of headlineLines) console.log(line);
|
||||
if (failureLines.length) {
|
||||
console.log('[test:paid] failures and split verdicts:');
|
||||
for (const line of failureLines) console.log(` ${line}`);
|
||||
}
|
||||
for (const panel of laterPanels) console.log(` attempt ${panel.attempt} (re-run; reported, never replacing attempt ${primary}): ${formatPanelLine(panel, manifest.tier)}`);
|
||||
const fence = (lines: string[]) => ['```', ...lines.map(line => line.replace(/```/g, "'''")), '```'];
|
||||
const fence = (lines: string[]) => publishedFence(lines);
|
||||
fs.writeFileSync(summaryMdPath, [
|
||||
...fence(headlineLines),
|
||||
...(failureLines.length ? ['', '**Failures and split verdicts**', '', ...fence(failureLines)] : []),
|
||||
...(censusLines.length ? ['', '**Skipped, deselected and unattributed testcases**', '', ...fence(censusLines)] : []),
|
||||
...(verdict.problems.length ? ['', `**ACTION REQUIRED (${verdict.problems.length})**`, '', ...fence(verdict.problems.map(p => sanitizeTrialError(p) ?? p))] : []),
|
||||
...(verdict.problems.length ? ['', `**ACTION REQUIRED (${verdict.problems.length})**`, '', ...fence(verdict.problems.map(p => sanitizeTrialError(p) ?? p)), '', `Guide: ${CENSUS_RED_GUIDE}`] : []),
|
||||
].join('\n') + '\n');
|
||||
if (!manualProblems.length) fs.writeFileSync(summaryPath, JSON.stringify({ version: 2, files, totals: {
|
||||
...evidence, total: evidence.passed + evidence.failed + evidence.manual_accepted,
|
||||
@@ -626,7 +741,8 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo
|
||||
marks: p.marks, split: p.split, quarantined: p.quarantined, failsLane: p.failsLane, redClass: p.redClass, reason: p.reason,
|
||||
trials: p.trials.map(t => ({ trial: t.trial, outcome: t.outcome, ...(t.failure_class ? { failure_class: t.failure_class } : {}),
|
||||
...(t.exit_reason ? { exit_reason: t.exit_reason } : {}), ...(t.error ? { error: t.error } : {}) })) })),
|
||||
failures: failureLines.map(line => sanitizeTrialError(line) ?? line) }, null, 2) + '\n');
|
||||
// evals.yml wraps these in a fixed fence for the PR comment.
|
||||
failures: failureLines.map(line => sanitizeFixedFenceLine(line)) }, null, 2) + '\n');
|
||||
if (verdict.problems.length) {
|
||||
console.error(`[test:paid] report: ${verdict.problems.length} problem(s):`);
|
||||
for (const problem of verdict.problems) console.error(` ✗ ${problem}`);
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
*/
|
||||
import * as path from 'node:path';
|
||||
import { normalizeRelativePath, type LanePolicy } from './shard-engine';
|
||||
import type { EvalCaseKind, PanelShape, TrialFailureClass, TrialOutcome } from '../../test/helpers/eval-store';
|
||||
import type { EvalCaseKind, PanelShape, TrialFailureCause, TrialFailureClass, TrialFailureDetail, TrialOutcome, TrialSessionSummary } from '../../test/helpers/eval-store';
|
||||
|
||||
export const ROOT = path.resolve(import.meta.dir, '..', '..');
|
||||
|
||||
@@ -119,6 +119,11 @@ export interface ShardTrialRecord {
|
||||
cost_usd: number;
|
||||
duration_ms: number;
|
||||
model?: string;
|
||||
failure_cause?: TrialFailureCause;
|
||||
failure_cause_evidence?: string;
|
||||
failure_detail?: TrialFailureDetail;
|
||||
sessions?: TrialSessionSummary[];
|
||||
cost_known?: false;
|
||||
}
|
||||
|
||||
/** Upload reserve between a slice's in-process deadline and its CI job ceiling (ENG-2). */
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
/**
|
||||
* The one sanitizer for model- or fixture-written text that reaches a public
|
||||
* issue, PR comment or committed file (census report, tracking issue,
|
||||
* eval:pass-rates output). Control and bidi characters are stripped, the
|
||||
* repo's credential-scan patterns (lib/redact-engine, the engine the quality
|
||||
* gate's credential scan runs) are redacted, @-mentions are neutralized and
|
||||
* length is capped. Inline text also has HTML and Markdown link brackets
|
||||
* escaped; blocks go inside a code fence longer than any backtick run they
|
||||
* contain, so their content stays literal and cannot close the fence.
|
||||
*/
|
||||
import { redactFindingSpans } from '../../lib/redact-engine';
|
||||
|
||||
export const PUBLISHED_TEXT_MAX = 1000;
|
||||
|
||||
const cap = (text: string, max: number) => (text.length > max ? `${text.slice(0, max - 1)}…` : text);
|
||||
|
||||
function clean(text: string): string {
|
||||
// eslint-disable-next-line no-control-regex
|
||||
const stripped = text.replace(/[\u0000-\u0008\u000a-\u001f\u007f-\u009f\u200b-\u200f\u202a-\u202e\u2066-\u2069]/g, ' ').slice(0, 64 * 1024);
|
||||
const redacted = redactFindingSpans(stripped) ?? '[redacted: credential-shaped text that could not be isolated]';
|
||||
return redacted.replace(/@(?=[A-Za-z0-9_-])/g, '@\u200b');
|
||||
}
|
||||
|
||||
/** One inline line: cleaned, HTML, Markdown link brackets and backticks neutralized, capped. */
|
||||
export function sanitizePublishedText(text: string, max = PUBLISHED_TEXT_MAX): string {
|
||||
return cap(clean(text).replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>')
|
||||
.replace(/\[/g, '[').replace(/\]/g, ']').replace(/`/g, "'"), max);
|
||||
}
|
||||
|
||||
/** One line for a consumer that wraps it in a fixed three-backtick fence (the PR comment): backticks become quotes. */
|
||||
export function sanitizeFixedFenceLine(text: string, max = PUBLISHED_TEXT_MAX): string {
|
||||
return cap(clean(text).replace(/`/g, "'"), max);
|
||||
}
|
||||
|
||||
/** Cleaned, capped lines inside a fence longer than the longest backtick run they contain. */
|
||||
export function publishedFence(lines: readonly string[], max = PUBLISHED_TEXT_MAX): string[] {
|
||||
const body = lines.map(line => cap(clean(line), max));
|
||||
const longest = Math.max(0, ...body.flatMap(line => (line.match(/`+/g) ?? []).map(run => run.length)));
|
||||
const fence = '`'.repeat(Math.max(3, longest + 1));
|
||||
return [fence, ...body, fence];
|
||||
}
|
||||
+140
-138
@@ -1,109 +1,109 @@
|
||||
{
|
||||
"version": 2,
|
||||
"recordedAt": "2026-09-29T19:29:33.835Z",
|
||||
"recordedAt": "2026-10-04T18:45:27.037Z",
|
||||
"tiers": {
|
||||
"gate": {
|
||||
"test/llm-judge-recommendation.test.ts": 1000,
|
||||
"test/skill-e2e-ask-user-question-format-compliance.test.ts": 63904,
|
||||
"test/skill-e2e-ask-user-question-format-compliance.test.ts": 54768,
|
||||
"test/skill-e2e-autoplan-dual-voice.test.ts": 1000,
|
||||
"test/skill-e2e-bws.test.ts": 101092,
|
||||
"test/skill-e2e-bws.test.ts": 74261,
|
||||
"test/skill-e2e-context-skills.test.ts": 1000,
|
||||
"test/skill-e2e-coverage-audit.test.ts": 50849,
|
||||
"test/skill-e2e-cso.test.ts": 227394,
|
||||
"test/skill-e2e-deploy.test.ts#benchmark-workflow": 62156,
|
||||
"test/skill-e2e-deploy.test.ts#canary-workflow": 50673,
|
||||
"test/skill-e2e-deploy.test.ts#land-and-deploy-first-run": 125350,
|
||||
"test/skill-e2e-deploy.test.ts#land-and-deploy-review-gate": 177755,
|
||||
"test/skill-e2e-deploy.test.ts#land-and-deploy-workflow": 200984,
|
||||
"test/skill-e2e-deploy.test.ts#setup-deploy-workflow": 51171,
|
||||
"test/skill-e2e-coverage-audit.test.ts": 65669,
|
||||
"test/skill-e2e-cso.test.ts": 229620,
|
||||
"test/skill-e2e-deploy.test.ts#benchmark-workflow": 120646,
|
||||
"test/skill-e2e-deploy.test.ts#canary-workflow": 42804,
|
||||
"test/skill-e2e-deploy.test.ts#land-and-deploy-first-run": 103709,
|
||||
"test/skill-e2e-deploy.test.ts#land-and-deploy-review-gate": 147499,
|
||||
"test/skill-e2e-deploy.test.ts#land-and-deploy-workflow": 148339,
|
||||
"test/skill-e2e-deploy.test.ts#setup-deploy-workflow": 36798,
|
||||
"test/skill-e2e-design.test.ts": 261000,
|
||||
"test/skill-e2e-design.test.ts#design-review-detector-shim": 42811,
|
||||
"test/skill-e2e-design.test.ts#design-review-detector-shim-dom": 81914,
|
||||
"test/skill-e2e-design.test.ts#design-review-plugin-handoff": 126642,
|
||||
"test/skill-e2e-design.test.ts#plan-design-review-no-ui-scope": 24431,
|
||||
"test/skill-e2e-diagram.test.ts": 29998,
|
||||
"test/skill-e2e-docsync-spawned.test.ts": 137929,
|
||||
"test/skill-e2e-design.test.ts#design-review-detector-shim": 45224,
|
||||
"test/skill-e2e-design.test.ts#design-review-detector-shim-dom": 87823,
|
||||
"test/skill-e2e-design.test.ts#design-review-plugin-handoff": 74874,
|
||||
"test/skill-e2e-design.test.ts#plan-design-review-no-ui-scope": 39970,
|
||||
"test/skill-e2e-diagram.test.ts": 30124,
|
||||
"test/skill-e2e-docsync-spawned.test.ts": 108078,
|
||||
"test/skill-e2e-gbrain-roundtrip-local.test.ts": 1000,
|
||||
"test/skill-e2e-hermetic-canary.test.ts": 8813,
|
||||
"test/skill-e2e-investigate-owned-completion.test.ts": 45178,
|
||||
"test/skill-e2e-investigate-owned-termination.test.ts": 48608,
|
||||
"test/skill-e2e-hermetic-canary.test.ts": 4951,
|
||||
"test/skill-e2e-investigate-owned-completion.test.ts": 36179,
|
||||
"test/skill-e2e-investigate-owned-termination.test.ts": 48428,
|
||||
"test/skill-e2e-ios-device.test.ts": 1000,
|
||||
"test/skill-e2e-learnings.test.ts": 28899,
|
||||
"test/skill-e2e-office-hours-auto-mode.test.ts": 83101,
|
||||
"test/skill-e2e-learnings.test.ts": 33590,
|
||||
"test/skill-e2e-office-hours-auto-mode.test.ts": 78399,
|
||||
"test/skill-e2e-office-hours-brain-writeback.test.ts": 1000,
|
||||
"test/skill-e2e-office-hours-phase4.test.ts": 1000,
|
||||
"test/skill-e2e-office-hours.test.ts": 1000,
|
||||
"test/skill-e2e-plan-ceo-finding-floor.test.ts": 297738,
|
||||
"test/skill-e2e-plan-ceo-plan-mode.test.ts": 36812,
|
||||
"test/skill-e2e-plan-ceo-finding-floor.test.ts": 248087,
|
||||
"test/skill-e2e-plan-ceo-plan-mode.test.ts": 112067,
|
||||
"test/skill-e2e-plan-decision-classification.test.ts": 1000,
|
||||
"test/skill-e2e-plan-design-with-ui.test.ts": 240000,
|
||||
"test/skill-e2e-plan-devex-finding-floor.test.ts": 207635,
|
||||
"test/skill-e2e-plan-design-with-ui.test.ts": 285788,
|
||||
"test/skill-e2e-plan-devex-finding-floor.test.ts": 181446,
|
||||
"test/skill-e2e-plan-devex-peer-comparison-classification.test.ts": 1000,
|
||||
"test/skill-e2e-plan-devex-plan-mode.test.ts": 115679,
|
||||
"test/skill-e2e-plan-devex-plan-mode.test.ts": 105953,
|
||||
"test/skill-e2e-plan-format.test.ts": 1000,
|
||||
"test/skill-e2e-plan-mode-no-op.test.ts": 217184,
|
||||
"test/skill-e2e-plan-mode-no-op.test.ts": 271596,
|
||||
"test/skill-e2e-plan-prosons.test.ts": 1000,
|
||||
"test/skill-e2e-plan-tune.test.ts": 63925,
|
||||
"test/skill-e2e-plan-tune.test.ts": 70702,
|
||||
"test/skill-e2e-plan.test.ts": 251000,
|
||||
"test/skill-e2e-plan.test.ts#plan-review-report": 76406,
|
||||
"test/skill-e2e-plan.test.ts#plan-review-report": 197666,
|
||||
"test/skill-e2e-qa-bugs.test.ts": 1000,
|
||||
"test/skill-e2e-qa-callers.test.ts": 737164,
|
||||
"test/skill-e2e-qa-callers.test.ts#review-exploratory-small-cli": 294865,
|
||||
"test/skill-e2e-qa-callers.test.ts#ship-exploratory-late-input": 294865,
|
||||
"test/skill-e2e-qa-callers.test.ts#ship-exploratory-plan-checks": 294865,
|
||||
"test/skill-e2e-qa-callers.test.ts#ship-exploratory-small-cli": 294865,
|
||||
"test/skill-e2e-qa-callers.test.ts#ship-exploratory-unavailable": 294865,
|
||||
"test/skill-e2e-qa-functional-fix.test.ts": 235133,
|
||||
"test/skill-e2e-qa-functional.test.ts": 356641,
|
||||
"test/skill-e2e-qa-callers.test.ts#review-exploratory-small-cli": 224426,
|
||||
"test/skill-e2e-qa-callers.test.ts#ship-exploratory-late-input": 234381,
|
||||
"test/skill-e2e-qa-callers.test.ts#ship-exploratory-plan-checks": 192841,
|
||||
"test/skill-e2e-qa-callers.test.ts#ship-exploratory-small-cli": 147399,
|
||||
"test/skill-e2e-qa-callers.test.ts#ship-exploratory-unavailable": 191231,
|
||||
"test/skill-e2e-qa-functional-fix.test.ts": 200616,
|
||||
"test/skill-e2e-qa-functional.test.ts": 188766,
|
||||
"test/skill-e2e-qa-workflow.test.ts#qa-bootstrap": 35863,
|
||||
"test/skill-e2e-qa-workflow.test.ts#qa-only-no-fix": 201333,
|
||||
"test/skill-e2e-qa-workflow.test.ts#qa-quick": 169852,
|
||||
"test/skill-e2e-retro.test.ts": 141804,
|
||||
"test/skill-e2e-retro.test.ts": 200434,
|
||||
"test/skill-e2e-review-army.test.ts": 520000,
|
||||
"test/skill-e2e-review-army.test.ts#review-army-delivery-audit": 50219,
|
||||
"test/skill-e2e-review-army.test.ts#review-army-json-findings": 21332,
|
||||
"test/skill-e2e-review-army.test.ts#review-army-migration-safety": 145112,
|
||||
"test/skill-e2e-review-army.test.ts#review-army-perf-n-plus-one": 241299,
|
||||
"test/skill-e2e-review-army.test.ts#review-army-quality-score": 86836,
|
||||
"test/skill-e2e-review-attribution.test.ts": 77592,
|
||||
"test/skill-e2e-review.test.ts": 136132,
|
||||
"test/skill-e2e-review-army.test.ts#review-army-delivery-audit": 39313,
|
||||
"test/skill-e2e-review-army.test.ts#review-army-json-findings": 22950,
|
||||
"test/skill-e2e-review-army.test.ts#review-army-migration-safety": 162617,
|
||||
"test/skill-e2e-review-army.test.ts#review-army-perf-n-plus-one": 114005,
|
||||
"test/skill-e2e-review-army.test.ts#review-army-quality-score": 55496,
|
||||
"test/skill-e2e-review-attribution.test.ts": 80562,
|
||||
"test/skill-e2e-review.test.ts": 132863,
|
||||
"test/skill-e2e-safety-codex-boundary.test.ts": 1000,
|
||||
"test/skill-e2e-safety-codex-consult.test.ts": 82940,
|
||||
"test/skill-e2e-safety-codex-consult.test.ts": 150155,
|
||||
"test/skill-e2e-safety-design-risk.test.ts": 1000,
|
||||
"test/skill-e2e-safety-ios-demo.test.ts": 1000,
|
||||
"test/skill-e2e-safety-pair-agent.test.ts": 39860,
|
||||
"test/skill-e2e-safety-ship-evidence.test.ts": 106130,
|
||||
"test/skill-e2e-session-intelligence.test.ts": 55332,
|
||||
"test/skill-e2e-safety-pair-agent.test.ts": 36028,
|
||||
"test/skill-e2e-safety-ship-evidence.test.ts": 100833,
|
||||
"test/skill-e2e-session-intelligence.test.ts": 53080,
|
||||
"test/skill-e2e-shared-libs-paths.test.ts": 680000,
|
||||
"test/skill-e2e-shared-libs-paths.test.ts#shared-libs-review-index-flags": 190202,
|
||||
"test/skill-e2e-shared-libs-paths.test.ts#shared-libs-review-path-eligibility": 205615,
|
||||
"test/skill-e2e-shared-libs-paths.test.ts#shared-libs-review-prior-coverage": 214728,
|
||||
"test/skill-e2e-shared-libs-paths.test.ts#shared-libs-review-index-flags": 145972,
|
||||
"test/skill-e2e-shared-libs-paths.test.ts#shared-libs-review-path-eligibility": 170775,
|
||||
"test/skill-e2e-shared-libs-paths.test.ts#shared-libs-review-prior-coverage": 172422,
|
||||
"test/skill-e2e-shared-libs.test.ts": 658000,
|
||||
"test/skill-e2e-shared-libs.test.ts#shared-libs-read-only": 197454,
|
||||
"test/skill-e2e-shared-libs.test.ts#shared-libs-review-lifecycle": 267080,
|
||||
"test/skill-e2e-shared-libs.test.ts#shared-libs-review-revalidation": 316752,
|
||||
"test/skill-e2e-shared-libs.test.ts#shared-libs-unsupported-git": 170776,
|
||||
"test/skill-e2e-shared-libs.test.ts#shared-libs-read-only": 160556,
|
||||
"test/skill-e2e-shared-libs.test.ts#shared-libs-review-lifecycle": 164017,
|
||||
"test/skill-e2e-shared-libs.test.ts#shared-libs-review-revalidation": 281097,
|
||||
"test/skill-e2e-shared-libs.test.ts#shared-libs-unsupported-git": 87581,
|
||||
"test/skill-e2e-ship-docsync.test.ts": 129000,
|
||||
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-completion": 366116,
|
||||
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-current": 434135,
|
||||
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-failure": 164667,
|
||||
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-late-result": 214218,
|
||||
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-launch-failure": 140987,
|
||||
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-missing-asset": 142564,
|
||||
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-missing-marker": 117648,
|
||||
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-recovery": 225432,
|
||||
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-stale-after": 269572,
|
||||
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-stale-before": 197633,
|
||||
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-store": 396588,
|
||||
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-timeout-unsettled": 152278,
|
||||
"test/skill-e2e-ship-hook-consent.test.ts": 69995,
|
||||
"test/skill-e2e-ship-hook-refresh.test.ts": 70007,
|
||||
"test/skill-e2e-ship-skip.test.ts": 80318,
|
||||
"test/skill-e2e-skillify.test.ts": 191127,
|
||||
"test/skill-e2e-test-value.test.ts": 332000,
|
||||
"test/skill-e2e-third-party-actions.test.ts": 82102,
|
||||
"test/skill-e2e-triage.test.ts": 54088,
|
||||
"test/skill-e2e-workflow.test.ts": 184925,
|
||||
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-completion": 398349,
|
||||
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-current": 382630,
|
||||
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-failure": 238690,
|
||||
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-late-result": 132034,
|
||||
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-launch-failure": 53611,
|
||||
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-missing-asset": 106550,
|
||||
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-missing-marker": 59963,
|
||||
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-recovery": 132137,
|
||||
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-stale-after": 155419,
|
||||
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-stale-before": 164443,
|
||||
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-store": 392835,
|
||||
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-timeout-unsettled": 63645,
|
||||
"test/skill-e2e-ship-hook-consent.test.ts": 49572,
|
||||
"test/skill-e2e-ship-hook-refresh.test.ts": 40853,
|
||||
"test/skill-e2e-ship-skip.test.ts": 81807,
|
||||
"test/skill-e2e-skillify.test.ts": 177262,
|
||||
"test/skill-e2e-test-value.test.ts": 216843,
|
||||
"test/skill-e2e-third-party-actions.test.ts": 71961,
|
||||
"test/skill-e2e-triage.test.ts": 64816,
|
||||
"test/skill-e2e-workflow.test.ts": 265175,
|
||||
"test/skill-llm-eval.test.ts": 354000,
|
||||
"test/skill-routing-e2e.test.ts": 1000
|
||||
},
|
||||
@@ -123,98 +123,100 @@
|
||||
"test/carve-section-loading.test.ts#carve-section-loading-review": 247902,
|
||||
"test/carve-section-loading.test.ts#carve-section-loading-setup-gbrain": 80994,
|
||||
"test/carve-section-loading.test.ts#carve-section-loading-spec": 176639,
|
||||
"test/codex-e2e-recommendation-substance.test.ts": 17984,
|
||||
"test/codex-e2e-shared-libs.test.ts": 53442,
|
||||
"test/codex-e2e-sol-scope.test.ts": 37063,
|
||||
"test/codex-e2e.test.ts": 120390,
|
||||
"test/llm-judge-recommendation.test.ts": 15334,
|
||||
"test/skill-e2e-arm-benchmark.test.ts": 75588,
|
||||
"test/codex-e2e-multiblock-live.test.ts": 180000,
|
||||
"test/codex-e2e-recommendation-substance.test.ts": 14617,
|
||||
"test/codex-e2e-shared-libs.test.ts": 49563,
|
||||
"test/codex-e2e-sol-scope.test.ts": 32228,
|
||||
"test/codex-e2e.test.ts": 110677,
|
||||
"test/llm-judge-recommendation.test.ts": 21562,
|
||||
"test/skill-e2e-arm-benchmark.test.ts": 72714,
|
||||
"test/skill-e2e-aside.test.ts": 1000,
|
||||
"test/skill-e2e-auq-consistency.test.ts": 70429,
|
||||
"test/skill-e2e-auq-matrix.test.ts": 165796,
|
||||
"test/skill-e2e-auq-verbose-vs-carved-ab.test.ts": 58547,
|
||||
"test/skill-e2e-auto-decide-preserved.test.ts": 154931,
|
||||
"test/skill-e2e-autoplan-dual-voice.test.ts": 167918,
|
||||
"test/skill-e2e-benchmark-providers.test.ts": 9495,
|
||||
"test/skill-e2e-auq-consistency.test.ts": 63226,
|
||||
"test/skill-e2e-auq-matrix.test.ts": 145925,
|
||||
"test/skill-e2e-auq-verbose-vs-carved-ab.test.ts": 65480,
|
||||
"test/skill-e2e-auto-decide-preserved.test.ts": 234151,
|
||||
"test/skill-e2e-autoplan-dual-voice.test.ts": 154207,
|
||||
"test/skill-e2e-benchmark-providers.test.ts": 8847,
|
||||
"test/skill-e2e-bws.test.ts": 1000,
|
||||
"test/skill-e2e-context-skills.test.ts": 169497,
|
||||
"test/skill-e2e-context-skills.test.ts": 144874,
|
||||
"test/skill-e2e-coverage-audit.test.ts": 1000,
|
||||
"test/skill-e2e-cso.test.ts": 253498,
|
||||
"test/skill-e2e-cso.test.ts": 303949,
|
||||
"test/skill-e2e-design-model-smoke.test.ts": 90000,
|
||||
"test/skill-e2e-design.test.ts": 817000,
|
||||
"test/skill-e2e-design.test.ts#design-consultation-core": 159257,
|
||||
"test/skill-e2e-design.test.ts#design-consultation-existing": 175607,
|
||||
"test/skill-e2e-design.test.ts#design-consultation-preview": 114638,
|
||||
"test/skill-e2e-design.test.ts#design-consultation-research": 109573,
|
||||
"test/skill-e2e-design.test.ts#design-html-slop-gate": 158970,
|
||||
"test/skill-e2e-design.test.ts#plan-design-review-plan-mode": 300166,
|
||||
"test/skill-e2e-diagram.test.ts": 55187,
|
||||
"test/skill-e2e-design.test.ts#design-consultation-core": 173162,
|
||||
"test/skill-e2e-design.test.ts#design-consultation-existing": 225306,
|
||||
"test/skill-e2e-design.test.ts#design-consultation-preview": 107037,
|
||||
"test/skill-e2e-design.test.ts#design-consultation-research": 99890,
|
||||
"test/skill-e2e-design.test.ts#design-html-slop-gate": 139668,
|
||||
"test/skill-e2e-design.test.ts#plan-design-review-plan-mode": 169183,
|
||||
"test/skill-e2e-diagram.test.ts": 52819,
|
||||
"test/skill-e2e-gbrain-roundtrip-local.test.ts": 1000,
|
||||
"test/skill-e2e-health.test.ts": 188584,
|
||||
"test/skill-e2e-health.test.ts": 164674,
|
||||
"test/skill-e2e-hermetic-canary.test.ts": 1000,
|
||||
"test/skill-e2e-ios-device.test.ts": 1000,
|
||||
"test/skill-e2e-learnings.test.ts": 1000,
|
||||
"test/skill-e2e-office-hours-brain-writeback.test.ts": 203044,
|
||||
"test/skill-e2e-office-hours-design-draft.test.ts": 285868,
|
||||
"test/skill-e2e-office-hours-design-draft.test.ts": 210162,
|
||||
"test/skill-e2e-office-hours-phase4.test.ts": 41421,
|
||||
"test/skill-e2e-office-hours-section-loading.test.ts": 1200000,
|
||||
"test/skill-e2e-office-hours.test.ts": 136970,
|
||||
"test/skill-e2e-outside-plan-disabled.test.ts": 54255,
|
||||
"test/skill-e2e-outside-voice.test.ts": 186881,
|
||||
"test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-opus-5-5.test.ts": 56080,
|
||||
"test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-sonnet.test.ts": 85282,
|
||||
"test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash.test.ts": 80465,
|
||||
"test/skill-e2e-overlay-harness-opus-4-7-effort-match-trivial.test.ts": 49282,
|
||||
"test/skill-e2e-overlay-harness-opus-4-7-literal-interpretation.test.ts": 318532,
|
||||
"test/skill-e2e-plan-ceo-mode-routing.test.ts": 443504,
|
||||
"test/skill-e2e-plan-ceo-review-section-loading.test.ts": 342250,
|
||||
"test/skill-e2e-outside-plan-disabled.test.ts": 72070,
|
||||
"test/skill-e2e-outside-voice.test.ts": 195221,
|
||||
"test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-opus-5-5.test.ts": 57374,
|
||||
"test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-sonnet.test.ts": 77984,
|
||||
"test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash.test.ts": 55431,
|
||||
"test/skill-e2e-overlay-harness-opus-4-7-effort-match-trivial.test.ts": 34905,
|
||||
"test/skill-e2e-overlay-harness-opus-4-7-literal-interpretation.test.ts": 226920,
|
||||
"test/skill-e2e-plan-ceo-mode-routing.test.ts": 378316,
|
||||
"test/skill-e2e-plan-ceo-review-section-loading.test.ts": 302668,
|
||||
"test/skill-e2e-plan-ceo-split-overflow.test.ts": 504266,
|
||||
"test/skill-e2e-plan-decision-classification.test.ts": 93469,
|
||||
"test/skill-e2e-plan-design-finding-floor.test.ts": 155538,
|
||||
"test/skill-e2e-plan-design-plan-mode.test.ts": 106608,
|
||||
"test/skill-e2e-plan-devex-peer-comparison-classification.test.ts": 79786,
|
||||
"test/skill-e2e-plan-decision-classification.test.ts": 88190,
|
||||
"test/skill-e2e-plan-design-finding-floor.test.ts": 131042,
|
||||
"test/skill-e2e-plan-design-plan-mode.test.ts": 111838,
|
||||
"test/skill-e2e-plan-devex-peer-comparison-classification.test.ts": 77201,
|
||||
"test/skill-e2e-plan-eng-artifact-full.test.ts": 595009,
|
||||
"test/skill-e2e-plan-eng-artifact.test.ts": 498474,
|
||||
"test/skill-e2e-plan-eng-finding-floor.test.ts": 388666,
|
||||
"test/skill-e2e-plan-eng-multi-finding-batching.test.ts": 597500,
|
||||
"test/skill-e2e-plan-eng-plan-mode.test.ts": 229226,
|
||||
"test/skill-e2e-plan-eng-artifact.test.ts": 406795,
|
||||
"test/skill-e2e-plan-eng-finding-floor.test.ts": 234397,
|
||||
"test/skill-e2e-plan-eng-multi-finding-batching.test.ts": 409161,
|
||||
"test/skill-e2e-plan-eng-plan-mode.test.ts": 170914,
|
||||
"test/skill-e2e-plan-format.test.ts": 219670,
|
||||
"test/skill-e2e-plan-prosons.test.ts": 163659,
|
||||
"test/skill-e2e-plan-tune.test.ts": 1000,
|
||||
"test/skill-e2e-plan.test.ts": 824000,
|
||||
"test/skill-e2e-plan.test.ts#plan-ceo-review": 136101,
|
||||
"test/skill-e2e-plan.test.ts#plan-ceo-review-expansion-energy": 91984,
|
||||
"test/skill-e2e-plan.test.ts#plan-ceo-review-selective": 267212,
|
||||
"test/skill-e2e-plan.test.ts#plan-eng-review": 128506,
|
||||
"test/skill-e2e-qa-bugs.test.ts": 333847,
|
||||
"test/skill-e2e-plan.test.ts#plan-ceo-review": 274619,
|
||||
"test/skill-e2e-plan.test.ts#plan-ceo-review-expansion-energy": 100000,
|
||||
"test/skill-e2e-plan.test.ts#plan-ceo-review-selective": 265163,
|
||||
"test/skill-e2e-plan.test.ts#plan-eng-review": 169400,
|
||||
"test/skill-e2e-qa-bugs.test.ts": 355966,
|
||||
"test/skill-e2e-qa-workflow.test.ts#qa-fix-loop": 411728,
|
||||
"test/skill-e2e-retro.test.ts": 166711,
|
||||
"test/skill-e2e-retro.test.ts": 150139,
|
||||
"test/skill-e2e-review-army.test.ts": 511000,
|
||||
"test/skill-e2e-review-army.test.ts#review-army-consensus": 277908,
|
||||
"test/skill-e2e-review-army.test.ts#review-army-red-team": 88882,
|
||||
"test/skill-e2e-review-army.test.ts#review-army-simplification": 136644,
|
||||
"test/skill-e2e-review-army.test.ts#review-army-simplification-precision": 23884,
|
||||
"test/skill-e2e-review-army.test.ts#review-army-consensus": 107375,
|
||||
"test/skill-e2e-review-army.test.ts#review-army-red-team": 110743,
|
||||
"test/skill-e2e-review-army.test.ts#review-army-simplification": 140434,
|
||||
"test/skill-e2e-review-army.test.ts#review-army-simplification-precision": 25020,
|
||||
"test/skill-e2e-review-attribution.test.ts": 1000,
|
||||
"test/skill-e2e-review.test.ts": 179362,
|
||||
"test/skill-e2e-safety-codex-boundary.test.ts": 100310,
|
||||
"test/skill-e2e-safety-codex-boundary.test.ts": 102851,
|
||||
"test/skill-e2e-safety-codex-consult.test.ts": 1000,
|
||||
"test/skill-e2e-safety-design-risk.test.ts": 52700,
|
||||
"test/skill-e2e-safety-ios-demo.test.ts": 53270,
|
||||
"test/skill-e2e-safety-design-risk.test.ts": 54889,
|
||||
"test/skill-e2e-safety-ios-demo.test.ts": 100092,
|
||||
"test/skill-e2e-safety-pair-agent.test.ts": 1000,
|
||||
"test/skill-e2e-safety-ship-evidence.test.ts": 1000,
|
||||
"test/skill-e2e-session-intelligence.test.ts": 1000,
|
||||
"test/skill-e2e-setup-gbrain-bad-token.test.ts": 42912,
|
||||
"test/skill-e2e-setup-gbrain-path4-local-pglite.test.ts": 165743,
|
||||
"test/skill-e2e-setup-gbrain-remote.test.ts": 121127,
|
||||
"test/skill-e2e-shared-libs-periodic.test.ts": 369660,
|
||||
"test/skill-e2e-ship-section-loading.test.ts": 255273,
|
||||
"test/skill-e2e-setup-gbrain-bad-token.test.ts": 39292,
|
||||
"test/skill-e2e-setup-gbrain-path4-local-pglite.test.ts": 128414,
|
||||
"test/skill-e2e-setup-gbrain-remote.test.ts": 89701,
|
||||
"test/skill-e2e-shared-libs-periodic.test.ts": 217221,
|
||||
"test/skill-e2e-ship-section-loading.test.ts": 189463,
|
||||
"test/skill-e2e-skillify.test.ts": 58522,
|
||||
"test/skill-e2e-sync-gbrain-readiness.test.ts": 60234,
|
||||
"test/skill-e2e-sync-gbrain-readiness.test.ts": 47502,
|
||||
"test/skill-e2e-test-value.test.ts": 332000,
|
||||
"test/skill-e2e-third-party-actions.test.ts": 1000,
|
||||
"test/skill-e2e-triage.test.ts": 1000,
|
||||
"test/skill-e2e-workflow.test.ts": 110357,
|
||||
"test/skill-llm-eval.test.ts": 383454,
|
||||
"test/skill-routing-e2e.test.ts": 68843
|
||||
"test/skill-e2e-workflow.test.ts": 121594,
|
||||
"test/skill-llm-eval.test.ts": 339704,
|
||||
"test/skill-routing-e2e.test.ts": 81360
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -133,7 +133,7 @@ If \`NEEDS_SETUP\`:
|
||||
3. If \`bun\` is not installed:
|
||||
\`\`\`bash
|
||||
if ! command -v bun >/dev/null 2>&1; then
|
||||
BUN_VERSION="1.3.10"
|
||||
BUN_VERSION="1.4.2"
|
||||
BUN_INSTALL_SHA="bab8acfb046aac8c72407bdcce903957665d655d7acaa3e11c7c4616beae68dd"
|
||||
tmpfile=$(mktemp "\${TMPDIR:-/tmp}/bun-install.XXXXXX")
|
||||
curl -fsSL "https://bun.sh/install" -o "$tmpfile"
|
||||
|
||||
@@ -140,6 +140,10 @@ export const RESOLVERS: Record<string, ResolverFn> = {
|
||||
DX_FRAMEWORK: generateDxFramework,
|
||||
TASTE_PROFILE: generateTasteProfile,
|
||||
BIN_DIR: (ctx) => ctx.paths.binDir,
|
||||
// Quotable runtime root for paths under gstack's install (freeze/bin, bin, lib):
|
||||
// $GSTACK_ROOT on env-var hosts (each fence gets the runtime prelude), the
|
||||
// literal $HOME path on Claude, so a quoted path keeps working with spaces.
|
||||
RUNTIME_ROOT: (ctx) => getHostConfig(ctx.host).usesEnvVars ? '$GSTACK_ROOT' : `$HOME/${getHostConfig(ctx.host).globalRoot}`,
|
||||
// Literal bin dir for skills that skip the shared preamble, where
|
||||
// $GSTACK_BIN is never set (#2906): the per-install root when rendering for
|
||||
// one, else the host's default global root.
|
||||
|
||||
@@ -1,29 +1,26 @@
|
||||
import { type TemplateContext, toShellPath } from './types';
|
||||
import { type TemplateContext } from './types';
|
||||
import { binaryAssignment } from './runtime-root';
|
||||
|
||||
/**
|
||||
* {{MAKE_PDF_SETUP}} — emits the shell preamble that resolves $P to the
|
||||
* make-pdf binary. Mirrors generateDesignSetup.
|
||||
* {{MAKE_PDF_SETUP}} — emits the readiness check that resolves $P to the
|
||||
* make-pdf binary (make-pdf/dist/pdf). Mirrors generateDesignSetup: the
|
||||
* assignment comes from binaryAssignment(ctx, 'make-pdf'), the same resolver
|
||||
* the runtime prelude puts into every later block that uses $P on env-var
|
||||
* hosts, so readiness and use always name the same binary.
|
||||
*
|
||||
* $P = make-pdf/dist/pdf.
|
||||
*
|
||||
* Resolution order:
|
||||
* 1. Local skill root: $_ROOT/{localSkillRoot}/make-pdf/dist/pdf
|
||||
* 2. Global: ~/{globalRoot}/make-pdf/dist/pdf
|
||||
* 3. Env override (MAKE_PDF_BIN) — for contributor dev builds
|
||||
* Resolution order: MAKE_PDF_BIN (contributor dev builds), then the
|
||||
* repo-local install, then the global install (env-var hosts: $GSTACK_ROOT,
|
||||
* which the prelude resolves repo-local first).
|
||||
*/
|
||||
export function generateMakePdfSetup(ctx: TemplateContext): string {
|
||||
return `## MAKE-PDF SETUP (run this check BEFORE any make-pdf command)
|
||||
|
||||
\`\`\`bash
|
||||
_ROOT=$(git rev-parse --show-toplevel 2>/dev/null)
|
||||
P=""
|
||||
[ -n "$MAKE_PDF_BIN" ] && [ -x "$MAKE_PDF_BIN" ] && P="$MAKE_PDF_BIN"
|
||||
[ -z "$P" ] && [ -n "$_ROOT" ] && [ -x "$_ROOT/${ctx.paths.localSkillRoot}/make-pdf/dist/pdf" ] && P="$_ROOT/${ctx.paths.localSkillRoot}/make-pdf/dist/pdf"
|
||||
[ -z "$P" ] && P="${toShellPath(ctx.paths.skillRoot)}/make-pdf/dist/pdf"
|
||||
${binaryAssignment(ctx, 'make-pdf')}
|
||||
if [ -x "$P" ]; then
|
||||
echo "MAKE_PDF_READY: $P"
|
||||
alias _p_="$P" # shellcheck alias helper (not exported)
|
||||
export P # available as $P in subsequent blocks within the same skill invocation
|
||||
export P # env-var hosts re-derive $P in every later block (runtime prelude)
|
||||
else
|
||||
echo "MAKE_PDF_NOT_AVAILABLE (run './setup' in the gstack repo to build it)"
|
||||
fi
|
||||
@@ -36,12 +33,12 @@ If \`MAKE_PDF_READY\` is printed: \`$P\` is the binary path for the rest of
|
||||
the skill. Use \`$P\` (not an explicit path) so the skill body stays portable.
|
||||
|
||||
Core commands:
|
||||
- \`$P generate <input.md> [output.pdf]\` — render markdown to PDF (80% use case)
|
||||
- \`$P generate --cover --toc essay.md out.pdf\` — full publication layout
|
||||
- \`$P generate --watermark DRAFT memo.md draft.pdf\` — diagonal DRAFT watermark
|
||||
- \`$P preview <input.md>\` — render HTML and open in browser (fast iteration)
|
||||
- \`$P setup\` — verify the browser (Aside, or gstack's own headless fallback) + pdftotext and run a smoke test
|
||||
- \`$P --help\` — full flag reference
|
||||
- \`"$P" generate <input.md> [output.pdf]\` — render markdown to PDF (80% use case)
|
||||
- \`"$P" generate --cover --toc essay.md out.pdf\` — full publication layout
|
||||
- \`"$P" generate --watermark DRAFT memo.md draft.pdf\` — diagonal DRAFT watermark
|
||||
- \`"$P" preview <input.md>\` — render HTML and open in browser (fast iteration)
|
||||
- \`"$P" setup\` — verify the browser (Aside, or gstack's own headless fallback) + pdftotext and run a smoke test
|
||||
- \`"$P" --help\` — full flag reference
|
||||
|
||||
Output contract:
|
||||
- \`stdout\`: ONLY the output path on success. One line.
|
||||
|
||||
@@ -176,7 +176,7 @@ ${outsideVoiceInvocation(ctx, { timeoutMs: 540000, nativeAlreadyRequired: true,
|
||||
${outsideVoiceFor(ctx).id === 'codex' ? 'The Codex backend uses `codex review --base` without a positional prompt: those arguments are mutually exclusive. Never drop --base to resolve an argv error; prompt-only review changes the diff scope.' : 'The Claude Code backend receives the parent-captured base diff, including committed and working-tree changes, because review mode cannot execute git.'}
|
||||
|
||||
Present output under \`${outsideVoiceFor(ctx).label.toUpperCase()} SAYS (code review):\` inside a \`tool-output\` fence.
|
||||
Only a completed response with severity tags or an explicit no-findings conclusion establishes the gate. P0/P1 findings (\`[P0]\`/\`[P1]\` or native \`P0:\`/\`P1:\` labels; \`VERDICT: findings\`) → GATE: FAIL. Completed without P0/P1 → GATE: PASS. Refusal, failure, missing markers or \`OUTSIDE_STATUS: unverified\` → GATE: MISSING COVERAGE; preserve the existing user decision flow.
|
||||
Only a completed response with severity tags or an explicit no-findings conclusion establishes the gate. P0/P1 findings (\`[P0]\`/\`[P1]\` or native \`P0:\`/\`P1:\` labels; \`VERDICT: findings\`) → GATE: FAIL. Completed without P0/P1 → GATE: PASS. Refusal, failure, missing markers or \`OUTSIDE_STATUS: unverified\` → GATE: MISSING COVERAGE; no fix question.
|
||||
|
||||
If GATE is FAIL, use AskUserQuestion:
|
||||
\`\`\`
|
||||
@@ -409,7 +409,7 @@ THE PLAN:
|
||||
function codexPlanReviewRun(ctx: TemplateContext, ceo: boolean, needsApprovalReadiness: boolean): string {
|
||||
return `**If \`CODEX_MODE: ready\` (or \`unverified\`) — run ${outsideVoiceFor(ctx).label}:**
|
||||
|
||||
${['plan-ceo-review', 'plan-eng-review'].includes(ctx.skillName) ? `Run this block only for \`ready\`, in the one foreground Bash call described below.
|
||||
${['plan-ceo-review', 'plan-eng-review'].includes(ctx.skillName) ? `Run this block for \`ready\` or \`unverified\`, in the one foreground Bash call described below.
|
||||
Its opening harness guard rechecks the fresh shell: exit 78 uses the same Native
|
||||
fallback below, never a replacement provider. Finish termination before fallback and consume only
|
||||
completed output. Use private temporary paths, with no background jobs.` : `Run the selected backend in the one foreground Bash invocation described below.
|
||||
|
||||
@@ -467,13 +467,13 @@ The plan completion results augment the existing Scope Drift Detection. If a pla
|
||||
|
||||
- **NOT DONE items** become additional evidence for **MISSING REQUIREMENTS** in the scope drift report.
|
||||
- **Items in the diff that don't match any plan item** become evidence for **SCOPE CREEP** detection.
|
||||
- **HIGH-impact plan-file discrepancies** trigger AskUserQuestion:
|
||||
- Show the investigation findings
|
||||
- **HIGH-impact plan-file discrepancies** trigger AskUserQuestion showing the investigation findings:
|
||||
- Options: A) Stop this review for implementation, B) Continue this review with P1 TODOs, C) Record the items as intentionally dropped
|
||||
- A ends this invocation before code review or implementation. List the missing work; after implementation, start a fresh /review.
|
||||
- B queues the approved TODO changes for Step 5, not this read-only audit. B/C continue to the final Scope Check and Step 2. None of these choices authorizes shipping or waives required verification.
|
||||
- Spawned or non-interactive: report REQUIREMENTS MISSING, continue; record nothing.
|
||||
|
||||
This is **INFORMATIONAL** unless HIGH-impact plan-file discrepancies are found (then it gates via AskUserQuestion).
|
||||
Otherwise the audit is **INFORMATIONAL**.
|
||||
Discrepancies derived only from fallback sources (commit messages, TODOS.md, PR description) never trigger
|
||||
this question, whatever their IMPACT: report them in the Scope Check as lower-confidence missing requirements.
|
||||
|
||||
|
||||
@@ -97,9 +97,6 @@ Never reset DEADLINE_FILE/bypass DEADLINE_TOOL. Expiry or an invalid/missing DEA
|
||||
|
||||
Each probe is one native command/interaction plus checks, excluding bookkeeping.
|
||||
Never batch probes.
|
||||
How one probe works: (1) if bounded, check the clock and choose the next command;
|
||||
(2) publish a checkpoint naming it (functional: \`capture --after\` publishes it);
|
||||
(3) run exactly that command via DEADLINE_TOOL or EVIDENCE_TOOL, read its result, repeat.
|
||||
|
||||
1. First demonstrate success: output AND durable effects. Guard if bounded; await completion.
|
||||
2. **Decide whether another probe is needed.** If bounded, run \`bun DEADLINE_TOOL status DEADLINE_FILE\`.
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
* Runtime root for env-var hosts (C1, #1159): the one owner of how generated
|
||||
* bash finds gstack on Codex and every other `usesEnvVars` host.
|
||||
*
|
||||
* Those hosts run each fenced block in a fresh shell, so `GSTACK_ROOT`, `B`
|
||||
* and `D` set by an earlier block are gone. The preamble's root resolution and
|
||||
* Those hosts run each fenced block in a fresh shell, so `GSTACK_ROOT`, `B`,
|
||||
* `D` and `P` set by an earlier block are gone. The preamble's root resolution and
|
||||
* the per-fence prelude come from `runtimeRootPrelude()`; `insertRuntimePreludes()`
|
||||
* is the single post-render pass (gen-skill-docs, before rewriteInstallRoot)
|
||||
* that puts it into every fence that uses these variables without assigning
|
||||
@@ -19,6 +19,13 @@ import { toShellPath, type TemplateContext } from './types';
|
||||
|
||||
/** Per-fence byte budget for the prelude (T-ENG1), asserted by the INV-3 test. */
|
||||
export const PRELUDE_BYTE_BUDGET = 400;
|
||||
/**
|
||||
* make-pdf's dev-build override (E2), appended to `P`'s assignment. It is the
|
||||
* one fixed segment outside PRELUDE_BYTE_BUDGET: root plus `P=...` fits the
|
||||
* budget like `B`/`D` do, and the override is needed in every block that runs
|
||||
* `$P` so readiness and use name the same binary.
|
||||
*/
|
||||
export const MAKE_PDF_OVERRIDE = ';[ -x "${MAKE_PDF_BIN:-}" ]&&P=$MAKE_PDF_BIN||:';
|
||||
|
||||
const ROOT_VARS = /\$\{?GSTACK_(?:ROOT|BIN|BROWSE|DESIGN|MAKE_PDF)\b/;
|
||||
const assigns = (name: string) => new RegExp(`(?:^|[\\s;&|(])${name}=`, 'm');
|
||||
@@ -48,13 +55,23 @@ export function runtimeRootPrelude(ctx: TemplateContext): string {
|
||||
}
|
||||
|
||||
/**
|
||||
* Lines that assign `B` (browse) or `D` (design). BROWSE SETUP, DESIGN SETUP and
|
||||
* the prelude all use this, so a later fence runs the binary setup checked.
|
||||
* Env-var hosts derive from `GSTACK_ROOT`, which the prelude already points at
|
||||
* the repo-local install when one exists; other hosts keep their
|
||||
* repo-local-first probe.
|
||||
* Lines that assign `B` (browse), `D` (design) or `P` (make-pdf). BROWSE SETUP,
|
||||
* DESIGN SETUP, MAKE-PDF SETUP and the prelude all use this, so a later fence
|
||||
* runs the binary setup checked. Env-var hosts derive from `GSTACK_ROOT`, which
|
||||
* the prelude already points at the repo-local install when one exists; other
|
||||
* hosts keep their repo-local-first probe. make-pdf's binary is
|
||||
* `make-pdf/dist/pdf`, and `MAKE_PDF_BIN` (a contributor dev build) wins on
|
||||
* every host.
|
||||
*/
|
||||
export function binaryAssignment(ctx: TemplateContext, tool: 'browse' | 'design'): string {
|
||||
export function binaryAssignment(ctx: TemplateContext, tool: 'browse' | 'design' | 'make-pdf'): string {
|
||||
if (tool === 'make-pdf') {
|
||||
if (getHostConfig(ctx.host).usesEnvVars) return `P=$GSTACK_ROOT/make-pdf/dist/pdf${MAKE_PDF_OVERRIDE}`;
|
||||
return `_ROOT=$(git rev-parse --show-toplevel 2>/dev/null)
|
||||
P=""
|
||||
[ -n "$MAKE_PDF_BIN" ] && [ -x "$MAKE_PDF_BIN" ] && P="$MAKE_PDF_BIN"
|
||||
[ -z "$P" ] && [ -n "$_ROOT" ] && [ -x "$_ROOT/${ctx.paths.localSkillRoot}/make-pdf/dist/pdf" ] && P="$_ROOT/${ctx.paths.localSkillRoot}/make-pdf/dist/pdf"
|
||||
[ -z "$P" ] && P="${toShellPath(ctx.paths.skillRoot)}/make-pdf/dist/pdf"`;
|
||||
}
|
||||
const v = tool === 'browse' ? 'B' : 'D';
|
||||
const dir = tool === 'browse' ? ctx.paths.browseDir : ctx.paths.designDir;
|
||||
if (getHostConfig(ctx.host).usesEnvVars) return `${v}=$GSTACK_ROOT/${tool}/dist/${tool}`;
|
||||
@@ -69,8 +86,9 @@ export function fencePrelude(ctx: TemplateContext, body: string): string {
|
||||
if (!getHostConfig(ctx.host).usesEnvVars) return '';
|
||||
const needB = uses('B').test(body) && !assigns('B').test(body);
|
||||
const needD = uses('D').test(body) && !assigns('D').test(body);
|
||||
const needP = uses('P').test(body) && !assigns('P').test(body);
|
||||
if (assigns('GSTACK_ROOT').test(body)) return '';
|
||||
const needRoot = needB || needD || ROOT_VARS.test(body);
|
||||
const needRoot = needB || needD || needP || ROOT_VARS.test(body);
|
||||
const derived = Object.entries(DERIVED)
|
||||
.filter(([name]) => uses(`GSTACK_${name}`).test(body) && !assigns(`GSTACK_${name}`).test(body))
|
||||
.map(([name, dir]) => `GSTACK_${name}=$GSTACK_ROOT/${dir}`);
|
||||
@@ -79,6 +97,7 @@ export function fencePrelude(ctx: TemplateContext, body: string): string {
|
||||
derived.join(' '),
|
||||
needB ? binaryAssignment(ctx, 'browse') : '',
|
||||
needD ? binaryAssignment(ctx, 'design') : '',
|
||||
needP ? binaryAssignment(ctx, 'make-pdf') : '',
|
||||
].filter(Boolean).join('\n');
|
||||
}
|
||||
|
||||
|
||||
@@ -47,37 +47,54 @@ all preceding valid round files in order (omit them for round 1):
|
||||
~/.claude/skills/gstack/bin/gstack-office-hours-review prepare --design "<design-path>" --out-dir "<review-directory>" "<round-1.json if present>" "<round-2.json if present>"
|
||||
\`\`\`
|
||||
|
||||
Omit absent arguments rather than passing placeholders. The helper chooses the next
|
||||
Omit absent arguments rather than passing placeholders. Users get 3 rounds; only
|
||||
when a caller sets a lower review round limit, add \`--max-rounds <N>\` with that
|
||||
same value to every prepare, check, and finalize command. The helper chooses the next
|
||||
round and writes \`round-N.prompt.md\`. It includes the full finding schema, all five
|
||||
review dimensions (Completeness, Consistency, Clarity, Scope, Feasibility), the
|
||||
office-hours coaching contract, and the COMPLETE preceding JSON verdict.
|
||||
office-hours coaching contract, and the COMPLETE preceding JSON verdict. It also
|
||||
saves the design as reviewed (\`round-N.design.md\`). Round 1 is a full review;
|
||||
rounds 2 and 3 are delta re-reviews of the exact design diff since the last round,
|
||||
so prepare every round in the same review directory and do not edit the design
|
||||
between prepare and its review.
|
||||
|
||||
Use the Agent tool with \`run_in_background: false\` and its returned \`dispatch\`
|
||||
string unchanged as the prompt. The reviewer must Read the entire prepared prompt
|
||||
file before reviewing the design. Do not recreate the prompt, copy selected fields,
|
||||
or summarize prior findings. A parent Read does not deliver the file to the reviewer.
|
||||
The reviewer has fresh context and cannot see the brainstorming conversation.
|
||||
Its prepared contract requires a complete JSON Write and an identical JSON response.
|
||||
Its prepared contract requires a complete JSON Write, sealed by the dispatch's
|
||||
\`Seal:\` command, and a one-line \`OFFICE_HOURS_VERDICT\` receipt as its entire response.
|
||||
It protects the required coaching and Assignment sections, distinguishes unknown
|
||||
customer facts from committed behavior, and requires evidence for every prior status.
|
||||
|
||||
**Step 2: Check stop conditions, then fix and re-dispatch**
|
||||
|
||||
After each verdict, BEFORE fixing any findings or dispatching again, validate the
|
||||
saved files with the helper (list every completed round in order):
|
||||
saved files with the helper. Pass the reviewer's entire response unchanged as the
|
||||
receipt and list every completed round in order:
|
||||
|
||||
\`\`\`bash
|
||||
~/.claude/skills/gstack/bin/gstack-office-hours-review check "<round-1.json>" "<round-2.json if present>" "<round-3.json if present>"
|
||||
~/.claude/skills/gstack/bin/gstack-office-hours-review check --receipt "<reviewer response>" "<round-1.json>" "<round-2.json if present>" "<round-3.json if present>"
|
||||
\`\`\`
|
||||
|
||||
A missing, malformed, or mismatched receipt fails the check: that attempt is a failed review.
|
||||
|
||||
Omit absent arguments rather than passing placeholders.
|
||||
**Convergence guard and stopping rules:** Read its stop reason:
|
||||
- PASS: no unresolved findings; proceed to Step 3.
|
||||
- CONVERGENCE: the reviewer explicitly marked a prior obligation persisting with
|
||||
a concrete prior/current finding pair and document evidence. Stop even if new
|
||||
findings appear. Shared topic labels or new refinements alone are insufficient.
|
||||
- MAX_ITERATIONS: round 3 completed; stop.
|
||||
- CONTINUE: fix the listed findings in the design, then return to Step 1 to prepare and dispatch the next review.
|
||||
**Convergence guard and stopping rules:** Each finding is blocking or minor. Only
|
||||
blocking findings require another round. Read its stop reason:
|
||||
- PASS: no blocking findings remain. Any minor findings are recorded, not fixed,
|
||||
and never justify another round; proceed to Step 3.
|
||||
- CONVERGENCE: the reviewer explicitly marked a blocking prior obligation
|
||||
persisting with a concrete prior/current finding pair and document evidence.
|
||||
Stop even if new findings appear. Shared topic labels or new refinements alone
|
||||
are insufficient.
|
||||
- MAX_ITERATIONS: round 3 completed (or the caller's lower round limit); stop.
|
||||
- CONTINUE: fix only the blocking findings in the design, then return to Step 1
|
||||
to prepare and dispatch the next review. Do not edit for minor findings
|
||||
mid-loop: they stay recorded for the user, and new text only gives the next
|
||||
diff more to review. Its reviewer checks every prior finding and raises new
|
||||
blocking findings only for problems your changes introduced or exposed.
|
||||
|
||||
On a stop, do not fix again or re-dispatch. Run the finalizer before approval:
|
||||
|
||||
@@ -119,7 +136,7 @@ mkdir -p "$GSTACK_STATE_ROOT/analytics"
|
||||
echo '{"skill":"office-hours","ts":"'$(date -u +%Y-%m-%dT%H:%M:%SZ)'","iterations":ITERATIONS,"issues_found":FOUND,"issues_fixed":FIXED,"remaining":REMAINING,"quality_score":SCORE}' >> "$GSTACK_STATE_ROOT/analytics/spec-review.jsonl" 2>/dev/null || true
|
||||
\`\`\`
|
||||
Use iterations, issues_found, issues_fixed, remaining, and quality_score from the
|
||||
helper. FOUND counts finding observations across rounds; FIXED counts only
|
||||
helper; its remaining_blocking and remaining_minor split the remaining count. FOUND counts finding observations across rounds; FIXED counts only
|
||||
reviewer-confirmed resolutions. An unavailable score is null, never invented.`;
|
||||
}
|
||||
|
||||
|
||||
@@ -60,13 +60,13 @@ backslashes serialize cleanly — never use hand-rolled \`echo\` / \`printf\`.
|
||||
|
||||
\`\`\`bash
|
||||
GSTACK_STATE_ROOT=$(~/.claude/skills/gstack/bin/gstack-paths --get GSTACK_STATE_ROOT); : "\${GSTACK_STATE_ROOT:?gstack-paths failed; reinstall with ./setup or /gstack-upgrade}"
|
||||
BRANCH=$(~/.claude/skills/gstack/bin/gstack-slug --get BRANCH 2>/dev/null)
|
||||
SLUG=$(~/.claude/skills/gstack/bin/gstack-slug --get SLUG 2>/dev/null)
|
||||
TASKS_DIR="$GSTACK_STATE_ROOT/projects/\${SLUG:-unknown}"
|
||||
mkdir -p "$TASKS_DIR"
|
||||
TASKS_FILE="$TASKS_DIR/tasks-${phase}-$(date +%Y%m%d-%H%M%S).jsonl"
|
||||
COMMIT=$(git rev-parse HEAD 2>/dev/null || echo unknown)
|
||||
BRANCH=$(git branch --show-current 2>/dev/null || echo unknown)
|
||||
# The task record keeps the raw git branch; artifact filenames keep gstack-slug's sanitized BRANCH.
|
||||
GIT_BRANCH=$(git branch --show-current 2>/dev/null || echo unknown)
|
||||
RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)-$$"
|
||||
|
||||
# Repeat ONE jq invocation per task identified during this review.
|
||||
@@ -77,7 +77,7 @@ RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)-$$"
|
||||
jq -nc \\
|
||||
--arg phase '${phase}' \\
|
||||
--arg run_id "$RUN_ID" \\
|
||||
--arg branch "$BRANCH" \\
|
||||
--arg branch "$GIT_BRANCH" \\
|
||||
--arg commit "$COMMIT" \\
|
||||
--arg id "$TASK_ID" \\
|
||||
--arg priority "$PRIORITY" \\
|
||||
|
||||
@@ -129,7 +129,7 @@ function cardRules(mode: TestValueBarMode, skillName: string): string {
|
||||
qa: skillName === 'qa-only' ? 'Put it under each proposed test.' : 'Put it in the 8e.5 record.',
|
||||
audit: 'Read cards from test header comments when present.',
|
||||
}[mode];
|
||||
return `Value card: \`Value: protects=<...>; fails_when=<...>; why_new=<...>; seam=none\` (seam: \`none\` or its name); each field at most ${CARD_FIELD_MAX_BYTES} UTF-8 bytes here (clamp to 157 plus \`...\`; JSON keeps full values). ${where} A missing upstream card never blocks: derive it; ignore unknown fields.
|
||||
return `Value card: \`Value: protects=<...>; fails_when=<...>; why_new=<...>; seam=none\` (seam: \`none\` or its name); each field at most ${CARD_FIELD_MAX_BYTES} UTF-8 bytes here (clamp to 157 plus \`...\`; written JSON keeps full values). ${where} A missing upstream card never blocks: derive it; ignore unknown fields.
|
||||
|
||||
Example: ${EXAMPLE_CARD}
|
||||
${EXAMPLE_REJECTED}`;
|
||||
|
||||
@@ -676,6 +676,8 @@ export interface FreeRunReport {
|
||||
inFlight: string[];
|
||||
/** Planned files never observed in the stream (silent passers + never-flushed wedges). */
|
||||
filesWithNoOutput: number;
|
||||
/** The last nonblank line either stream printed: a native abort's message when bun dies without a summary. */
|
||||
lastOutputLine: string | null;
|
||||
}
|
||||
|
||||
interface FileProgress {
|
||||
@@ -711,6 +713,7 @@ export class FreeRunReporter {
|
||||
private testsRan: number | null = null;
|
||||
private filesRan: number | null = null;
|
||||
private sawSummary = false;
|
||||
private lastOutputLine: string | null = null;
|
||||
|
||||
constructor(
|
||||
private readonly plannedFiles: string[],
|
||||
@@ -758,6 +761,7 @@ export class FreeRunReporter {
|
||||
unhandledErrors: [...this.unhandled],
|
||||
inFlight,
|
||||
filesWithNoOutput: this.plannedFiles.filter((f) => !this.progress.has(normalizeRelativePath(f))).length,
|
||||
lastOutputLine: this.lastOutputLine,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -768,6 +772,7 @@ export class FreeRunReporter {
|
||||
// (fail) lines land under a second phantom file (observed on the first
|
||||
// Linux run: 5 real failures reported as 10 across 2 files).
|
||||
const line = stripAnsiLine(rawLine).replace(/^::group::/, '');
|
||||
if (line.trim() !== '') this.lastOutputLine = line.trim();
|
||||
let visible = false;
|
||||
const failedCount = this.failureSummary.consume(line, origin);
|
||||
if (failedCount !== null) {
|
||||
@@ -903,6 +908,10 @@ export function buildRunEpilogue(
|
||||
for (const u of report.unhandledErrors) {
|
||||
lines.push(` ⚠ unhandled error between tests (around ${u.file ?? 'unknown file'})`);
|
||||
}
|
||||
if (status === 'failed' && !report.sawTerminalSummary) {
|
||||
lines.push(` ⚠ test process ended before its summary; in flight: ${report.inFlight.length > 0 ? report.inFlight.join(', ') : 'unknown'}`);
|
||||
if (report.lastOutputLine) lines.push(` ⚠ last output: ${report.lastOutputLine}`);
|
||||
}
|
||||
if (status === 'timed-out') {
|
||||
if (report.inFlight.length > 0) {
|
||||
lines.push(` ⏱ in flight at kill: ${report.inFlight.join(', ')}`);
|
||||
|
||||
+25
-16
@@ -66,13 +66,16 @@ import { EVAL_POLICY, PERIODIC_CI_EXCLUDE } from '../test/helpers/periodic-exclu
|
||||
import {
|
||||
getProjectEvalDir, getClaudeCliVersion, isFinalizedEvalResultFile, failureClassOf, panelVerdict,
|
||||
sanitizeTrialError, CONTRACT_VIOLATIONS_FILE, TRIAL_ENV, type TrialFailureClass,
|
||||
trialCostKnown, trialFailureFields, trialSessions,
|
||||
} from '../test/helpers/eval-store';
|
||||
import { STALL_WINDOW_MS } from '../test/helpers/eval-budgets';
|
||||
import { readSessionLedger, type SessionLedgerRow } from '../test/helpers/session-ledger';
|
||||
import { preflightAnthropicApi } from '../test/helpers/anthropic-preflight';
|
||||
import { e2eReuseLaneProblem, prepareE2EShardReuse, selectPlanReceipts } from './e2e-shard-reuse';
|
||||
import { E2E_TOUCHFILES, E2E_TIERS } from '../test/helpers/touchfiles';
|
||||
import { scopeCodexAccess, shardFile, shardCaseId, shardTrial, trialShardKey, type CaseTrialPlan, caseTrialPlan, excludedCasesNamePattern, caseTestNamePattern, expandCaseShards, expandTrialShards, fileCaseRegistration, partitionCaseExclusions } from './lib/paid-cases';
|
||||
import { retriesForFiles, trialPanelKey, sliceExecutionOrder, buildRunManifest, parseRunManifest, type SliceResult, sliceExitCode, guardTrialRecords, formatSlicePlan, formatCapacityPreflight, sliceDeadlineMs } from './lib/paid-plan';
|
||||
import { caseFile, runCaseDiagnosis, formatPanelLine, runPaidReport } from './lib/paid-report';
|
||||
import { caseSelection, runCaseDiagnosis, formatPanelLine, runPaidReport } from './lib/paid-report';
|
||||
import {
|
||||
DEFAULT_JOBS, DEFAULT_MAX_FILES_PER_SHARD, DEFAULT_SHARD_TIMEOUT_MS, DEFAULT_TIER, DEFAULT_WITHIN_SHARD_CONCURRENCY, OVERLAY_MAX_ACTIVE_SHARDS,
|
||||
PAID_LANE_POLICY, PAID_TIERS, ROOT, SLICE_JOB_STARTED_AT_ENV, SLICE_UPLOAD_RESERVE_MS, isOverlayTestFile,
|
||||
@@ -94,8 +97,10 @@ export * from './lib/paid-cases';
|
||||
export * from './lib/paid-plan';
|
||||
export * from './lib/paid-report';
|
||||
|
||||
/** Records and contract evidence an isolated shard left in its eval dir. */
|
||||
export function readTrialEvidence(evalDir: string | undefined): { records: any[]; contract: string | null } {
|
||||
type TrialEvidence = { records: any[]; contract: string | null; sessions?: SessionLedgerRow[] };
|
||||
|
||||
/** Records, contract evidence and session-ledger rows an isolated shard left in its eval dir. */
|
||||
export function readTrialEvidence(evalDir: string | undefined): TrialEvidence {
|
||||
if (!evalDir || !fs.existsSync(evalDir)) return { records: [], contract: null };
|
||||
const names = fs.readdirSync(evalDir);
|
||||
const parse = (name: string) => { try { return JSON.parse(fs.readFileSync(path.join(evalDir, name), 'utf8')); } catch { return null; } };
|
||||
@@ -108,28 +113,32 @@ export function readTrialEvidence(evalDir: string | undefined): { records: any[]
|
||||
const line = fs.readFileSync(path.join(evalDir, CONTRACT_VIOLATIONS_FILE), 'utf8').split('\n').find(l => l.trim());
|
||||
if (line) contract = String(JSON.parse(line).message ?? 'contract violation');
|
||||
} catch { /* no sidecar */ }
|
||||
return { records, contract };
|
||||
return { records, contract, sessions: readSessionLedger(evalDir) };
|
||||
}
|
||||
|
||||
/** Classify one isolated trial shard. Contract evidence always fails the trial. */
|
||||
export function classifyTrialShard(
|
||||
outcome: Pick<ShardOutcome, 'status' | 'executedTests' | 'skippedTests' | 'elapsedMs' | 'runnerError'>,
|
||||
caseId: string, trial: number, plan: CaseTrialPlan,
|
||||
evidence: { records: any[]; contract: string | null },
|
||||
evidence: TrialEvidence,
|
||||
): ShardTrialRecord {
|
||||
const failedRecord = evidence.records.find(record => record.passed === false) ?? evidence.records[0];
|
||||
const base: ShardTrialRecord = {
|
||||
case: caseId, trial, kind: plan.kind, panel: plan.panel, quarantined: plan.quarantined, outcome: null,
|
||||
cost_usd: Math.round(evidence.records.reduce((sum, record) => sum + (Number(record.cost_usd) || 0), 0) * 100) / 100,
|
||||
duration_ms: outcome.elapsedMs,
|
||||
...(typeof failedRecord?.model === 'string' ? { model: failedRecord.model } : {}),
|
||||
...(typeof failedRecord?.model === 'string' ? { model: failedRecord.model } : {}), ...trialSessions(evidence.sessions ?? []),
|
||||
...trialCostKnown(evidence.records, evidence.sessions),
|
||||
};
|
||||
const failed = (failureClass: TrialFailureClass, error?: string): ShardTrialRecord => {
|
||||
const cls = evidence.contract !== null ? 'contract' : failureClass;
|
||||
const raw = evidence.contract ?? failedRecord?.error ?? error;
|
||||
return { ...base, outcome: 'failed', failure_class: cls,
|
||||
...(failedRecord?.exit_reason ? { exit_reason: String(failedRecord.exit_reason) } : {}),
|
||||
...(Number.isInteger(failedRecord?.timeout_at_turn) ? { timeout_at_turn: failedRecord.timeout_at_turn } : {}),
|
||||
...(sanitizeTrialError(raw) ? { error: sanitizeTrialError(raw) } : {}),
|
||||
...trialFailureFields({ failure_class: cls, exit_reason: failedRecord?.exit_reason, error: raw, sessions: evidence.sessions, record: failedRecord }, STALL_WINDOW_MS) };
|
||||
};
|
||||
const failed = (failureClass: TrialFailureClass, error?: string): ShardTrialRecord => ({
|
||||
...base, outcome: 'failed', failure_class: evidence.contract !== null ? 'contract' : failureClass,
|
||||
...(failedRecord?.exit_reason ? { exit_reason: String(failedRecord.exit_reason) } : {}),
|
||||
...(Number.isInteger(failedRecord?.timeout_at_turn) ? { timeout_at_turn: failedRecord.timeout_at_turn } : {}),
|
||||
...(sanitizeTrialError(evidence.contract ?? failedRecord?.error ?? error) ? { error: sanitizeTrialError(evidence.contract ?? failedRecord?.error ?? error) } : {}),
|
||||
});
|
||||
if (outcome.runnerError !== undefined) return { ...base, harness: `runner error: ${sanitizeTrialError(outcome.runnerError) ?? 'unknown'}` };
|
||||
if (outcome.status === 'never-started') return { ...base, harness: 'never started' };
|
||||
if (outcome.status === 'passed-empty') return { ...base, harness: 'hollow: executed no case' };
|
||||
@@ -873,16 +882,16 @@ async function main(): Promise<number> {
|
||||
if (options.reportDir) return runPaidReport(options.reportDir, { writeDurations: options.writeDurations });
|
||||
|
||||
if (options.caseId && options.listOnly) {
|
||||
const file = caseFile(options.caseId);
|
||||
const { file, mode, reason } = caseSelection(options.caseId);
|
||||
const plan = caseTrialPlan(options.caseId);
|
||||
const n = options.trials ?? plan.panel.n;
|
||||
console.log(`[test:paid] --case ${options.caseId}: ${n} trial(s) of ${file} (kind ${plan.kind}), list only`);
|
||||
for (let trial = 1; trial <= n; trial++) console.log(` ${trialShardKey(file, options.caseId, trial)}`);
|
||||
console.log(`[test:paid] --case ${options.caseId}: ${n} trial(s) of ${file} (kind ${plan.kind}), selects ${reason}, list only`);
|
||||
for (let trial = 1; trial <= n; trial++) console.log(` ${mode === 'file' ? `${file} (trial ${trial})` : trialShardKey(file, options.caseId, trial)}`);
|
||||
return 0;
|
||||
}
|
||||
if (options.caseId) {
|
||||
preflightAnthropicApi(process.env);
|
||||
const verdict = await runCaseDiagnosis(options.caseId, { trials: options.trials ?? undefined, jobs: options.jobs,
|
||||
const verdict = await runCaseDiagnosis(options.caseId, { trials: options.trials ?? undefined, jobs: options.jobs, runShards: runPaidShards,
|
||||
withinShardConcurrency: options.withinShardConcurrency, timeoutMs: timeoutOverride,
|
||||
evalDirBase: process.env.GSTACK_EVAL_DIR || getProjectEvalDir() });
|
||||
return verdict.status === 'PASS' ? 0 : 1;
|
||||
|
||||
@@ -265,7 +265,6 @@
|
||||
"test/model-overlays.test.ts\tTS2322\tType 'string' is not assignable to type '\"claude\" | \"fable-5\" | \"gemini\" | \"gpt\" | \"gpt-5.4\" | \"gpt-5.6-sol\" | \"gpt-6-astra\" | \"o-series\" | \"opus-4-7\" | \"opus-4-8\" | \"sonnet-5\" | undefined'.": 4,
|
||||
"test/native-auto-decide-pty.test.ts\tTS2345\tArgument of type 'string | undefined' is not assignable to parameter of type 'string'. Type 'undefined' is not assignable to type 'string'.": 1,
|
||||
"test/native-auto-decide.test.ts\tTS2345\tArgument of type '{ status: string; calls: never[]; assistantMessages: { sessionId: string; text: string; timestamp: string; }[]; }' is not assignable to parameter of type 'PlanCountTranscript'. Types of property 'status' are incompatible. Type 'string' is not assignable to type '\"error\" | \"missing\" | \"ready\"'.": 1,
|
||||
"test/office-hours-completion.test.ts\tTS18046\t'reordered.findings' is of type 'unknown'.": 1,
|
||||
"test/office-hours-review.test.ts\tTS18048\t'check.stderr' is possibly 'undefined'.": 1,
|
||||
"test/office-hours-review.test.ts\tTS18048\t'check.stdout' is possibly 'undefined'.": 1,
|
||||
"test/office-hours-review.test.ts\tTS18048\t'direct.stderr' is possibly 'undefined'.": 1,
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
# Runs as user `ubi` (passwordless sudo) from the synced checkout.
|
||||
set -euo pipefail
|
||||
|
||||
BUN_VERSION=1.4.0
|
||||
BUN_VERSION=1.4.2
|
||||
NODE_VERSION=22.20.0
|
||||
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
Reference in new issue
Block a user