diff --git a/scripts/lib/paid-report.ts b/scripts/lib/paid-report.ts index 748f778ef..8e2bf6492 100644 --- a/scripts/lib/paid-report.ts +++ b/scripts/lib/paid-report.ts @@ -64,14 +64,39 @@ import { DEFAULT_JOBS, type PaidCaseSelection, type PaidTier, ROOT, type RunShar // ─── Local diagnosis: one case through the CI panel runner (A9) ──────────── -/** The one paid file that statically registers `id`, else a thrown reason. */ +/** How --case runs exactly one case: trials selected by its Bun test name, or its whole file when that file registers no other case. */ +export interface CaseSelection { file: string; mode: 'name' | 'file'; reason: string } + +const escapeRe = (text: string) => text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + +/** + * The paid file that registers `id` and how a trial selects it. A file whose + * registration is complete wins over one that also lists the id; a test + * registered literally under the id (or its CASE_TEST_NAMES label) is + * selected by name; a loop that registers ids at runtime is selected by the + * id it names its test; a file registering only this case runs whole (its + * test names are not the id). Anything else throws before any process starts. + */ +export function caseSelection(id: string, rootDir = ROOT, discovered = collectPaidTestFiles(rootDir)): CaseSelection { + const files = discovered.map(file => { + const source = fs.readFileSync(path.join(rootDir, file), 'utf8'); + return { file, source, ...fileCaseRegistration(file, source) }; + }).filter(f => f.registered.includes(id)); + const owners = files.filter(f => f.known).length ? files.filter(f => f.known) : files; + if (owners.length !== 1) throw new Error(`--case ${id}: ${owners.length ? `registered by ${owners.map(f => f.file).join(', ')}` : 'no paid file registers it'}; it needs exactly one`); + const owner = owners[0]!; + const name = escapeRe(CASE_TEST_NAMES[id] ?? id); + if (new RegExp(`\\b(?:test|it|test\\.\\w+|test(?:Concurrent)?IfSelected)\\s*\\(\\s*(['"\`])${name}\\1|testName\\s*:\\s*(['"\`])${name}\\2`).test(owner.source)) { + return { file: owner.file, mode: 'name', reason: `its Bun test is named ${CASE_TEST_NAMES[id] ?? id}` }; + } + if (owner.registered.length === 1) return { file: owner.file, mode: 'file', reason: 'the whole file (it registers no other case)' }; + if (owner.computed) return { file: owner.file, mode: 'name', reason: `its loop names the Bun test ${id} at runtime` }; + throw new Error(`--case ${id}: ${owner.file} registers several cases and none of its tests is named ${id}; add its test name to CASE_TEST_NAMES`); +} + +/** The one paid file that registers `id`, else a thrown reason. */ export function caseFile(id: string, rootDir = ROOT, discovered = collectPaidTestFiles(rootDir)): string { - const owners = discovered.filter(file => { - const { registered, known } = fileCaseRegistration(file, fs.readFileSync(path.join(rootDir, file), 'utf8')); - return known && registered.includes(id); - }); - if (owners.length !== 1) throw new Error(`--case ${id}: ${owners.length ? `registered by ${owners.join(', ')}` : 'no paid file statically registers it'}; it needs exactly one`); - return owners[0]!; + return caseSelection(id, rootDir, discovered).file; } /** @@ -87,25 +112,34 @@ export async function runCaseDiagnosis(id: string, options: { } = {}): Promise { const rootDir = options.rootDir ?? ROOT; const log = options.log ?? ((line: string) => console.log(line)); - const file = options.file ?? caseFile(id, rootDir); - const testName = CASE_TEST_NAMES[id] ?? id; - if (!options.commandFor && !fs.readFileSync(path.join(rootDir, file), 'utf8').includes(testName)) { - throw new Error(`--case ${id}: ${file} has no literal Bun test named "${testName}", so a trial could not select it. ` - + 'Run the whole file (bun test with EVALS=1) or add its literal name to CASE_TEST_NAMES.'); - } + const selection = options.file ? { file: options.file, mode: 'name' as const } : caseSelection(id, rootDir); + const file = selection.file; const policy = caseTrialPlan(id); const n = options.trials ?? policy.panel.n; const plan: CaseTrialPlan = { ...policy, panel: { n, k: policy.panel.k === policy.panel.n ? n : Math.min(policy.panel.k, n) } }; const keys = Array.from({ length: n }, (_, i) => trialShardKey(file, id, i + 1)); const tier = E2E_TIERS[id] as PaidTier; log(`[test:paid] --case ${id}: ${n} trial(s) of ${file} (kind ${plan.kind}, PASS at ${plan.panel.k}/${n}${plan.quarantined ? ', quarantined' : ''}), tier=${tier}`); - const summary = await runPaidShards(keys.map(key => [key]), { - jobs: Math.min(options.jobs ?? DEFAULT_JOBS, n), withinShardConcurrency: options.withinShardConcurrency, timeoutMs: options.timeoutMs, - rootDir, log, commandFor: options.commandFor, evalDirBase: options.evalDirBase, - trials: Object.fromEntries(keys.map(key => [key, plan])), + const shardOptions = { withinShardConcurrency: options.withinShardConcurrency, timeoutMs: options.timeoutMs, rootDir, log, commandFor: options.commandFor, env: { ...(options.env ?? process.env), EVALS: '1', EVALS_TIER: tier, EVALS_PREFLIGHT_OK: '1', EVALS_ALL: '1', - ...paidSelectionEnv('full', { e2e: [id], judges: [] }, `--case ${id}`) }, - }); + ...paidSelectionEnv('full', { e2e: [id], judges: [] }, `--case ${id}`) } }; + if (selection.mode === 'file') { + // The whole file is the case: one file shard per trial, each with its own eval dir, judged by shard status as CI does. + const fileTrials = []; + for (let trial = 1; trial <= n; trial++) { + const [outcome] = (await runPaidShards([[file]], { ...shardOptions, jobs: 1, + ...(options.evalDirBase ? { evalDirBase: path.join(options.evalDirBase, `t${trial}`) } : {}) })).outcomes; + if (!outcome || outcome.runnerError !== undefined || isAllSkippedPass(outcome) || !['passed', 'failed', 'timed-out'].includes(outcome.status)) continue; + fileTrials.push({ trial, outcome: outcome.status === 'passed' ? 'passed' as const : 'failed' as const, + ...(outcome.status === 'passed' ? {} : { failure_class: outcome.status === 'timed-out' ? 'timeout' as const : 'assertion' as const }) }); + } + const verdict = panelVerdict({ case: id, kind: plan.kind, panel: plan.panel, trials: fileTrials, quarantined: plan.quarantined }); + log(formatPanelLine({ ...verdict, file, slices: {} }, tier)); + return verdict; + } + const summary = await runPaidShards(keys.map(key => [key]), { ...shardOptions, + jobs: Math.min(options.jobs ?? DEFAULT_JOBS, n), evalDirBase: options.evalDirBase, + trials: Object.fromEntries(keys.map(key => [key, plan])) }); const trials = summary.outcomes.flatMap(outcome => outcome.trial && outcome.trial.outcome !== null ? [{ trial: outcome.trial.trial, outcome: outcome.trial.outcome, ...(outcome.trial.failure_class ? { failure_class: outcome.trial.failure_class } : {}), ...(outcome.trial.exit_reason ? { exit_reason: outcome.trial.exit_reason } : {}), ...(outcome.trial.error ? { error: outcome.trial.error } : {}), diff --git a/scripts/test-paid-shards.ts b/scripts/test-paid-shards.ts index 63ce29518..8fcecd0ee 100644 --- a/scripts/test-paid-shards.ts +++ b/scripts/test-paid-shards.ts @@ -109,7 +109,7 @@ export { PERIODIC_CI_EXCLUDE }; import { scopeCodexAccess, shardFile, shardCaseId, shardTrial, trialShardKey, type CaseTrialPlan, caseTrialPlan, excludedCasesNamePattern, caseTestNamePattern, expandCaseShards, expandTrialShards, partitionCaseExclusions } from './lib/paid-cases'; import { retriesForFiles, trialPanelKey, sliceExecutionOrder, buildRunManifest, parseRunManifest, type SliceResult, sliceExitCode, guardTrialRecords, formatSlicePlan, formatCapacityPreflight } from './lib/paid-plan'; -import { caseFile, runCaseDiagnosis, formatPanelLine, runPaidReport } from './lib/paid-report'; +import { caseSelection, runCaseDiagnosis, formatPanelLine, runPaidReport } from './lib/paid-report'; export * from './lib/paid-cases'; export * from './lib/paid-plan'; export * from './lib/paid-report'; @@ -1497,11 +1497,11 @@ async function main(): Promise { if (options.reportDir) return runPaidReport(options.reportDir, { writeDurations: options.writeDurations }); if (options.caseId && options.listOnly) { - const file = caseFile(options.caseId); + const { file, mode, reason } = caseSelection(options.caseId); const plan = caseTrialPlan(options.caseId); const n = options.trials ?? plan.panel.n; - console.log(`[test:paid] --case ${options.caseId}: ${n} trial(s) of ${file} (kind ${plan.kind}), list only`); - for (let trial = 1; trial <= n; trial++) console.log(` ${trialShardKey(file, options.caseId, trial)}`); + console.log(`[test:paid] --case ${options.caseId}: ${n} trial(s) of ${file} (kind ${plan.kind}), selects ${reason}, list only`); + for (let trial = 1; trial <= n; trial++) console.log(` ${mode === 'file' ? `${file} (trial ${trial})` : trialShardKey(file, options.caseId, trial)}`); return 0; } if (options.caseId) { diff --git a/test/paid-shards.test.ts b/test/paid-shards.test.ts index 6d94b3554..bec8a1f82 100644 --- a/test/paid-shards.test.ts +++ b/test/paid-shards.test.ts @@ -13,6 +13,7 @@ import { describe, test, expect } from 'bun:test'; import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; +import { spawnSync } from 'child_process'; import { E2E_TIERS, E2E_TOUCHFILES } from './helpers/touchfiles'; const ROOT = path.resolve(import.meta.dir, '..'); @@ -56,6 +57,7 @@ import { excludedCasesNamePattern, runCaseDiagnosis, caseFile, + caseSelection, parseCliOptions, loadPaidTestDurations, shardDurationViolations, @@ -777,6 +779,49 @@ console.log("Ran 1 tests across 1 files. [1ms]"); process.exit(${fail ? 1 : 0}); expect(caseFile('review-sql-injection')).toBe('test/skill-e2e-review.test.ts'); }); + test('B4: every registered E2E case resolves to exactly itself, loop-registered cases included', () => { + const unresolved: string[] = []; + for (const id of Object.keys(E2E_TIERS)) { + let selection: ReturnType; + try { selection = caseSelection(id); } catch (error) { unresolved.push(`${id}: ${(error as Error).message}`); continue; } + const { registered, computed } = fileCaseRegistration(selection.file, fs.readFileSync(path.join(ROOT, selection.file), 'utf8')); + expect(registered, id).toContain(id); + if (selection.mode === 'file') expect(registered, id).toEqual([id]); + else expect(computed || fs.readFileSync(path.join(ROOT, selection.file), 'utf8').includes(CASE_TEST_NAMES[id] ?? id), id).toBe(true); + } + // Fifteen carve files each register this one case; CI runs each file as its own shard. + expect(unresolved).toEqual([expect.stringMatching(/^carve-section-loading: --case carve-section-loading: registered by test\/carve-section-loading-/)]); + expect(caseSelection('plan-mode-no-op')).toEqual({ file: 'test/skill-e2e-plan-mode-no-op.test.ts', mode: 'file', reason: 'the whole file (it registers no other case)' }); + expect(caseSelection('review-coverage-audit')).toMatchObject({ file: 'test/skill-e2e-coverage-audit.test.ts', mode: 'name' }); + expect(caseSelection('plan-eng-multi-finding-batching').file).toBe('test/skill-e2e-plan-eng-multi-finding-batching.test.ts'); + expect(caseSelection('review-sql-injection')).toMatchObject({ file: 'test/skill-e2e-review.test.ts', mode: 'name' }); + }); + + test('B4: --list prints the selection; an unknown id exits non-zero before any process starts', () => { + const run = (args: string[]) => spawnSync(process.execPath, [path.join(ROOT, 'scripts/test-paid-shards.ts'), ...args], { encoding: 'utf8', timeout: 60_000, cwd: ROOT }); + const listed = run(['--tier', 'gate', '--case', 'plan-mode-no-op', '--list']); + expect(listed.status, listed.stderr).toBe(0); + expect(listed.stdout).toContain('--case plan-mode-no-op: 1 trial(s) of test/skill-e2e-plan-mode-no-op.test.ts (kind rule), selects the whole file (it registers no other case), list only'); + const unknown = run(['--tier', 'gate', '--case', 'no-such-case']); + expect(unknown.status).toBe(1); + expect(unknown.stderr).toContain('--case needs a live E2E case id. Received: no-such-case'); + expect(unknown.stdout).not.toContain('shard'); + }); + + test('B4: a whole-file case runs one file shard per trial and judges each by shard status', async () => { + const evalDirBase = fs.mkdtempSync(path.join(os.tmpdir(), 'case-file-diagnosis-')); + const lines: string[] = []; + let call = 0; + try { + const verdict = await runCaseDiagnosis('plan-mode-no-op', { trials: 2, evalDirBase, log: line => lines.push(line), + commandFor: files => { call++; expect(files).toEqual(['test/skill-e2e-plan-mode-no-op.test.ts']); + return { command: process.execPath, args: ['-e', `console.log("Ran 5 tests across 1 files. [1ms]"); process.exit(${call === 2 ? 1 : 0});`] }; } }); + expect(call).toBe(2); + expect(verdict).toMatchObject({ case: 'plan-mode-no-op', kind: 'rule', panel: { n: 2, k: 2 }, passed: 1, status: 'FAIL' }); + expect(lines.join('\n')).toContain('FAIL 1/2 (✓✗)'); + } finally { fs.rmSync(evalDirBase, { recursive: true, force: true }); } + }); + test('--case runs the CI panel runner and prints its panelVerdict', async () => { const evalDirBase = fs.mkdtempSync(path.join(os.tmpdir(), 'case-diagnosis-')); const lines: string[] = [];