diff --git a/scripts/paid-test-durations.json b/scripts/paid-test-durations.json index ca65aceb3..c46c2a434 100644 --- a/scripts/paid-test-durations.json +++ b/scripts/paid-test-durations.json @@ -32,7 +32,7 @@ "test/skill-e2e-plan-ceo-finding-floor.test.ts": 297738, "test/skill-e2e-plan-ceo-plan-mode.test.ts": 36812, "test/skill-e2e-plan-decision-classification.test.ts": 1000, - "test/skill-e2e-plan-design-with-ui.test.ts": 595151, + "test/skill-e2e-plan-design-with-ui.test.ts": 240000, "test/skill-e2e-plan-devex-finding-floor.test.ts": 207635, "test/skill-e2e-plan-devex-peer-comparison-classification.test.ts": 1000, "test/skill-e2e-plan-devex-plan-mode.test.ts": 115679, @@ -50,6 +50,11 @@ "test/skill-e2e-plan.test.ts#plan-review-report": 76406, "test/skill-e2e-qa-bugs.test.ts": 1000, "test/skill-e2e-qa-callers.test.ts": 737164, + "test/skill-e2e-qa-callers.test.ts#review-exploratory-small-cli": 294865, + "test/skill-e2e-qa-callers.test.ts#ship-exploratory-late-input": 294865, + "test/skill-e2e-qa-callers.test.ts#ship-exploratory-plan-checks": 294865, + "test/skill-e2e-qa-callers.test.ts#ship-exploratory-small-cli": 294865, + "test/skill-e2e-qa-callers.test.ts#ship-exploratory-unavailable": 294865, "test/skill-e2e-qa-functional-fix.test.ts": 235133, "test/skill-e2e-qa-functional.test.ts": 356641, "test/skill-e2e-qa-workflow.test.ts": 427303, @@ -162,7 +167,7 @@ "test/skill-e2e-plan-design-plan-mode.test.ts": 106608, "test/skill-e2e-plan-devex-peer-comparison-classification.test.ts": 79786, "test/skill-e2e-plan-eng-finding-floor.test.ts": 388666, - "test/skill-e2e-plan-eng-multi-finding-batching.test.ts": 1318488, + "test/skill-e2e-plan-eng-multi-finding-batching.test.ts": 597500, "test/skill-e2e-plan-eng-plan-mode.test.ts": 229226, "test/skill-e2e-plan-format.test.ts": 219670, "test/skill-e2e-plan-prosons.test.ts": 163659, diff --git a/scripts/test-paid-shards.ts b/scripts/test-paid-shards.ts index cf853b04f..ee2cbfa9b 100644 --- a/scripts/test-paid-shards.ts +++ b/scripts/test-paid-shards.ts @@ -142,6 +142,7 @@ export const CASE_SHARDED_FILES: readonly string[] = [ 'test/skill-e2e-shared-libs-paths.test.ts', 'test/skill-e2e-shared-libs.test.ts', 'test/skill-e2e-ship-docsync.test.ts', + 'test/skill-e2e-qa-callers.test.ts', ]; /** Bun test names that differ from their E2E id. */ @@ -2268,6 +2269,11 @@ export async function runCaseDiagnosis(id: string, options: { const rootDir = options.rootDir ?? ROOT; const log = options.log ?? ((line: string) => console.log(line)); const file = options.file ?? caseFile(id, rootDir); + const testName = CASE_TEST_NAMES[id] ?? id; + if (!options.commandFor && !fs.readFileSync(path.join(rootDir, file), 'utf8').includes(testName)) { + throw new Error(`--case ${id}: ${file} has no literal Bun test named "${testName}", so a trial could not select it. ` + + 'Run the whole file (bun test with EVALS=1) or add its literal name to CASE_TEST_NAMES.'); + } const policy = caseTrialPlan(id); const n = options.trials ?? policy.panel.n; const plan: CaseTrialPlan = { ...policy, panel: { n, k: policy.panel.k === policy.panel.n ? n : Math.min(policy.panel.k, n) } }; @@ -2924,6 +2930,14 @@ async function main(): Promise { // a slice whose artifact never landed is a FAILURE, not an absence. if (options.reportDir) return runPaidReport(options.reportDir, { writeDurations: options.writeDurations }); + if (options.caseId && options.listOnly) { + const file = caseFile(options.caseId); + const plan = caseTrialPlan(options.caseId); + const n = options.trials ?? plan.panel.n; + console.log(`[test:paid] --case ${options.caseId}: ${n} trial(s) of ${file} (kind ${plan.kind}), list only`); + for (let trial = 1; trial <= n; trial++) console.log(` ${trialShardKey(file, options.caseId, trial)}`); + return 0; + } if (options.caseId) { preflightAnthropicApi(process.env); const verdict = await runCaseDiagnosis(options.caseId, { trials: options.trials ?? undefined, jobs: options.jobs, diff --git a/test/cookie-validation-phases.test.ts b/test/cookie-validation-phases.test.ts index c32f247b3..25fdc8ad5 100644 --- a/test/cookie-validation-phases.test.ts +++ b/test/cookie-validation-phases.test.ts @@ -45,11 +45,12 @@ test('the existing quality and behavior phases retain their complete separate sh expect(behavior.evalsAll).toBe(true); expect(qualityFiles).toHaveLength(1); // 44 files (first-task-scaffold registers no gate case, so the gate lane - // skips it); the six case-sharded files contribute one shard per gate case. + // skips it); the seven case-sharded files contribute one shard per gate case. expect(new Set(behaviorFiles.map(file => file.split('#')[0])).size).toBe(44); - expect(behaviorFiles).toHaveLength(73); + expect(behaviorFiles).toHaveLength(77); expect(behaviorFiles).toEqual(expect.arrayContaining([ - 'test/skill-e2e-qa-callers.test.ts', + ...['review-exploratory-small-cli', 'ship-exploratory-small-cli', 'ship-exploratory-unavailable', + 'ship-exploratory-plan-checks', 'ship-exploratory-late-input'].map(id => `test/skill-e2e-qa-callers.test.ts#${id}`), 'test/skill-e2e-qa-functional-fix.test.ts', 'test/skill-e2e-qa-functional.test.ts', 'test/skill-e2e-ship-skip.test.ts', diff --git a/test/paid-retry-supervision.test.ts b/test/paid-retry-supervision.test.ts index 7f0e93796..66abbe259 100644 --- a/test/paid-retry-supervision.test.ts +++ b/test/paid-retry-supervision.test.ts @@ -156,8 +156,8 @@ test('fixed AUQ count remains strict while mixed-tier files keep ordinary case h [AUQ_CONSISTENCY_RETRY_BUDGET.file, 0, false], [AUQ_CONSISTENCY_RETRY_BUDGET.file, 1, true], [AUQ_CONSISTENCY_RETRY_BUDGET.file, 2, false], - ['test/skill-e2e-qa-callers.test.ts', 3, true], - ['test/skill-e2e-qa-callers.test.ts', 5, true], + ['test/skill-e2e-plan-format.test.ts', 2, true], + ['test/skill-e2e-plan-format.test.ts', 4, true], // A case shard of a case-sharded registered file executes exactly its case. [plannedKey('test/skill-e2e-plan.test.ts'), 1, true], [plannedKey('test/skill-e2e-plan.test.ts'), 2, false], @@ -211,7 +211,7 @@ test('detached PR fallback and release commands cover their actual default worke const prFloor = Math.ceil((Math.ceil(fullGateFiles.length / prWorkers) * 1_800_000 + fullGateFiles.reduce( (total, file) => total + Math.max(0, resolvePaidShardBudget([file]).timeoutMs - 1_800_000), 0, )) / 1000 * 1.05); - expect(prFloor).toBe(72_755); + expect(prFloor).toBe(76_535); expect(prWall).toBe(92_820_000); expect(prWall).toBeGreaterThanOrEqual(paidShardWallUpperBoundMs(files, prWorkers) + 120_000); diff --git a/test/skill-e2e-arm-benchmark.test.ts b/test/skill-e2e-arm-benchmark.test.ts index 01e3d7ee9..c831ee793 100644 --- a/test/skill-e2e-arm-benchmark.test.ts +++ b/test/skill-e2e-arm-benchmark.test.ts @@ -21,7 +21,7 @@ * Failure taxonomy (CEO review finding 2): * - zero-diff arm -> VALID scored cell (LOC 0, judge scores it "none"). * - harvest failure -> cell FAILED, harvest: null recorded. - * - judge still malformed after armJudge's bounded retries -> judge_error + * - judge output malformed (armJudge never re-asks) -> judge_error * cell: excluded from aggregates, surfaced in the run report, never * silently dropped. * @@ -108,7 +108,7 @@ async function runArmCell(task: ArmTask, arm: Arm): Promise { } const checks = runChecks(task, dirs.dir); - // Judge taxonomy: still malformed after armJudge's bounded retries -> + // Judge taxonomy: malformed armJudge output (no re-ask) -> // judge_error cell (excluded from aggregates, surfaced in the report). let judge: ArmJudgeScore | null = null; let judgeError: string | null = null; diff --git a/test/skill-e2e-qa-callers.test.ts b/test/skill-e2e-qa-callers.test.ts index 0b37e43bb..e441e1cbf 100644 --- a/test/skill-e2e-qa-callers.test.ts +++ b/test/skill-e2e-qa-callers.test.ts @@ -79,10 +79,17 @@ async function capture(caseId: QaCallerCase) { } } -describeIfSelected('Automatic parent exploratory QA', [...QA_CALLER_CASES], () => { - for (const caseId of QA_CALLER_CASES) { - testConcurrentIfSelected(caseId, () => capture(caseId), QA_CALLER_TEST_MS); - } +const LITERAL_CASES = ['review-exploratory-small-cli', 'ship-exploratory-small-cli', 'ship-exploratory-unavailable', + 'ship-exploratory-plan-checks', 'ship-exploratory-late-input'] as const; +if (LITERAL_CASES.join() !== QA_CALLER_CASES.join()) throw new Error('qa-callers literal registration drifted from QA_CALLER_CASES'); + +describeIfSelected('Automatic parent exploratory QA', ['review-exploratory-small-cli', 'ship-exploratory-small-cli', + 'ship-exploratory-unavailable', 'ship-exploratory-plan-checks', 'ship-exploratory-late-input'], () => { + testConcurrentIfSelected('review-exploratory-small-cli', () => capture('review-exploratory-small-cli'), QA_CALLER_TEST_MS); + testConcurrentIfSelected('ship-exploratory-small-cli', () => capture('ship-exploratory-small-cli'), QA_CALLER_TEST_MS); + testConcurrentIfSelected('ship-exploratory-unavailable', () => capture('ship-exploratory-unavailable'), QA_CALLER_TEST_MS); + testConcurrentIfSelected('ship-exploratory-plan-checks', () => capture('ship-exploratory-plan-checks'), QA_CALLER_TEST_MS); + testConcurrentIfSelected('ship-exploratory-late-input', () => capture('ship-exploratory-late-input'), QA_CALLER_TEST_MS); }); afterAll(() => finalizeEvalCollector(collector));