fix(evals): --case list mode and name precheck; case-shard qa-callers; refresh batching and design-with-ui seeds

scripts/paid-test-durations.json

@@ -32,7 +32,7 @@
       "test/skill-e2e-plan-ceo-finding-floor.test.ts": 297738,
       "test/skill-e2e-plan-ceo-plan-mode.test.ts": 36812,
       "test/skill-e2e-plan-decision-classification.test.ts": 1000,
-      "test/skill-e2e-plan-design-with-ui.test.ts": 595151,
+      "test/skill-e2e-plan-design-with-ui.test.ts": 240000,
       "test/skill-e2e-plan-devex-finding-floor.test.ts": 207635,
       "test/skill-e2e-plan-devex-peer-comparison-classification.test.ts": 1000,
       "test/skill-e2e-plan-devex-plan-mode.test.ts": 115679,
@@ -50,6 +50,11 @@
       "test/skill-e2e-plan.test.ts#plan-review-report": 76406,
       "test/skill-e2e-qa-bugs.test.ts": 1000,
       "test/skill-e2e-qa-callers.test.ts": 737164,
+      "test/skill-e2e-qa-callers.test.ts#review-exploratory-small-cli": 294865,
+      "test/skill-e2e-qa-callers.test.ts#ship-exploratory-late-input": 294865,
+      "test/skill-e2e-qa-callers.test.ts#ship-exploratory-plan-checks": 294865,
+      "test/skill-e2e-qa-callers.test.ts#ship-exploratory-small-cli": 294865,
+      "test/skill-e2e-qa-callers.test.ts#ship-exploratory-unavailable": 294865,
       "test/skill-e2e-qa-functional-fix.test.ts": 235133,
       "test/skill-e2e-qa-functional.test.ts": 356641,
       "test/skill-e2e-qa-workflow.test.ts": 427303,
@@ -162,7 +167,7 @@
       "test/skill-e2e-plan-design-plan-mode.test.ts": 106608,
       "test/skill-e2e-plan-devex-peer-comparison-classification.test.ts": 79786,
       "test/skill-e2e-plan-eng-finding-floor.test.ts": 388666,
-      "test/skill-e2e-plan-eng-multi-finding-batching.test.ts": 1318488,
+      "test/skill-e2e-plan-eng-multi-finding-batching.test.ts": 597500,
       "test/skill-e2e-plan-eng-plan-mode.test.ts": 229226,
       "test/skill-e2e-plan-format.test.ts": 219670,
       "test/skill-e2e-plan-prosons.test.ts": 163659,

scripts/test-paid-shards.ts

@@ -142,6 +142,7 @@ export const CASE_SHARDED_FILES: readonly string[] = [
   'test/skill-e2e-shared-libs-paths.test.ts',
   'test/skill-e2e-shared-libs.test.ts',
   'test/skill-e2e-ship-docsync.test.ts',
+  'test/skill-e2e-qa-callers.test.ts',
 ];
 
 /** Bun test names that differ from their E2E id. */
@@ -2268,6 +2269,11 @@ export async function runCaseDiagnosis(id: string, options: {
   const rootDir = options.rootDir ?? ROOT;
   const log = options.log ?? ((line: string) => console.log(line));
   const file = options.file ?? caseFile(id, rootDir);
+  const testName = CASE_TEST_NAMES[id] ?? id;
+  if (!options.commandFor && !fs.readFileSync(path.join(rootDir, file), 'utf8').includes(testName)) {
+    throw new Error(`--case ${id}: ${file} has no literal Bun test named "${testName}", so a trial could not select it. `
+      + 'Run the whole file (bun test <file> with EVALS=1) or add its literal name to CASE_TEST_NAMES.');
+  }
   const policy = caseTrialPlan(id);
   const n = options.trials ?? policy.panel.n;
   const plan: CaseTrialPlan = { ...policy, panel: { n, k: policy.panel.k === policy.panel.n ? n : Math.min(policy.panel.k, n) } };
@@ -2924,6 +2930,14 @@ async function main(): Promise<number> {
   // a slice whose artifact never landed is a FAILURE, not an absence.
   if (options.reportDir) return runPaidReport(options.reportDir, { writeDurations: options.writeDurations });
 
+  if (options.caseId && options.listOnly) {
+    const file = caseFile(options.caseId);
+    const plan = caseTrialPlan(options.caseId);
+    const n = options.trials ?? plan.panel.n;
+    console.log(`[test:paid] --case ${options.caseId}: ${n} trial(s) of ${file} (kind ${plan.kind}), list only`);
+    for (let trial = 1; trial <= n; trial++) console.log(`  ${trialShardKey(file, options.caseId, trial)}`);
+    return 0;
+  }
   if (options.caseId) {
     preflightAnthropicApi(process.env);
     const verdict = await runCaseDiagnosis(options.caseId, { trials: options.trials ?? undefined, jobs: options.jobs,

test/cookie-validation-phases.test.ts

@@ -45,11 +45,12 @@ test('the existing quality and behavior phases retain their complete separate sh
   expect(behavior.evalsAll).toBe(true);
   expect(qualityFiles).toHaveLength(1);
   // 44 files (first-task-scaffold registers no gate case, so the gate lane
-  // skips it); the six case-sharded files contribute one shard per gate case.
+  // skips it); the seven case-sharded files contribute one shard per gate case.
   expect(new Set(behaviorFiles.map(file => file.split('#')[0])).size).toBe(44);
-  expect(behaviorFiles).toHaveLength(73);
+  expect(behaviorFiles).toHaveLength(77);
   expect(behaviorFiles).toEqual(expect.arrayContaining([
-    'test/skill-e2e-qa-callers.test.ts',
+    ...['review-exploratory-small-cli', 'ship-exploratory-small-cli', 'ship-exploratory-unavailable',
+      'ship-exploratory-plan-checks', 'ship-exploratory-late-input'].map(id => `test/skill-e2e-qa-callers.test.ts#${id}`),
     'test/skill-e2e-qa-functional-fix.test.ts',
     'test/skill-e2e-qa-functional.test.ts',
     'test/skill-e2e-ship-skip.test.ts',

test/paid-retry-supervision.test.ts

@@ -156,8 +156,8 @@ test('fixed AUQ count remains strict while mixed-tier files keep ordinary case h
     [AUQ_CONSISTENCY_RETRY_BUDGET.file, 0, false],
     [AUQ_CONSISTENCY_RETRY_BUDGET.file, 1, true],
     [AUQ_CONSISTENCY_RETRY_BUDGET.file, 2, false],
-    ['test/skill-e2e-qa-callers.test.ts', 3, true],
-    ['test/skill-e2e-qa-callers.test.ts', 5, true],
+    ['test/skill-e2e-plan-format.test.ts', 2, true],
+    ['test/skill-e2e-plan-format.test.ts', 4, true],
     // A case shard of a case-sharded registered file executes exactly its case.
     [plannedKey('test/skill-e2e-plan.test.ts'), 1, true],
     [plannedKey('test/skill-e2e-plan.test.ts'), 2, false],
@@ -211,7 +211,7 @@ test('detached PR fallback and release commands cover their actual default worke
   const prFloor = Math.ceil((Math.ceil(fullGateFiles.length / prWorkers) * 1_800_000 + fullGateFiles.reduce(
     (total, file) => total + Math.max(0, resolvePaidShardBudget([file]).timeoutMs - 1_800_000), 0,
   )) / 1000 * 1.05);
-  expect(prFloor).toBe(72_755);
+  expect(prFloor).toBe(76_535);
   expect(prWall).toBe(92_820_000);
   expect(prWall).toBeGreaterThanOrEqual(paidShardWallUpperBoundMs(files, prWorkers) + 120_000);
 

test/skill-e2e-arm-benchmark.test.ts

@@ -21,7 +21,7 @@
  * Failure taxonomy (CEO review finding 2):
  *   - zero-diff arm  -> VALID scored cell (LOC 0, judge scores it "none").
  *   - harvest failure -> cell FAILED, harvest: null recorded.
- *   - judge still malformed after armJudge's bounded retries -> judge_error
+ *   - judge output malformed (armJudge never re-asks) -> judge_error
  *     cell: excluded from aggregates, surfaced in the run report, never
  *     silently dropped.
  *
@@ -108,7 +108,7 @@ async function runArmCell(task: ArmTask, arm: Arm): Promise<CellResult> {
     }
     const checks = runChecks(task, dirs.dir);
 
-    // Judge taxonomy: still malformed after armJudge's bounded retries ->
+    // Judge taxonomy: malformed armJudge output (no re-ask) ->
     // judge_error cell (excluded from aggregates, surfaced in the report).
     let judge: ArmJudgeScore | null = null;
     let judgeError: string | null = null;

test/skill-e2e-qa-callers.test.ts

@@ -79,10 +79,17 @@ async function capture(caseId: QaCallerCase) {
   }
 }
 
-describeIfSelected('Automatic parent exploratory QA', [...QA_CALLER_CASES], () => {
-  for (const caseId of QA_CALLER_CASES) {
-    testConcurrentIfSelected(caseId, () => capture(caseId), QA_CALLER_TEST_MS);
-  }
+const LITERAL_CASES = ['review-exploratory-small-cli', 'ship-exploratory-small-cli', 'ship-exploratory-unavailable',
+  'ship-exploratory-plan-checks', 'ship-exploratory-late-input'] as const;
+if (LITERAL_CASES.join() !== QA_CALLER_CASES.join()) throw new Error('qa-callers literal registration drifted from QA_CALLER_CASES');
+
+describeIfSelected('Automatic parent exploratory QA', ['review-exploratory-small-cli', 'ship-exploratory-small-cli',
+  'ship-exploratory-unavailable', 'ship-exploratory-plan-checks', 'ship-exploratory-late-input'], () => {
+  testConcurrentIfSelected('review-exploratory-small-cli', () => capture('review-exploratory-small-cli'), QA_CALLER_TEST_MS);
+  testConcurrentIfSelected('ship-exploratory-small-cli', () => capture('ship-exploratory-small-cli'), QA_CALLER_TEST_MS);
+  testConcurrentIfSelected('ship-exploratory-unavailable', () => capture('ship-exploratory-unavailable'), QA_CALLER_TEST_MS);
+  testConcurrentIfSelected('ship-exploratory-plan-checks', () => capture('ship-exploratory-plan-checks'), QA_CALLER_TEST_MS);
+  testConcurrentIfSelected('ship-exploratory-late-input', () => capture('ship-exploratory-late-input'), QA_CALLER_TEST_MS);
 });
 
 afterAll(() => finalizeEvalCollector(collector));