Merge remote-tracking branch 'origin/capy/audit-fix-wave' into capy/fixwave-baseline-repairs

test/helpers/periodic-exclude-data.ts

@@ -44,3 +44,18 @@ export const PERIODIC_CI_EXCLUDE: Record<string, { reason: string; tracking: str
     tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)',
   },
 };
+
+/**
+ * Case-level exclusions for case-sharded files (`<file>#<case id>`), same
+ * contract as above: a case lands here only when a CI runner cannot execute it
+ * (it self-skips), with reason + tracking. The planner records each as an
+ * excluded manifest entry instead of an empty case shard, so the exact
+ * one-case check stays strict for every planned case. Pinned by
+ * test/periodic-exclude-policy.test.ts.
+ */
+export const CASE_CI_EXCLUDE: Record<string, { reason: string; tracking: string }> = {
+  'test/skill-e2e-design.test.ts#design-review-fix': {
+    reason: '/design-review drives the Aside browser; CI runners are Linux without Aside, so the case registers test.skip("needs Aside")',
+    tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)',
+  },
+};

test/periodic-exclude-policy.test.ts

@@ -9,9 +9,10 @@ import { describe, expect, test } from 'bun:test';
 import * as fs from 'node:fs';
 import * as path from 'node:path';
 
-import { PERIODIC_CI_EXCLUDE } from './helpers/periodic-exclude-data';
+import { CASE_CI_EXCLUDE, PERIODIC_CI_EXCLUDE } from './helpers/periodic-exclude-data';
+import { E2E_TOUCHFILES } from './helpers/touchfiles';
 import { isPaidTestFile } from './helpers/paid-test-set';
-import { selectPaidTestFiles } from '../scripts/test-paid-shards';
+import { buildRunManifest, CASE_SHARDED_FILES, expandCaseShards, partitionCaseExclusions, selectPaidTestFiles, shardCaseId, shardFile } from '../scripts/test-paid-shards';
 
 const ROOT = path.resolve(__dirname, '..');
 
@@ -42,4 +43,32 @@ describe('periodic exclude policy', () => {
       expect(reason).not.toStartWith('excluded: ');
     }
   });
+
+  test('case exclusions name a registered case of a case-sharded file and carry reason + tracking', () => {
+    const entries = Object.entries(CASE_CI_EXCLUDE);
+    expect(entries.length).toBeGreaterThan(0);
+    for (const [key, meta] of entries) {
+      const file = shardFile(key), id = shardCaseId(key);
+      expect(CASE_SHARDED_FILES, `${key}: not a case-sharded file`).toContain(file);
+      expect(id !== null && E2E_TOUCHFILES[id]?.includes(file), `${key}: not a registered case of ${file}`).toBe(true);
+      expect(meta.reason.length, `${key}: empty reason`).toBeGreaterThan(20);
+      expect(meta.tracking.length, `${key}: empty tracking pointer`).toBeGreaterThan(5);
+    }
+  });
+
+  test('an excluded case is an excluded manifest entry with its reason, never a planned empty case shard', () => {
+    for (const [key] of Object.entries(CASE_CI_EXCLUDE)) {
+      const tiers = (['gate', 'periodic', 'marathon'] as const).filter(tier => expandCaseShards([shardFile(key)], tier).includes(key));
+      expect(tiers.length, `${key} belongs to no tier`).toBeGreaterThan(0);
+      for (const tier of tiers) {
+        const { runnable, excluded } = partitionCaseExclusions(expandCaseShards([shardFile(key)], tier));
+        expect(runnable).not.toContain(key);
+        expect(excluded.find(entry => entry.file === key)?.reason).toStartWith('excluded: ');
+        const manifest = buildRunManifest({ tier, sliceBudgetMs: 540_000, jobs: 2, evalsAll: true, env: { EVALS_ALL: '1' } });
+        const entry = manifest.entries.find(entry => entry.file === key)!;
+        expect(entry).toMatchObject({ status: 'excluded', slice: 0 });
+        expect(entry.reason).toContain(CASE_CI_EXCLUDE[key]!.tracking);
+      }
+    }
+  });
 });