test: skip hollow tier shards and census judges in the paid planner (B5)

A paid file is now skipped for a tier lane only when every E2E id it registers
is known statically and none has that tier; ids come from the touchfile
registrations and literal testName/*IfSelected arguments, so a comment or
skill path that quotes another id cannot unschedule it, and computed names
keep today's scheduling. --list and the manifest show each skip as
"skipped: no E2E_TIERS id has tier <tier>". The weekly gate census drops the
LLM judges (--skip-judges); they still run in the periodic census and PR gate
lanes. Gate lane 52 -> 42 files, census 41; periodic 77 -> 69.

.github/workflows/evals-periodic.yml

@@ -107,7 +107,7 @@ jobs:
       - name: Emit gate census manifest (ALL gate tests)
         env:
           EVALS_ALL: "1"
-        run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --emit-plan /tmp/gate-census-plan/manifest.json --slices 7
+        run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --emit-plan /tmp/gate-census-plan/manifest.json --slices 7 --skip-judges
 
       - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         with:

scripts/test-paid-shards.ts

@@ -165,6 +165,39 @@ export function classifyPaidTestFile(source: string, tier: PaidTier): TierClassi
   return { included: true, reason: 'no whole-file tier guard — runtime E2E_TIERS filter decides' };
 }
 
+/**
+ * A file is skipped for a tier lane only when its registered E2E ids are fully
+ * known and none of them has that tier. Ids are the touchfile registrations that
+ * list the file plus literal registration arguments (testName, *IfSelected);
+ * quoted strings elsewhere (comments, skill paths) never count. Any computed
+ * registration, an id missing from the file's touchfile registration, or no id at
+ * all keeps today's scheduling (the child's runtime filter decides).
+ */
+export function tierSkipReason(
+  file: string, source: string, tier: PaidTier,
+  touchfiles: Record<string, string[]> = E2E_TOUCHFILES,
+  tiers: Record<string, string> = E2E_TIERS,
+): string | null {
+  const rel = normalizeRelativePath(file);
+  const registered = Object.keys(touchfiles).filter(key => touchfiles[key]!.includes(rel));
+  if (!registered.length) return null;
+  const computed = /testName\s*:\s*(?:`[^`]*\$\{|[A-Za-z_$])/.test(source)
+    || /\btest(?:Concurrent)?IfSelected\s*\(\s*(?:`[^`]*\$\{|[A-Za-z_$])/.test(source)
+    || /\bdescribeIfSelected\s*\([^,]*,(?!\s*\[)/.test(source)
+    || [...source.matchAll(/\bdescribeIfSelected\s*\([^,]*,\s*\[([^\]]*)\]/g)].some(m => m[1]!.split(',')
+      .map(item => item.trim()).some(item => item && !/^(['"`])[^'"`$]*\1$/.test(item)));
+  if (computed) return null;
+  const literal = [
+    ...[...source.matchAll(/testName\s*:\s*(['"`])([^'"`]+)\1/g)].map(m => m[2]!),
+    ...[...source.matchAll(/\btest(?:Concurrent)?IfSelected\s*\(\s*(['"`])([^'"`]+)\1/g)].map(m => m[2]!),
+    ...[...source.matchAll(/\bdescribeIfSelected\s*\([^,]*,\s*\[([^\]]*)\]/g)]
+      .flatMap(m => [...m[1]!.matchAll(/(['"`])([^'"`]+)\1/g)].map(n => n[2]!)),
+  ].filter(id => id in tiers);
+  if (literal.some(id => !registered.includes(id))) return null;
+  if (registered.some(id => tiers[id] === tier)) return null;
+  return `skipped: no E2E_TIERS id has tier ${tier}`;
+}
+
 export interface TierSelection {
   selected: string[];
   excluded: Array<{ file: string; reason: string }>;
@@ -198,8 +231,9 @@ export function selectPaidTestFiles(files: string[], tier: PaidTier, rootDir = R
     }
     const source = fs.readFileSync(path.join(rootDir, file), 'utf8');
     const classification = classifyPaidTestFile(source, tier);
-    if (classification.included) selected.push(file);
-    else excluded.push({ file, reason: classification.reason });
+    const skip = classification.included ? tierSkipReason(file, source, tier) : null;
+    if (classification.included && !skip) selected.push(file);
+    else excluded.push({ file, reason: skip ?? classification.reason });
   }
   return { selected, excluded };
 }
@@ -1063,6 +1097,8 @@ export function buildRunManifest(opts: {
   changedFiles?: string[];
   /** Recorded per-file durations; defaults to the committed seed under rootDir. */
   durations?: Record<string, number>;
+  /** Weekly gate census only: LLM judges already run in the periodic census and PR gate lanes. */
+  skipJudges?: boolean;
 }): PaidRunManifest {
   if (!Number.isInteger(opts.sliceCount) || opts.sliceCount <= 0) {
     throw new Error(`--slices needs a positive integer. Received: ${opts.sliceCount}`);
@@ -1072,7 +1108,11 @@ export function buildRunManifest(opts: {
   const profile = opts.profile ?? validatedProfile(env.EVALS_PROFILE, 'EVALS_PROFILE');
   if (profile === 'pr' && opts.tier !== 'gate') throw new Error('PR profile requires gate tier; use --profile full for periodic coverage');
   const discovered = opts.discovered ?? collectPaidTestFiles(rootDir);
-  const { selected, excluded } = selectPaidTestFiles(discovered, opts.tier, rootDir, env);
+  const tierSelection = selectPaidTestFiles(discovered, opts.tier, rootDir, env);
+  const judge = (file: string) => /^test\/skill-llm-eval[^/]*\.test\.ts$/.test(normalizeRelativePath(file));
+  const selected = opts.skipJudges ? tierSelection.selected.filter(file => !judge(file)) : tierSelection.selected;
+  const excluded = [...tierSelection.excluded, ...(opts.skipJudges ? tierSelection.selected.filter(judge)
+    .map(file => ({ file, reason: 'skipped: LLM judges run in the periodic census and PR gate lanes' })) : [])];
   const shards = planPaidShards(selected, { maxFilesPerShard: 1 });
   const cases = computePaidCaseSelection({ profile, env, rootDir, changedFiles: opts.changedFiles });
   const fast = cases.coverage?.mode === 'pr';
@@ -1382,6 +1422,7 @@ type CliOptions = {
   profile: PaidProfile;
   profileExplicit: boolean;
   listOnly: boolean;
+  skipJudges: boolean;
   timeoutMs: number;
   timeoutExplicit: boolean;
   jobs: number;
@@ -1431,6 +1472,7 @@ export function parseCliOptions(argv: string[], env: NodeJS.ProcessEnv = process
     profile: validatedProfile(env.EVALS_PROFILE, 'EVALS_PROFILE'),
     profileExplicit: !!env.EVALS_PROFILE,
     listOnly: false,
+    skipJudges: false,
     timeoutExplicit: !!env.EVALS_SHARD_TIMEOUT_MS,
     timeoutMs: env.EVALS_SHARD_TIMEOUT_MS
       ? parsePositiveInt(env.EVALS_SHARD_TIMEOUT_MS, 'EVALS_SHARD_TIMEOUT_MS')
@@ -1474,6 +1516,7 @@ export function parseCliOptions(argv: string[], env: NodeJS.ProcessEnv = process
       if (!value) throw new Error('--emit-plan needs a file path');
       options.emitPlanPath = value; continue;
     }
+    if (arg === '--skip-judges') { options.skipJudges = true; continue; }
     if (arg === '--slices') { options.slices = parsePositiveInt(argv[index += 1], '--slices'); continue; }
     if (arg === '--plan') {
       const value = argv[index += 1];
@@ -1490,6 +1533,7 @@ export function parseCliOptions(argv: string[], env: NodeJS.ProcessEnv = process
     throw new Error(`Unknown argument: ${arg}`);
   }
   if (options.writeDurations && !options.reportDir) throw new Error('--write-durations requires --report');
+  if (options.skipJudges && (!options.emitPlanPath || options.tier !== 'gate')) throw new Error('--skip-judges applies only to an emitted gate census plan');
   if (options.profile === 'pr' && options.tier !== 'gate') throw new Error('PR profile requires gate tier');
   if (options.profile === 'pr' && options.maxFilesPerShard !== 1) throw new Error('PR profile requires one file per shard to preserve case accounting');
   return options;
@@ -1507,6 +1551,7 @@ async function main(): Promise<number> {
       sliceCount: options.slices,
       timeoutMs: options.timeoutExplicit ? options.timeoutMs : undefined,
       evalsAll: process.env.EVALS_ALL === '1',
+      skipJudges: options.skipJudges,
     });
     fs.mkdirSync(path.dirname(path.resolve(options.emitPlanPath)), { recursive: true });
     fs.writeFileSync(options.emitPlanPath, `${JSON.stringify(manifest, null, 2)}\n`);

test/cookie-validation-phases.test.ts

@@ -42,7 +42,7 @@ test('the existing quality and behavior phases retain their complete separate sh
   expect(quality.evalsAll).toBe(true);
   expect(behavior.evalsAll).toBe(true);
   expect(qualityFiles).toHaveLength(1);
-  expect(behaviorFiles).toHaveLength(51);
+  expect(behaviorFiles).toHaveLength(41);
   expect(qualityFiles.every(file => file.startsWith('test/skill-llm-eval'))).toBe(true);
   expect(behaviorFiles.every(file => !qualityFiles.includes(file))).toBe(true);
 });

test/eng-finding-retry-budget.test.ts

@@ -124,7 +124,7 @@ test('live periodic census fits the declared CI wall including setup', () => {
     return paidShardWallUpperBoundMs(files, workers);
   });
   expect(Math.max(...walls) + 20 * 60_000).toBeLessThanOrEqual(periodicJob['timeout-minutes'] * 60_000);
-  expect(m.entries.filter(e => e.status === 'planned')).toHaveLength(77);
+  expect(m.entries.filter(e => e.status === 'planned')).toHaveLength(69);
   const overlays = m.entries.filter(e => e.status === 'planned' && e.slice === periodicSliceCount);
   expect(overlays).toHaveLength(4);
   expect(overlays.every(e => isOverlayTestFile(e.file))).toBe(true);
@@ -171,7 +171,7 @@ test('current detach supervision covers the live-census floor', () => {
   const floor = Math.ceil((Math.ceil(files.length / DEFAULT_JOBS) * DEFAULT_SHARD_TIMEOUT_MS + excess) / 1000 * 1.05);
   const pkg = JSON.parse(fs.readFileSync(path.join(import.meta.dir, '../package.json'), 'utf8'));
   const configured = Number(pkg.scripts['eval:bg:periodic'].match(/--timeout\s+(\d+)/)[1]);
-  expect(floor).toBe(37800);
+  expect(floor).toBe(35910);
   expect(configured).toBeGreaterThanOrEqual(floor);
   expect(pkg.scripts['eval:bg:gate']).toContain('--timeout 36000');
 });

test/paid-retry-supervision.test.ts

@@ -210,9 +210,9 @@ test('both gate executors cover the complete census without increasing aggregate
     expect(executor.strategy.matrix.slice).toEqual(Array.from({ length: slices }, (_, i) => i + 1));
     expect(planned.slices).toBe(slices);
     const manifest = buildRunManifest({ tier: 'gate', sliceCount: planned.slices, evalsAll: true, env: { EVALS_ALL: '1' } });
-    expect(manifest.entries.filter(row => row.status === 'planned')).toHaveLength(52);
+    expect(manifest.entries.filter(row => row.status === 'planned')).toHaveLength(42);
     const files = manifest.entries.filter(row => row.status === 'planned').map(row => row.file);
-    expect(new Set(files).size).toBe(52);
+    expect(new Set(files).size).toBe(42);
     expect(files.sort()).toEqual(selectPaidTestFiles(collectPaidTestFiles(), 'gate').selected.sort());
     const walls = executor.strategy.matrix.slice.map((slice: number) => paidShardWallUpperBoundMs(
       manifest.entries.filter(row => row.status === 'planned' && row.slice === slice).map(row => row.file), workers,
@@ -246,7 +246,7 @@ test('the periodic executor supervises every actual case and retry within its CI
   const manifest = buildRunManifest({ tier: 'periodic', sliceCount: planned.slices,
     evalsAll: true, env: { EVALS_ALL: '1' } });
   const census = manifest.entries.filter(row => row.status === 'planned');
-  expect(census).toHaveLength(77);
+  expect(census).toHaveLength(69);
   expect(census.find(row => row.file === 'test/skill-llm-eval.test.ts')?.budget?.timeoutMs).toBe(5_960_000);
   const walls = executor.strategy.matrix.slice.map((slice: number) => paidShardWallUpperBoundMs(
     census.filter(row => row.slice === slice).map(row => row.file), active.jobs,

test/paid-shards.test.ts

@@ -30,6 +30,8 @@ import {
   runPaidShards,
   summarize,
   summaryExitCode,
+  tierSkipReason,
+  buildRunManifest,
   type ShardOutcome,
 } from '../scripts/test-paid-shards';
 
@@ -68,6 +70,57 @@ describe('paid test enumeration', () => {
   });
 });
 
+describe('tier lane skip (B5)', () => {
+  const file = 'test/skill-e2e-sample.test.ts';
+  const touchfiles = { 'sample-gate': [file, 'sample/**'], 'sample-periodic': [file], other: ['x/**'] };
+  const tiers = { 'sample-gate': 'gate', 'sample-periodic': 'periodic', other: 'periodic' };
+  const reg = { 'sample-gate': [file] } as Record<string, string[]>;
+
+  test('a fully static file with only other-tier ids is skipped with its reason', () => {
+    const source = "describeIfSelected('Sample', ['sample-gate'], () => { testIfSelected('sample-gate', async () => {}); });";
+    expect(tierSkipReason(file, source, 'periodic', reg, tiers)).toBe('skipped: no E2E_TIERS id has tier periodic');
+    expect(tierSkipReason(file, source, 'gate', reg, tiers)).toBeNull();
+  });
+
+  test('a comment or path that quotes another-tier id cannot change the decision', () => {
+    const source = "// see 'other' and 'sample-periodic' for context\nconst dir = 'sample-periodic/fixtures';\n" +
+      "testIfSelected('sample-gate', async () => {});";
+    expect(tierSkipReason(file, source, 'periodic', reg, tiers)).toBe('skipped: no E2E_TIERS id has tier periodic');
+    expect(tierSkipReason(file, source, 'gate', reg, tiers)).toBeNull();
+  });
+
+  test('computed registrations, unregistered literal ids and id-less files keep today\'s scheduling', () => {
+    for (const source of [
+      "describeIfSelected(name, keys, () => {});",
+      "describeIfSelected('Sample', [...keys], () => {});",
+      "testConcurrentIfSelected(`sample-${label}`, async () => {});",
+      "runSkillTest({ testName: `sample-${label}` });",
+      "testIfSelected(caseName, async () => {});",
+    ]) expect(tierSkipReason(file, source, 'periodic', reg, tiers)).toBeNull();
+    expect(tierSkipReason(file, "testIfSelected('sample-periodic', async () => {});", 'periodic', reg, tiers)).toBeNull();
+    expect(tierSkipReason(file, "testIfSelected('sample-gate', async () => {});", 'periodic', {}, tiers)).toBeNull();
+    expect(tierSkipReason(file, "testIfSelected('sample-gate', async () => {});", 'gate', touchfiles, tiers)).toBeNull();
+  });
+
+  test('the real constructed-name diagram file stays scheduled in both lanes', () => {
+    const source = fs.readFileSync(path.join(ROOT, 'test/skill-e2e-diagram.test.ts'), 'utf8');
+    for (const tier of ['gate', 'periodic'] as const) expect(tierSkipReason('test/skill-e2e-diagram.test.ts', source, tier)).toBeNull();
+  });
+
+  test('the weekly gate census alone drops the LLM judges, with the reason in the manifest', () => {
+    const census = buildRunManifest({ tier: 'gate', sliceCount: 7, evalsAll: true, skipJudges: true, env: { EVALS_ALL: '1' } });
+    const judge = census.entries.find(entry => entry.file === 'test/skill-llm-eval.test.ts');
+    expect(judge).toMatchObject({ status: 'excluded', reason: 'skipped: LLM judges run in the periodic census and PR gate lanes' });
+    for (const tier of ['gate', 'periodic'] as const) {
+      const lane = buildRunManifest({ tier, sliceCount: 7, evalsAll: true, env: { EVALS_ALL: '1' } });
+      expect(lane.entries.find(entry => entry.file === 'test/skill-llm-eval.test.ts')?.status).toBe('planned');
+    }
+    const workflow = fs.readFileSync(path.join(ROOT, '.github/workflows/evals-periodic.yml'), 'utf8');
+    expect(workflow.match(/--skip-judges/g)).toHaveLength(1);
+    expect(workflow).toMatch(/--tier gate --emit-plan \/tmp\/gate-census-plan\/manifest\.json --slices 7 --skip-judges/);
+  });
+});
+
 describe('tier classification', () => {
   test('excludes only on an explicit other-tier guard', () => {
     const gateGuard = "const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'gate';";