diff --git a/.github/workflows/evals-periodic.yml b/.github/workflows/evals-periodic.yml index 0f3ad63a5..6fa19d47b 100644 --- a/.github/workflows/evals-periodic.yml +++ b/.github/workflows/evals-periodic.yml @@ -107,7 +107,7 @@ jobs: - name: Emit gate census manifest (ALL gate tests) env: EVALS_ALL: "1" - run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --emit-plan /tmp/gate-census-plan/manifest.json --slices 7 + run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --emit-plan /tmp/gate-census-plan/manifest.json --slices 7 --skip-judges - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: diff --git a/scripts/test-paid-shards.ts b/scripts/test-paid-shards.ts index f92a15701..f7d98868c 100644 --- a/scripts/test-paid-shards.ts +++ b/scripts/test-paid-shards.ts @@ -165,6 +165,39 @@ export function classifyPaidTestFile(source: string, tier: PaidTier): TierClassi return { included: true, reason: 'no whole-file tier guard — runtime E2E_TIERS filter decides' }; } +/** + * A file is skipped for a tier lane only when its registered E2E ids are fully + * known and none of them has that tier. Ids are the touchfile registrations that + * list the file plus literal registration arguments (testName, *IfSelected); + * quoted strings elsewhere (comments, skill paths) never count. Any computed + * registration, an id missing from the file's touchfile registration, or no id at + * all keeps today's scheduling (the child's runtime filter decides). + */ +export function tierSkipReason( + file: string, source: string, tier: PaidTier, + touchfiles: Record = E2E_TOUCHFILES, + tiers: Record = E2E_TIERS, +): string | null { + const rel = normalizeRelativePath(file); + const registered = Object.keys(touchfiles).filter(key => touchfiles[key]!.includes(rel)); + if (!registered.length) return null; + const computed = /testName\s*:\s*(?:`[^`]*\$\{|[A-Za-z_$])/.test(source) + || /\btest(?:Concurrent)?IfSelected\s*\(\s*(?:`[^`]*\$\{|[A-Za-z_$])/.test(source) + || /\bdescribeIfSelected\s*\([^,]*,(?!\s*\[)/.test(source) + || [...source.matchAll(/\bdescribeIfSelected\s*\([^,]*,\s*\[([^\]]*)\]/g)].some(m => m[1]!.split(',') + .map(item => item.trim()).some(item => item && !/^(['"`])[^'"`$]*\1$/.test(item))); + if (computed) return null; + const literal = [ + ...[...source.matchAll(/testName\s*:\s*(['"`])([^'"`]+)\1/g)].map(m => m[2]!), + ...[...source.matchAll(/\btest(?:Concurrent)?IfSelected\s*\(\s*(['"`])([^'"`]+)\1/g)].map(m => m[2]!), + ...[...source.matchAll(/\bdescribeIfSelected\s*\([^,]*,\s*\[([^\]]*)\]/g)] + .flatMap(m => [...m[1]!.matchAll(/(['"`])([^'"`]+)\1/g)].map(n => n[2]!)), + ].filter(id => id in tiers); + if (literal.some(id => !registered.includes(id))) return null; + if (registered.some(id => tiers[id] === tier)) return null; + return `skipped: no E2E_TIERS id has tier ${tier}`; +} + export interface TierSelection { selected: string[]; excluded: Array<{ file: string; reason: string }>; @@ -198,8 +231,9 @@ export function selectPaidTestFiles(files: string[], tier: PaidTier, rootDir = R } const source = fs.readFileSync(path.join(rootDir, file), 'utf8'); const classification = classifyPaidTestFile(source, tier); - if (classification.included) selected.push(file); - else excluded.push({ file, reason: classification.reason }); + const skip = classification.included ? tierSkipReason(file, source, tier) : null; + if (classification.included && !skip) selected.push(file); + else excluded.push({ file, reason: skip ?? classification.reason }); } return { selected, excluded }; } @@ -1063,6 +1097,8 @@ export function buildRunManifest(opts: { changedFiles?: string[]; /** Recorded per-file durations; defaults to the committed seed under rootDir. */ durations?: Record; + /** Weekly gate census only: LLM judges already run in the periodic census and PR gate lanes. */ + skipJudges?: boolean; }): PaidRunManifest { if (!Number.isInteger(opts.sliceCount) || opts.sliceCount <= 0) { throw new Error(`--slices needs a positive integer. Received: ${opts.sliceCount}`); @@ -1072,7 +1108,11 @@ export function buildRunManifest(opts: { const profile = opts.profile ?? validatedProfile(env.EVALS_PROFILE, 'EVALS_PROFILE'); if (profile === 'pr' && opts.tier !== 'gate') throw new Error('PR profile requires gate tier; use --profile full for periodic coverage'); const discovered = opts.discovered ?? collectPaidTestFiles(rootDir); - const { selected, excluded } = selectPaidTestFiles(discovered, opts.tier, rootDir, env); + const tierSelection = selectPaidTestFiles(discovered, opts.tier, rootDir, env); + const judge = (file: string) => /^test\/skill-llm-eval[^/]*\.test\.ts$/.test(normalizeRelativePath(file)); + const selected = opts.skipJudges ? tierSelection.selected.filter(file => !judge(file)) : tierSelection.selected; + const excluded = [...tierSelection.excluded, ...(opts.skipJudges ? tierSelection.selected.filter(judge) + .map(file => ({ file, reason: 'skipped: LLM judges run in the periodic census and PR gate lanes' })) : [])]; const shards = planPaidShards(selected, { maxFilesPerShard: 1 }); const cases = computePaidCaseSelection({ profile, env, rootDir, changedFiles: opts.changedFiles }); const fast = cases.coverage?.mode === 'pr'; @@ -1382,6 +1422,7 @@ type CliOptions = { profile: PaidProfile; profileExplicit: boolean; listOnly: boolean; + skipJudges: boolean; timeoutMs: number; timeoutExplicit: boolean; jobs: number; @@ -1431,6 +1472,7 @@ export function parseCliOptions(argv: string[], env: NodeJS.ProcessEnv = process profile: validatedProfile(env.EVALS_PROFILE, 'EVALS_PROFILE'), profileExplicit: !!env.EVALS_PROFILE, listOnly: false, + skipJudges: false, timeoutExplicit: !!env.EVALS_SHARD_TIMEOUT_MS, timeoutMs: env.EVALS_SHARD_TIMEOUT_MS ? parsePositiveInt(env.EVALS_SHARD_TIMEOUT_MS, 'EVALS_SHARD_TIMEOUT_MS') @@ -1474,6 +1516,7 @@ export function parseCliOptions(argv: string[], env: NodeJS.ProcessEnv = process if (!value) throw new Error('--emit-plan needs a file path'); options.emitPlanPath = value; continue; } + if (arg === '--skip-judges') { options.skipJudges = true; continue; } if (arg === '--slices') { options.slices = parsePositiveInt(argv[index += 1], '--slices'); continue; } if (arg === '--plan') { const value = argv[index += 1]; @@ -1490,6 +1533,7 @@ export function parseCliOptions(argv: string[], env: NodeJS.ProcessEnv = process throw new Error(`Unknown argument: ${arg}`); } if (options.writeDurations && !options.reportDir) throw new Error('--write-durations requires --report'); + if (options.skipJudges && (!options.emitPlanPath || options.tier !== 'gate')) throw new Error('--skip-judges applies only to an emitted gate census plan'); if (options.profile === 'pr' && options.tier !== 'gate') throw new Error('PR profile requires gate tier'); if (options.profile === 'pr' && options.maxFilesPerShard !== 1) throw new Error('PR profile requires one file per shard to preserve case accounting'); return options; @@ -1507,6 +1551,7 @@ async function main(): Promise { sliceCount: options.slices, timeoutMs: options.timeoutExplicit ? options.timeoutMs : undefined, evalsAll: process.env.EVALS_ALL === '1', + skipJudges: options.skipJudges, }); fs.mkdirSync(path.dirname(path.resolve(options.emitPlanPath)), { recursive: true }); fs.writeFileSync(options.emitPlanPath, `${JSON.stringify(manifest, null, 2)}\n`); diff --git a/test/cookie-validation-phases.test.ts b/test/cookie-validation-phases.test.ts index d463a7731..0cf99ce84 100644 --- a/test/cookie-validation-phases.test.ts +++ b/test/cookie-validation-phases.test.ts @@ -42,7 +42,7 @@ test('the existing quality and behavior phases retain their complete separate sh expect(quality.evalsAll).toBe(true); expect(behavior.evalsAll).toBe(true); expect(qualityFiles).toHaveLength(1); - expect(behaviorFiles).toHaveLength(51); + expect(behaviorFiles).toHaveLength(41); expect(qualityFiles.every(file => file.startsWith('test/skill-llm-eval'))).toBe(true); expect(behaviorFiles.every(file => !qualityFiles.includes(file))).toBe(true); }); diff --git a/test/eng-finding-retry-budget.test.ts b/test/eng-finding-retry-budget.test.ts index 2fb5a166d..9a25d240b 100644 --- a/test/eng-finding-retry-budget.test.ts +++ b/test/eng-finding-retry-budget.test.ts @@ -124,7 +124,7 @@ test('live periodic census fits the declared CI wall including setup', () => { return paidShardWallUpperBoundMs(files, workers); }); expect(Math.max(...walls) + 20 * 60_000).toBeLessThanOrEqual(periodicJob['timeout-minutes'] * 60_000); - expect(m.entries.filter(e => e.status === 'planned')).toHaveLength(77); + expect(m.entries.filter(e => e.status === 'planned')).toHaveLength(69); const overlays = m.entries.filter(e => e.status === 'planned' && e.slice === periodicSliceCount); expect(overlays).toHaveLength(4); expect(overlays.every(e => isOverlayTestFile(e.file))).toBe(true); @@ -171,7 +171,7 @@ test('current detach supervision covers the live-census floor', () => { const floor = Math.ceil((Math.ceil(files.length / DEFAULT_JOBS) * DEFAULT_SHARD_TIMEOUT_MS + excess) / 1000 * 1.05); const pkg = JSON.parse(fs.readFileSync(path.join(import.meta.dir, '../package.json'), 'utf8')); const configured = Number(pkg.scripts['eval:bg:periodic'].match(/--timeout\s+(\d+)/)[1]); - expect(floor).toBe(37800); + expect(floor).toBe(35910); expect(configured).toBeGreaterThanOrEqual(floor); expect(pkg.scripts['eval:bg:gate']).toContain('--timeout 36000'); }); diff --git a/test/paid-retry-supervision.test.ts b/test/paid-retry-supervision.test.ts index f04cc2bf1..f09ab3067 100644 --- a/test/paid-retry-supervision.test.ts +++ b/test/paid-retry-supervision.test.ts @@ -210,9 +210,9 @@ test('both gate executors cover the complete census without increasing aggregate expect(executor.strategy.matrix.slice).toEqual(Array.from({ length: slices }, (_, i) => i + 1)); expect(planned.slices).toBe(slices); const manifest = buildRunManifest({ tier: 'gate', sliceCount: planned.slices, evalsAll: true, env: { EVALS_ALL: '1' } }); - expect(manifest.entries.filter(row => row.status === 'planned')).toHaveLength(52); + expect(manifest.entries.filter(row => row.status === 'planned')).toHaveLength(42); const files = manifest.entries.filter(row => row.status === 'planned').map(row => row.file); - expect(new Set(files).size).toBe(52); + expect(new Set(files).size).toBe(42); expect(files.sort()).toEqual(selectPaidTestFiles(collectPaidTestFiles(), 'gate').selected.sort()); const walls = executor.strategy.matrix.slice.map((slice: number) => paidShardWallUpperBoundMs( manifest.entries.filter(row => row.status === 'planned' && row.slice === slice).map(row => row.file), workers, @@ -246,7 +246,7 @@ test('the periodic executor supervises every actual case and retry within its CI const manifest = buildRunManifest({ tier: 'periodic', sliceCount: planned.slices, evalsAll: true, env: { EVALS_ALL: '1' } }); const census = manifest.entries.filter(row => row.status === 'planned'); - expect(census).toHaveLength(77); + expect(census).toHaveLength(69); expect(census.find(row => row.file === 'test/skill-llm-eval.test.ts')?.budget?.timeoutMs).toBe(5_960_000); const walls = executor.strategy.matrix.slice.map((slice: number) => paidShardWallUpperBoundMs( census.filter(row => row.slice === slice).map(row => row.file), active.jobs, diff --git a/test/paid-shards.test.ts b/test/paid-shards.test.ts index b4577d0c4..8760d7947 100644 --- a/test/paid-shards.test.ts +++ b/test/paid-shards.test.ts @@ -30,6 +30,8 @@ import { runPaidShards, summarize, summaryExitCode, + tierSkipReason, + buildRunManifest, type ShardOutcome, } from '../scripts/test-paid-shards'; @@ -68,6 +70,57 @@ describe('paid test enumeration', () => { }); }); +describe('tier lane skip (B5)', () => { + const file = 'test/skill-e2e-sample.test.ts'; + const touchfiles = { 'sample-gate': [file, 'sample/**'], 'sample-periodic': [file], other: ['x/**'] }; + const tiers = { 'sample-gate': 'gate', 'sample-periodic': 'periodic', other: 'periodic' }; + const reg = { 'sample-gate': [file] } as Record; + + test('a fully static file with only other-tier ids is skipped with its reason', () => { + const source = "describeIfSelected('Sample', ['sample-gate'], () => { testIfSelected('sample-gate', async () => {}); });"; + expect(tierSkipReason(file, source, 'periodic', reg, tiers)).toBe('skipped: no E2E_TIERS id has tier periodic'); + expect(tierSkipReason(file, source, 'gate', reg, tiers)).toBeNull(); + }); + + test('a comment or path that quotes another-tier id cannot change the decision', () => { + const source = "// see 'other' and 'sample-periodic' for context\nconst dir = 'sample-periodic/fixtures';\n" + + "testIfSelected('sample-gate', async () => {});"; + expect(tierSkipReason(file, source, 'periodic', reg, tiers)).toBe('skipped: no E2E_TIERS id has tier periodic'); + expect(tierSkipReason(file, source, 'gate', reg, tiers)).toBeNull(); + }); + + test('computed registrations, unregistered literal ids and id-less files keep today\'s scheduling', () => { + for (const source of [ + "describeIfSelected(name, keys, () => {});", + "describeIfSelected('Sample', [...keys], () => {});", + "testConcurrentIfSelected(`sample-${label}`, async () => {});", + "runSkillTest({ testName: `sample-${label}` });", + "testIfSelected(caseName, async () => {});", + ]) expect(tierSkipReason(file, source, 'periodic', reg, tiers)).toBeNull(); + expect(tierSkipReason(file, "testIfSelected('sample-periodic', async () => {});", 'periodic', reg, tiers)).toBeNull(); + expect(tierSkipReason(file, "testIfSelected('sample-gate', async () => {});", 'periodic', {}, tiers)).toBeNull(); + expect(tierSkipReason(file, "testIfSelected('sample-gate', async () => {});", 'gate', touchfiles, tiers)).toBeNull(); + }); + + test('the real constructed-name diagram file stays scheduled in both lanes', () => { + const source = fs.readFileSync(path.join(ROOT, 'test/skill-e2e-diagram.test.ts'), 'utf8'); + for (const tier of ['gate', 'periodic'] as const) expect(tierSkipReason('test/skill-e2e-diagram.test.ts', source, tier)).toBeNull(); + }); + + test('the weekly gate census alone drops the LLM judges, with the reason in the manifest', () => { + const census = buildRunManifest({ tier: 'gate', sliceCount: 7, evalsAll: true, skipJudges: true, env: { EVALS_ALL: '1' } }); + const judge = census.entries.find(entry => entry.file === 'test/skill-llm-eval.test.ts'); + expect(judge).toMatchObject({ status: 'excluded', reason: 'skipped: LLM judges run in the periodic census and PR gate lanes' }); + for (const tier of ['gate', 'periodic'] as const) { + const lane = buildRunManifest({ tier, sliceCount: 7, evalsAll: true, env: { EVALS_ALL: '1' } }); + expect(lane.entries.find(entry => entry.file === 'test/skill-llm-eval.test.ts')?.status).toBe('planned'); + } + const workflow = fs.readFileSync(path.join(ROOT, '.github/workflows/evals-periodic.yml'), 'utf8'); + expect(workflow.match(/--skip-judges/g)).toHaveLength(1); + expect(workflow).toMatch(/--tier gate --emit-plan \/tmp\/gate-census-plan\/manifest\.json --slices 7 --skip-judges/); + }); +}); + describe('tier classification', () => { test('excludes only on an explicit other-tier guard', () => { const gateGuard = "const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'gate';";