mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-16 01:45:29 +02:00
evals: parent-side shard skipping — a one-test diff runs 3 of 44 shards
The sharded runner spawned every shard regardless of diff; only the
child self-skipped, so a typical single-skill change still paid 44 Bun
boots + container-equivalent setup for shards with zero selected tests.
The parent now computes selection once (mirroring e2e-helpers exactly:
EVALS_ALL -> run-all, empty union -> run-all, git errors propagate the
fail-closed throw) and drops shards where no selected test name maps in.
Mapping = quoted E2E map keys in the file's source UNION keys whose dep
list registers the file (constructed-name families need the second
direction). FAIL-OPEN everywhere it matters: run-all, non-skill-e2e
files, unreadable source, zero mapped names all keep the shard — the
child filter stays authoritative, so a parent bug can only run extra.
New taxonomy status skipped-by-diff (never conflated with
never-started); selection banner prints once; --list is selection-aware.
C6 lands in the same commit: a HARD tier-alignment test — every paid
skill-e2e file must be parent-mappable or provably fail-open-safe.
Note: this change-set's 14 dep-list registrations in touchfiles-data.ts
rode along in f945c841 (concurrent-agent staging); they belong to this
change logically.
Demo: selection of one test -> 'running 3 of 44 shards, 41
skipped-by-diff'. 13 new $0 tests via injected seams.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
0337aaf05d
commit
9b9623e30d
@@ -20,6 +20,8 @@ import { describe, test, expect } from 'bun:test';
|
||||
import { readdirSync, readFileSync } from 'fs';
|
||||
import * as path from 'path';
|
||||
import { E2E_TOUCHFILES, E2E_TIERS, LLM_JUDGE_TOUCHFILES } from './helpers/touchfiles';
|
||||
import { isPaidTestFile } from './helpers/paid-test-set';
|
||||
import { knownTestNamesInSource, PARENT_MAPPER_TEST_NAMES } from '../scripts/test-paid-shards';
|
||||
|
||||
const TEST_DIR = import.meta.dir;
|
||||
// Both quote styles — a mechanical refactor to double quotes must not
|
||||
@@ -88,4 +90,47 @@ describe('E2E tier alignment (touchfiles declaration vs test self-gate)', () =>
|
||||
|
||||
expect(misaligned).toEqual([]);
|
||||
});
|
||||
|
||||
// HARD invariant (C6): the paid sharded runner skips a skill-e2e shard when
|
||||
// none of the file's MAPPED test names (E2E map keys quoted in its source,
|
||||
// union E2E map keys whose dep list registers the file) are diff-selected.
|
||||
// A skill-e2e file the mapper cannot see at all is only safe if it provably
|
||||
// opts out of name-based selection: it must not touch the e2e-helpers
|
||||
// selection surface (describeIfSelected / runSkillTest / selectedTests) AND
|
||||
// it must carry an explicit whole-file EVALS_TIER self-gate (the child-side
|
||||
// gate that makes the parent's fail-open keep semantically correct).
|
||||
//
|
||||
// Anything else is an invisible-test-names hole: the parent could drop a
|
||||
// shard whose child would have run real work. Fix by either quoting the
|
||||
// test's E2E map key as a string literal in the file, or adding the file's
|
||||
// path to its key's dep list in test/helpers/touchfiles-data.ts.
|
||||
test('every paid skill-e2e file is visible to the parent diff mapper (or provably fail-open-safe)', () => {
|
||||
const invisible: string[] = [];
|
||||
|
||||
for (const file of testFiles) {
|
||||
const repoPath = `test/${file}`;
|
||||
if (!isPaidTestFile(repoPath)) continue;
|
||||
const content = readFileSync(path.join(TEST_DIR, file), 'utf-8');
|
||||
|
||||
const quoted = knownTestNamesInSource(content, PARENT_MAPPER_TEST_NAMES);
|
||||
const registered = Object.keys(E2E_TOUCHFILES).filter((k) => E2E_TOUCHFILES[k].includes(repoPath));
|
||||
if (quoted.length + registered.length > 0) continue; // parent-mappable
|
||||
|
||||
const usesNameSelection = /\b(describeIfSelected|runSkillTest|selectedTests)\b/.test(content);
|
||||
const selfGated = /EVALS_TIER\s*===\s*['"](gate|periodic)['"]/.test(content);
|
||||
if (!usesNameSelection && selfGated) continue; // fail-open-safe standalone
|
||||
|
||||
invisible.push(
|
||||
`${repoPath}: invisible to the parent diff mapper — no E2E map key quoted in the file, `
|
||||
+ 'not registered in any E2E_TOUCHFILES dep list, and it '
|
||||
+ (usesNameSelection
|
||||
? 'uses name-based selection (describeIfSelected/runSkillTest/selectedTests)'
|
||||
: 'has no whole-file EVALS_TIER self-gate')
|
||||
+ '. Quote the test\'s E2E map key as a string literal, or add this file path to its '
|
||||
+ 'key\'s dep list in test/helpers/touchfiles-data.ts.',
|
||||
);
|
||||
}
|
||||
|
||||
expect(invisible).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user