test: clean up the paid eval lane (B1-B4, B6, B7)

- B1: delete paid files that assert nothing or cannot pass meaningfully:
  skill-llm-eval-spec and skill-e2e-spec-execute (test.todo), gemini-e2e
  (+ gemini-session-runner; no gemini CLI in CI), ship-idempotency (red
  since v1.63), the two opus-4-7 *-sonnet overlay wrappers, conductor-prose
  (+ its source-evaluation replay), codex-e2e-plan-format; drop their keys,
  scripts and census rows.
- B2: skill-llm-eval grades browse/sections/command-list.md with one union
  judge that also carries the baseline score pin; regression-vs-baseline
  deleted (paid run: pass, c4/c4/a4).
- B3: memory-pipeline, ios-qa, ios-qa-swift-build and plan-tune-cathedral
  make no model calls; renamed out of the paid glob so they run on every
  PR. Swift builds need GSTACK_TEST_SWIFT=1; device stub deleted.
- B4: codex-e2e*, outside-voice, aside and ios-device cannot run in the CI
  image; excluded from the weekly lane with a tracked re-entry condition.
- B6: fold opus-47's negative routing controls into skill-routing-e2e
  journey-negatives (paid run: 3/3 unrouted) and delete the file.
- B7: delete the never-green brain-privacy-gate eval; a free
  gstack-skill-start test now proves consent precedes artifacts egress.
This commit is contained in:
garrytan committed 2026-09-29 06:08:49 +00:00
1 parent 5d032ef299
commit 53e7f3212f
58 files changed
+273 -2594

No files matched your search

+8 -8
View File
@@ -52,7 +52,7 @@ describe('overlay file policy', () => {
});
test('only the exact wrapper family gets one attempt and the extra process grace', () => {
expect(overlayFiles).toHaveLength(6);
expect(overlayFiles).toHaveLength(4);
expect(OVERLAY_MAX_ACTIVE_SHARDS).toBe(1);
expect(OVERLAY_MIN_FILE_WALL_MS).toBe(1_830_000);
for (const file of overlayFiles) {
@@ -123,16 +123,16 @@ describe('overlay file policy', () => {
});
describe('overlay manifest affinity and CI capacity', () => {
test('actual lifecycle wrapper guards include all six in periodic and exclude all six from gate', () => {
test('actual lifecycle wrapper guards include all four in periodic and exclude all four from gate', () => {
for (const tier of ['periodic', 'gate'] as const) {
const manifest = buildRunManifest({ tier, sliceCount: 6, evalsAll: true, env: { EVALS_ALL: '1' } });
const entries = manifest.entries.filter(entry => isOverlayTestFile(entry.file));
expect(entries).toHaveLength(6);
expect(entries).toHaveLength(4);
expect(entries.every(entry => entry.status === (tier === 'periodic' ? 'planned' : 'excluded'))).toBe(true);
}
});
test('95 files retain every case, reserve slice six, and fit 330 minutes with actual family walls', () => {
test('93 files retain every case, reserve slice six, and fit 330 minutes with actual family walls', () => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'overlay-affinity-'));
const normalFiles = Array.from({ length: 89 }, (_, i) => `test/skill-e2e-normal-${i.toString().padStart(2, '0')}.test.ts`);
const discovered = [...normalFiles, ...overlayFiles];
@@ -144,11 +144,11 @@ describe('overlay manifest affinity and CI capacity', () => {
}
const opts = { tier: 'periodic' as const, sliceCount: 6, evalsAll: true, discovered, rootDir: dir, env: { EVALS_ALL: '1' } };
const manifest = buildRunManifest(opts);
expect(manifest.entries).toHaveLength(95);
expect(new Set(manifest.entries.map(e => e.file)).size).toBe(95);
expect(manifest.entries).toHaveLength(93);
expect(new Set(manifest.entries.map(e => e.file)).size).toBe(93);
expect(manifest.entries.every(e => e.status === 'planned')).toBe(true);
const counts = [1, 2, 3, 4, 5, 6].map(slice => manifest.entries.filter(e => e.slice === slice).length);
expect(counts).toEqual([18, 18, 18, 18, 17, 6]);
expect(counts).toEqual([18, 18, 18, 18, 17, 4]);
expect(manifest.entries.filter(e => e.slice === 6).map(e => e.file).sort()).toEqual([...overlayFiles].sort());
expect(buildRunManifest({ ...opts, discovered: [...discovered].reverse() })).toEqual(manifest);
expect(parseRunManifest(JSON.stringify(manifest))).toEqual(manifest);
@@ -172,7 +172,7 @@ describe('overlay manifest affinity and CI capacity', () => {
const overlayMinutes = Math.ceil(overlayFiles.length / OVERLAY_MAX_ACTIVE_SHARDS)
* Math.max(...overlayFiles.map(file => resolvePaidShardTimeoutMs([file]))) / 60_000;
expect(normalMinutes).toBe(270);
expect(overlayMinutes).toBe(183);
expect(overlayMinutes).toBe(122);
expect(job['timeout-minutes']).toBeGreaterThanOrEqual(Math.max(normalMinutes, overlayMinutes) + 20);
expect(job['timeout-minutes'] * 60_000).toBeGreaterThanOrEqual(AUTOPLAN_CHAIN_BUDGET.ciJobMs);