test: clean up the paid eval lane (B1-B4, B6, B7)

- B1: delete paid files that assert nothing or cannot pass meaningfully:
  skill-llm-eval-spec and skill-e2e-spec-execute (test.todo), gemini-e2e
  (+ gemini-session-runner; no gemini CLI in CI), ship-idempotency (red
  since v1.63), the two opus-4-7 *-sonnet overlay wrappers, conductor-prose
  (+ its source-evaluation replay), codex-e2e-plan-format; drop their keys,
  scripts and census rows.
- B2: skill-llm-eval grades browse/sections/command-list.md with one union
  judge that also carries the baseline score pin; regression-vs-baseline
  deleted (paid run: pass, c4/c4/a4).
- B3: memory-pipeline, ios-qa, ios-qa-swift-build and plan-tune-cathedral
  make no model calls; renamed out of the paid glob so they run on every
  PR. Swift builds need GSTACK_TEST_SWIFT=1; device stub deleted.
- B4: codex-e2e*, outside-voice, aside and ios-device cannot run in the CI
  image; excluded from the weekly lane with a tracked re-entry condition.
- B6: fold opus-47's negative routing controls into skill-routing-e2e
  journey-negatives (paid run: 3/3 unrouted) and delete the file.
- B7: delete the never-green brain-privacy-gate eval; a free
  gstack-skill-start test now proves consent precedes artifacts egress.
This commit is contained in:
garrytan committed 2026-09-29 06:08:49 +00:00
1 parent 5d032ef299
commit 53e7f3212f
58 files changed
+273 -2594

No files matched your search

+6 -14
View File
@@ -83,8 +83,7 @@ describe('selectTests', () => {
// These cases use an outside-only/Code Quality excerpt or descriptive metadata.
.filter(id => !['outside-plan-disabled-no-fallback', 'plan-ceo-review-prosons-cadence',
'plan-review-prosons-format', 'shared-libs-plan-callers'].includes(id));
const existing = ['learnings-show', 'codex-plan-ceo-format-mode', 'codex-plan-ceo-format-approach',
'codex-plan-eng-format-coverage', 'codex-plan-eng-format-kind'];
const existing = ['learnings-show'];
const result = selectTests(['scripts/resolvers/learnings.ts'], E2E_TOUCHFILES);
expect(result.reason).toBe('diff');
expect(result.selected.sort()).toEqual([...new Set([...consumers, ...existing])].sort());
@@ -161,12 +160,8 @@ describe('selectTests', () => {
expect(fs.readFileSync(path.join(ROOT, `${output}.tmpl`), 'utf8')).toContain(`{{${token}}}`);
}
const generated = selectTests(consumers.map(([output]) => output), E2E_TOUCHFILES);
// These two CEO-format cases already depend on every resolver through
// scripts/resolvers/**; keep that existing selection alongside consumers.
// The bounded Code Quality fixture stops before Test review.
const expected = [...new Set([...generated.selected.filter(id => id !== 'shared-libs-plan-callers' && !SHIP_GUARD_ONLY.includes(id)),
'codex-plan-ceo-format-mode', 'codex-plan-ceo-format-approach',
])].sort();
const expected = [...new Set(generated.selected.filter(id => id !== 'shared-libs-plan-callers' && !SHIP_GUARD_ONLY.includes(id)))].sort();
const actual = selectTests(['scripts/resolvers/testing.ts'], E2E_TOUCHFILES);
expect(actual.reason).toBe('diff');
expect(actual.selected.sort()).toEqual(expected);
@@ -373,11 +368,9 @@ describe('selectTests', () => {
expect(result.selected).toContain('plan-ceo-split-overflow');
// v2 plan Phase B carve: the section-loading E2E depends on plan-ceo-review/**.
expect(result.selected).toContain('plan-ceo-section-loading');
expect(result.selected).toContain('codex-plan-ceo-format-mode');
expect(result.selected).toContain('codex-plan-ceo-format-approach');
expect(result.selected).toContain('outside-plan-disabled-no-fallback');
expect(result.selected.length).toBe(25);
expect(result.skipped.length).toBe(Object.keys(E2E_TOUCHFILES).length - 25);
expect(result.selected.length).toBe(23);
expect(result.skipped.length).toBe(Object.keys(E2E_TOUCHFILES).length - 23);
});
test('global touchfile triggers ALL tests', () => {
@@ -398,7 +391,6 @@ describe('selectTests', () => {
expect(result.reason).toBe('diff');
expect(result.selected).toContain('autoplan-chain-pty');
expect(result.selected).toContain('plan-ceo-mode-routing');
expect(result.selected).not.toContain('codex-plan-ceo-format-mode');
expect(result.selected).not.toContain('retro');
});
@@ -604,7 +596,7 @@ describe('TOUCHFILES completeness', () => {
);
const unique = registeredJudgeTestNames(llmContent);
expect(unique).toHaveLength(27);
expect(unique).toHaveLength(23);
const missing = unique.filter(name => !(name in LLM_JUDGE_TOUCHFILES));
if (missing.length > 0) {
@@ -623,7 +615,7 @@ describe('TOUCHFILES completeness', () => {
testIfSelected('unmapped judge case', async () => {}, 120_000);
`;
const names = registeredJudgeTestNames(withUnmappedCase);
expect(names).toHaveLength(28);
expect(names).toHaveLength(24);
expect(names.filter(name => !(name in LLM_JUDGE_TOUCHFILES))).toEqual(['unmapped judge case']);
});