mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-03 18:06:54 +02:00
test: clean up the paid eval lane (B1-B4, B6, B7)
- B1: delete paid files that assert nothing or cannot pass meaningfully: skill-llm-eval-spec and skill-e2e-spec-execute (test.todo), gemini-e2e (+ gemini-session-runner; no gemini CLI in CI), ship-idempotency (red since v1.63), the two opus-4-7 *-sonnet overlay wrappers, conductor-prose (+ its source-evaluation replay), codex-e2e-plan-format; drop their keys, scripts and census rows. - B2: skill-llm-eval grades browse/sections/command-list.md with one union judge that also carries the baseline score pin; regression-vs-baseline deleted (paid run: pass, c4/c4/a4). - B3: memory-pipeline, ios-qa, ios-qa-swift-build and plan-tune-cathedral make no model calls; renamed out of the paid glob so they run on every PR. Swift builds need GSTACK_TEST_SWIFT=1; device stub deleted. - B4: codex-e2e*, outside-voice, aside and ios-device cannot run in the CI image; excluded from the weekly lane with a tracked re-entry condition. - B6: fold opus-47's negative routing controls into skill-routing-e2e journey-negatives (paid run: 3/3 unrouted) and delete the file. - B7: delete the never-green brain-privacy-gate eval; a free gstack-skill-start test now proves consent precedes artifacts egress.
This commit is contained in:
1 parent
5d032ef299
commit
53e7f3212f
58 files changed
+273
-2594
No files matched your search
+6
-14
@@ -83,8 +83,7 @@ describe('selectTests', () => {
|
||||
// These cases use an outside-only/Code Quality excerpt or descriptive metadata.
|
||||
.filter(id => !['outside-plan-disabled-no-fallback', 'plan-ceo-review-prosons-cadence',
|
||||
'plan-review-prosons-format', 'shared-libs-plan-callers'].includes(id));
|
||||
const existing = ['learnings-show', 'codex-plan-ceo-format-mode', 'codex-plan-ceo-format-approach',
|
||||
'codex-plan-eng-format-coverage', 'codex-plan-eng-format-kind'];
|
||||
const existing = ['learnings-show'];
|
||||
const result = selectTests(['scripts/resolvers/learnings.ts'], E2E_TOUCHFILES);
|
||||
expect(result.reason).toBe('diff');
|
||||
expect(result.selected.sort()).toEqual([...new Set([...consumers, ...existing])].sort());
|
||||
@@ -161,12 +160,8 @@ describe('selectTests', () => {
|
||||
expect(fs.readFileSync(path.join(ROOT, `${output}.tmpl`), 'utf8')).toContain(`{{${token}}}`);
|
||||
}
|
||||
const generated = selectTests(consumers.map(([output]) => output), E2E_TOUCHFILES);
|
||||
// These two CEO-format cases already depend on every resolver through
|
||||
// scripts/resolvers/**; keep that existing selection alongside consumers.
|
||||
// The bounded Code Quality fixture stops before Test review.
|
||||
const expected = [...new Set([...generated.selected.filter(id => id !== 'shared-libs-plan-callers' && !SHIP_GUARD_ONLY.includes(id)),
|
||||
'codex-plan-ceo-format-mode', 'codex-plan-ceo-format-approach',
|
||||
])].sort();
|
||||
const expected = [...new Set(generated.selected.filter(id => id !== 'shared-libs-plan-callers' && !SHIP_GUARD_ONLY.includes(id)))].sort();
|
||||
const actual = selectTests(['scripts/resolvers/testing.ts'], E2E_TOUCHFILES);
|
||||
expect(actual.reason).toBe('diff');
|
||||
expect(actual.selected.sort()).toEqual(expected);
|
||||
@@ -373,11 +368,9 @@ describe('selectTests', () => {
|
||||
expect(result.selected).toContain('plan-ceo-split-overflow');
|
||||
// v2 plan Phase B carve: the section-loading E2E depends on plan-ceo-review/**.
|
||||
expect(result.selected).toContain('plan-ceo-section-loading');
|
||||
expect(result.selected).toContain('codex-plan-ceo-format-mode');
|
||||
expect(result.selected).toContain('codex-plan-ceo-format-approach');
|
||||
expect(result.selected).toContain('outside-plan-disabled-no-fallback');
|
||||
expect(result.selected.length).toBe(25);
|
||||
expect(result.skipped.length).toBe(Object.keys(E2E_TOUCHFILES).length - 25);
|
||||
expect(result.selected.length).toBe(23);
|
||||
expect(result.skipped.length).toBe(Object.keys(E2E_TOUCHFILES).length - 23);
|
||||
});
|
||||
|
||||
test('global touchfile triggers ALL tests', () => {
|
||||
@@ -398,7 +391,6 @@ describe('selectTests', () => {
|
||||
expect(result.reason).toBe('diff');
|
||||
expect(result.selected).toContain('autoplan-chain-pty');
|
||||
expect(result.selected).toContain('plan-ceo-mode-routing');
|
||||
expect(result.selected).not.toContain('codex-plan-ceo-format-mode');
|
||||
expect(result.selected).not.toContain('retro');
|
||||
});
|
||||
|
||||
@@ -604,7 +596,7 @@ describe('TOUCHFILES completeness', () => {
|
||||
);
|
||||
|
||||
const unique = registeredJudgeTestNames(llmContent);
|
||||
expect(unique).toHaveLength(27);
|
||||
expect(unique).toHaveLength(23);
|
||||
|
||||
const missing = unique.filter(name => !(name in LLM_JUDGE_TOUCHFILES));
|
||||
if (missing.length > 0) {
|
||||
@@ -623,7 +615,7 @@ describe('TOUCHFILES completeness', () => {
|
||||
testIfSelected('unmapped judge case', async () => {}, 120_000);
|
||||
`;
|
||||
const names = registeredJudgeTestNames(withUnmappedCase);
|
||||
expect(names).toHaveLength(28);
|
||||
expect(names).toHaveLength(24);
|
||||
expect(names.filter(name => !(name in LLM_JUDGE_TOUCHFILES))).toEqual(['unmapped judge case']);
|
||||
});
|
||||
|
||||
|
||||
Reference in new issue
Block a user