mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-08 04:11:18 +02:00
test: clean up the paid eval lane (B1-B4, B6, B7)
- B1: delete paid files that assert nothing or cannot pass meaningfully: skill-llm-eval-spec and skill-e2e-spec-execute (test.todo), gemini-e2e (+ gemini-session-runner; no gemini CLI in CI), ship-idempotency (red since v1.63), the two opus-4-7 *-sonnet overlay wrappers, conductor-prose (+ its source-evaluation replay), codex-e2e-plan-format; drop their keys, scripts and census rows. - B2: skill-llm-eval grades browse/sections/command-list.md with one union judge that also carries the baseline score pin; regression-vs-baseline deleted (paid run: pass, c4/c4/a4). - B3: memory-pipeline, ios-qa, ios-qa-swift-build and plan-tune-cathedral make no model calls; renamed out of the paid glob so they run on every PR. Swift builds need GSTACK_TEST_SWIFT=1; device stub deleted. - B4: codex-e2e*, outside-voice, aside and ios-device cannot run in the CI image; excluded from the weekly lane with a tracked re-entry condition. - B6: fold opus-47's negative routing controls into skill-routing-e2e journey-negatives (paid run: 3/3 unrouted) and delete the file. - B7: delete the never-green brain-privacy-gate eval; a free gstack-skill-start test now proves consent precedes artifacts egress.
This commit is contained in:
1 parent
5d032ef299
commit
53e7f3212f
58 files changed
+273
-2594
No files matched your search
@@ -52,7 +52,7 @@ describe('overlay file policy', () => {
|
||||
});
|
||||
|
||||
test('only the exact wrapper family gets one attempt and the extra process grace', () => {
|
||||
expect(overlayFiles).toHaveLength(6);
|
||||
expect(overlayFiles).toHaveLength(4);
|
||||
expect(OVERLAY_MAX_ACTIVE_SHARDS).toBe(1);
|
||||
expect(OVERLAY_MIN_FILE_WALL_MS).toBe(1_830_000);
|
||||
for (const file of overlayFiles) {
|
||||
@@ -123,16 +123,16 @@ describe('overlay file policy', () => {
|
||||
});
|
||||
|
||||
describe('overlay manifest affinity and CI capacity', () => {
|
||||
test('actual lifecycle wrapper guards include all six in periodic and exclude all six from gate', () => {
|
||||
test('actual lifecycle wrapper guards include all four in periodic and exclude all four from gate', () => {
|
||||
for (const tier of ['periodic', 'gate'] as const) {
|
||||
const manifest = buildRunManifest({ tier, sliceCount: 6, evalsAll: true, env: { EVALS_ALL: '1' } });
|
||||
const entries = manifest.entries.filter(entry => isOverlayTestFile(entry.file));
|
||||
expect(entries).toHaveLength(6);
|
||||
expect(entries).toHaveLength(4);
|
||||
expect(entries.every(entry => entry.status === (tier === 'periodic' ? 'planned' : 'excluded'))).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
test('95 files retain every case, reserve slice six, and fit 330 minutes with actual family walls', () => {
|
||||
test('93 files retain every case, reserve slice six, and fit 330 minutes with actual family walls', () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'overlay-affinity-'));
|
||||
const normalFiles = Array.from({ length: 89 }, (_, i) => `test/skill-e2e-normal-${i.toString().padStart(2, '0')}.test.ts`);
|
||||
const discovered = [...normalFiles, ...overlayFiles];
|
||||
@@ -144,11 +144,11 @@ describe('overlay manifest affinity and CI capacity', () => {
|
||||
}
|
||||
const opts = { tier: 'periodic' as const, sliceCount: 6, evalsAll: true, discovered, rootDir: dir, env: { EVALS_ALL: '1' } };
|
||||
const manifest = buildRunManifest(opts);
|
||||
expect(manifest.entries).toHaveLength(95);
|
||||
expect(new Set(manifest.entries.map(e => e.file)).size).toBe(95);
|
||||
expect(manifest.entries).toHaveLength(93);
|
||||
expect(new Set(manifest.entries.map(e => e.file)).size).toBe(93);
|
||||
expect(manifest.entries.every(e => e.status === 'planned')).toBe(true);
|
||||
const counts = [1, 2, 3, 4, 5, 6].map(slice => manifest.entries.filter(e => e.slice === slice).length);
|
||||
expect(counts).toEqual([18, 18, 18, 18, 17, 6]);
|
||||
expect(counts).toEqual([18, 18, 18, 18, 17, 4]);
|
||||
expect(manifest.entries.filter(e => e.slice === 6).map(e => e.file).sort()).toEqual([...overlayFiles].sort());
|
||||
expect(buildRunManifest({ ...opts, discovered: [...discovered].reverse() })).toEqual(manifest);
|
||||
expect(parseRunManifest(JSON.stringify(manifest))).toEqual(manifest);
|
||||
@@ -172,7 +172,7 @@ describe('overlay manifest affinity and CI capacity', () => {
|
||||
const overlayMinutes = Math.ceil(overlayFiles.length / OVERLAY_MAX_ACTIVE_SHARDS)
|
||||
* Math.max(...overlayFiles.map(file => resolvePaidShardTimeoutMs([file]))) / 60_000;
|
||||
expect(normalMinutes).toBe(270);
|
||||
expect(overlayMinutes).toBe(183);
|
||||
expect(overlayMinutes).toBe(122);
|
||||
expect(job['timeout-minutes']).toBeGreaterThanOrEqual(Math.max(normalMinutes, overlayMinutes) + 20);
|
||||
expect(job['timeout-minutes'] * 60_000).toBeGreaterThanOrEqual(AUTOPLAN_CHAIN_BUDGET.ciJobMs);
|
||||
|
||||
|
||||
Reference in new issue
Block a user