E5: design live-model check as a periodic case

test/skill-e2e-design-model-smoke.test.ts runs design/scripts/live-model-check.ts
(default models only) under a temp HOME. Without OPENAI_API_KEY the case is a
Bun skip that the census lists as SKIPPED with no coverage credit; with a key,
exit 0 passes and any other exit fails with the script output (exit 2 under a
present key fails as 'nothing was checked'). paid-test-durations.json carries
a 90 s packing estimate. Registration (E2E_TOUCHFILES, E2E_TIERS periodic,
E2E_KINDS rule) is the integrator's.
This commit is contained in:
garrytan committed 2026-10-04 18:40:06 +00:00
1 parent 446da35002
commit a0939d6593
2 files changed
+42

No files matched your search

+1
View File
@@ -141,6 +141,7 @@
"test/skill-e2e-coverage-audit.test.ts": 1000,
"test/skill-e2e-cso.test.ts": 253498,
"test/skill-e2e-deploy.test.ts": 1000,
"test/skill-e2e-design-model-smoke.test.ts": 90000,
"test/skill-e2e-design.test.ts": 817000,
"test/skill-e2e-design.test.ts#design-consultation-core": 159257,
"test/skill-e2e-design.test.ts#design-consultation-existing": 175607,
+41
View File
@@ -0,0 +1,41 @@
/**
* Design model smoke (E5, periodic): runs design/scripts/live-model-check.ts,
* which sends the design binary's real request bodies with the DEFAULT
* models (overrides are ignored), so a retired or unentitled default fails
* here before users hit it. Without OPENAI_API_KEY the case is a Bun skip:
* the report lists it as SKIPPED with zero coverage credit, never a pass.
* With a key, exit 0 passes; exit 2 (the script saw no key) and every other
* exit fail with the script's output attached.
*/
import { describe, test, expect } from 'bun:test';
import { spawnSync } from 'child_process';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import { e2eTierEnabled } from './helpers/e2e-gate';
const ROOT = path.resolve(import.meta.dir, '..');
const CASE = 'design-model-smoke';
const KEY_PRESENT = !!process.env.OPENAI_API_KEY?.trim();
const CHECK_MS = 400_000;
if (e2eTierEnabled('periodic') && !KEY_PRESENT) {
process.stderr.write(`\n${CASE}: SKIPPED: OPENAI_API_KEY is not set; the design model defaults were not checked (no coverage)\n`);
}
describe.skipIf(!e2eTierEnabled('periodic') || !KEY_PRESENT)('design live model defaults (E5)', () => {
test(CASE, () => {
const home = fs.mkdtempSync(path.join(os.tmpdir(), 'design-model-smoke-'));
try {
const run = spawnSync(process.execPath, ['run', 'design/scripts/live-model-check.ts'], {
cwd: ROOT, encoding: 'utf8', timeout: CHECK_MS,
env: { ...process.env, HOME: home, GSTACK_HOME: path.join(home, '.gstack') },
});
const output = `${run.stdout ?? ''}${run.stderr ?? ''}`.trim();
console.log(output);
if (run.status === 2) throw new Error(`live-model-check saw no OPENAI_API_KEY although the test had one; nothing was checked:\n${output}`);
expect(run.status, output).toBe(0);
} finally {
fs.rmSync(home, { recursive: true, force: true });
}
}, CHECK_MS + 10_000);
});