mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-01 14:34:42 +01:00
Merge wave/w-followups
This commit is contained in:
garrytan committed
commit
4ea2c6eed7
16 files changed
+1056
-177
No files matched your search
@@ -0,0 +1,45 @@
|
||||
/**
|
||||
* The red-census docs stay true to the code they describe: the failure-cause
|
||||
* glossary in docs/TESTING_INTERNALS.md lists exactly FAILURE_CAUSES, the
|
||||
* guide's commands use flags eval:pass-rates and test-paid-shards accept, and
|
||||
* the guide is linked from CONTRIBUTING.md and both census reports.
|
||||
*/
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { FAILURE_CAUSES } from './helpers/eval-store';
|
||||
import { parsePassRatesArgs } from '../scripts/lib/eval-history';
|
||||
|
||||
const ROOT = path.join(import.meta.dir, '..');
|
||||
const read = (rel: string) => fs.readFileSync(path.join(ROOT, rel), 'utf8');
|
||||
const internals = read('docs/TESTING_INTERNALS.md');
|
||||
const guide = read('docs/evals/census-red.md');
|
||||
|
||||
describe('red-census docs', () => {
|
||||
test('the failure-cause glossary has one row per FAILURE_CAUSES value, in precedence order', () => {
|
||||
const section = internals.slice(internals.indexOf('<a id="failure-causes"></a>'));
|
||||
const rows = [...section.slice(0, section.indexOf('\n\n', section.indexOf('| `failure_cause` |'))).matchAll(/^\| `([a-z_]+)` \|/gm)]
|
||||
.map(m => m[1]).filter(cause => cause !== 'failure_cause');
|
||||
expect(rows).toEqual([...FAILURE_CAUSES]);
|
||||
});
|
||||
|
||||
test('every eval:pass-rates command in the guide parses', () => {
|
||||
const commands = [...guide.matchAll(/^bun run eval:pass-rates ([^\n#]*)/gm)].map(m => m[1]!.trim());
|
||||
expect(commands.length).toBeGreaterThanOrEqual(6);
|
||||
for (const command of commands) {
|
||||
const args = command.replace(/<[^>]+>/g, 'x').split(/\s+/).filter(Boolean)
|
||||
.map((arg, i, all) => (all[i - 1] === '--run' || all[i - 1] === '--runs' ? '10' : arg));
|
||||
const parsed = parsePassRatesArgs(args, () => true);
|
||||
expect('error' in parsed ? parsed.error : null, command).toBeNull();
|
||||
}
|
||||
});
|
||||
|
||||
test('the guide is linked from CONTRIBUTING, TESTING_INTERNALS and both census reports', () => {
|
||||
expect(read('CONTRIBUTING.md')).toContain('docs/evals/census-red.md');
|
||||
expect(internals).toContain('(evals/census-red.md)');
|
||||
for (const workflow of ['evals-periodic.yml', 'evals-marathon.yml']) {
|
||||
expect(read(`.github/workflows/${workflow}`)).toContain('/docs/evals/census-red.md');
|
||||
}
|
||||
for (const anchor of ['failure-causes', 'eval-verdict-policy']) expect(internals).toContain(`<a id="${anchor}"></a>`);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,116 @@
|
||||
/**
|
||||
* Live Codex multi-block check (E6, periodic, host-run Codex job).
|
||||
*
|
||||
* Codex runs every shell block of a skill in a fresh shell. This case runs
|
||||
* `./setup --host codex` once from the job's own checkout into a fresh HOME
|
||||
* and CODEX_HOME, then checks what a Codex user gets:
|
||||
* 1. `codex debug prompt-input` lists the `gstack` router, and the router
|
||||
* it names is a real file (Codex skips a symlinked SKILL.md);
|
||||
* 2. `codex exec` running /learn's "Show recent" block, a later block that
|
||||
* only its own prelude can root, runs the installed helper and prints the
|
||||
* learning seeded before the run; no block reports a missing install or
|
||||
* an unresolved gstack helper path.
|
||||
* Auth is the CI login home's auth.json only (scopeCodexAccess hands this
|
||||
* file the CLI on PATH and that home as CODEX_HOME).
|
||||
*/
|
||||
import { describe, test, afterAll, expect } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { spawn, spawnSync } from 'child_process';
|
||||
import { CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { e2eTierEnabled } from './helpers/e2e-gate';
|
||||
import { hermeticChildEnv } from './helpers/hermetic-env';
|
||||
import { parseCodexJSONL, type CodexResult } from './helpers/codex-session-runner';
|
||||
import { CODEX_EVAL_FINALIZE_MS, createCodexEvalCollector, runRecordedCodexEval } from './helpers/codex-eval';
|
||||
import { MULTIBLOCK_SENTINEL_KEY, multiblockProblems, routerPathFromPromptInput } from './helpers/codex-multiblock';
|
||||
import { CODEX_FRONTIER_MODEL } from '../scripts/resolvers/constants';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const CASE = 'codex-multiblock-live';
|
||||
const SETUP_MS = 300_000;
|
||||
const CODEX_AVAILABLE = spawnSync('which', ['codex'], { timeout: 30_000 }).status === 0;
|
||||
const ENABLED = e2eTierEnabled('periodic') && CODEX_AVAILABLE;
|
||||
if (e2eTierEnabled('periodic') && !CODEX_AVAILABLE) process.stderr.write(`\n${CASE}: SKIPPED: codex binary not on PATH\n`);
|
||||
|
||||
const collector = ENABLED ? createCodexEvalCollector('codex-e2e-multiblock-live') : null;
|
||||
afterAll(async () => { await collector?.finalize(); });
|
||||
|
||||
/** `codex exec --json` with stdout/stderr in files (no pipe can outlive the process group), killed on abort. */
|
||||
function codexExec(args: string[], env: NodeJS.ProcessEnv, cwd: string, scratch: string, signal: AbortSignal): Promise<CodexResult> {
|
||||
const started = Date.now();
|
||||
const outFile = path.join(scratch, 'exec.jsonl');
|
||||
const errFile = path.join(scratch, 'exec.stderr');
|
||||
const out = fs.openSync(outFile, 'w');
|
||||
const err = fs.openSync(errFile, 'w');
|
||||
const proc = spawn('codex', args, { cwd, env, stdio: ['ignore', out, err], detached: true });
|
||||
fs.closeSync(out);
|
||||
fs.closeSync(err);
|
||||
const kill = () => { try { process.kill(-proc.pid!, 'SIGKILL'); } catch { /* already gone */ } };
|
||||
signal.addEventListener('abort', kill, { once: true });
|
||||
return new Promise(resolve => {
|
||||
const finish = (exitCode: number) => {
|
||||
signal.removeEventListener('abort', kill);
|
||||
kill();
|
||||
const rawLines = fs.readFileSync(outFile, 'utf8').split('\n').filter(line => line.trim());
|
||||
resolve({ ...parseCodexJSONL(rawLines), rawLines, exitCode: signal.aborted ? 124 : exitCode,
|
||||
durationMs: Date.now() - started, stderr: fs.readFileSync(errFile, 'utf8') });
|
||||
};
|
||||
proc.once('error', () => finish(1));
|
||||
proc.once('exit', (code, sig) => finish(code ?? (sig ? 128 + (os.constants.signals[sig] ?? 0) : 1)));
|
||||
});
|
||||
}
|
||||
|
||||
(ENABLED ? describe : describe.skip)('live Codex multi-block install (E6)', () => {
|
||||
test(CASE, async () => {
|
||||
const scratch = fs.mkdtempSync(path.join(os.tmpdir(), 'e6-'));
|
||||
try {
|
||||
const home = path.join(scratch, 'home');
|
||||
const codexHome = path.join(home, '.codex');
|
||||
const repo = path.join(scratch, 'repo');
|
||||
// Codex will not create its sandbox helper aliases under its temp dir, so TMPDIR sits beside CODEX_HOME.
|
||||
const tmp = path.join(home, 'tmp');
|
||||
for (const dir of [codexHome, repo, tmp]) fs.mkdirSync(dir, { recursive: true });
|
||||
const ciHome = process.env.CODEX_HOME ?? path.join(os.homedir(), '.codex');
|
||||
if (fs.existsSync(path.join(ciHome, 'auth.json'))) fs.copyFileSync(path.join(ciHome, 'auth.json'), path.join(codexHome, 'auth.json'));
|
||||
const env = hermeticChildEnv({ HOME: home, CODEX_HOME: codexHome, TMPDIR: tmp }, { extraAllow: ['OPENAI_API_KEY', 'CODEX_*'] });
|
||||
|
||||
const setupStarted = Date.now();
|
||||
const setup = spawnSync('./setup', ['--host', 'codex'], { cwd: ROOT, env, encoding: 'utf8', timeout: SETUP_MS });
|
||||
console.log(`${CASE}: ./setup --host codex exit ${setup.status} in ${Math.round((Date.now() - setupStarted) / 1000)}s`);
|
||||
expect(setup.status, `${setup.stdout.slice(-2000)}\n${setup.stderr.slice(-2000)}`).toBe(0);
|
||||
|
||||
const promptInput = spawnSync('codex', ['debug', 'prompt-input'], { cwd: repo, env, encoding: 'utf8', timeout: 60_000 });
|
||||
expect(promptInput.status, promptInput.stderr).toBe(0);
|
||||
const router = routerPathFromPromptInput(promptInput.stdout);
|
||||
expect(router, 'codex debug prompt-input does not list the gstack router').toBe(path.join(codexHome, 'skills', 'gstack', 'SKILL.md'));
|
||||
expect(fs.lstatSync(router!).isFile()).toBe(true);
|
||||
|
||||
for (const args of [['init', '-q'], ['remote', 'add', 'origin', 'https://github.com/example/e6-fixture.git']]) {
|
||||
expect(spawnSync('git', args, { cwd: repo, env, timeout: 30_000 }).status).toBe(0);
|
||||
}
|
||||
const seed = spawnSync(path.join(codexHome, 'skills', 'gstack', 'bin', 'gstack-learnings-log'), [JSON.stringify({ skill: 'learn',
|
||||
type: 'pitfall', key: MULTIBLOCK_SENTINEL_KEY, insight: 'Seeded by the E6 live Codex check', confidence: 9, source: 'user-stated' })],
|
||||
{ cwd: repo, env, encoding: 'utf8', timeout: 30_000 });
|
||||
expect(seed.status, seed.stderr).toBe(0);
|
||||
|
||||
const result = await runRecordedCodexEval({
|
||||
name: CASE,
|
||||
suite: 'codex-e2e-multiblock-live',
|
||||
budgetMs: CAPTURE_MS,
|
||||
run: signal => codexExec(['exec', '--json', '-s', 'read-only', '--skip-git-repo-check',
|
||||
'--model', process.env.GSTACK_CODEX_MODEL ?? CODEX_FRONTIER_MODEL,
|
||||
'Use the gstack learn skill to show this project\'s recent learnings. Run its "Show recent" step, then list what it printed.'],
|
||||
env, repo, scratch, signal),
|
||||
validate: run => {
|
||||
const problems = multiblockProblems(run.rawLines);
|
||||
if (problems.length) throw new Error(problems.join('\n'));
|
||||
},
|
||||
record: entry => collector?.addTest(entry),
|
||||
});
|
||||
console.log(`${CASE}: ${result.toolCalls.length} commands, ${result.tokens} tokens, ${Math.round(result.durationMs / 1000)}s`);
|
||||
} finally {
|
||||
fs.rmSync(scratch, { recursive: true, force: true });
|
||||
}
|
||||
}, SETUP_MS + 120_000 + CAPTURE_MS + CODEX_EVAL_FINALIZE_MS);
|
||||
});
|
||||
@@ -0,0 +1,61 @@
|
||||
/**
|
||||
* Free replays for the live Codex multi-block oracles (E6): synthetic
|
||||
* `codex exec --json` events in the shape codex-cli 0.160 writes (see
|
||||
* test/fixtures/codex-sandbox/exec-json-healthy.jsonl) and a
|
||||
* `codex debug prompt-input` skills block.
|
||||
*/
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { MULTIBLOCK_SENTINEL_KEY, multiblockProblems, routerPathFromPromptInput } from './helpers/codex-multiblock';
|
||||
|
||||
const PRELUDE = '[ -d "${GSTACK_ROOT:-/-}/bin" ]&&[ -d "$GSTACK_ROOT/lib" ]||{ _r=$(git rev-parse --show-toplevel 2>/dev/null)/.agents/skills/gstack;'
|
||||
+ '[ -d "$_r/bin" ]||_r=${CODEX_HOME:-~/.codex}/skills/gstack;[ -d "$_r/bin" ]||{ echo "gstack: no install found (tried $_r). Fix: ./setup --host codex from your gstack checkout; ./setup --status shows it.">&2;exit 1;};GSTACK_ROOT=$_r;}';
|
||||
const SHOW_RECENT = `${PRELUDE}\nSLUG=$($GSTACK_ROOT/bin/gstack-slug --get SLUG 2>/dev/null)\n$GSTACK_ROOT/bin/gstack-learnings-search --limit 20 2>/dev/null || echo "No learnings yet."`;
|
||||
const LISTING = `LEARNINGS: 1 loaded (1 pitfall)\n\n## Pitfalls\n- [${MULTIBLOCK_SENTINEL_KEY}] (confidence: 9/10, user-stated, 2026-10-04)\n Seeded by the E6 live Codex check\n`;
|
||||
|
||||
const command = (cmd: string, output: string, exitCode: number) => JSON.stringify({ type: 'item.completed',
|
||||
item: { id: 'item_2', type: 'command_execution', command: `/bin/bash -lc ${JSON.stringify(cmd)}`, aggregated_output: output, exit_code: exitCode, status: exitCode === 0 ? 'completed' : 'failed' } });
|
||||
const run = (...lines: string[]) => ['{"type":"thread.started","thread_id":"t"}', ...lines, '{"type":"turn.completed","usage":{"input_tokens":1,"output_tokens":1}}'];
|
||||
const preamble = command(`${PRELUDE}\nGSTACK_BIN=$GSTACK_ROOT/bin\n"$GSTACK_BIN/gstack-skill-start" --skill "learn"`, 'BRANCH: main\n', 0);
|
||||
|
||||
describe('multiblockProblems', () => {
|
||||
test('a later block rooted by its own prelude that prints the seeded learning passes', () => {
|
||||
expect(multiblockProblems(run(preamble, command(SHOW_RECENT, LISTING, 0)))).toEqual([]);
|
||||
});
|
||||
|
||||
test('a block that cannot find the install fails, even when another block succeeded', () => {
|
||||
const missing = command(SHOW_RECENT, 'gstack: no install found (tried /root/.codex/skills/gstack). Fix: ./setup --host codex', 1);
|
||||
const problems = multiblockProblems(run(preamble, missing, command(SHOW_RECENT, LISTING, 0)));
|
||||
expect(problems).toHaveLength(1);
|
||||
expect(problems[0]).toStartWith('a block could not find the install');
|
||||
});
|
||||
|
||||
test('the pre-prelude symptom (an empty root, /bin/gstack-slug) fails', () => {
|
||||
const problems = multiblockProblems(run(command('SLUG=$($GSTACK_ROOT/bin/gstack-slug --get SLUG)\n$GSTACK_ROOT/bin/gstack-learnings-search',
|
||||
'bash: line 1: /bin/gstack-slug: No such file or directory\nbash: line 2: /bin/gstack-learnings-search: No such file or directory\n', 127)));
|
||||
expect(problems.some(p => p.startsWith('a gstack helper path did not resolve: /bin/gstack-slug: No such file or directory'))).toBe(true);
|
||||
expect(problems.some(p => p.startsWith('no learnings-search block rooted by its own prelude printed the seeded learning'))).toBe(true);
|
||||
});
|
||||
|
||||
test('a preamble-only run, a run without the seeded learning, or a hand-set root does not count', () => {
|
||||
expect(multiblockProblems(run(preamble))).toEqual(['no block ran gstack-learnings-search (the skill\'s "Show recent" block)']);
|
||||
expect(multiblockProblems(run(preamble, command(SHOW_RECENT, 'No learnings yet.\n', 0)))[0]).toContain('exit 0, sentinel absent');
|
||||
const handSet = command('GSTACK_ROOT=/home/u/.codex/skills/gstack\n$GSTACK_ROOT/bin/gstack-learnings-search --limit 20', LISTING, 0);
|
||||
expect(multiblockProblems(run(preamble, handSet))[0]).toContain('GSTACK_ROOT set by hand');
|
||||
});
|
||||
});
|
||||
|
||||
describe('routerPathFromPromptInput', () => {
|
||||
const promptInput = (entries: string[]) => JSON.stringify([{ type: 'message', role: 'developer', content: [{ type: 'input_text',
|
||||
text: ['<skills_instructions>', '## Skills', '### Skill roots', '- `r0` = `/h/.codex/skills`', '- `r1` = `/h/.codex/skills/.system`',
|
||||
'### Available skills', '- skill-creator: Create or update a Codex skill. (file: r1/skill-creator/SKILL.md)', ...entries].join('\n') }] }], null, 2);
|
||||
|
||||
test('names the router file through the skill roots table', () => {
|
||||
const listed = promptInput(['- gstack: Route a task to the right gstack skill. (gstack) (file: r0/gstack/SKILL.md)',
|
||||
'- learn: Manage project learnings. (gstack) (file: r0/gstack-learn/SKILL.md)']);
|
||||
expect(routerPathFromPromptInput(`WARNING: proceeding without PATH aliases\n${listed}`)).toBe('/h/.codex/skills/gstack/SKILL.md');
|
||||
});
|
||||
|
||||
test('is null when Codex lists gstack skills but not the router', () => {
|
||||
expect(routerPathFromPromptInput(promptInput(['- learn: Manage project learnings. (gstack) (file: r0/gstack-learn/SKILL.md)']))).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,193 @@
|
||||
/**
|
||||
* The weekly tracking issues (evals-periodic.yml, evals-marathon.yml).
|
||||
*
|
||||
* Only main posts to or closes a tracking issue: a branch-dispatched census
|
||||
* writes the same report to its step summary and a report artifact instead.
|
||||
* The periodic report carries the full pass-rate alarm lines, the red ledger,
|
||||
* session headroom and the all-green estimate, read with an explicit
|
||||
* --branch, and every report-derived line passes the published-text
|
||||
* sanitizer. The write steps run here as shell scripts with a fake
|
||||
* `eval:pass-rates`; the sanitizer is the real one.
|
||||
*/
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { spawnSync } from 'child_process';
|
||||
|
||||
const ROOT = path.join(import.meta.dir, '..');
|
||||
type Step = { name?: string; if?: string; run?: string; uses?: string; with?: Record<string, string> };
|
||||
type Workflow = { jobs: Record<string, { steps?: Step[] }> };
|
||||
const WORKFLOWS = ['evals-periodic.yml', 'evals-marathon.yml'] as const;
|
||||
const parse = (name: string) => Bun.YAML.parse(fs.readFileSync(path.join(ROOT, '.github/workflows', name), 'utf8')) as Workflow;
|
||||
const MAIN_ONLY = "github.ref == 'refs/heads/main'";
|
||||
|
||||
describe('tracking issues are main-only', () => {
|
||||
for (const name of WORKFLOWS) {
|
||||
test(`${name}: only the upsert and close steps call gh issue, and both run on main only`, () => {
|
||||
const issueSteps = Object.values(parse(name).jobs).flatMap(job => job.steps ?? []).filter(step => /\bgh issue\b/.test(step.run ?? ''));
|
||||
expect(issueSteps.map(step => step.name).sort()).toEqual(['Close the tracking issue on a green run', 'Upsert tracking issue on failure']);
|
||||
for (const step of issueSteps) expect(step.if, step.name).toContain(`always() && ${MAIN_ONLY} && `);
|
||||
const upsert = issueSteps.find(step => step.name === 'Upsert tracking issue on failure')!;
|
||||
expect(upsert.run).toMatch(/gh issue comment .*--body-file "\$BODY_FILE"/);
|
||||
expect(upsert.run).toMatch(/gh issue create .*--body-file "\$BODY_FILE"/);
|
||||
expect(issueSteps.find(step => step.name === 'Close the tracking issue on a green run')!.run).toContain('gh issue close');
|
||||
});
|
||||
|
||||
test(`${name}: every ref writes the report to the step summary and an artifact`, () => {
|
||||
const steps = parse(name).jobs.report!.steps!;
|
||||
const write = steps.find(step => /^Write the (census|marathon) report$/.test(step.name ?? ''))!;
|
||||
const upload = steps.find(step => /^Upload the (census|marathon) report$/.test(step.name ?? ''))!;
|
||||
const upsert = steps.find(step => step.name === 'Upsert tracking issue on failure')!;
|
||||
expect(write.if).not.toContain('github.ref');
|
||||
expect(upload.if).toBe(write.if);
|
||||
expect(upsert.if).toBe(write.if!.replace('always() && ', `always() && ${MAIN_ONLY} && `));
|
||||
expect(write.run).toContain('>> "$GITHUB_STEP_SUMMARY"');
|
||||
expect(write.run).toContain('docs/evals/census-red.md');
|
||||
expect(write.run).toContain('./scripts/lib/published-text');
|
||||
const reportFile = /> (\/tmp\/[\w-]+\.md)\n/.exec(write.run!)![1]!;
|
||||
expect(upload.uses).toStartWith('actions/upload-artifact@');
|
||||
expect(upload.with!.path).toBe(reportFile);
|
||||
expect(upload.with!.name).toMatch(/^(census|marathon)-report-a\$\{\{ github\.run_attempt \}\}$/);
|
||||
expect(upsert.run).toContain(`BODY_FILE=${reportFile}`);
|
||||
});
|
||||
}
|
||||
|
||||
test('evals-periodic.yml: every pass-rates read names its branch explicitly', () => {
|
||||
const runs = parse('evals-periodic.yml').jobs.report!.steps!.map(step => step.run ?? '').join('\n');
|
||||
const calls = [...runs.matchAll(/^\s*bun run eval:pass-rates [^\n>]*/gm)].map(m => m[0].trim());
|
||||
expect(calls.length).toBe(3);
|
||||
for (const call of calls) expect(call).toContain('--runs 10 --branch "$GITHUB_REF_NAME"');
|
||||
expect(calls.some(call => call.includes('--gate'))).toBe(true);
|
||||
expect(calls.some(call => call.includes('--reds'))).toBe(true);
|
||||
expect(calls.some(call => call.includes('--headroom'))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
/** Run a workflow step's script with its expressions and /tmp paths bound to a scratch dir. */
|
||||
function runStep(workflow: string, stepName: string, files: Record<string, string>, expressions: Record<string, string>) {
|
||||
const step = parse(workflow).jobs.report!.steps!.find(s => s.name === stepName)!;
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'tracking-issue-'));
|
||||
const work = path.join(dir, 'tmp');
|
||||
const bin = path.join(dir, 'bin');
|
||||
fs.mkdirSync(bin);
|
||||
for (const [rel, body] of Object.entries(files)) {
|
||||
fs.mkdirSync(path.dirname(path.join(work, rel)), { recursive: true });
|
||||
fs.writeFileSync(path.join(work, rel), body);
|
||||
}
|
||||
const log = path.join(dir, 'pass-rates-calls.log');
|
||||
fs.writeFileSync(path.join(bin, 'bun'), [
|
||||
'#!/bin/sh',
|
||||
'if [ "$1" = run ] && [ "$2" = eval:pass-rates ]; then',
|
||||
` echo "$*" >> '${log}'`,
|
||||
` case " $* " in *" --reds "*) cat '${work}/fake-reds.txt';; *" --headroom "*) cat '${work}/fake-headroom.txt';; esac`,
|
||||
' exit 0',
|
||||
'fi',
|
||||
`exec '${process.execPath}' "$@"`,
|
||||
].join('\n'), { mode: 0o755 });
|
||||
const script = step.run!
|
||||
.replace(/\$\{\{ ([^}]+) \}\}/g, (_, expr: string) => expressions[expr.trim()] ?? `<${expr.trim()}>`)
|
||||
.replaceAll('/tmp/', `${work}/`);
|
||||
const summary = path.join(dir, 'step-summary.md');
|
||||
const result = spawnSync('bash', ['-c', script], {
|
||||
cwd: ROOT, encoding: 'utf8', timeout: 60_000,
|
||||
env: { PATH: `${bin}:${process.env.PATH}`, HOME: dir, GITHUB_STEP_SUMMARY: summary, GITHUB_REF_NAME: 'feature/x',
|
||||
GITHUB_SHA: 'abc1234', GITHUB_EVENT_NAME: 'workflow_dispatch', GITHUB_SERVER_URL: 'https://github.com',
|
||||
GITHUB_REPOSITORY: 'garrytan/gstack', GITHUB_RUN_ID: '42', REDISPATCH: 'false', REDISPATCH_OF: '' },
|
||||
});
|
||||
const read = (file: string) => (fs.existsSync(file) ? fs.readFileSync(file, 'utf8') : '');
|
||||
const reports = Object.fromEntries(fs.readdirSync(work).filter(name => name.endsWith('.md')).map(name => [name, read(path.join(work, name))]));
|
||||
const out = { status: result.status, stderr: result.stderr, summary: read(summary), calls: read(log), body: (name: string) => reports[name] ?? '' };
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
return out;
|
||||
}
|
||||
|
||||
const PERIODIC_EXPRESSIONS = {
|
||||
'steps.reconcile.outputs.exit': '1', 'steps.gate-reconcile.outputs.exit': '0', 'steps.pass-rates.outputs.exit': '1',
|
||||
'needs.eval-slices.result': 'failure', 'needs.eval-codex-slices.result': 'success', 'needs.gate-census.result': 'success',
|
||||
};
|
||||
|
||||
describe('the periodic census report', () => {
|
||||
const passRates = [
|
||||
'scope: garrytan/gstack evals-periodic.yml on feature/x + main, last 10 completed run(s) per branch: 1, 2',
|
||||
'pass-rates: policy v1, 40 post-policy trial(s), 0 pre-policy (display only)',
|
||||
' label kind tier current series pre-policy manual case',
|
||||
' ACTION rule gate 9/10 [59.6%–98.2%] - 0 noisy-case',
|
||||
'ACTION REQUIRED (3):',
|
||||
' [drift] noisy-case passes 9/10 (below 95% over >= 10 trials): fix it, or propose a CASE_QUARANTINE entry with a written diagnosis',
|
||||
' [headroom] slow-case session sdk:slow-case#1: max 520s of 600s (87%) above the 85% cap; cut work, never raise the budget',
|
||||
' [drift] mention @attacker ``` </details><img src=x onerror=alert(1)> ghp_' + 'a'.repeat(36),
|
||||
'free-suite flaky-passes (/home/x/.gstack/flake-ledger.jsonl):',
|
||||
' 2x test/unrelated.test.ts',
|
||||
].join('\n');
|
||||
const reds = [
|
||||
'scope: garrytan/gstack evals-periodic.yml on feature/x + main, last 10 completed run(s) per branch: 1, 2',
|
||||
'reds: verdict reds per census lane by failure class and machine cause',
|
||||
' 1 periodic/full census: 1 red of 146 verdicts — class: assertion 1; cause: provider_stall 1',
|
||||
' ✗ noisy-case FAIL t1 assertion/provider_stall — Error: expect(received).toBe(expected)',
|
||||
'pooled: 1 red of 146 verdicts = 0.7% per verdict [95% 0.1%–3.8%]',
|
||||
'all-green probability (approximation, assumes independent verdicts): 81.2% for the latest census\'s 146 verdicts [55%–95% with the pooled rate\'s 95% interval as prior]; formula Π(1 - p_i) over its cases; rerun: bun run eval:pass-rates --reds',
|
||||
].join('\n');
|
||||
const headroom = [
|
||||
'scope: garrytan/gstack evals-periodic.yml on feature/x + main',
|
||||
'headroom: slowest armed session per case (warn > 75%, alarm > 85%; insufficient below 3 samples; censored = timed out)',
|
||||
' ALARM 520s/600s (87%) 3 540s slow-case',
|
||||
].join('\n');
|
||||
const files = {
|
||||
'pass-rates.txt': passRates, 'fake-reds.txt': reds, 'fake-headroom.txt': headroom,
|
||||
'paid-report/report-summary.md': '## Periodic summary\n✗ noisy-case', 'report.txt': 'reconcile tail',
|
||||
};
|
||||
|
||||
test('carries the full alarm lines, red ledger, headroom and all-green estimate, sanitized, for an explicit branch', () => {
|
||||
const run = runStep('evals-periodic.yml', 'Write the census report', files, PERIODIC_EXPRESSIONS);
|
||||
expect(run.stderr).toBe('');
|
||||
expect(run.status).toBe(0);
|
||||
const body = run.body('census-report.md');
|
||||
expect(run.summary).toBe(body);
|
||||
expect(body).toStartWith('Periodic census report: branch feature/x @ abc1234, event workflow_dispatch, run https://github.com/garrytan/gstack/actions/runs/42');
|
||||
expect(body).toContain('triage with https://github.com/garrytan/gstack/blob/abc1234/docs/evals/census-red.md');
|
||||
expect(body).toContain('- periodic reconciliation exit: 1 (slices job: failure, Codex slices job: success)');
|
||||
expect(body).toContain('- all-green probability (approximation, assumes independent verdicts): 81.2% for the latest census');
|
||||
expect(body).toContain('## Periodic summary');
|
||||
// Every alarm line, not just the header; the flaky-pass block is not an alarm.
|
||||
expect(body).toContain('ACTION REQUIRED (3):');
|
||||
expect(body).toContain('[drift] noisy-case passes 9/10 (below 95% over >= 10 trials): fix it');
|
||||
expect(body).toContain('[headroom] slow-case session sdk:slow-case#1: max 520s of 600s (87%) above the 85% cap');
|
||||
expect(body).not.toContain('test/unrelated.test.ts');
|
||||
expect(body).toContain('✗ noisy-case FAIL t1 assertion/provider_stall — Error: expect(received).toBe(expected)');
|
||||
expect(body).toContain('ALARM 520s/600s (87%)');
|
||||
// Hostile alarm text stays literal inside a fence it cannot close; mentions and tokens are neutralized.
|
||||
const hostile = body.split('\n').findIndex(line => line.includes('mention @'));
|
||||
const fence = body.split('\n').slice(0, hostile).reverse().find(line => /^`{3,}$/.test(line))!;
|
||||
expect(fence.length).toBeGreaterThan(3);
|
||||
expect(body).toContain('@\u200battacker');
|
||||
expect(body).not.toContain('ghp_' + 'a'.repeat(36));
|
||||
expect(run.calls.trim().split('\n')).toEqual([
|
||||
'run eval:pass-rates --reds --runs 10 --branch feature/x',
|
||||
'run eval:pass-rates --headroom --runs 10 --branch feature/x',
|
||||
]);
|
||||
});
|
||||
|
||||
test('missing history becomes a visible line, never a failed step', () => {
|
||||
const run = runStep('evals-periodic.yml', 'Write the census report', { 'fake-reds.txt': '', 'fake-headroom.txt': '' }, PERIODIC_EXPRESSIONS);
|
||||
expect(run.status).toBe(0);
|
||||
const body = run.body('census-report.md');
|
||||
expect(body).toContain('all-green probability: unavailable (see the red ledger)');
|
||||
expect(body).toContain('(no pass-rate alarms)');
|
||||
expect(body).toContain('(no periodic report summary)');
|
||||
});
|
||||
});
|
||||
|
||||
describe('the marathon report', () => {
|
||||
test('fences the sanitized reconciliation output and links the guide', () => {
|
||||
const run = runStep('evals-marathon.yml', 'Write the marathon report', { 'report.txt': 'red slice\nping @someone ```' },
|
||||
{ 'steps.reconcile.outputs.exit': '1', 'needs.eval-slices.result': 'success' });
|
||||
expect(run.status).toBe(0);
|
||||
const body = run.body('marathon-report.md');
|
||||
expect(run.summary).toBe(body);
|
||||
expect(body).toStartWith('Marathon report (non-blocking lane): branch feature/x @ abc1234, event workflow_dispatch');
|
||||
expect(body).toContain('docs/evals/census-red.md');
|
||||
expect(body).toContain('- reconciliation exit: 1');
|
||||
expect(body).toContain('````\nred slice\nping @\u200bsomeone ```\n````');
|
||||
});
|
||||
});
|
||||
@@ -377,11 +377,11 @@ describe('panel verdict surfaces (eval reliability policy)', () => {
|
||||
const gate = report.steps.find(step => step.id === 'pass-rates')!;
|
||||
expect(gate.run).toContain('bun run eval:pass-rates --gate --runs 10');
|
||||
expect(gate.if).toBe('always()');
|
||||
for (const name of ['Upsert tracking issue on failure', 'Fail the workflow when reconciliation failed']) {
|
||||
for (const name of ['Write the census report', 'Upsert tracking issue on failure', 'Fail the workflow when reconciliation failed']) {
|
||||
expect(report.steps.find(step => step.name === name)!.if).toContain("steps.pass-rates.outputs.exit != '0'");
|
||||
}
|
||||
const upsert = report.steps.find(step => step.name === 'Upsert tracking issue on failure')!;
|
||||
expect(upsert.run).toContain('report-summary.md');
|
||||
// The report body is written once on every ref; the main-only upsert posts it (test/evals-tracking-issue.test.ts).
|
||||
expect(report.steps.find(step => step.name === 'Write the census report')!.run).toContain('report-summary.md');
|
||||
expect(report.steps.find(step => step.name === 'Close the tracking issue on a green run')!.run).toContain('gh issue close');
|
||||
expect(report.steps.filter(step => step.with?.name?.startsWith('trial-outcomes-')).length).toBe(2);
|
||||
const redispatch = jobs.redispatch!;
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
/**
|
||||
* Oracles for the live Codex multi-block case (E6,
|
||||
* test/codex-e2e-multiblock-live.test.ts). Codex runs every shell block in a
|
||||
* fresh shell, so a gstack skill works there only when each later block
|
||||
* resolves the install on its own (the env-var host prelude). The case
|
||||
* installs gstack with `./setup --host codex` into a fresh CODEX_HOME, seeds
|
||||
* one learning, and runs /learn's "Show recent" block through `codex exec`.
|
||||
* Pure functions: the free test replays synthetic `exec --json` events.
|
||||
*/
|
||||
|
||||
export const MULTIBLOCK_SENTINEL_KEY = 'e6-multiblock-sentinel';
|
||||
|
||||
interface CommandEvent { command: string; output: string; exitCode: number | null }
|
||||
|
||||
function completedCommands(rawLines: readonly string[]): CommandEvent[] {
|
||||
const commands: CommandEvent[] = [];
|
||||
for (const line of rawLines) {
|
||||
try {
|
||||
const event = JSON.parse(line);
|
||||
if (event?.type !== 'item.completed' || event.item?.type !== 'command_execution') continue;
|
||||
commands.push({ command: String(event.item.command ?? ''), output: String(event.item.aggregated_output ?? ''),
|
||||
exitCode: typeof event.item.exit_code === 'number' ? event.item.exit_code : null });
|
||||
} catch { /* not an event line */ }
|
||||
}
|
||||
return commands;
|
||||
}
|
||||
|
||||
/**
|
||||
* Problems with a live run: any block that could not find the install, a
|
||||
* gstack helper path that did not resolve, or no later block (the skill's
|
||||
* learnings-search block, which only the prelude can root) that ran the
|
||||
* installed helper and printed the seeded learning.
|
||||
*/
|
||||
export function multiblockProblems(rawLines: readonly string[], sentinel = MULTIBLOCK_SENTINEL_KEY): string[] {
|
||||
const commands = completedCommands(rawLines);
|
||||
const problems: string[] = [];
|
||||
for (const { command, output } of commands) {
|
||||
if (output.includes('gstack: no install found')) problems.push(`a block could not find the install: ${output.trim().slice(0, 200)}`);
|
||||
const unresolved = /\S*\/bin\/gstack-[\w-]+: (?:No such file or directory|command not found|not found)/.exec(output);
|
||||
if (unresolved) problems.push(`a gstack helper path did not resolve: ${unresolved[0]} (in: ${command.slice(0, 120)})`);
|
||||
}
|
||||
const search = commands.filter(c => c.command.includes('gstack-learnings-search'));
|
||||
if (search.length === 0) problems.push('no block ran gstack-learnings-search (the skill\'s "Show recent" block)');
|
||||
else if (!search.some(c => c.exitCode === 0 && c.output.includes(sentinel) && !/GSTACK_ROOT=["']?\//.test(c.command))) {
|
||||
problems.push(`no learnings-search block rooted by its own prelude printed the seeded learning ${sentinel}: `
|
||||
+ search.map(c => `exit ${c.exitCode}, ${c.output.includes(sentinel) ? 'sentinel present' : 'sentinel absent'}`
|
||||
+ `${/GSTACK_ROOT=["']?\//.test(c.command) ? ', GSTACK_ROOT set by hand' : ''}`).join('; '));
|
||||
}
|
||||
return problems;
|
||||
}
|
||||
|
||||
/**
|
||||
* The `gstack` router entry from `codex debug prompt-input` output (a JSON
|
||||
* array of input items): the skill path it names, resolved through the skill
|
||||
* roots table, or null when Codex does not list the router.
|
||||
*/
|
||||
export function routerPathFromPromptInput(promptInput: string): string | null {
|
||||
let text = promptInput;
|
||||
try {
|
||||
const items = JSON.parse(promptInput.slice(promptInput.indexOf('[')));
|
||||
text = (Array.isArray(items) ? items : []).flatMap((item: { content?: Array<{ text?: string }> }) => item.content ?? [])
|
||||
.map(part => part.text ?? '').join('\n');
|
||||
} catch { /* scan the raw text */ }
|
||||
const entry = /^- gstack: .*\(file: (r\d+)\/gstack\/SKILL\.md\)$/m.exec(text);
|
||||
if (!entry) return null;
|
||||
const root = new RegExp(`^- \`${entry[1]}\` = \`([^\`]+)\`$`, 'm').exec(text);
|
||||
return root ? `${root[1]}/gstack/SKILL.md` : null;
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
/**
|
||||
* Design model smoke (E5, periodic): runs design/scripts/live-model-check.ts,
|
||||
* which sends the design binary's real request bodies with the DEFAULT
|
||||
* models (overrides are ignored), so a retired or unentitled default fails
|
||||
* here before users hit it. Without OPENAI_API_KEY the case is a Bun skip:
|
||||
* the report lists it as SKIPPED with zero coverage credit, never a pass.
|
||||
* With a key, exit 0 passes; exit 2 (the script saw no key) and every other
|
||||
* exit fail with the script's output attached.
|
||||
*/
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { e2eTierEnabled } from './helpers/e2e-gate';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const CASE = 'design-model-smoke';
|
||||
const KEY_PRESENT = !!process.env.OPENAI_API_KEY?.trim();
|
||||
const CHECK_MS = 400_000;
|
||||
if (e2eTierEnabled('periodic') && !KEY_PRESENT) {
|
||||
process.stderr.write(`\n${CASE}: SKIPPED: OPENAI_API_KEY is not set; the design model defaults were not checked (no coverage)\n`);
|
||||
}
|
||||
|
||||
describe.skipIf(!e2eTierEnabled('periodic') || !KEY_PRESENT)('design live model defaults (E5)', () => {
|
||||
test(CASE, () => {
|
||||
const home = fs.mkdtempSync(path.join(os.tmpdir(), 'design-model-smoke-'));
|
||||
try {
|
||||
const run = spawnSync(process.execPath, ['run', 'design/scripts/live-model-check.ts'], {
|
||||
cwd: ROOT, encoding: 'utf8', timeout: CHECK_MS,
|
||||
env: { ...process.env, HOME: home, GSTACK_HOME: path.join(home, '.gstack') },
|
||||
});
|
||||
const output = `${run.stdout ?? ''}${run.stderr ?? ''}`.trim();
|
||||
console.log(output);
|
||||
if (run.status === 2) throw new Error(`live-model-check saw no OPENAI_API_KEY although the test had one; nothing was checked:\n${output}`);
|
||||
expect(run.status, output).toBe(0);
|
||||
} finally {
|
||||
fs.rmSync(home, { recursive: true, force: true });
|
||||
}
|
||||
}, CHECK_MS + 10_000);
|
||||
});
|
||||
Reference in new issue
Block a user