mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-11 19:20:26 +01:00
Merge commit '16db02a13f143cee232a0abb03ca7e911b41e69f' into HEAD
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> # Conflicts: # bin/gstack-config # bin/gstack-doctor-components.sh
This commit is contained in:
garrytan committed
commit
a683bb4738
26 files changed
+1774
-113
No files matched your search
@@ -0,0 +1,411 @@
|
||||
/**
|
||||
* GST-537 PR 3, shadow mode through the driver (bin/gstack-autoplan,
|
||||
* lib/autoplan-shadow.ts): every gate writer records the shadow before it
|
||||
* prints GATE, the lock is not held during the call, a live lease makes a
|
||||
* second writer pay nothing and answer wait, a crashed call settles unknown
|
||||
* (never released) and answer records the revision retrospectively, the
|
||||
* checkpoint taken at the gate holds the record, owner provenance comes from
|
||||
* the owner's own words, and the reader exports answered revisions only.
|
||||
* Model calls go to test/fixtures/shadow-fake-model.ts: never a real network call.
|
||||
*/
|
||||
import { afterAll, describe, expect, test } from 'bun:test';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { releaseUndispatched } from '../lib/autoplan-budget';
|
||||
import { readState } from '../lib/autoplan-state';
|
||||
import { readLedger } from '../lib/spend-ledger';
|
||||
import { AUTOPLAN, ROOT, SMALL_PLAN, env as baseEnv, journal, readRun, review } from './helpers/autoplan-run-fixture';
|
||||
|
||||
const TMP = fs.mkdtempSync(path.join(os.tmpdir(), 'ap-shadow-drv-'));
|
||||
const FAKE = path.join(ROOT, 'test', 'fixtures', 'shadow-fake-model.ts');
|
||||
const servers: Array<{ kill: (sig?: number) => void }> = [];
|
||||
afterAll(() => { for (const s of servers) s.kill(); fs.rmSync(TMP, { recursive: true, force: true }); });
|
||||
const rows = (file: string) => (fs.existsSync(file) ? fs.readFileSync(file, 'utf8').split('\n').filter(Boolean).map(l => JSON.parse(l)) : []);
|
||||
const record = (out: string, rev = 1) => path.join(out, 'shadow', `ceo-only.r${rev}.jsonl`);
|
||||
const events = (out: string) => journal(out).map(r => r.event);
|
||||
|
||||
interface Fake { env: Record<string, string>; requests: () => any[]; receipts: () => any[] }
|
||||
async function fake(name: string, script: Array<Record<string, unknown>>, extra: Record<string, string> = {}): Promise<Fake> {
|
||||
const dir = path.join(TMP, `fake-${name}`);
|
||||
fs.mkdirSync(path.join(dir, 'state'), { recursive: true });
|
||||
fs.writeFileSync(path.join(dir, 'script.json'), JSON.stringify(script));
|
||||
servers.push(Bun.spawn(['bun', FAKE, dir], { stdout: 'ignore', stderr: 'inherit' }));
|
||||
for (let i = 0; i < 200 && !fs.existsSync(path.join(dir, 'port')); i++) await Bun.sleep(25);
|
||||
const port = fs.readFileSync(path.join(dir, 'port'), 'utf8');
|
||||
const env = baseEnv({ GSTACK_STATE_ROOT: path.join(dir, 'state'), GSTACK_CAPY: '', GSTACK_AUTOPLAN_SHADOW: '1', ANTHROPIC_API_KEY: 'sk-fake-anthropic', OPENAI_API_KEY: 'sk-fake-openai',
|
||||
ANTHROPIC_BASE_URL: `http://127.0.0.1:${port}`, OPENAI_BASE_URL: `http://127.0.0.1:${port}/v1`, ...extra });
|
||||
return { env, requests: () => rows(path.join(dir, 'requests.jsonl')), receipts: () => rows(path.join(dir, 'state', 'security', 'egress.jsonl')) };
|
||||
}
|
||||
const ap = (args: string[], e: Record<string, string>) => spawnSync('bun', [AUTOPLAN, ...args], { cwd: ROOT, env: e, encoding: 'utf8', timeout: 120_000 });
|
||||
const apAsync = (args: string[], e: Record<string, string>) => Bun.spawn(['bun', AUTOPLAN, ...args], { cwd: ROOT, env: e, stdout: 'pipe', stderr: 'pipe' });
|
||||
function ok(r: { status: number | null; stdout: string; stderr: string }, what: string) { if (r.status !== 0) throw new Error(`${what} failed (${r.status}): ${r.stderr}\n${r.stdout}`); return r; }
|
||||
|
||||
function start(name: string, e: Record<string, string>, extra: string[] = []) {
|
||||
const dir = path.join(TMP, name);
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
fs.writeFileSync(path.join(dir, 'plan.md'), SMALL_PLAN);
|
||||
const out = path.join(dir, 'run');
|
||||
const first = ok(ap(['next', '--out', out, '--plan', path.join(dir, 'plan.md'), '--no-ui', ...extra], e), 'first next');
|
||||
return { out, first };
|
||||
}
|
||||
const attempt = (out: string, phase: string, voice: string) => readRun(out).phases[phase][voice];
|
||||
const resultFile = (out: string, phase: string, voice: string) => attempt(out, phase, voice).attempts.at(-1).result;
|
||||
/** next, both fixture reviews, the native submit; returns the outside submit's argv so a test can run it its own way. */
|
||||
function toOutsideSubmit(out: string, phase: string, e: Record<string, string>): string[] {
|
||||
ok(ap(['next', '--out', out], e), `next ${phase}`);
|
||||
for (const voice of ['native', 'outside']) ok(review(path.join(out, `${phase}-${voice}-prompt.md`)), `review ${voice}`);
|
||||
ok(ap(['submit', '--out', out, '--phase', phase, '--voice', 'native', '--result', resultFile(out, phase, 'native'), '--model', 'anthropic/claude-opus-4-7', '--attempt', attempt(out, phase, 'native').attempt, '--runner', 'host-subagent'], e), 'submit native');
|
||||
return ['submit', '--out', out, '--phase', phase, '--voice', 'outside', '--result', resultFile(out, phase, 'outside'), '--model', 'openai/gpt-6-astra', '--attempt', attempt(out, phase, 'outside').attempt, '--runner', 'host-subagent'];
|
||||
}
|
||||
const phase = (out: string, name: string, e: Record<string, string>) => ok(ap(toOutsideSubmit(out, name, e), e), `submit outside ${name}`);
|
||||
const lineIndex = (stdout: string, prefix: string) => stdout.split('\n').findIndex(l => l.startsWith(prefix));
|
||||
const exported = (args: string[], e: Record<string, string>) => ap(['shadow', ...args, '--json'], e);
|
||||
|
||||
describe('submit records the shadow before GATE; the record stays private and checkpointed', async () => {
|
||||
const f = await fake('main', [{ mode: 'valid' }]);
|
||||
const ckpt = path.join(TMP, 'ckpt-main');
|
||||
const e = { ...f.env, GSTACK_AUTOPLAN_CHECKPOINT_DIR: ckpt };
|
||||
const { out, first } = start('main', e);
|
||||
phase(out, 'ceo', e);
|
||||
const gate = phase(out, 'eng', e);
|
||||
const shadowRows = rows(record(out));
|
||||
|
||||
test('init prints SHADOW: on and stores the switch in run.json and the init row', () => {
|
||||
expect(first.stdout).toContain('SHADOW: on model=claude-opus-5-5 (GSTACK_AUTOPLAN_SHADOW=1)');
|
||||
expect(readRun(out).shadow).toEqual({ enabled: true, model: 'claude-opus-5-5', source: 'env' });
|
||||
expect(journal(out).find(r => r.event === 'init').shadow.enabled).toBe(true);
|
||||
});
|
||||
test('SHADOW: recorded prints before GATE, and GATE_SPEND shows the shadow spend', () => {
|
||||
expect(lineIndex(gate.stdout, 'SHADOW: recorded model=claude-opus-5-5 usd=0.0140')).toBeGreaterThan(-1);
|
||||
expect(lineIndex(gate.stdout, 'SHADOW: ')).toBeLessThan(lineIndex(gate.stdout, 'GATE: gate_rev=1'));
|
||||
expect(gate.stdout).toMatch(/GATE_SPEND: .* attempts=4 shadow=0\.0140/);
|
||||
const ev = events(out);
|
||||
expect(ev.indexOf('gate_written')).toBeLessThan(ev.indexOf('shadow_pending'));
|
||||
expect(ev.indexOf('shadow_pending')).toBeLessThan(ev.indexOf('shadow_dispatched'));
|
||||
expect(ev.indexOf('shadow_dispatched')).toBeLessThan(ev.indexOf('shadow_recorded'));
|
||||
expect(f.requests().length).toBe(1);
|
||||
});
|
||||
test('one row per item with the frozen item, the rule pick, eligibility, class and hashes', () => {
|
||||
expect(shadowRows.map(r => [r.item, r.item_class, r.ceo_eligible, r.rule_pick, r.status, r.retrospective])).toEqual([
|
||||
['uc1', 'challenge2', true, 'a', 'recorded', false], ['d1', 'finding', false, null, 'recorded', false], ['d2', 'finding', false, null, 'recorded', false],
|
||||
['uc2', 'challenge2', false, 'a', 'recorded', false], ['d3', 'finding', false, null, 'recorded', false], ['d4', 'finding', false, null, 'recorded', false], ['p1', 'p1', true, 'a', 'recorded', false]]);
|
||||
expect(shadowRows[0].decision).toBe(`${readRun(out).run}-uc1`);
|
||||
expect(shadowRows[0].item_frozen.options.map((o: { key: string }) => o.key)).toEqual(['a', 'b']);
|
||||
expect(typeof shadowRows[0].audit).toBe('boolean');
|
||||
});
|
||||
test('a finding item is never ceo_eligible and has no rule pick: a CEO-only run leaves an open Critical or High finding to the owner', () => {
|
||||
const findings = shadowRows.filter(r => r.item_class === 'finding');
|
||||
expect(findings.map(r => [r.item, r.phase])).toEqual([['d1', 'ceo'], ['d2', 'ceo'], ['d3', 'eng'], ['d4', 'eng']]);
|
||||
expect(findings.every(r => r.ceo_eligible === false && r.rule_pick === null)).toBe(true);
|
||||
});
|
||||
test('exactly one egress receipt per send, with the shadow sink and consent', () => {
|
||||
expect(f.receipts().map(r => [r.sink, r.consent])).toEqual([['autoplan-shadow', 'autoplan_shadow on']]);
|
||||
});
|
||||
test('the checkpoint taken at the gate holds the shadow record; a restore keeps it byte for byte', () => {
|
||||
const collection = path.join(ckpt, readRun(out).run);
|
||||
const current = JSON.parse(fs.readFileSync(path.join(collection, 'current.json'), 'utf8'));
|
||||
const gen = path.join(collection, 'generations', current.generation, 'run', 'shadow', 'ceo-only.r1.jsonl');
|
||||
expect(fs.readFileSync(gen).equals(fs.readFileSync(record(out)))).toBe(true);
|
||||
const restored = path.join(TMP, 'main-restored');
|
||||
ok(ap(['resume', '--out', restored, '--from', collection], e), 'resume --from');
|
||||
expect(fs.readFileSync(record(restored)).equals(fs.readFileSync(record(out)))).toBe(true);
|
||||
expect(f.requests().length).toBe(1);
|
||||
});
|
||||
test('the gate render, owner brief, decisions and public export carry no pick, reason or salt', () => {
|
||||
const salt = fs.readFileSync(path.join(out, 'shadow', 'salt'), 'utf8').trim();
|
||||
const render = spawnSync('bun', [path.join(ROOT, 'bin', 'gstack-gate'), 'render', path.join(out, 'gate.json'), '--all'], { cwd: ROOT, env: e, encoding: 'utf8', timeout: 60_000 });
|
||||
const brief = spawnSync('bun', [path.join(ROOT, 'bin', 'gstack-owner-brief'), 'render', '--out', out], { cwd: ROOT, env: e, encoding: 'utf8', timeout: 60_000 });
|
||||
expect(render.status).toBe(0);
|
||||
for (const text of [render.stdout, brief.stdout, fs.readFileSync(path.join(out, 'decisions.jsonl'), 'utf8'), fs.readFileSync(path.join(out, 'gate.json'), 'utf8')]) {
|
||||
expect(text).not.toContain('fake prediction');
|
||||
expect(text).not.toContain('confidence');
|
||||
expect(text).not.toContain(salt);
|
||||
}
|
||||
const pub = path.join(TMP, 'main-public');
|
||||
ok(ap(['export', '--out', out, '--public', pub], e), 'export --public');
|
||||
const all = fs.readdirSync(pub, { recursive: true }).map(String);
|
||||
expect(all.some(p => p.includes('shadow'))).toBe(false);
|
||||
for (const p of all) if (fs.statSync(path.join(pub, p)).isFile()) expect(fs.readFileSync(path.join(pub, p), 'utf8')).not.toContain(salt);
|
||||
expect(readRun(out).salt).toBeUndefined();
|
||||
});
|
||||
test('gstack-artifact validate checks the record against its journaled sha256', () => {
|
||||
const validate = () => spawnSync('bun', [path.join(ROOT, 'bin', 'gstack-artifact'), 'validate', path.join(out, 'run.json')], { cwd: ROOT, env: e, encoding: 'utf8', timeout: 60_000 });
|
||||
expect(validate().status).toBe(0);
|
||||
const good = fs.readFileSync(record(out), 'utf8');
|
||||
fs.writeFileSync(record(out), good.replace('"pick":"a"', '"pick":"b"'));
|
||||
const bad = validate();
|
||||
expect(bad.status).toBe(1);
|
||||
expect(bad.stderr).toContain('(ARTIFACT_STALE)');
|
||||
expect(bad.stderr).toContain('shadow/ceo-only.r1.jsonl');
|
||||
fs.writeFileSync(record(out), good);
|
||||
});
|
||||
test('printing the gate again pays nothing', () => {
|
||||
const again = ok(ap(['next', '--out', out], e), 'next again');
|
||||
expect(again.stdout).toContain('GATE: gate_rev=1');
|
||||
expect(again.stdout).not.toContain('SHADOW:');
|
||||
expect(f.requests().length).toBe(1);
|
||||
});
|
||||
test('the reader refuses an unanswered revision', () => {
|
||||
const r = exported(['--out', out], e);
|
||||
expect(r.status).toBe(3);
|
||||
expect(r.stderr).toContain('(SHADOW_BEFORE_ANSWER)');
|
||||
});
|
||||
test('"Do it" relayed as tokens records owner_mode rec, owner_named false, the rec_hash and the agent role', () => {
|
||||
ok(ap(['answer', '--out', out, '--gate-rev', '1', '--reply', 'uc1a d1a d2a uc2a d3a d4a p1b', '--by', 'agent:capy', '--owner-reply', 'Do it'], e), 'answer');
|
||||
const d = rows(path.join(out, 'decisions.jsonl'));
|
||||
const gateRec = shadowRows[0].rec_hash;
|
||||
expect(d.map(r => [r.label, r.owner_mode, r.owner_named, r.answered_role, r.rec_hash === gateRec, r.answered_by])).toEqual([
|
||||
['UC1', 'rec', false, 'agent', true, 'agent:capy'], ['D1', 'rec', false, 'agent', true, 'agent:capy'], ['D2', 'rec', false, 'agent', true, 'agent:capy'],
|
||||
['UC2', 'rec', false, 'agent', true, 'agent:capy'], ['D3', 'rec', false, 'agent', true, 'agent:capy'], ['D4', 'rec', false, 'agent', true, 'agent:capy'], ['P1', 'rec', false, 'agent', true, 'agent:capy']]);
|
||||
});
|
||||
test('the reader exports the answered revision with the owner provenance', () => {
|
||||
const r = exported(['--out', out], e);
|
||||
expect(r.status).toBe(0);
|
||||
const x = JSON.parse(r.stdout);
|
||||
expect(x.rows.map((row: any) => [row.item, row.status, row.shadow.pick, row.owner.chosen, row.owner.owner_mode, row.owner.rec_hash_at_answer === row.owner.rec_hash_at_record])).toEqual([
|
||||
['uc1', 'recorded', 'a', 'a', 'rec', true], ['d1', 'recorded', 'a', 'a', 'rec', true], ['d2', 'recorded', 'a', 'a', 'rec', true],
|
||||
['uc2', 'recorded', 'a', 'a', 'rec', true], ['d3', 'recorded', 'a', 'a', 'rec', true], ['d4', 'recorded', 'a', 'a', 'rec', true], ['p1', 'recorded', 'a', 'b', 'rec', true]]);
|
||||
expect(x.counts.status).toEqual({ recorded: 7 });
|
||||
expect(x.counts.ceo_eligible).toEqual({ true: 2, false: 5 });
|
||||
});
|
||||
});
|
||||
|
||||
describe('a failed shadow never blocks: the gate prints and answers; owner words name items', async () => {
|
||||
const f = await fake('fail', [{ mode: 'http500' }]);
|
||||
const e = f.env;
|
||||
const { out } = start('fail', e);
|
||||
phase(out, 'ceo', e);
|
||||
const gate = phase(out, 'eng', e);
|
||||
test('a 500 journals shadow_failed http, prints SHADOW: failed http, and still prints the gate', () => {
|
||||
expect(gate.stdout).toContain('SHADOW: failed http');
|
||||
expect(gate.stdout).toContain('GATE: gate_rev=1');
|
||||
expect(journal(out).find(r => r.event === 'shadow_failed')).toMatchObject({ gate_rev: 1, code: 'http' });
|
||||
expect(fs.existsSync(record(out))).toBe(false);
|
||||
});
|
||||
test('owner words "uc1a" record tokens and owner_named true for uc1 only', () => {
|
||||
const r = ok(ap(['answer', '--out', out, '--gate-rev', '1', '--reply', 'uc1a', '--by', 'agent:capy', '--owner-reply', 'uc1a'], e), 'answer uc1a');
|
||||
expect(r.stdout).toContain('GATE_PENDING:');
|
||||
const uc1 = rows(path.join(out, 'decisions.jsonl')).find(r => r.label === 'UC1');
|
||||
expect(uc1).toMatchObject({ owner_mode: 'tokens', owner_named: true, answered_role: 'agent' });
|
||||
expect(rows(path.join(out, 'decisions.jsonl')).find(r => r.label === 'UC2').owner_mode).toBeUndefined();
|
||||
});
|
||||
test('an interrogated plan approval is refused by the reader', () => {
|
||||
ok(ap(['answer', '--out', out, '--gate-rev', '1', '--reply', 'p1c', '--by', 'human'], e), 'answer p1c');
|
||||
const r = exported(['--out', out], e);
|
||||
expect(r.status).toBe(3);
|
||||
expect(r.stderr).toContain('answered c (interrogate)');
|
||||
});
|
||||
test('an unparsed owner reply records owner_mode unparsed; the reader shows failed with its code', () => {
|
||||
ok(ap(['answer', '--out', out, '--gate-rev', '1', '--reply', 'uc2a p1a', '--by', 'agent:capy', '--owner-reply', 'sounds good to me'], e), 'answer');
|
||||
const d = rows(path.join(out, 'decisions.jsonl'));
|
||||
expect(d.find(r => r.label === 'UC2')).toMatchObject({ owner_mode: 'unparsed', owner_named: false, owner_reply_parsed: false });
|
||||
const x = JSON.parse(exported(['--out', out], e).stdout);
|
||||
expect(x.rows.map((row: any) => [row.item, row.status, row.code])).toEqual([['uc1', 'failed', 'http'], ['d1', 'failed', 'http'], ['d2', 'failed', 'http'], ['uc2', 'failed', 'http'], ['d3', 'failed', 'http'], ['d4', 'failed', 'http'], ['p1', 'failed', 'http']]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('the lease: the lock is free during the call, a second writer pays nothing, answer waits', async () => {
|
||||
const f = await fake('lease', [{ mode: 'valid', delay_ms: 2500 }]);
|
||||
const e = f.env;
|
||||
const { out } = start('lease', e);
|
||||
phase(out, 'ceo', e);
|
||||
const writer = apAsync(toOutsideSubmit(out, 'eng', e), e);
|
||||
for (let i = 0; i < 400 && !fs.existsSync(path.join(out, 'shadow', 'r1.pending')); i++) await Bun.sleep(25);
|
||||
const lease = JSON.parse(fs.readFileSync(path.join(out, 'shadow', 'r1.pending'), 'utf8'));
|
||||
const second = ap(['next', '--out', out], e);
|
||||
const answer = apAsync(['answer', '--out', out, '--gate-rev', '1', '--reply', 'uc1a uc2a p1a', '--by', 'human'], e);
|
||||
const [writerCode, answerCode] = await Promise.all([writer.exited, answer.exited]);
|
||||
const writerOut = await new Response(writer.stdout).text();
|
||||
test('the lease names the writer, and the run lock is free while it calls', () => {
|
||||
expect(lease).toMatchObject({ gate_rev: 1, pid: writer.pid, host: os.hostname(), retrospective: false });
|
||||
expect(second.status).toBe(0);
|
||||
});
|
||||
test('a second writer during the call prints SHADOW: pending and pays nothing; two writers pay once', () => {
|
||||
expect(second.stdout).toContain('SHADOW: pending (gate_rev 1 is being recorded by pid');
|
||||
expect(f.requests().length).toBe(1);
|
||||
expect(writerCode).toBe(0);
|
||||
expect(writerOut).toContain('SHADOW: recorded');
|
||||
});
|
||||
test('answer waits for the record, then answers; the record is prospective', () => {
|
||||
expect(answerCode).toBe(0);
|
||||
const ev = events(out);
|
||||
expect(ev.indexOf('shadow_recorded')).toBeLessThan(ev.indexOf('answered'));
|
||||
expect(rows(record(out)).every(r => r.retrospective === false)).toBe(true);
|
||||
expect(readRun(out).status).toBe('complete');
|
||||
});
|
||||
});
|
||||
|
||||
describe('a crash mid-call: the dispatched call settles unknown and answer records the gate retrospectively', async () => {
|
||||
const f = await fake('crash', [{ mode: 'valid', delay_ms: 30000 }, { mode: 'valid' }]);
|
||||
const e = f.env;
|
||||
const { out } = start('crash', e);
|
||||
phase(out, 'ceo', e);
|
||||
const writer = apAsync(toOutsideSubmit(out, 'eng', e), e);
|
||||
for (let i = 0; i < 400 && !(events(out).includes('shadow_dispatched') && f.requests().length > 0); i++) await Bun.sleep(25);
|
||||
writer.kill(9);
|
||||
await writer.exited;
|
||||
const ledgerBefore = readLedger(path.join(out, 'spend.json'));
|
||||
const released = releaseUndispatched(readState(out));
|
||||
const answer = ap(['answer', '--out', out, '--gate-rev', '1', '--reply', 'uc1a uc2a p1a', '--by', 'human'], e);
|
||||
const ledger = readLedger(path.join(out, 'spend.json'));
|
||||
test('a dispatched shadow call is never released as undispatched', () => {
|
||||
expect(ledgerBefore.reserved.map(r => [r.attempt, r.label])).toEqual([['shadow-r1-1', 'shadow']]);
|
||||
expect(released).toEqual([]);
|
||||
});
|
||||
test('answer settles the dead call unknown, records every row retrospective, then answers', () => {
|
||||
expect(answer.status).toBe(0);
|
||||
expect(answer.stdout).toMatch(/SHADOW: recorded model=claude-opus-5-5 usd=0\.0140 retrospective/);
|
||||
expect(ledger.settled.find(s => s.attempt === 'shadow-r1-1')).toMatchObject({ usd: 'unknown', label: 'shadow' });
|
||||
expect(ledger.released.some(r => r.attempt === 'shadow-r1-1')).toBe(false);
|
||||
expect(journal(out).find(r => r.event === 'shadow_expired').settled_unknown).toEqual(['shadow-r1-1']);
|
||||
expect(rows(record(out)).every(r => r.retrospective === true)).toBe(true);
|
||||
expect(f.requests().length).toBe(2);
|
||||
});
|
||||
test('the reader labels those rows retrospective', () => {
|
||||
const x = JSON.parse(exported(['--out', out], e).stdout);
|
||||
expect(x.counts.status).toEqual({ retrospective: 7 });
|
||||
});
|
||||
});
|
||||
|
||||
describe('every gate writer records the shadow before it prints GATE', async () => {
|
||||
const f = await fake('writers', [{ mode: 'valid' }]);
|
||||
const off = baseEnv({ GSTACK_CAPY: '' });
|
||||
/** A gate written with shadow off, then switched on: the next command that prints it records the shadow first. */
|
||||
const unshadowed = (name: string) => {
|
||||
const { out } = start(name, off);
|
||||
phase(out, 'ceo', off); phase(out, 'eng', off);
|
||||
const run = readRun(out);
|
||||
run.shadow = { enabled: true, model: 'claude-opus-5-5', source: 'env' };
|
||||
fs.writeFileSync(path.join(out, 'run.json'), JSON.stringify(run, null, 2) + '\n');
|
||||
return out;
|
||||
};
|
||||
for (const verb of ['next', 'resume'] as const) test(`${verb}`, () => {
|
||||
const out = unshadowed(`writer-${verb}`);
|
||||
const r = ok(ap([verb, '--out', out], f.env), verb);
|
||||
expect(lineIndex(r.stdout, 'SHADOW: recorded')).toBeGreaterThan(-1);
|
||||
expect(lineIndex(r.stdout, 'SHADOW: recorded')).toBeLessThan(lineIndex(r.stdout, 'GATE: gate_rev=1'));
|
||||
expect(fs.existsSync(record(out))).toBe(true);
|
||||
});
|
||||
test('close --deferred (and a malformed item records malformed)', async () => {
|
||||
const g = await fake('close', [{ mode: 'bad_option' }, { mode: 'bad_option' }]);
|
||||
const { out } = start('writer-close', g.env);
|
||||
phase(out, 'ceo', g.env);
|
||||
ok(ap(['next', '--out', out], g.env), 'next eng');
|
||||
const r = ok(ap(['close', '--out', out, '--phase', 'eng', '--deferred', 'out of time', '--by', 'human'], g.env), 'close');
|
||||
expect(lineIndex(r.stdout, 'SHADOW: recorded')).toBeLessThan(lineIndex(r.stdout, 'GATE: gate_rev=1'));
|
||||
expect(rows(record(out)).map(x => [x.item, x.item_class, x.status])).toEqual([['uc1', 'challenge2', 'malformed'], ['d1', 'finding', 'recorded'], ['d2', 'finding', 'recorded'], ['d3', 'cap_deferred', 'recorded'], ['p1', 'p1', 'recorded']]);
|
||||
ok(ap(['answer', '--out', out, '--gate-rev', '1', '--reply', 'rec', '--by', 'human', '--rec-hash', rows(record(out))[0].rec_hash], g.env), 'answer');
|
||||
closeOut = out;
|
||||
});
|
||||
test('submit, revise, check and run share the same publish step (source ratchet; submit runs above)', () => {
|
||||
const bin = fs.readFileSync(AUTOPLAN, 'utf8');
|
||||
for (const verb of ['next', 'submit', 'revise', 'close', 'check']) expect(bin).toMatch(new RegExp(`if \\(cmd === '${verb}'\\) return await publish\\(out, json, mutating\\(`));
|
||||
expect(bin).toContain('return await publish(out, json, withRunLock(out, () => {\n const r = resumeRun(out);');
|
||||
const gateLineCalls = bin.split('\n').filter(l => /gateLines\((state|at), out\)/.test(l) && !l.includes('function gateLines'));
|
||||
expect(gateLineCalls.length).toBe(2);
|
||||
expect(gateLineCalls.every(l => l.includes('line ? [line] : []'))).toBe(true);
|
||||
expect(bin).toContain("const step = withRunLock(out, () => { const r = doNext(out, rest, price); return r.gate ? claimed(out, r) : r; });");
|
||||
});
|
||||
});
|
||||
let closeOut = '';
|
||||
|
||||
describe('the record: once per revision, a reopen writes r2, temp files and symlinks', async () => {
|
||||
const f = await fake('reopen', [{ mode: 'valid' }]);
|
||||
const e = f.env;
|
||||
const { out } = start('reopen', e);
|
||||
phase(out, 'ceo', e); phase(out, 'eng', e);
|
||||
const r1 = fs.readFileSync(record(out));
|
||||
ok(ap(['answer', '--out', out, '--gate-rev', '1', '--reply', 'uc1a uc2a p1d', '--by', 'human'], e), 'answer p1d');
|
||||
const orphan = path.join(out, 'shadow', 'ceo-only.r2.jsonl.99999.dead.tmp');
|
||||
fs.writeFileSync(orphan, 'partial');
|
||||
phase(out, 'ceo', e);
|
||||
const gate2 = phase(out, 'eng', e);
|
||||
test('a reopen writes r2 and leaves r1 byte-identical; the orphan temp file is removed', () => {
|
||||
expect(gate2.stdout).toContain('GATE: gate_rev=2');
|
||||
expect(fs.readFileSync(record(out)).equals(r1)).toBe(true);
|
||||
expect(rows(record(out, 2)).map(r => r.decision)).toEqual(rows(record(out, 2)).map(r => `${readRun(out).run}-r2-${r.item}`));
|
||||
expect(fs.existsSync(orphan)).toBe(false);
|
||||
expect(f.requests().length).toBe(2);
|
||||
});
|
||||
test('re-recording keeps the audit marks: the same salt and decision ids mark the same rows', () => {
|
||||
const salt = fs.readFileSync(path.join(out, 'shadow', 'salt'), 'utf8').trim();
|
||||
for (const r of [...rows(record(out)), ...rows(record(out, 2))]) {
|
||||
const mark = BigInt('0x' + new Bun.CryptoHasher('sha256').update(salt + r.decision).digest('hex')) % 1000n < 100n;
|
||||
expect(r.audit).toBe(r.action === 'decide' && mark);
|
||||
}
|
||||
});
|
||||
test('a symlinked shadow folder is refused, nothing is written through it, and the gate still prints', async () => {
|
||||
const g = await fake('symlink', [{ mode: 'valid' }]);
|
||||
const { out: o2 } = start('symlink', g.env);
|
||||
const target = path.join(TMP, 'symlink-target');
|
||||
fs.mkdirSync(target);
|
||||
fs.symlinkSync(target, path.join(o2, 'shadow'));
|
||||
phase(o2, 'ceo', g.env);
|
||||
const r = phase(o2, 'eng', g.env);
|
||||
expect(r.stdout).toContain('SHADOW: failed unsafe_dir');
|
||||
expect(r.stdout).toContain('GATE: gate_rev=1');
|
||||
expect(fs.readdirSync(target)).toEqual([]);
|
||||
expect(g.requests().length).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('the switch through the driver, and the spend cap', async () => {
|
||||
const f = await fake('switch', [{ mode: 'valid' }]);
|
||||
test('off by default: no SHADOW line, run.json says off, and no shadow folder at the gate', () => {
|
||||
const e = { ...f.env, GSTACK_AUTOPLAN_SHADOW: '' };
|
||||
const { out, first } = start('switch-default', e);
|
||||
expect(first.stdout).not.toContain('SHADOW:');
|
||||
expect(readRun(out).shadow).toEqual({ enabled: false, reason: 'off by default', source: 'default' });
|
||||
phase(out, 'ceo', e);
|
||||
expect(phase(out, 'eng', e).stdout).not.toContain('SHADOW:');
|
||||
expect(fs.existsSync(path.join(out, 'shadow'))).toBe(false);
|
||||
});
|
||||
test('the env var wins over config; --no-outside gives off with its reason; resume cannot flip it', () => {
|
||||
fs.writeFileSync(path.join(f.env.GSTACK_STATE_ROOT!, 'config.yaml'), 'autoplan_shadow: on\n');
|
||||
const { out: offRun } = start('switch-env', { ...f.env, GSTACK_AUTOPLAN_SHADOW: '0' });
|
||||
expect(readRun(offRun).shadow).toMatchObject({ enabled: false, source: 'env' });
|
||||
const { out: cfgRun, first } = start('switch-config', { ...f.env, GSTACK_AUTOPLAN_SHADOW: '' });
|
||||
expect(first.stdout).toContain('SHADOW: on model=claude-opus-5-5 (autoplan_shadow on)');
|
||||
ok(ap(['resume', '--out', cfgRun], { ...f.env, GSTACK_AUTOPLAN_SHADOW: '0' }), 'resume');
|
||||
expect(readRun(cfgRun).shadow).toMatchObject({ enabled: true, source: 'config' });
|
||||
fs.rmSync(path.join(f.env.GSTACK_STATE_ROOT!, 'config.yaml'));
|
||||
const { first: noOutside } = start('switch-no-outside', f.env, ['--no-outside']);
|
||||
expect(noOutside.stdout).toContain('SHADOW: off (--no-outside)');
|
||||
});
|
||||
test('a cap that cannot cover the call journals shadow_failed budget and the gate prints', () => {
|
||||
const { out } = start('switch-cap', f.env, ['--spend-cap', '0.05', '--estimate-usd', '0.001']);
|
||||
phase(out, 'ceo', f.env);
|
||||
const r = phase(out, 'eng', f.env);
|
||||
expect(r.stdout).toContain('SHADOW: failed budget');
|
||||
expect(r.stdout).toContain('GATE: gate_rev=1');
|
||||
expect(journal(out).find(x => x.event === 'shadow_failed').code).toBe('budget');
|
||||
expect(f.requests().length).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('the reader over many runs', () => {
|
||||
test('missing, failed, malformed, retrospective and recorded rows; dedupe across a repeated entry; CEO_ONLY_NO_GATE', () => {
|
||||
const e = baseEnv({ GSTACK_CAPY: '' });
|
||||
const { out: missing } = start('reader-missing', e);
|
||||
phase(missing, 'ceo', e); phase(missing, 'eng', e);
|
||||
const run = readRun(missing);
|
||||
run.shadow = { enabled: true, model: 'claude-opus-5-5', source: 'env' };
|
||||
fs.writeFileSync(path.join(missing, 'run.json'), JSON.stringify(run, null, 2) + '\n');
|
||||
ok(ap(['answer', '--out', missing, '--gate-rev', '1', '--reply', 'uc1a uc2a p1a', '--by', 'human'], e), 'answer missing');
|
||||
const { out: ceoOnly } = start('reader-ceo-only', e, ['--ceo-only']);
|
||||
const list = path.join(TMP, 'runs.txt');
|
||||
fs.writeFileSync(list, [missing, path.join(TMP, 'fail', 'run'), path.join(TMP, 'crash', 'run'), closeOut, path.join(TMP, 'main', 'run'), path.join(TMP, 'main', 'run'), ceoOnly].join('\n') + '\n');
|
||||
const r = exported(['--runs', list], e);
|
||||
expect(r.status).toBe(0);
|
||||
expect(r.stderr).toContain('(CEO_ONLY_NO_GATE)');
|
||||
const x = JSON.parse(r.stdout);
|
||||
expect(x.counts.status).toEqual({ missing: 7, failed: 7, retrospective: 7, malformed: 1, recorded: 11 });
|
||||
expect(x.rows.length).toBe(33);
|
||||
expect(x.counts.item_class.cap_deferred).toBe(1);
|
||||
expect(x.rows.filter((row: any) => row.item_class === 'finding').every((row: any) => row.ceo_eligible === false)).toBe(true);
|
||||
const text = ap(['shadow', '--runs', list], e);
|
||||
expect(text.stdout).toContain('SHADOW_EXPORT: rows=33 refused=1');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,312 @@
|
||||
/**
|
||||
* GST-537 PR 3, shadow mode for the CEO-only decider (lib/autoplan-shadow.ts):
|
||||
* the frozen decider prompt, the switch, the reply contract, the 10% audit
|
||||
* mark, the call (truncation, the OpenAI output bound, the 90 s deadline, the
|
||||
* shadow's own egress receipt), and the gate-cost exclusion of shadow spend.
|
||||
* Model calls go to test/fixtures/shadow-fake-model.ts through
|
||||
* ANTHROPIC_BASE_URL and OPENAI_BASE_URL: never a real network call.
|
||||
*/
|
||||
import { afterAll, describe, expect, test } from 'bun:test';
|
||||
import * as crypto from 'node:crypto';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { SHADOW_PROMPT_SHA256, SHADOW_PROMPT_VERSION, auditMark, buildPage, claimShadow, parsePage, recordFile, runShadow, shadowPromptFile, shadowSwitch } from '../lib/autoplan-shadow';
|
||||
import { callModel } from '../lib/outside-voice-api';
|
||||
import { gateCost, renderGateCost } from '../lib/autoplan-gate-cost';
|
||||
import { readState } from '../lib/autoplan-state';
|
||||
import { reserve, settle, readLedger } from '../lib/spend-ledger';
|
||||
import type { GateList } from '../lib/gate-list';
|
||||
import { ROOT, readRun, runPhase, startSmallRun } from './helpers/autoplan-run-fixture';
|
||||
|
||||
const TMP = fs.mkdtempSync(path.join(os.tmpdir(), 'ap-shadow-'));
|
||||
const FAKE = path.join(ROOT, 'test', 'fixtures', 'shadow-fake-model.ts');
|
||||
const servers: Array<{ kill: () => void }> = [];
|
||||
afterAll(() => { for (const s of servers) s.kill(); fs.rmSync(TMP, { recursive: true, force: true }); });
|
||||
const rows = (file: string) => (fs.existsSync(file) ? fs.readFileSync(file, 'utf8').split('\n').filter(Boolean).map(l => JSON.parse(l)) : []);
|
||||
|
||||
/** A fake model server in its own process, answering step n of its script for the n-th request. */
|
||||
async function fake(name: string, script: Array<Record<string, unknown>>) {
|
||||
const dir = path.join(TMP, `fake-${name}`);
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
fs.writeFileSync(path.join(dir, 'script.json'), JSON.stringify(script));
|
||||
const proc = Bun.spawn(['bun', FAKE, dir], { stdout: 'ignore', stderr: 'inherit' });
|
||||
servers.push(proc);
|
||||
for (let i = 0; i < 200 && !fs.existsSync(path.join(dir, 'port')); i++) await Bun.sleep(25);
|
||||
const port = Number(fs.readFileSync(path.join(dir, 'port'), 'utf8'));
|
||||
const state = path.join(dir, 'state');
|
||||
fs.mkdirSync(state, { recursive: true });
|
||||
const env = { HOME: state, GSTACK_HOME: state, GSTACK_STATE_ROOT: state, ANTHROPIC_API_KEY: 'sk-fake-anthropic', OPENAI_API_KEY: 'sk-fake-openai', ANTHROPIC_BASE_URL: `http://127.0.0.1:${port}`, OPENAI_BASE_URL: `http://127.0.0.1:${port}/v1` };
|
||||
return { dir, env, requests: () => rows(path.join(dir, 'requests.jsonl')), receipts: () => rows(path.join(state, 'security', 'egress.jsonl')) };
|
||||
}
|
||||
|
||||
// One run at its gate (uc1 from CEO, uc2 from Eng, p1), shadow off; each case copies it and turns shadow on in run.json.
|
||||
const BASE = startSmallRun(path.join(TMP, 'base'));
|
||||
runPhase(BASE, 'ceo');
|
||||
runPhase(BASE, 'eng');
|
||||
let copies = 0;
|
||||
function gateRun(model = 'claude-opus-5-5'): string {
|
||||
const out = path.join(TMP, `run-${++copies}`);
|
||||
fs.cpSync(BASE, out, { recursive: true });
|
||||
const run = readRun(out);
|
||||
run.shadow = { enabled: true, model, source: 'env' };
|
||||
fs.writeFileSync(path.join(out, 'run.json'), JSON.stringify(run, null, 2) + '\n');
|
||||
return out;
|
||||
}
|
||||
const gateOf = (out: string) => JSON.parse(fs.readFileSync(path.join(out, 'gate.json'), 'utf8')) as GateList;
|
||||
const decider = () => fs.readFileSync(shadowPromptFile(), 'utf8');
|
||||
|
||||
describe('the decider prompt is frozen and predicts; it imports no mandate', () => {
|
||||
test('its sha256 is pinned beside its version, and every row stores it', () => {
|
||||
expect(SHADOW_PROMPT_VERSION).toBe(1);
|
||||
expect(crypto.createHash('sha256').update(fs.readFileSync(shadowPromptFile())).digest('hex')).toBe(SHADOW_PROMPT_SHA256);
|
||||
});
|
||||
test('no take-the-recommendation or keep-the-direction rule; the recommendation is a labeled input', () => {
|
||||
const text = decider();
|
||||
expect(text).not.toMatch(/\(recommended\)/i);
|
||||
expect(text).not.toMatch(/keep (the|your|its) (plan's )?direction/i);
|
||||
expect(text).not.toMatch(/sections\/ceo-only\.md/);
|
||||
expect(text).toContain("The reviewers' recommendation is one input among several");
|
||||
for (const slot of ['{{PLAN}}', '{{QUESTIONS}}', '{{IDS}}']) expect(text.split(slot).length).toBe(2);
|
||||
expect(text.trimEnd().endsWith('```')).toBe(true);
|
||||
});
|
||||
test('the CEO-only driver module does not name the CEO-only skill section', () => {
|
||||
expect(fs.readFileSync(path.join(ROOT, 'lib', 'autoplan-ceo-only.ts'), 'utf8')).not.toContain('sections/ceo-only.md');
|
||||
});
|
||||
});
|
||||
|
||||
describe('the switch is off by default and fixed at init', () => {
|
||||
const state = fs.mkdtempSync(path.join(TMP, 'cfg-'));
|
||||
const keys = { GSTACK_STATE_ROOT: state, ANTHROPIC_API_KEY: 'sk-fake', OPENAI_API_KEY: '' };
|
||||
const on = { outsideEnabled: true, ceoOnly: false };
|
||||
test('off by default, with no line', () => {
|
||||
expect(shadowSwitch(on, keys)).toEqual({ shadow: { enabled: false, reason: 'off by default', source: 'default' } });
|
||||
});
|
||||
test('config on turns it on with the default model; the env var wins over config', () => {
|
||||
fs.writeFileSync(path.join(state, 'config.yaml'), 'autoplan_shadow: on\n');
|
||||
const r = shadowSwitch(on, keys);
|
||||
expect(r.shadow).toEqual({ enabled: true, model: 'claude-opus-5-5', source: 'config' });
|
||||
expect(r.line).toStartWith('SHADOW: on model=claude-opus-5-5');
|
||||
expect(shadowSwitch(on, { ...keys, GSTACK_AUTOPLAN_SHADOW: '0' }).shadow).toMatchObject({ enabled: false, source: 'env' });
|
||||
fs.rmSync(path.join(state, 'config.yaml'));
|
||||
expect(shadowSwitch(on, { ...keys, GSTACK_AUTOPLAN_SHADOW: '1' }).shadow).toMatchObject({ enabled: true, source: 'env' });
|
||||
});
|
||||
test('no key, --no-outside, a CEO-only run, an unpriced model or a bad value give off with the reason', () => {
|
||||
const env = { ...keys, GSTACK_AUTOPLAN_SHADOW: 'on' };
|
||||
expect(shadowSwitch(on, { ...env, ANTHROPIC_API_KEY: '' }).line).toBe('SHADOW: off (no API key for claude-opus-5-5 (ANTHROPIC_API_KEY))');
|
||||
expect(shadowSwitch({ ...on, outsideEnabled: false }, env).line).toBe('SHADOW: off (--no-outside)');
|
||||
expect(shadowSwitch({ ...on, ceoOnly: true }, env).line).toBe('SHADOW: off (a CEO-only run writes no gate)');
|
||||
fs.writeFileSync(path.join(state, 'config.yaml'), 'autoplan_shadow_model: claude-unpriced-9\n');
|
||||
expect(shadowSwitch(on, env).line).toBe('SHADOW: off (claude-unpriced-9 has no price in lib/pricing.ts, so --spend-cap cannot bound it)');
|
||||
fs.rmSync(path.join(state, 'config.yaml'));
|
||||
expect(shadowSwitch(on, { ...keys, GSTACK_AUTOPLAN_SHADOW: 'yes' }).line).toBe('SHADOW: off (GSTACK_AUTOPLAN_SHADOW=yes is not on or off)');
|
||||
});
|
||||
test('a run.json without the field reads off: nothing is claimed', () => {
|
||||
const out = path.join(TMP, 'no-field');
|
||||
fs.cpSync(BASE, out, { recursive: true });
|
||||
const run = readRun(out);
|
||||
expect(run.shadow).toEqual({ enabled: false, reason: 'off by default', source: 'default' });
|
||||
delete run.shadow;
|
||||
fs.writeFileSync(path.join(out, 'run.json'), JSON.stringify(run, null, 2) + '\n');
|
||||
expect(claimShadow(out)).toEqual({});
|
||||
expect(fs.existsSync(path.join(out, 'shadow'))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('the reply contract: exactly one valid object per requested id', () => {
|
||||
const out = gateRun();
|
||||
const page = buildPage(readState(out), gateOf(out), decider());
|
||||
const valid = page.items.map(i => ({ id: i.item.id, action: 'decide', pick: 'a', confidence: 0.6, cites: i.findings.slice(0, 1).map(f => f.id), reason: 'r' }));
|
||||
const P1 = valid.findIndex(v => v.id === 'p1');
|
||||
const reply = (objs: unknown[]) => '```json\n' + JSON.stringify(objs) + '\n```';
|
||||
test('the page carries each item as rendered, with its findings, and p1 lists every finding the others name', () => {
|
||||
expect(page.items.map(i => i.item.id)).toEqual(['uc1', 'd1', 'd2', 'uc2', 'd3', 'd4', 'p1']);
|
||||
expect(page.items[0]!.frozen).toMatchObject({ item_class: 'challenge2', kind: 'approval', recommended: 'a' });
|
||||
expect(page.items[0]!.findings.map(f => f.severity).every(Boolean)).toBe(true);
|
||||
expect(page.items[P1]!.findings.map(f => f.id).sort()).toEqual(page.items.filter((_, i) => i !== P1).flatMap(i => i.findings).map(f => f.id).sort());
|
||||
expect(page.prompt).toContain('"id": "uc1"');
|
||||
expect(page.prompt).not.toContain('{{');
|
||||
});
|
||||
test('slot names inside the plan text stay plan text; the template slots are still filled', () => {
|
||||
const planFile = path.join(TMP, 'slot-plan.md');
|
||||
fs.writeFileSync(planFile, '## Implementation plan\n\nfill {{QUESTIONS}} and {{IDS}} in {{PLAN}}\n');
|
||||
const p = buildPage({ ...readState(out), active_plan: planFile }, gateOf(out), decider());
|
||||
expect(p.prompt).toContain('fill {{QUESTIONS}} and {{IDS}} in {{PLAN}}');
|
||||
expect(p.prompt).toContain('"id": "uc1"');
|
||||
expect(p.prompt).toContain('uc1, d1, d2, uc2, d3, d4, p1');
|
||||
expect(p.prompt.split('{{').length).toBe(4);
|
||||
});
|
||||
test('a good reply is clean', () => {
|
||||
expect(parsePage(reply(valid), page.items).clean).toBe(true);
|
||||
});
|
||||
const cases: Array<[string, unknown[] | string, string]> = [
|
||||
['missing', valid.slice(0, 2), 'missing'],
|
||||
['duplicate', [...valid, valid[P1]], 'duplicate'],
|
||||
['unknown option', valid.map((v, i) => (i === P1 ? { ...v, pick: 'z' } : v)), 'pick "z" is not an option of p1'],
|
||||
['foreign cite', valid.map((v, i) => (i === P1 ? { ...v, cites: ['nope'] } : v)), 'cites ["nope"] name a finding outside p1\'s origin'],
|
||||
['confidence above 1', valid.map((v, i) => (i === P1 ? { ...v, confidence: 1.2 } : v)), 'confidence 1.2 is not a number from 0 to 1'],
|
||||
['bad action', valid.map((v, i) => (i === P1 ? { ...v, action: 'approve' } : v)), 'action "approve" is not decide, send_back or hold'],
|
||||
['long reason', valid.map((v, i) => (i === P1 ? { ...v, reason: 'x'.repeat(401) } : v)), 'reason is not a string of at most 400 characters'],
|
||||
['no JSON', 'I would approve.', 'no JSON array in the reply'],
|
||||
];
|
||||
for (const [name, objs, why] of cases) test(`${name} marks that item malformed`, () => {
|
||||
const p = parsePage(typeof objs === 'string' ? objs : reply(objs), page.items);
|
||||
expect(p.clean).toBe(false);
|
||||
expect(p.byId.get('p1')).toEqual({ ok: false, why });
|
||||
});
|
||||
test('a foreign id leaves the items valid but the page unclean, so it is asked again', () => {
|
||||
const p = parsePage(reply([...valid, { ...valid[0], id: 'zz9' }]), page.items);
|
||||
expect(p.foreign).toEqual(['zz9']);
|
||||
expect([...p.byId.values()].every(v => v.ok)).toBe(true);
|
||||
expect(p.clean).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('the 10% audit mark', () => {
|
||||
test('on a fixed 2,000-row corpus the marked share is what the hash gives', () => {
|
||||
const marked = Array.from({ length: 2000 }, (_, i) => auditMark('fixed-salt-0001', `run-fixture-r${1 + (i % 3)}-d${i}`)).filter(Boolean).length;
|
||||
expect(marked).toBe(Array.from({ length: 2000 }, (_, i) => BigInt('0x' + crypto.createHash('sha256').update(`fixed-salt-0001run-fixture-r${1 + (i % 3)}-d${i}`).digest('hex')) % 1000n < 100n).filter(Boolean).length);
|
||||
expect(marked / 2000).toBeGreaterThanOrEqual(0.08);
|
||||
expect(marked / 2000).toBeLessThanOrEqual(0.12);
|
||||
});
|
||||
});
|
||||
|
||||
describe('callModel: the shadow receipt, stop_reason and the OpenAI output bound (existing callers unchanged)', async () => {
|
||||
const f = await fake('transport', [{ mode: 'valid' }, { mode: 'truncated' }, { mode: 'valid' }, { mode: 'truncated' }]);
|
||||
test('a receipt override writes exactly one receipt with its sink and consent; the default stays the outside-voice receipt', async () => {
|
||||
await callModel({ model: 'claude-opus-5-5', prompt: 'p', timeoutMs: 5000, env: f.env, maxTokens: 10, receipt: { sink: 'autoplan-shadow', consent: 'autoplan_shadow on', payloadClass: 'plan-and-gate' } });
|
||||
expect(f.receipts().map(r => [r.sink, r.consent, r.payload_class ?? r.payloadClass])).toEqual([['autoplan-shadow', 'autoplan_shadow on', 'plan-and-gate']]);
|
||||
const cut = await callModel({ model: 'claude-opus-5-5', prompt: 'p', timeoutMs: 5000, env: f.env });
|
||||
expect(cut.stop_reason).toBe('max_tokens');
|
||||
expect(f.receipts().at(-1)!.sink).toBe('outside-voice-api');
|
||||
expect(f.receipts().length).toBe(2);
|
||||
});
|
||||
test('OpenAI gets max_completion_tokens only when the caller set an output limit, and its finish_reason comes back', async () => {
|
||||
const plain = await callModel({ model: 'gpt-6.1-sol', prompt: 'p', timeoutMs: 5000, env: f.env });
|
||||
const bounded = await callModel({ model: 'gpt-6.1-sol', prompt: 'p', timeoutMs: 5000, env: f.env, maxTokens: 4000 });
|
||||
const [a, b] = f.requests().slice(2).map(r => r.body);
|
||||
expect(Object.keys(a)).toEqual(['model', 'messages']);
|
||||
expect(b.max_completion_tokens).toBe(4000);
|
||||
expect(plain.stop_reason).toBe('stop');
|
||||
expect(bounded.stop_reason).toBe('length');
|
||||
});
|
||||
});
|
||||
|
||||
describe('runShadow: retry, truncation, failures and the deadline', async () => {
|
||||
const ids = (out: string) => rows(recordFile(out, 1));
|
||||
const shadowed = async (name: string, script: Array<Record<string, unknown>>, model?: string, deadlineMs?: number) => {
|
||||
const f = await fake(name, script);
|
||||
const out = gateRun(model);
|
||||
const { claim } = claimShadow(out);
|
||||
expect(claim).toEqual({ gate_rev: 1, retrospective: false });
|
||||
const started = Date.now();
|
||||
const outcome = await runShadow(out, claim!, { env: { ...process.env, ...f.env }, deadlineMs });
|
||||
return { f, out, outcome, ms: Date.now() - started };
|
||||
};
|
||||
test('a good reply records every item, keeps the prompt and the raw reply, and settles one shadow-labeled call', async () => {
|
||||
const { f, out, outcome } = await shadowed('good', [{ mode: 'valid' }]);
|
||||
expect(outcome.status).toBe('recorded');
|
||||
expect(outcome.line).toMatch(/^SHADOW: recorded model=claude-opus-5-5 usd=0\.0140$/);
|
||||
expect(ids(out).map(r => [r.item, r.status, r.pick, r.rule_pick, r.prompt_sha256 === SHADOW_PROMPT_SHA256])).toEqual([['uc1', 'recorded', 'a', 'a', true], ['d1', 'recorded', 'a', null, true], ['d2', 'recorded', 'a', null, true],
|
||||
['uc2', 'recorded', 'a', 'a', true], ['d3', 'recorded', 'a', null, true], ['d4', 'recorded', 'a', null, true], ['p1', 'recorded', 'a', 'a', true]]);
|
||||
expect(f.requests().map(r => r.body.max_tokens)).toEqual([4000]);
|
||||
expect(fs.existsSync(path.join(out, 'shadow', 'r1.prompt.md'))).toBe(true);
|
||||
expect(fs.readFileSync(path.join(out, 'shadow', 'shadow-r1-1.reply.txt'), 'utf8')).toContain('fake prediction');
|
||||
expect(readLedger(path.join(out, 'spend.json')).settled.filter(s => s.label === 'shadow').map(s => [s.attempt, s.usd])).toEqual([['shadow-r1-1', 0.014]]);
|
||||
expect(fs.existsSync(path.join(out, 'shadow', 'r1.pending'))).toBe(false);
|
||||
});
|
||||
test('a link at a reply path is replaced, never written through', async () => {
|
||||
const f = await fake('linked-reply', [{ mode: 'valid' }]);
|
||||
const out = gateRun();
|
||||
const { claim } = claimShadow(out);
|
||||
const sentinel = path.join(TMP, 'sentinel.txt');
|
||||
fs.writeFileSync(sentinel, 'keep');
|
||||
const reply = path.join(out, 'shadow', 'shadow-r1-1.reply.txt');
|
||||
fs.symlinkSync(sentinel, reply);
|
||||
expect((await runShadow(out, claim!, { env: { ...process.env, ...f.env } })).status).toBe('recorded');
|
||||
expect(fs.readFileSync(sentinel, 'utf8')).toBe('keep');
|
||||
expect(fs.lstatSync(reply).isSymbolicLink()).toBe(false);
|
||||
expect(fs.readFileSync(reply, 'utf8')).toContain('fake prediction');
|
||||
});
|
||||
test('a cap changed during the call admits the retry against the new cap and is not written back over', async () => {
|
||||
const f = await fake('cap-moved', [{ mode: 'truncated', delay_ms: 400 }, { mode: 'valid' }]);
|
||||
const out = gateRun();
|
||||
const { claim } = claimShadow(out);
|
||||
const pending = runShadow(out, claim!, { env: { ...process.env, ...f.env } });
|
||||
await Bun.sleep(150);
|
||||
const run = readRun(out);
|
||||
run.spend_cap_usd = 2;
|
||||
fs.writeFileSync(path.join(out, 'run.json'), JSON.stringify(run, null, 2) + '\n');
|
||||
expect((await pending).status).toBe('recorded');
|
||||
expect(f.requests().length).toBe(2);
|
||||
expect(readLedger(path.join(out, 'spend.json')).cap_usd).toBe(2);
|
||||
});
|
||||
test('a malformed page is asked once more; items valid in either reply record, the rest record malformed', async () => {
|
||||
const fixed = await shadowed('missing-then-valid', [{ mode: 'missing' }, { mode: 'valid' }]);
|
||||
expect(fixed.f.requests().length).toBe(2);
|
||||
expect(ids(fixed.out).every(r => r.status === 'recorded')).toBe(true);
|
||||
const still = await shadowed('bad-twice', [{ mode: 'bad_option' }, { mode: 'bad_option' }]);
|
||||
expect(still.f.requests().length).toBe(2);
|
||||
expect(ids(still.out).map(r => r.status)).toEqual(['malformed', 'recorded', 'recorded', 'recorded', 'recorded', 'recorded', 'recorded']);
|
||||
expect(ids(still.out)[0]).toMatchObject({ pick: null, audit: false, malformed: 'pick "z" is not an option of uc1' });
|
||||
expect(still.outcome.line).toContain('malformed=1/7');
|
||||
});
|
||||
test('a truncated reply raises max_tokens once to 8,000 (no page retry); truncated twice fails truncated', async () => {
|
||||
const raised = await shadowed('truncated-once', [{ mode: 'truncated' }, { mode: 'valid' }]);
|
||||
expect(raised.f.requests().map(r => r.body.max_tokens)).toEqual([4000, 8000]);
|
||||
expect(raised.outcome.status).toBe('recorded');
|
||||
const twice = await shadowed('truncated-twice', [{ mode: 'truncated' }]);
|
||||
expect(twice.f.requests().length).toBe(2);
|
||||
expect(twice.outcome.line).toBe('SHADOW: failed truncated');
|
||||
expect(fs.existsSync(recordFile(twice.out, 1))).toBe(false);
|
||||
});
|
||||
test('an OpenAI shadow model sends max_completion_tokens and treats finish_reason length as truncated', async () => {
|
||||
const r = await shadowed('openai', [{ mode: 'truncated' }, { mode: 'valid' }], 'gpt-6.1-sol');
|
||||
expect(r.f.requests().map(x => [x.path, x.body.max_completion_tokens])).toEqual([['/v1/chat/completions', 4000], ['/v1/chat/completions', 8000]]);
|
||||
expect(r.outcome.status).toBe('recorded');
|
||||
});
|
||||
test('a 500 fails http and settles the call at 0; the journal names the code', async () => {
|
||||
const r = await shadowed('http500', [{ mode: 'http500' }]);
|
||||
expect(r.outcome.line).toBe('SHADOW: failed http');
|
||||
expect(readLedger(path.join(r.out, 'spend.json')).settled.filter(s => s.label === 'shadow').map(s => s.usd)).toEqual([0]);
|
||||
});
|
||||
test('one deadline covers the call and its retry: a slow malformed reply leaves the retry no time', async () => {
|
||||
const r = await shadowed('deadline-retry', [{ mode: 'missing', delay_ms: 1200 }, { mode: 'valid', delay_ms: 1200 }], undefined, 1500);
|
||||
expect(r.ms).toBeLessThan(2400);
|
||||
expect(r.outcome.status).toBe('recorded');
|
||||
expect(ids(r.out).map(x => x.status)).toEqual(['recorded', 'recorded', 'recorded', 'recorded', 'recorded', 'recorded', 'malformed']);
|
||||
const slow = await shadowed('deadline-call', [{ mode: 'valid', delay_ms: 3000 }], undefined, 800);
|
||||
expect(slow.outcome.line).toBe('SHADOW: failed timeout');
|
||||
expect(slow.ms).toBeLessThan(2500);
|
||||
expect(readLedger(path.join(slow.out, 'spend.json')).settled.filter(s => s.label === 'shadow').map(s => s.usd)).toEqual(['unknown']);
|
||||
});
|
||||
test('a record already published makes a later claim a no-op: no second call is paid', async () => {
|
||||
const { f, out } = await shadowed('noop', [{ mode: 'valid' }]);
|
||||
const before = fs.readFileSync(recordFile(out, 1));
|
||||
expect(claimShadow(out)).toEqual({});
|
||||
expect(f.requests().length).toBe(1);
|
||||
expect(fs.readFileSync(recordFile(out, 1)).equals(before)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('gate cost: shadow spend counts against the cap and shows in GATE_SPEND, never in the per-voice price', () => {
|
||||
test('a $5 voice settle followed by a shadow settle keeps GATE_REC_COST at x 5.00', () => {
|
||||
const out = path.join(TMP, 'cost');
|
||||
fs.mkdirSync(out, { recursive: true });
|
||||
const file = path.join(out, 'spend.json');
|
||||
reserve(file, 'r-a1', 5, { cap: 5.2, label: 'ceo/native' }); settle(file, 'r-a1', 'unknown');
|
||||
reserve(file, 'shadow-r1-1', 0.15, { label: 'shadow' }); settle(file, 'shadow-r1-1', 0.014);
|
||||
const list: GateList = { schema_version: 1, run: 'r', gate_rev: 1, items: [
|
||||
{ id: 'd1', title: 't', kind: 'auto', recommended: 'b', phase: 'ceo', options: [{ key: 'a', text: 'keep', changes_plan: false }, { key: 'b', text: 'change', changes_plan: true }] },
|
||||
{ id: 'p1', title: 'Plan approval', kind: 'approval', recommended: 'b', options: [{ key: 'a', text: 'x' }, { key: 'b', text: 'y' }] }] };
|
||||
const c = gateCost(out, list, { phase_order: ['ceo', 'eng'], outside_enabled: true, spend_cap_usd: 5.2 });
|
||||
expect(c.per_voice_usd).toBe(5);
|
||||
expect(c.attempts).toBe(1);
|
||||
expect(c.shadow_usd).toBe(0.014);
|
||||
expect(c.summary.worst_case_usd).toBe(5.014);
|
||||
const [spend, rec] = renderGateCost(c);
|
||||
expect(spend).toContain('attempts=1 shadow=0.0140');
|
||||
expect(rec).toContain('× 5.00');
|
||||
expect(rec).toContain('(over the cap: raise it with gstack-autoplan budget before answering rec)');
|
||||
});
|
||||
});
|
||||
+2
@@ -137,6 +137,8 @@
|
||||
{ "file": "lib/autoplan-run.ts", "movedFrom": "(new: next/submit/resume over lib/autoplan-state.ts, B2)" },
|
||||
{ "file": "lib/autoplan-gate.ts", "movedFrom": "(new: phase close, gate build, answer and reopen, B2/B9)" },
|
||||
{ "file": "lib/autoplan-ceo-only.ts", "movedFrom": "(new: the CEO-only driver run's finish, report and refusals, GST-537)" },
|
||||
{ "file": "lib/autoplan-shadow.ts", "movedFrom": "(new: shadow mode for the CEO-only decider: switch, lease, call and record, GST-537 PR 3)" },
|
||||
{ "file": "lib/autoplan-shadow-reader.ts", "movedFrom": "(new: the shadow-mode export for taste-replay R2 and its private validate check, GST-537 PR 3)" },
|
||||
{ "file": "lib/autoplan-export.ts", "movedFrom": "(new: the unattended export file set, B2/B12)" },
|
||||
{ "file": "lib/spec-review-cap.ts", "movedFrom": "(new: the spec-review loop at the cap: numbered gaps and deterministic fix verification, B8)" },
|
||||
{ "file": "bin/gstack-autoplan", "movedFrom": "(new: thin entrypoint over lib/autoplan-run.ts, B2)" },
|
||||
|
||||
Vendored
+59
@@ -0,0 +1,59 @@
|
||||
#!/usr/bin/env -S bun --no-env-file --config=/dev/null
|
||||
/**
|
||||
* A local stand-in for the Anthropic Messages and OpenAI Chat Completions APIs,
|
||||
* so the shadow-mode tests (test/autoplan-shadow*.test.ts) never make a real
|
||||
* network call: the tests point ANTHROPIC_BASE_URL and OPENAI_BASE_URL at it.
|
||||
* Usage: bun test/fixtures/shadow-fake-model.ts <dir>. It writes <dir>/port,
|
||||
* appends every request to <dir>/requests.jsonl, and answers the n-th request
|
||||
* with step n of <dir>/script.json (the last step repeats; default valid):
|
||||
* { "mode": valid | missing | duplicate | foreign | bad_option | bad_cite | bad_conf | garbage | truncated | send_back | http500,
|
||||
* "delay_ms"?: n }. A valid reply picks each question's first option with the
|
||||
* question's first finding as its cite; usage is 1,000 input and 500 output tokens.
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
|
||||
const dir = process.argv[2]!;
|
||||
interface Step { mode: string; delay_ms?: number }
|
||||
interface Question { id: string; options: Array<{ key: string }>; findings: Array<{ id: string }> }
|
||||
let served = 0;
|
||||
|
||||
function questions(prompt: string): Question[] {
|
||||
const at = prompt.indexOf('## The questions');
|
||||
const block = /```json\n([\s\S]*?)\n```/.exec(prompt.slice(at))?.[1];
|
||||
return block ? JSON.parse(block) : [];
|
||||
}
|
||||
|
||||
function answer(mode: string, qs: Question[]): string {
|
||||
const rows: Array<Record<string, unknown>> = qs.map(q => ({ id: q.id, action: mode === 'send_back' ? 'send_back' : 'decide', pick: q.options[0]!.key, confidence: 0.75, cites: q.findings.slice(0, 1).map(f => f.id), reason: 'fake prediction' }));
|
||||
if (mode === 'missing') rows.pop();
|
||||
if (mode === 'duplicate') rows.push({ ...rows[0]! });
|
||||
if (mode === 'foreign') rows.push({ ...rows[0]!, id: 'zz9' });
|
||||
if (mode === 'bad_option') rows[0]!.pick = 'z';
|
||||
if (mode === 'bad_cite') rows[0]!.cites = ['not-a-finding'];
|
||||
if (mode === 'bad_conf') rows[0]!.confidence = 1.5;
|
||||
if (mode === 'garbage') return 'I would approve it.';
|
||||
if (mode === 'truncated') return '```json\n[{"id": "' + (qs[0]?.id ?? 'p1') + '", "act';
|
||||
return 'Here is my prediction.\n```json\n' + JSON.stringify(rows, null, 2) + '\n```\n';
|
||||
}
|
||||
|
||||
const server = Bun.serve({
|
||||
port: 0, hostname: '127.0.0.1',
|
||||
async fetch(req) {
|
||||
const url = new URL(req.url);
|
||||
const body = await req.json() as Record<string, any>;
|
||||
let steps: Step[] = [{ mode: 'valid' }];
|
||||
try { steps = JSON.parse(fs.readFileSync(path.join(dir, 'script.json'), 'utf8')); } catch { /* default */ }
|
||||
const step = steps[Math.min(served, steps.length - 1)]!;
|
||||
served++;
|
||||
fs.appendFileSync(path.join(dir, 'requests.jsonl'), JSON.stringify({ n: served, path: url.pathname, mode: step.mode, body, at: Date.now() }) + '\n');
|
||||
if (step.delay_ms) await Bun.sleep(step.delay_ms);
|
||||
if (step.mode === 'http500') return Response.json({ error: { message: 'fake overloaded' } }, { status: 500 });
|
||||
const text = answer(step.mode, questions(String(body.messages?.[0]?.content ?? '')));
|
||||
const cut = step.mode === 'truncated';
|
||||
if (url.pathname.endsWith('/v1/messages')) return Response.json({ model: body.model, content: [{ type: 'text', text }], stop_reason: cut ? 'max_tokens' : 'end_turn', usage: { input_tokens: 1000, output_tokens: 500 } });
|
||||
return Response.json({ model: body.model, choices: [{ message: { content: text }, finish_reason: cut ? 'length' : 'stop' }], usage: { prompt_tokens: 1000, completion_tokens: 500 } });
|
||||
},
|
||||
});
|
||||
fs.writeFileSync(path.join(dir, 'port.tmp'), String(server.port));
|
||||
fs.renameSync(path.join(dir, 'port.tmp'), path.join(dir, 'port'));
|
||||
@@ -211,6 +211,7 @@ export const FREE_FIXTURES: Readonly<Record<string, readonly string[]>> = {
|
||||
'test/fixtures/review-gate-integrity-opus-reports.json': ['test/gate-integrity-templates.test.ts'],
|
||||
'test/fixtures/review-wrong-tree/**': ['test/review-dispatch.test.ts'],
|
||||
'test/fixtures/session-update-heartbeat.sh': ['test/session-update-heartbeat.test.ts'],
|
||||
'test/fixtures/shadow-fake-model.ts': ['test/autoplan-shadow-driver.test.ts', 'test/autoplan-shadow.test.ts'],
|
||||
'test/fixtures/shard-cli-parity/**': ['test/shard-cli-parity.test.ts'],
|
||||
'test/fixtures/shard-equivalence/**': ['test/shard-engine-equivalence.test.ts'],
|
||||
'test/fixtures/shared-libs-claude-version-probe.json': ['test/shared-libs-fixture.test.ts'],
|
||||
|
||||
Reference in new issue
Block a user