fix(autoplan): shadow pages carry main's finding items; a finding item is never ceo_eligible

After merging #3767 (main 90a170c), open Critical/High findings are gate items
(item_class finding). The shadow records one row per gate item, so its pages now
include them. A CEO-only run leaves those items to the owner, so their rows are
ceo_eligible false (rule_pick was already null). The shadow fixtures follow
main's gate (items, counts, "Do it" tokens); the two merge adapters follow
main's doNext(out, rest, price) and its per-voice reservation (--estimate-usd).

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed 2026-11-02 02:26:56 +00:00
1 parent a119b08394
commit bfb878e691
5 files changed
+42 -30

No files matched your search

+2 -1
View File
@@ -712,7 +712,8 @@ predictions with his answers.
prospective evidence.
- **The record.** `shadow/ceo-only.r<rev>.jsonl`, one row per gate item: the
frozen item, `item_class` (`challenge2`, `challenge1`, `disagree`, `agent`,
`cap_deferred`, `cap_blocker`, `cap_delta`, `p1`), `ceo_eligible`, the
`cap_deferred`, `cap_blocker`, `cap_delta`, `finding`, `p1`), `ceo_eligible`
(false for a `finding` item, which a CEO-only run leaves to the owner), the
prediction, `rule_pick` (the driver's keep-direction pick, the free
baseline), `rec_hash`, the plan and prompt hashes, the model, the gstack
version, the page's tokens and cost, and `audit`. It is written once per
+1 -1
View File
@@ -59,7 +59,7 @@ export function exportShadow(dirs: string[]): ShadowExport {
const status: ShadowStatus = s ? (s.retrospective ? 'retrospective' : s.status) : failed ? 'failed' : state.shadow?.enabled ? 'missing' : 'off';
const gateItem = gate?.items.find(i => i.id === item);
const row: ShadowExportRow = {
run: state.run, gate_rev: rev, item, decision: d.id, item_class: s?.item_class ?? gateItem?.item_class ?? null, ceo_eligible: s?.ceo_eligible ?? (item === 'p1' || d.phase === 'ceo'), status,
run: state.run, gate_rev: rev, item, decision: d.id, item_class: s?.item_class ?? gateItem?.item_class ?? null, ceo_eligible: s?.ceo_eligible ?? ((s?.item_class ?? gateItem?.item_class) !== 'finding' && (item === 'p1' || d.phase === 'ceo')), status,
...(status === 'failed' ? { code: String(failed!.code) } : {}),
shadow: s ? Object.fromEntries(['action', 'pick', 'confidence', 'cites', 'reason', 'malformed', 'rule_pick', 'audit', 'model', 'prompt_version', 'prompt_sha256', 'rec_hash', 'rec_rev', 'recorded_at', 'retrospective'].filter(k => k in s).map(k => [k, s[k]])) : null,
owner: { chosen: d.chosen ?? null, status: d.status, owner_mode: d.owner_mode ?? null, owner_named: d.owner_named ?? null, carried: !!(d.carried_from || d.carried_gate_from), carried_from: d.carried_from ?? d.carried_gate_from ?? null,
+1 -1
View File
@@ -362,7 +362,7 @@ function shadowRows(out: string, state: RunState, list: GateList, page: Page, pa
const decision = decisionId(list.run, list.gate_rev, item.id);
const base = {
schema_version: 1, run: list.run, gate_rev: list.gate_rev, decision, item: item.id, item_class: item.item_class ?? null, phase: item.phase ?? null,
ceo_eligible: item.id === 'p1' || item.phase === 'ceo', item_frozen: frozen, rec_hash: recHash(list), rec_rev: list.rec_rev ?? 0,
ceo_eligible: item.item_class !== 'finding' && (item.id === 'p1' || item.phase === 'ceo'), item_frozen: frozen, rec_hash: recHash(list), rec_rev: list.rec_rev ?? 0,
plan_sha256: page.plan_sha256, prompt_version: SHADOW_PROMPT_VERSION, prompt_sha256: page.prompt_sha256, request_sha256: page.request_sha256,
model: replies[replies.length - 1]?.result.model ?? state.shadow?.model, gstack_version: GSTACK_VERSION, rule_pick: ruleKept(item),
retrospective: false, page_usage: usage, page_usd: usdOf(replies), recorded_at: at,
+23 -14
View File
@@ -89,11 +89,17 @@ describe('submit records the shadow before GATE; the record stays private and ch
});
test('one row per item with the frozen item, the rule pick, eligibility, class and hashes', () => {
expect(shadowRows.map(r => [r.item, r.item_class, r.ceo_eligible, r.rule_pick, r.status, r.retrospective])).toEqual([
['uc1', 'challenge2', true, 'a', 'recorded', false], ['uc2', 'challenge2', false, 'a', 'recorded', false], ['p1', 'p1', true, 'a', 'recorded', false]]);
['uc1', 'challenge2', true, 'a', 'recorded', false], ['d1', 'finding', false, null, 'recorded', false], ['d2', 'finding', false, null, 'recorded', false],
['uc2', 'challenge2', false, 'a', 'recorded', false], ['d3', 'finding', false, null, 'recorded', false], ['d4', 'finding', false, null, 'recorded', false], ['p1', 'p1', true, 'a', 'recorded', false]]);
expect(shadowRows[0].decision).toBe(`${readRun(out).run}-uc1`);
expect(shadowRows[0].item_frozen.options.map((o: { key: string }) => o.key)).toEqual(['a', 'b']);
expect(typeof shadowRows[0].audit).toBe('boolean');
});
test('a finding item is never ceo_eligible and has no rule pick: a CEO-only run leaves an open Critical or High finding to the owner', () => {
const findings = shadowRows.filter(r => r.item_class === 'finding');
expect(findings.map(r => [r.item, r.phase])).toEqual([['d1', 'ceo'], ['d2', 'ceo'], ['d3', 'eng'], ['d4', 'eng']]);
expect(findings.every(r => r.ceo_eligible === false && r.rule_pick === null)).toBe(true);
});
test('exactly one egress receipt per send, with the shadow sink and consent', () => {
expect(f.receipts().map(r => [r.sink, r.consent])).toEqual([['autoplan-shadow', 'autoplan_shadow on']]);
});
@@ -147,20 +153,22 @@ describe('submit records the shadow before GATE; the record stays private and ch
expect(r.stderr).toContain('(SHADOW_BEFORE_ANSWER)');
});
test('"Do it" relayed as tokens records owner_mode rec, owner_named false, the rec_hash and the agent role', () => {
ok(ap(['answer', '--out', out, '--gate-rev', '1', '--reply', 'uc1a uc2a p1a', '--by', 'agent:capy', '--owner-reply', 'Do it'], e), 'answer');
ok(ap(['answer', '--out', out, '--gate-rev', '1', '--reply', 'uc1a d1a d2a uc2a d3a d4a p1b', '--by', 'agent:capy', '--owner-reply', 'Do it'], e), 'answer');
const d = rows(path.join(out, 'decisions.jsonl'));
const gateRec = shadowRows[0].rec_hash;
expect(d.map(r => [r.label, r.owner_mode, r.owner_named, r.answered_role, r.rec_hash === gateRec, r.answered_by])).toEqual([
['UC1', 'rec', false, 'agent', true, 'agent:capy'], ['UC2', 'rec', false, 'agent', true, 'agent:capy'], ['P1', 'rec', false, 'agent', true, 'agent:capy']]);
['UC1', 'rec', false, 'agent', true, 'agent:capy'], ['D1', 'rec', false, 'agent', true, 'agent:capy'], ['D2', 'rec', false, 'agent', true, 'agent:capy'],
['UC2', 'rec', false, 'agent', true, 'agent:capy'], ['D3', 'rec', false, 'agent', true, 'agent:capy'], ['D4', 'rec', false, 'agent', true, 'agent:capy'], ['P1', 'rec', false, 'agent', true, 'agent:capy']]);
});
test('the reader exports the answered revision with the owner provenance', () => {
const r = exported(['--out', out], e);
expect(r.status).toBe(0);
const x = JSON.parse(r.stdout);
expect(x.rows.map((row: any) => [row.item, row.status, row.shadow.pick, row.owner.chosen, row.owner.owner_mode, row.owner.rec_hash_at_answer === row.owner.rec_hash_at_record])).toEqual([
['uc1', 'recorded', 'a', 'a', 'rec', true], ['uc2', 'recorded', 'a', 'a', 'rec', true], ['p1', 'recorded', 'a', 'a', 'rec', true]]);
expect(x.counts.status).toEqual({ recorded: 3 });
expect(x.counts.ceo_eligible).toEqual({ true: 2, false: 1 });
['uc1', 'recorded', 'a', 'a', 'rec', true], ['d1', 'recorded', 'a', 'a', 'rec', true], ['d2', 'recorded', 'a', 'a', 'rec', true],
['uc2', 'recorded', 'a', 'a', 'rec', true], ['d3', 'recorded', 'a', 'a', 'rec', true], ['d4', 'recorded', 'a', 'a', 'rec', true], ['p1', 'recorded', 'a', 'b', 'rec', true]]);
expect(x.counts.status).toEqual({ recorded: 7 });
expect(x.counts.ceo_eligible).toEqual({ true: 2, false: 5 });
});
});
@@ -194,7 +202,7 @@ describe('a failed shadow never blocks: the gate prints and answers; owner words
const d = rows(path.join(out, 'decisions.jsonl'));
expect(d.find(r => r.label === 'UC2')).toMatchObject({ owner_mode: 'unparsed', owner_named: false, owner_reply_parsed: false });
const x = JSON.parse(exported(['--out', out], e).stdout);
expect(x.rows.map((row: any) => [row.item, row.status, row.code])).toEqual([['uc1', 'failed', 'http'], ['uc2', 'failed', 'http'], ['p1', 'failed', 'http']]);
expect(x.rows.map((row: any) => [row.item, row.status, row.code])).toEqual([['uc1', 'failed', 'http'], ['d1', 'failed', 'http'], ['d2', 'failed', 'http'], ['uc2', 'failed', 'http'], ['d3', 'failed', 'http'], ['d4', 'failed', 'http'], ['p1', 'failed', 'http']]);
});
});
@@ -257,7 +265,7 @@ describe('a crash mid-call: the dispatched call settles unknown and answer recor
});
test('the reader labels those rows retrospective', () => {
const x = JSON.parse(exported(['--out', out], e).stdout);
expect(x.counts.status).toEqual({ retrospective: 3 });
expect(x.counts.status).toEqual({ retrospective: 7 });
});
});
@@ -287,7 +295,7 @@ describe('every gate writer records the shadow before it prints GATE', async ()
ok(ap(['next', '--out', out], g.env), 'next eng');
const r = ok(ap(['close', '--out', out, '--phase', 'eng', '--deferred', 'out of time', '--by', 'human'], g.env), 'close');
expect(lineIndex(r.stdout, 'SHADOW: recorded')).toBeLessThan(lineIndex(r.stdout, 'GATE: gate_rev=1'));
expect(rows(record(out)).map(x => [x.item, x.item_class, x.status])).toEqual([['uc1', 'challenge2', 'malformed'], ['d1', 'cap_deferred', 'recorded'], ['p1', 'p1', 'recorded']]);
expect(rows(record(out)).map(x => [x.item, x.item_class, x.status])).toEqual([['uc1', 'challenge2', 'malformed'], ['d1', 'finding', 'recorded'], ['d2', 'finding', 'recorded'], ['d3', 'cap_deferred', 'recorded'], ['p1', 'p1', 'recorded']]);
ok(ap(['answer', '--out', out, '--gate-rev', '1', '--reply', 'rec', '--by', 'human', '--rec-hash', rows(record(out))[0].rec_hash], g.env), 'answer');
closeOut = out;
});
@@ -298,7 +306,7 @@ describe('every gate writer records the shadow before it prints GATE', async ()
const gateLineCalls = bin.split('\n').filter(l => /gateLines\((state|at), out\)/.test(l) && !l.includes('function gateLines'));
expect(gateLineCalls.length).toBe(2);
expect(gateLineCalls.every(l => l.includes('line ? [line] : []'))).toBe(true);
expect(bin).toContain("const step = withRunLock(out, () => { const r = doNext(out, rest); return r.gate ? claimed(out, r) : r; });");
expect(bin).toContain("const step = withRunLock(out, () => { const r = doNext(out, rest, price); return r.gate ? claimed(out, r) : r; });");
});
});
let closeOut = '';
@@ -367,7 +375,7 @@ describe('the switch through the driver, and the spend cap', async () => {
expect(noOutside.stdout).toContain('SHADOW: off (--no-outside)');
});
test('a cap that cannot cover the call journals shadow_failed budget and the gate prints', () => {
const { out } = start('switch-cap', f.env, ['--spend-cap', '0.05']);
const { out } = start('switch-cap', f.env, ['--spend-cap', '0.05', '--estimate-usd', '0.001']);
phase(out, 'ceo', f.env);
const r = phase(out, 'eng', f.env);
expect(r.stdout).toContain('SHADOW: failed budget');
@@ -393,10 +401,11 @@ describe('the reader over many runs', () => {
expect(r.status).toBe(0);
expect(r.stderr).toContain('(CEO_ONLY_NO_GATE)');
const x = JSON.parse(r.stdout);
expect(x.counts.status).toEqual({ missing: 3, failed: 3, retrospective: 3, malformed: 1, recorded: 5 });
expect(x.rows.length).toBe(15);
expect(x.counts.status).toEqual({ missing: 7, failed: 7, retrospective: 7, malformed: 1, recorded: 11 });
expect(x.rows.length).toBe(33);
expect(x.counts.item_class.cap_deferred).toBe(1);
expect(x.rows.filter((row: any) => row.item_class === 'finding').every((row: any) => row.ceo_eligible === false)).toBe(true);
const text = ap(['shadow', '--runs', list], e);
expect(text.stdout).toContain('SHADOW_EXPORT: rows=15 refused=1');
expect(text.stdout).toContain('SHADOW_EXPORT: rows=33 refused=1');
});
});
+15 -13
View File
@@ -117,12 +117,13 @@ describe('the reply contract: exactly one valid object per requested id', () =>
const out = gateRun();
const page = buildPage(readState(out), gateOf(out), decider());
const valid = page.items.map(i => ({ id: i.item.id, action: 'decide', pick: 'a', confidence: 0.6, cites: i.findings.slice(0, 1).map(f => f.id), reason: 'r' }));
const P1 = valid.findIndex(v => v.id === 'p1');
const reply = (objs: unknown[]) => '```json\n' + JSON.stringify(objs) + '\n```';
test('the page carries each item as rendered, with its findings, and p1 lists every finding the others name', () => {
expect(page.items.map(i => i.item.id)).toEqual(['uc1', 'uc2', 'p1']);
expect(page.items.map(i => i.item.id)).toEqual(['uc1', 'd1', 'd2', 'uc2', 'd3', 'd4', 'p1']);
expect(page.items[0]!.frozen).toMatchObject({ item_class: 'challenge2', kind: 'approval', recommended: 'a' });
expect(page.items[0]!.findings.map(f => f.severity).every(Boolean)).toBe(true);
expect(page.items[2]!.findings.map(f => f.id).sort()).toEqual([...page.items[0]!.findings, ...page.items[1]!.findings].map(f => f.id).sort());
expect(page.items[P1]!.findings.map(f => f.id).sort()).toEqual(page.items.filter((_, i) => i !== P1).flatMap(i => i.findings).map(f => f.id).sort());
expect(page.prompt).toContain('"id": "uc1"');
expect(page.prompt).not.toContain('{{');
});
@@ -132,7 +133,7 @@ describe('the reply contract: exactly one valid object per requested id', () =>
const p = buildPage({ ...readState(out), active_plan: planFile }, gateOf(out), decider());
expect(p.prompt).toContain('fill {{QUESTIONS}} and {{IDS}} in {{PLAN}}');
expect(p.prompt).toContain('"id": "uc1"');
expect(p.prompt).toContain('uc1, uc2, p1');
expect(p.prompt).toContain('uc1, d1, d2, uc2, d3, d4, p1');
expect(p.prompt.split('{{').length).toBe(4);
});
test('a good reply is clean', () => {
@@ -140,12 +141,12 @@ describe('the reply contract: exactly one valid object per requested id', () =>
});
const cases: Array<[string, unknown[] | string, string]> = [
['missing', valid.slice(0, 2), 'missing'],
['duplicate', [...valid, valid[2]], 'duplicate'],
['unknown option', valid.map((v, i) => (i === 2 ? { ...v, pick: 'z' } : v)), 'pick "z" is not an option of p1'],
['foreign cite', valid.map((v, i) => (i === 2 ? { ...v, cites: ['nope'] } : v)), 'cites ["nope"] name a finding outside p1\'s origin'],
['confidence above 1', valid.map((v, i) => (i === 2 ? { ...v, confidence: 1.2 } : v)), 'confidence 1.2 is not a number from 0 to 1'],
['bad action', valid.map((v, i) => (i === 2 ? { ...v, action: 'approve' } : v)), 'action "approve" is not decide, send_back or hold'],
['long reason', valid.map((v, i) => (i === 2 ? { ...v, reason: 'x'.repeat(401) } : v)), 'reason is not a string of at most 400 characters'],
['duplicate', [...valid, valid[P1]], 'duplicate'],
['unknown option', valid.map((v, i) => (i === P1 ? { ...v, pick: 'z' } : v)), 'pick "z" is not an option of p1'],
['foreign cite', valid.map((v, i) => (i === P1 ? { ...v, cites: ['nope'] } : v)), 'cites ["nope"] name a finding outside p1\'s origin'],
['confidence above 1', valid.map((v, i) => (i === P1 ? { ...v, confidence: 1.2 } : v)), 'confidence 1.2 is not a number from 0 to 1'],
['bad action', valid.map((v, i) => (i === P1 ? { ...v, action: 'approve' } : v)), 'action "approve" is not decide, send_back or hold'],
['long reason', valid.map((v, i) => (i === P1 ? { ...v, reason: 'x'.repeat(401) } : v)), 'reason is not a string of at most 400 characters'],
['no JSON', 'I would approve.', 'no JSON array in the reply'],
];
for (const [name, objs, why] of cases) test(`${name} marks that item malformed`, () => {
@@ -206,7 +207,8 @@ describe('runShadow: retry, truncation, failures and the deadline', async () =>
const { f, out, outcome } = await shadowed('good', [{ mode: 'valid' }]);
expect(outcome.status).toBe('recorded');
expect(outcome.line).toMatch(/^SHADOW: recorded model=claude-opus-5-5 usd=0\.0140$/);
expect(ids(out).map(r => [r.item, r.status, r.pick, r.rule_pick, r.prompt_sha256 === SHADOW_PROMPT_SHA256])).toEqual([['uc1', 'recorded', 'a', 'a', true], ['uc2', 'recorded', 'a', 'a', true], ['p1', 'recorded', 'a', 'a', true]]);
expect(ids(out).map(r => [r.item, r.status, r.pick, r.rule_pick, r.prompt_sha256 === SHADOW_PROMPT_SHA256])).toEqual([['uc1', 'recorded', 'a', 'a', true], ['d1', 'recorded', 'a', null, true], ['d2', 'recorded', 'a', null, true],
['uc2', 'recorded', 'a', 'a', true], ['d3', 'recorded', 'a', null, true], ['d4', 'recorded', 'a', null, true], ['p1', 'recorded', 'a', 'a', true]]);
expect(f.requests().map(r => r.body.max_tokens)).toEqual([4000]);
expect(fs.existsSync(path.join(out, 'shadow', 'r1.prompt.md'))).toBe(true);
expect(fs.readFileSync(path.join(out, 'shadow', 'shadow-r1-1.reply.txt'), 'utf8')).toContain('fake prediction');
@@ -245,9 +247,9 @@ describe('runShadow: retry, truncation, failures and the deadline', async () =>
expect(ids(fixed.out).every(r => r.status === 'recorded')).toBe(true);
const still = await shadowed('bad-twice', [{ mode: 'bad_option' }, { mode: 'bad_option' }]);
expect(still.f.requests().length).toBe(2);
expect(ids(still.out).map(r => r.status)).toEqual(['malformed', 'recorded', 'recorded']);
expect(ids(still.out).map(r => r.status)).toEqual(['malformed', 'recorded', 'recorded', 'recorded', 'recorded', 'recorded', 'recorded']);
expect(ids(still.out)[0]).toMatchObject({ pick: null, audit: false, malformed: 'pick "z" is not an option of uc1' });
expect(still.outcome.line).toContain('malformed=1/3');
expect(still.outcome.line).toContain('malformed=1/7');
});
test('a truncated reply raises max_tokens once to 8,000 (no page retry); truncated twice fails truncated', async () => {
const raised = await shadowed('truncated-once', [{ mode: 'truncated' }, { mode: 'valid' }]);
@@ -272,7 +274,7 @@ describe('runShadow: retry, truncation, failures and the deadline', async () =>
const r = await shadowed('deadline-retry', [{ mode: 'missing', delay_ms: 1200 }, { mode: 'valid', delay_ms: 1200 }], undefined, 1500);
expect(r.ms).toBeLessThan(2400);
expect(r.outcome.status).toBe('recorded');
expect(ids(r.out).map(x => x.status)).toEqual(['recorded', 'recorded', 'malformed']);
expect(ids(r.out).map(x => x.status)).toEqual(['recorded', 'recorded', 'recorded', 'recorded', 'recorded', 'recorded', 'malformed']);
const slow = await shadowed('deadline-call', [{ mode: 'valid', delay_ms: 3000 }], undefined, 800);
expect(slow.outcome.line).toBe('SHADOW: failed timeout');
expect(slow.ms).toBeLessThan(2500);