From bfb878e6912fd91a69894a11b271abe6b350ea35 Mon Sep 17 00:00:00 2001 From: garrytan Date: Mon, 2 Nov 2026 02:26:56 +0000 Subject: [PATCH] fix(autoplan): shadow pages carry main's finding items; a finding item is never ceo_eligible After merging #3767 (main 90a170c), open Critical/High findings are gate items (item_class finding). The shadow records one row per gate item, so its pages now include them. A CEO-only run leaves those items to the owner, so their rows are ceo_eligible false (rule_pick was already null). The shadow fixtures follow main's gate (items, counts, "Do it" tokens); the two merge adapters follow main's doNext(out, rest, price) and its per-voice reservation (--estimate-usd). Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- docs/unattended.md | 3 ++- lib/autoplan-shadow-reader.ts | 2 +- lib/autoplan-shadow.ts | 2 +- test/autoplan-shadow-driver.test.ts | 37 ++++++++++++++++++----------- test/autoplan-shadow.test.ts | 28 ++++++++++++---------- 5 files changed, 42 insertions(+), 30 deletions(-) diff --git a/docs/unattended.md b/docs/unattended.md index a04149ba5..0741274c1 100644 --- a/docs/unattended.md +++ b/docs/unattended.md @@ -712,7 +712,8 @@ predictions with his answers. prospective evidence. - **The record.** `shadow/ceo-only.r.jsonl`, one row per gate item: the frozen item, `item_class` (`challenge2`, `challenge1`, `disagree`, `agent`, - `cap_deferred`, `cap_blocker`, `cap_delta`, `p1`), `ceo_eligible`, the + `cap_deferred`, `cap_blocker`, `cap_delta`, `finding`, `p1`), `ceo_eligible` + (false for a `finding` item, which a CEO-only run leaves to the owner), the prediction, `rule_pick` (the driver's keep-direction pick, the free baseline), `rec_hash`, the plan and prompt hashes, the model, the gstack version, the page's tokens and cost, and `audit`. It is written once per diff --git a/lib/autoplan-shadow-reader.ts b/lib/autoplan-shadow-reader.ts index 1be2e4df0..32f1934eb 100644 --- a/lib/autoplan-shadow-reader.ts +++ b/lib/autoplan-shadow-reader.ts @@ -59,7 +59,7 @@ export function exportShadow(dirs: string[]): ShadowExport { const status: ShadowStatus = s ? (s.retrospective ? 'retrospective' : s.status) : failed ? 'failed' : state.shadow?.enabled ? 'missing' : 'off'; const gateItem = gate?.items.find(i => i.id === item); const row: ShadowExportRow = { - run: state.run, gate_rev: rev, item, decision: d.id, item_class: s?.item_class ?? gateItem?.item_class ?? null, ceo_eligible: s?.ceo_eligible ?? (item === 'p1' || d.phase === 'ceo'), status, + run: state.run, gate_rev: rev, item, decision: d.id, item_class: s?.item_class ?? gateItem?.item_class ?? null, ceo_eligible: s?.ceo_eligible ?? ((s?.item_class ?? gateItem?.item_class) !== 'finding' && (item === 'p1' || d.phase === 'ceo')), status, ...(status === 'failed' ? { code: String(failed!.code) } : {}), shadow: s ? Object.fromEntries(['action', 'pick', 'confidence', 'cites', 'reason', 'malformed', 'rule_pick', 'audit', 'model', 'prompt_version', 'prompt_sha256', 'rec_hash', 'rec_rev', 'recorded_at', 'retrospective'].filter(k => k in s).map(k => [k, s[k]])) : null, owner: { chosen: d.chosen ?? null, status: d.status, owner_mode: d.owner_mode ?? null, owner_named: d.owner_named ?? null, carried: !!(d.carried_from || d.carried_gate_from), carried_from: d.carried_from ?? d.carried_gate_from ?? null, diff --git a/lib/autoplan-shadow.ts b/lib/autoplan-shadow.ts index ca57f99ba..0cf988551 100644 --- a/lib/autoplan-shadow.ts +++ b/lib/autoplan-shadow.ts @@ -362,7 +362,7 @@ function shadowRows(out: string, state: RunState, list: GateList, page: Page, pa const decision = decisionId(list.run, list.gate_rev, item.id); const base = { schema_version: 1, run: list.run, gate_rev: list.gate_rev, decision, item: item.id, item_class: item.item_class ?? null, phase: item.phase ?? null, - ceo_eligible: item.id === 'p1' || item.phase === 'ceo', item_frozen: frozen, rec_hash: recHash(list), rec_rev: list.rec_rev ?? 0, + ceo_eligible: item.item_class !== 'finding' && (item.id === 'p1' || item.phase === 'ceo'), item_frozen: frozen, rec_hash: recHash(list), rec_rev: list.rec_rev ?? 0, plan_sha256: page.plan_sha256, prompt_version: SHADOW_PROMPT_VERSION, prompt_sha256: page.prompt_sha256, request_sha256: page.request_sha256, model: replies[replies.length - 1]?.result.model ?? state.shadow?.model, gstack_version: GSTACK_VERSION, rule_pick: ruleKept(item), retrospective: false, page_usage: usage, page_usd: usdOf(replies), recorded_at: at, diff --git a/test/autoplan-shadow-driver.test.ts b/test/autoplan-shadow-driver.test.ts index fae0be880..df5985cd7 100644 --- a/test/autoplan-shadow-driver.test.ts +++ b/test/autoplan-shadow-driver.test.ts @@ -89,11 +89,17 @@ describe('submit records the shadow before GATE; the record stays private and ch }); test('one row per item with the frozen item, the rule pick, eligibility, class and hashes', () => { expect(shadowRows.map(r => [r.item, r.item_class, r.ceo_eligible, r.rule_pick, r.status, r.retrospective])).toEqual([ - ['uc1', 'challenge2', true, 'a', 'recorded', false], ['uc2', 'challenge2', false, 'a', 'recorded', false], ['p1', 'p1', true, 'a', 'recorded', false]]); + ['uc1', 'challenge2', true, 'a', 'recorded', false], ['d1', 'finding', false, null, 'recorded', false], ['d2', 'finding', false, null, 'recorded', false], + ['uc2', 'challenge2', false, 'a', 'recorded', false], ['d3', 'finding', false, null, 'recorded', false], ['d4', 'finding', false, null, 'recorded', false], ['p1', 'p1', true, 'a', 'recorded', false]]); expect(shadowRows[0].decision).toBe(`${readRun(out).run}-uc1`); expect(shadowRows[0].item_frozen.options.map((o: { key: string }) => o.key)).toEqual(['a', 'b']); expect(typeof shadowRows[0].audit).toBe('boolean'); }); + test('a finding item is never ceo_eligible and has no rule pick: a CEO-only run leaves an open Critical or High finding to the owner', () => { + const findings = shadowRows.filter(r => r.item_class === 'finding'); + expect(findings.map(r => [r.item, r.phase])).toEqual([['d1', 'ceo'], ['d2', 'ceo'], ['d3', 'eng'], ['d4', 'eng']]); + expect(findings.every(r => r.ceo_eligible === false && r.rule_pick === null)).toBe(true); + }); test('exactly one egress receipt per send, with the shadow sink and consent', () => { expect(f.receipts().map(r => [r.sink, r.consent])).toEqual([['autoplan-shadow', 'autoplan_shadow on']]); }); @@ -147,20 +153,22 @@ describe('submit records the shadow before GATE; the record stays private and ch expect(r.stderr).toContain('(SHADOW_BEFORE_ANSWER)'); }); test('"Do it" relayed as tokens records owner_mode rec, owner_named false, the rec_hash and the agent role', () => { - ok(ap(['answer', '--out', out, '--gate-rev', '1', '--reply', 'uc1a uc2a p1a', '--by', 'agent:capy', '--owner-reply', 'Do it'], e), 'answer'); + ok(ap(['answer', '--out', out, '--gate-rev', '1', '--reply', 'uc1a d1a d2a uc2a d3a d4a p1b', '--by', 'agent:capy', '--owner-reply', 'Do it'], e), 'answer'); const d = rows(path.join(out, 'decisions.jsonl')); const gateRec = shadowRows[0].rec_hash; expect(d.map(r => [r.label, r.owner_mode, r.owner_named, r.answered_role, r.rec_hash === gateRec, r.answered_by])).toEqual([ - ['UC1', 'rec', false, 'agent', true, 'agent:capy'], ['UC2', 'rec', false, 'agent', true, 'agent:capy'], ['P1', 'rec', false, 'agent', true, 'agent:capy']]); + ['UC1', 'rec', false, 'agent', true, 'agent:capy'], ['D1', 'rec', false, 'agent', true, 'agent:capy'], ['D2', 'rec', false, 'agent', true, 'agent:capy'], + ['UC2', 'rec', false, 'agent', true, 'agent:capy'], ['D3', 'rec', false, 'agent', true, 'agent:capy'], ['D4', 'rec', false, 'agent', true, 'agent:capy'], ['P1', 'rec', false, 'agent', true, 'agent:capy']]); }); test('the reader exports the answered revision with the owner provenance', () => { const r = exported(['--out', out], e); expect(r.status).toBe(0); const x = JSON.parse(r.stdout); expect(x.rows.map((row: any) => [row.item, row.status, row.shadow.pick, row.owner.chosen, row.owner.owner_mode, row.owner.rec_hash_at_answer === row.owner.rec_hash_at_record])).toEqual([ - ['uc1', 'recorded', 'a', 'a', 'rec', true], ['uc2', 'recorded', 'a', 'a', 'rec', true], ['p1', 'recorded', 'a', 'a', 'rec', true]]); - expect(x.counts.status).toEqual({ recorded: 3 }); - expect(x.counts.ceo_eligible).toEqual({ true: 2, false: 1 }); + ['uc1', 'recorded', 'a', 'a', 'rec', true], ['d1', 'recorded', 'a', 'a', 'rec', true], ['d2', 'recorded', 'a', 'a', 'rec', true], + ['uc2', 'recorded', 'a', 'a', 'rec', true], ['d3', 'recorded', 'a', 'a', 'rec', true], ['d4', 'recorded', 'a', 'a', 'rec', true], ['p1', 'recorded', 'a', 'b', 'rec', true]]); + expect(x.counts.status).toEqual({ recorded: 7 }); + expect(x.counts.ceo_eligible).toEqual({ true: 2, false: 5 }); }); }); @@ -194,7 +202,7 @@ describe('a failed shadow never blocks: the gate prints and answers; owner words const d = rows(path.join(out, 'decisions.jsonl')); expect(d.find(r => r.label === 'UC2')).toMatchObject({ owner_mode: 'unparsed', owner_named: false, owner_reply_parsed: false }); const x = JSON.parse(exported(['--out', out], e).stdout); - expect(x.rows.map((row: any) => [row.item, row.status, row.code])).toEqual([['uc1', 'failed', 'http'], ['uc2', 'failed', 'http'], ['p1', 'failed', 'http']]); + expect(x.rows.map((row: any) => [row.item, row.status, row.code])).toEqual([['uc1', 'failed', 'http'], ['d1', 'failed', 'http'], ['d2', 'failed', 'http'], ['uc2', 'failed', 'http'], ['d3', 'failed', 'http'], ['d4', 'failed', 'http'], ['p1', 'failed', 'http']]); }); }); @@ -257,7 +265,7 @@ describe('a crash mid-call: the dispatched call settles unknown and answer recor }); test('the reader labels those rows retrospective', () => { const x = JSON.parse(exported(['--out', out], e).stdout); - expect(x.counts.status).toEqual({ retrospective: 3 }); + expect(x.counts.status).toEqual({ retrospective: 7 }); }); }); @@ -287,7 +295,7 @@ describe('every gate writer records the shadow before it prints GATE', async () ok(ap(['next', '--out', out], g.env), 'next eng'); const r = ok(ap(['close', '--out', out, '--phase', 'eng', '--deferred', 'out of time', '--by', 'human'], g.env), 'close'); expect(lineIndex(r.stdout, 'SHADOW: recorded')).toBeLessThan(lineIndex(r.stdout, 'GATE: gate_rev=1')); - expect(rows(record(out)).map(x => [x.item, x.item_class, x.status])).toEqual([['uc1', 'challenge2', 'malformed'], ['d1', 'cap_deferred', 'recorded'], ['p1', 'p1', 'recorded']]); + expect(rows(record(out)).map(x => [x.item, x.item_class, x.status])).toEqual([['uc1', 'challenge2', 'malformed'], ['d1', 'finding', 'recorded'], ['d2', 'finding', 'recorded'], ['d3', 'cap_deferred', 'recorded'], ['p1', 'p1', 'recorded']]); ok(ap(['answer', '--out', out, '--gate-rev', '1', '--reply', 'rec', '--by', 'human', '--rec-hash', rows(record(out))[0].rec_hash], g.env), 'answer'); closeOut = out; }); @@ -298,7 +306,7 @@ describe('every gate writer records the shadow before it prints GATE', async () const gateLineCalls = bin.split('\n').filter(l => /gateLines\((state|at), out\)/.test(l) && !l.includes('function gateLines')); expect(gateLineCalls.length).toBe(2); expect(gateLineCalls.every(l => l.includes('line ? [line] : []'))).toBe(true); - expect(bin).toContain("const step = withRunLock(out, () => { const r = doNext(out, rest); return r.gate ? claimed(out, r) : r; });"); + expect(bin).toContain("const step = withRunLock(out, () => { const r = doNext(out, rest, price); return r.gate ? claimed(out, r) : r; });"); }); }); let closeOut = ''; @@ -367,7 +375,7 @@ describe('the switch through the driver, and the spend cap', async () => { expect(noOutside.stdout).toContain('SHADOW: off (--no-outside)'); }); test('a cap that cannot cover the call journals shadow_failed budget and the gate prints', () => { - const { out } = start('switch-cap', f.env, ['--spend-cap', '0.05']); + const { out } = start('switch-cap', f.env, ['--spend-cap', '0.05', '--estimate-usd', '0.001']); phase(out, 'ceo', f.env); const r = phase(out, 'eng', f.env); expect(r.stdout).toContain('SHADOW: failed budget'); @@ -393,10 +401,11 @@ describe('the reader over many runs', () => { expect(r.status).toBe(0); expect(r.stderr).toContain('(CEO_ONLY_NO_GATE)'); const x = JSON.parse(r.stdout); - expect(x.counts.status).toEqual({ missing: 3, failed: 3, retrospective: 3, malformed: 1, recorded: 5 }); - expect(x.rows.length).toBe(15); + expect(x.counts.status).toEqual({ missing: 7, failed: 7, retrospective: 7, malformed: 1, recorded: 11 }); + expect(x.rows.length).toBe(33); expect(x.counts.item_class.cap_deferred).toBe(1); + expect(x.rows.filter((row: any) => row.item_class === 'finding').every((row: any) => row.ceo_eligible === false)).toBe(true); const text = ap(['shadow', '--runs', list], e); - expect(text.stdout).toContain('SHADOW_EXPORT: rows=15 refused=1'); + expect(text.stdout).toContain('SHADOW_EXPORT: rows=33 refused=1'); }); }); diff --git a/test/autoplan-shadow.test.ts b/test/autoplan-shadow.test.ts index 069eda18d..5784fa857 100644 --- a/test/autoplan-shadow.test.ts +++ b/test/autoplan-shadow.test.ts @@ -117,12 +117,13 @@ describe('the reply contract: exactly one valid object per requested id', () => const out = gateRun(); const page = buildPage(readState(out), gateOf(out), decider()); const valid = page.items.map(i => ({ id: i.item.id, action: 'decide', pick: 'a', confidence: 0.6, cites: i.findings.slice(0, 1).map(f => f.id), reason: 'r' })); + const P1 = valid.findIndex(v => v.id === 'p1'); const reply = (objs: unknown[]) => '```json\n' + JSON.stringify(objs) + '\n```'; test('the page carries each item as rendered, with its findings, and p1 lists every finding the others name', () => { - expect(page.items.map(i => i.item.id)).toEqual(['uc1', 'uc2', 'p1']); + expect(page.items.map(i => i.item.id)).toEqual(['uc1', 'd1', 'd2', 'uc2', 'd3', 'd4', 'p1']); expect(page.items[0]!.frozen).toMatchObject({ item_class: 'challenge2', kind: 'approval', recommended: 'a' }); expect(page.items[0]!.findings.map(f => f.severity).every(Boolean)).toBe(true); - expect(page.items[2]!.findings.map(f => f.id).sort()).toEqual([...page.items[0]!.findings, ...page.items[1]!.findings].map(f => f.id).sort()); + expect(page.items[P1]!.findings.map(f => f.id).sort()).toEqual(page.items.filter((_, i) => i !== P1).flatMap(i => i.findings).map(f => f.id).sort()); expect(page.prompt).toContain('"id": "uc1"'); expect(page.prompt).not.toContain('{{'); }); @@ -132,7 +133,7 @@ describe('the reply contract: exactly one valid object per requested id', () => const p = buildPage({ ...readState(out), active_plan: planFile }, gateOf(out), decider()); expect(p.prompt).toContain('fill {{QUESTIONS}} and {{IDS}} in {{PLAN}}'); expect(p.prompt).toContain('"id": "uc1"'); - expect(p.prompt).toContain('uc1, uc2, p1'); + expect(p.prompt).toContain('uc1, d1, d2, uc2, d3, d4, p1'); expect(p.prompt.split('{{').length).toBe(4); }); test('a good reply is clean', () => { @@ -140,12 +141,12 @@ describe('the reply contract: exactly one valid object per requested id', () => }); const cases: Array<[string, unknown[] | string, string]> = [ ['missing', valid.slice(0, 2), 'missing'], - ['duplicate', [...valid, valid[2]], 'duplicate'], - ['unknown option', valid.map((v, i) => (i === 2 ? { ...v, pick: 'z' } : v)), 'pick "z" is not an option of p1'], - ['foreign cite', valid.map((v, i) => (i === 2 ? { ...v, cites: ['nope'] } : v)), 'cites ["nope"] name a finding outside p1\'s origin'], - ['confidence above 1', valid.map((v, i) => (i === 2 ? { ...v, confidence: 1.2 } : v)), 'confidence 1.2 is not a number from 0 to 1'], - ['bad action', valid.map((v, i) => (i === 2 ? { ...v, action: 'approve' } : v)), 'action "approve" is not decide, send_back or hold'], - ['long reason', valid.map((v, i) => (i === 2 ? { ...v, reason: 'x'.repeat(401) } : v)), 'reason is not a string of at most 400 characters'], + ['duplicate', [...valid, valid[P1]], 'duplicate'], + ['unknown option', valid.map((v, i) => (i === P1 ? { ...v, pick: 'z' } : v)), 'pick "z" is not an option of p1'], + ['foreign cite', valid.map((v, i) => (i === P1 ? { ...v, cites: ['nope'] } : v)), 'cites ["nope"] name a finding outside p1\'s origin'], + ['confidence above 1', valid.map((v, i) => (i === P1 ? { ...v, confidence: 1.2 } : v)), 'confidence 1.2 is not a number from 0 to 1'], + ['bad action', valid.map((v, i) => (i === P1 ? { ...v, action: 'approve' } : v)), 'action "approve" is not decide, send_back or hold'], + ['long reason', valid.map((v, i) => (i === P1 ? { ...v, reason: 'x'.repeat(401) } : v)), 'reason is not a string of at most 400 characters'], ['no JSON', 'I would approve.', 'no JSON array in the reply'], ]; for (const [name, objs, why] of cases) test(`${name} marks that item malformed`, () => { @@ -206,7 +207,8 @@ describe('runShadow: retry, truncation, failures and the deadline', async () => const { f, out, outcome } = await shadowed('good', [{ mode: 'valid' }]); expect(outcome.status).toBe('recorded'); expect(outcome.line).toMatch(/^SHADOW: recorded model=claude-opus-5-5 usd=0\.0140$/); - expect(ids(out).map(r => [r.item, r.status, r.pick, r.rule_pick, r.prompt_sha256 === SHADOW_PROMPT_SHA256])).toEqual([['uc1', 'recorded', 'a', 'a', true], ['uc2', 'recorded', 'a', 'a', true], ['p1', 'recorded', 'a', 'a', true]]); + expect(ids(out).map(r => [r.item, r.status, r.pick, r.rule_pick, r.prompt_sha256 === SHADOW_PROMPT_SHA256])).toEqual([['uc1', 'recorded', 'a', 'a', true], ['d1', 'recorded', 'a', null, true], ['d2', 'recorded', 'a', null, true], + ['uc2', 'recorded', 'a', 'a', true], ['d3', 'recorded', 'a', null, true], ['d4', 'recorded', 'a', null, true], ['p1', 'recorded', 'a', 'a', true]]); expect(f.requests().map(r => r.body.max_tokens)).toEqual([4000]); expect(fs.existsSync(path.join(out, 'shadow', 'r1.prompt.md'))).toBe(true); expect(fs.readFileSync(path.join(out, 'shadow', 'shadow-r1-1.reply.txt'), 'utf8')).toContain('fake prediction'); @@ -245,9 +247,9 @@ describe('runShadow: retry, truncation, failures and the deadline', async () => expect(ids(fixed.out).every(r => r.status === 'recorded')).toBe(true); const still = await shadowed('bad-twice', [{ mode: 'bad_option' }, { mode: 'bad_option' }]); expect(still.f.requests().length).toBe(2); - expect(ids(still.out).map(r => r.status)).toEqual(['malformed', 'recorded', 'recorded']); + expect(ids(still.out).map(r => r.status)).toEqual(['malformed', 'recorded', 'recorded', 'recorded', 'recorded', 'recorded', 'recorded']); expect(ids(still.out)[0]).toMatchObject({ pick: null, audit: false, malformed: 'pick "z" is not an option of uc1' }); - expect(still.outcome.line).toContain('malformed=1/3'); + expect(still.outcome.line).toContain('malformed=1/7'); }); test('a truncated reply raises max_tokens once to 8,000 (no page retry); truncated twice fails truncated', async () => { const raised = await shadowed('truncated-once', [{ mode: 'truncated' }, { mode: 'valid' }]); @@ -272,7 +274,7 @@ describe('runShadow: retry, truncation, failures and the deadline', async () => const r = await shadowed('deadline-retry', [{ mode: 'missing', delay_ms: 1200 }, { mode: 'valid', delay_ms: 1200 }], undefined, 1500); expect(r.ms).toBeLessThan(2400); expect(r.outcome.status).toBe('recorded'); - expect(ids(r.out).map(x => x.status)).toEqual(['recorded', 'recorded', 'malformed']); + expect(ids(r.out).map(x => x.status)).toEqual(['recorded', 'recorded', 'recorded', 'recorded', 'recorded', 'recorded', 'malformed']); const slow = await shadowed('deadline-call', [{ mode: 'valid', delay_ms: 3000 }], undefined, 800); expect(slow.outcome.line).toBe('SHADOW: failed timeout'); expect(slow.ms).toBeLessThan(2500);