mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-18 06:04:33 +01:00
Approval page re-reviews two-reviewer scope changes and checks quotes only against this repository (#4441)
/autoplan's approval page now re-reviews a scope change both reviewers raise, asks a plan with no Owner's words to confirm its direction, and checks owner quotes only against this repository's approvals record. This keeps the rule that only the owner's own words set a plan's direction, and there is nothing for the reader to do. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> Review-Finding: p1/red-team#1 reader=red-team Review-Finding: p1/specialist-batched#1 reader=specialist-batched Review-Finding: p1/specialist-batched#2 reader=specialist-batched
This commit is contained in:
1 parent
c53b15c48d
commit
169eb9c8a5
10 files changed
+124
-39
No files matched your search
@@ -0,0 +1 @@
|
||||
- **/autoplan's approval page closes three gaps left by the last change.** When both reviewers challenge the plan and one calls it a build-order change while the other calls it a scope change, the page now treats it as a scope change, so taking it sends the plan back for review instead of settling it without one. A plan with no Owner's words section now gets the "confirm the direction is yours" question when both reviewers challenge it and the page recommends keeping the plan, as it already did for a challenge from one reviewer. And a quote is checked only against this repository's approvals record, so another repository's record for a branch with the same name can no longer verify it.
|
||||
@@ -15,6 +15,7 @@ import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { branchKey, captainApprovalsDir } from './approval-record';
|
||||
import { readVersionedSlugCache, sanitizeSlug } from './bin-context';
|
||||
import { cutDrops, parseCuts, parseDayOne, parseOwnerWords, wordKey, type DayOneTable, type OwnerQuote, type OwnerWordsSection } from './autoplan-owner-text';
|
||||
import { SETTLEABLE_CHANGES } from './autoplan-reconcile';
|
||||
import { readDecisions, readJournal, type RunState } from './autoplan-state';
|
||||
@@ -28,19 +29,24 @@ export interface GateOwnerWords { status: OwnerWordsSection['status']; quotes: A
|
||||
export interface OwnerContext { words: OwnerWordsSection; verified: Set<number>; dayOne: DayOneTable; cuts: string[]; app: boolean; lean: boolean }
|
||||
|
||||
/**
|
||||
* The owner's words in every approvals record already on disk for the branch checked out at `cwd` (lib/approval-record.ts
|
||||
* layout: <state root>/projects/<slug>/approvals/<branch key>.jsonl and its mirrors). Read-only: the slug is not resolved,
|
||||
* because resolving it writes the slug cache, and building a gate must not write outside the run.
|
||||
* The owner's words in this repository's approvals record for the branch checked out at `cwd` (lib/approval-record.ts
|
||||
* layout: <state root>/projects/<slug>/approvals/<branch key>.jsonl and <mirror>/<slug>/<branch key>.jsonl). Read-only:
|
||||
* the slug comes from GSTACK_PROJECT_SLUG or the repository's slug cache entry gstack-slug already wrote (repository
|
||||
* scope, which ignores project markers, as the writer resolves it), never a fresh resolution (that
|
||||
* writes the cache, and building a gate must not write outside the run). No known slug reads no record, so a quote stays
|
||||
* unverified rather than matching another repository's branch of the same name.
|
||||
*/
|
||||
export function ledgerWords(cwd: string, env: NodeJS.ProcessEnv = process.env): string[] {
|
||||
const head = spawnSync('git', ['symbolic-ref', '--short', '-q', 'HEAD'], { cwd, encoding: 'utf8', timeout: 10_000 });
|
||||
const branch = head.status === 0 ? head.stdout.trim() : '';
|
||||
if (!branch) return [];
|
||||
const stateRoot = resolveStateRoot(env);
|
||||
const pinned = sanitizeSlug((env.GSTACK_PROJECT_SLUG ?? '').trim());
|
||||
const slug = pinned && pinned !== '.' && pinned !== '..' ? pinned : readVersionedSlugCache(stateRoot, cwd, { scope: 'repository' });
|
||||
if (!slug) return [];
|
||||
const name = `${branchKey(branch)}.jsonl`;
|
||||
const files: string[] = [];
|
||||
const under = (root: string, sub: string[]) => { try { for (const d of fs.readdirSync(root)) files.push(path.join(root, d, ...sub, name)); } catch { /* no such root */ } };
|
||||
under(path.join(resolveStateRoot(env), 'projects'), ['approvals']);
|
||||
for (const root of [env.GSTACK_APPROVALS_DIR, captainApprovalsDir(env)]) if (root) under(root, []);
|
||||
const files = [path.join(stateRoot, 'projects', slug, 'approvals', name),
|
||||
...[env.GSTACK_APPROVALS_DIR, captainApprovalsDir(env)].flatMap(root => (root ? [path.join(root, slug, name)] : []))];
|
||||
const words: string[] = [];
|
||||
for (const file of files) {
|
||||
let text: string;
|
||||
@@ -92,13 +98,18 @@ export function proposalCost(ctx: OwnerContext, phase: string, changes: string):
|
||||
/** A proposal behind a Critical or High row keeps the finding path's b: the draft stands and the finding becomes a blocking build task. */
|
||||
export const blockingProposal = (rows: ChallengeRow[]): boolean => rows.some(r => r.severity === 'Critical' || r.severity === 'High');
|
||||
|
||||
/** The heaviest `changes` any row names: scope, cost or direction before packaging or ordering, so a pair never settles a scope change. */
|
||||
export function heaviestChanges(rows: Array<{ changes?: string } | undefined>): string | undefined {
|
||||
const named = rows.flatMap(r => (r?.changes ? [r.changes] : []));
|
||||
return ['scope', 'cost', 'direction'].find(c => named.includes(c)) ?? named[0];
|
||||
}
|
||||
|
||||
/**
|
||||
* One proposal from one or both voices (a paired twin) against agent-written plan text: a takes the change, b keeps the draft.
|
||||
* Its `changes` is the heaviest any row names (scope, cost, direction, then packaging or ordering), so a pair never settles a scope change.
|
||||
*/
|
||||
export function proposalItem(ctx: OwnerContext, phase: string, rows: ChallengeRow[], o: { carried?: boolean } = {}): NewItem {
|
||||
const named: string[] = rows.flatMap(r => (r.changes ? [r.changes] : []));
|
||||
const changes = ['scope', 'cost', 'direction'].find(c => named.includes(c)) ?? named[0] ?? 'direction';
|
||||
const changes = heaviestChanges(rows) ?? 'direction';
|
||||
const keep = blockingProposal(rows) ? "keep the plan's draft; carry it into the build as a blocking task (no plan change)" : "keep the plan's draft";
|
||||
const fixes = [...new Set(rows.map(r => r.resolution).filter((r): r is string => !!r))];
|
||||
const voices = rows.map(r => r.voice).join(' + ');
|
||||
|
||||
@@ -27,7 +27,7 @@ import { applyCapChoices, pendingDeltas, releaseUndispatched, roundCap, roundOf
|
||||
import { evaluateCarry, gateEvidence, voidPendingCheck } from './autoplan-carry';
|
||||
import { buildEli5, capyRelayLine } from './captain-brief';
|
||||
import { FINDING_ORIGIN, openEarlier, previousReview, reviewRows, settledIds } from './autoplan-confirm';
|
||||
import { blockingProposal, confirmDirectionItem, dayOneItems, directionIsOwners, gateOwnerWords, ownerContext, ownerQuoteOf, proposalItem, type ChallengeRow, type NewItem } from './autoplan-gate-items';
|
||||
import { blockingProposal, confirmDirectionItem, dayOneItems, directionIsOwners, gateOwnerWords, heaviestChanges, ownerContext, ownerQuoteOf, proposalItem, type ChallengeRow, type NewItem } from './autoplan-gate-items';
|
||||
import { phaseClose } from './autoplan-timing';
|
||||
import { agentWords, ANSWERED_BY_RE, applyReply, autoDefault, decisionId, decisionRows, keepDirection, legacyRecHash, parseReply, pendingHint, recHash, replyShape, validateGateList, type GateChoice, type GateItem, type GateList, type GateOption, type ParsedReply } from './gate-list';
|
||||
import type { DecisionRow, FindingRow } from './headless-artifacts';
|
||||
@@ -235,7 +235,8 @@ export function buildGate(state: RunState, gateRev: number): GateList {
|
||||
const title = `${n.title} [${row.native_id} + ${row.outside_id}]`;
|
||||
if (reran && !n.changes && !o.changes) { push(agentDecision(`d${++d}`, title, phase, 'both voices recommend changing your stated direction'), ids); continue; }
|
||||
const def = challengeDefault(row, n, o);
|
||||
push({ id: `uc${++uc}`, title, kind: 'approval', recommended: def.recommended, recommended_basis: def.basis, phase, item_class: 'challenge2', ...(n.changes ?? o.changes ? { changes: n.changes ?? o.changes } : {}),
|
||||
if (!quoted && def.recommended === 'a') { kept.push(`uc${uc + 1}`); keptIds.push(n.id, o.id); }
|
||||
push({ id: `uc${++uc}`, title, kind: 'approval', recommended: def.recommended, recommended_basis: def.basis, phase, item_class: 'challenge2', ...(heaviestChanges([n, o]) ? { changes: heaviestChanges([n, o]) } : {}),
|
||||
why: `both voices recommend changing your stated direction; blind spot: ${o.title}; cost if wrong: ${o.resolution ?? n.resolution ?? 'unstated'}`,
|
||||
options: [KEEP('keep your direction (the plan as reviewed)'), { key: 'b', text: def.text, changes_plan: true }] }, ids);
|
||||
continue;
|
||||
|
||||
+6
-4
@@ -12,7 +12,7 @@ import { resolveStateRoot } from "./state-root";
|
||||
import { legacyRemoteSlug, remoteSlug } from "./remote-identity";
|
||||
|
||||
/** Keep the slug inside the [a-zA-Z0-9._-] alphabet gstack-slug promises (`tr -cd`). */
|
||||
function sanitizeSlug(s: string): string {
|
||||
export function sanitizeSlug(s: string): string {
|
||||
return s.replace(/[^a-zA-Z0-9._-]/g, "");
|
||||
}
|
||||
|
||||
@@ -386,10 +386,12 @@ function markerSlug(stateRoot: string, cwd: string, root: string): string {
|
||||
* no entry, the entry predates the versioned format, or its value is unusable
|
||||
* (empty, "." or ".." after sanitizing). Never spawns git: hot-path readers
|
||||
* (the AskUserQuestion preference hook) rely on gstack-slug, which every skill
|
||||
* preamble runs, having already written the entry.
|
||||
* preamble runs, having already written the entry. Scope "repository" skips the
|
||||
* project marker's entry, as slugFromEnvironment's does, so approvals readers and
|
||||
* writers agree on the slug.
|
||||
*/
|
||||
export function readVersionedSlugCache(stateRoot: string, cwd: string): string | undefined {
|
||||
const marker = projectMarkerRoot(cwd);
|
||||
export function readVersionedSlugCache(stateRoot: string, cwd: string, opts: { scope?: "memory" | "repository" } = {}): string | undefined {
|
||||
const marker = opts.scope === "repository" ? "" : projectMarkerRoot(cwd);
|
||||
const file = marker ? markerCacheFile(stateRoot, marker) : slugCacheReadOrder(stateRoot, cwd).find((f) => existsSync(f));
|
||||
if (!file) return undefined;
|
||||
let raw: string;
|
||||
|
||||
@@ -64,7 +64,7 @@ describe('answer', () => {
|
||||
const out = gatedRun('shape');
|
||||
const gate = JSON.parse(fs.readFileSync(path.join(out, 'gate.json'), 'utf8'));
|
||||
expect(gate.gate_rev).toBe(1);
|
||||
expect(gate.items.map((i: any) => [i.id, i.kind, i.phase ?? null])).toEqual([['d1', 'auto', 'ceo'], ['uc1', 'approval', 'ceo'], ['d2', 'auto', 'ceo'], ['uc2', 'approval', 'eng'], ['d3', 'auto', 'eng'], ['d4', 'auto', 'eng'], ['p1', 'approval', null]]);
|
||||
expect(gate.items.map((i: any) => [i.id, i.kind, i.phase ?? null])).toEqual([['d1', 'auto', 'ceo'], ['uc1', 'approval', 'ceo'], ['d2', 'auto', 'ceo'], ['uc2', 'approval', 'eng'], ['d3', 'auto', 'eng'], ['d4', 'auto', 'eng'], ['d5', 'auto', null], ['p1', 'approval', null]]);
|
||||
expect(gate.items.filter((i: any) => i.finding).map((i: any) => i.id)).toEqual(['d2', 'd3', 'd4']);
|
||||
expect(gate.items[0].why).toMatch(/voices disagree: severity: native High, outside Low/);
|
||||
}, 30_000);
|
||||
@@ -147,7 +147,7 @@ describe('answer', () => {
|
||||
expect(eng.outside.stdout).toContain('status=gate_pending');
|
||||
expect(readRun(out).gate_rev).toBe(2);
|
||||
const rows = decisions(out);
|
||||
expect(rows.filter((x: any) => x.gate_rev === 1)).toHaveLength(7);
|
||||
expect(rows.filter((x: any) => x.gate_rev === 1)).toHaveLength(8);
|
||||
expect(rows.filter((x: any) => x.gate_rev === 2 && x.status === 'pending').length).toBeGreaterThan(0);
|
||||
const staleAgain = autoplan(['answer', '--by', 'human', '--out', out, '--gate-rev', '1', '--reply', 'all']);
|
||||
expect(staleAgain.status).toBe(3);
|
||||
|
||||
@@ -20,6 +20,7 @@ import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { applyGuidance, buildGate } from '../lib/autoplan-gate';
|
||||
import { branchKey } from '../lib/approval-record';
|
||||
import { slugCacheReadOrder } from '../lib/bin-context';
|
||||
import { proposalCost, proposalItem, ownerContext, recordedOwnerWords } from '../lib/autoplan-gate-items';
|
||||
import { confirmBlock, openEarlier, settledIds } from '../lib/autoplan-confirm';
|
||||
import { buildTasks } from '../lib/autoplan-export';
|
||||
@@ -297,6 +298,35 @@ describe('owner_quote downgrade, the day-one page requirement and the cost line'
|
||||
});
|
||||
});
|
||||
|
||||
describe('two-voice challenges (known gaps after #4431)', () => {
|
||||
const pair = (o: { native: 'ordering' | 'scope'; outside: 'ordering' | 'scope'; fix?: [string, string] }) => ({
|
||||
native: [row('ceo', 'native', { id: `${RUN}-ceo-r2-native-1`, severity: 'High', title: 'Ship the before-meeting card before the live lane', user_challenge: true, changes: o.native, resolution: o.fix?.[0] ?? 'build the card first' })],
|
||||
outside: [row('ceo', 'outside', { id: `${RUN}-ceo-r2-outside-1`, severity: 'High', title: 'Ship the before-meeting card before the live lane', user_challenge: true, changes: o.outside, resolution: o.fix?.[1] ?? 'build the card first' })],
|
||||
}) as unknown as ReturnType<typeof round2>;
|
||||
|
||||
test('an ordering plus scope pair is a scope change, so taking it is re-reviewed, never settled', () => {
|
||||
const list = buildGate(makeRun('pair-scope', { plan: planText({ owner: null }), round2: pair({ native: 'ordering', outside: 'scope' }), round1: false }), 1);
|
||||
const item = byClass(list, 'challenge2')[0]!;
|
||||
expect(item.changes).toBe('scope');
|
||||
expect(applyGuidance({ changes: item.changes, item: item.id })).toBe('edit the plan, then revise');
|
||||
});
|
||||
|
||||
test("with no Owner's words section, a two-voice challenge that keeps the plan's direction asks the owner to confirm it", () => {
|
||||
const list = buildGate(makeRun('pair-keep', { plan: planText({ owner: null }), round2: pair({ native: 'scope', outside: 'scope', fix: ['build the card first', 'drop the live lane'] }), round1: false }), 1);
|
||||
const item = byClass(list, 'challenge2')[0]!;
|
||||
expect(item.recommended).toBe('a');
|
||||
const confirm = byClass(list, 'confirm_direction');
|
||||
expect(confirm.length).toBe(1);
|
||||
expect(confirm[0]!.title).toContain(item.id);
|
||||
});
|
||||
|
||||
test('a two-voice challenge that recommends the change adds no confirm item', () => {
|
||||
const list = buildGate(makeRun('pair-change', { plan: planText({ owner: null }), round2: pair({ native: 'scope', outside: 'scope' }), round1: false }), 1);
|
||||
expect(byClass(list, 'challenge2')[0]!.recommended).toBe('b');
|
||||
expect(byClass(list, 'confirm_direction')).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('parsers, render and redaction', () => {
|
||||
test("Owner's words: ok, none and malformed, with the OWNER_WORDS line", () => {
|
||||
expect(parseOwnerWords(planText())).toEqual({ status: 'ok', quotes: [{ n: 1, who: 'The owner, day 0', text: 'I want to remember what I promised people after every meeting' }] });
|
||||
@@ -349,12 +379,52 @@ describe('parsers, render and redaction', () => {
|
||||
const mirror = path.join(TMP, 'ledger-mirror', 'some-slug');
|
||||
fs.mkdirSync(mirror, { recursive: true });
|
||||
fs.writeFileSync(path.join(mirror, `${branchKey('day-one-ledger')}.jsonl`), JSON.stringify({ schema_version: 1, id: 'x', words: 'The owner said: I want to remember what I promised people after every meeting' }) + '\n');
|
||||
const env = { HOME: home, GSTACK_APPROVALS_DIR: path.dirname(mirror) };
|
||||
const env = { HOME: home, GSTACK_APPROVALS_DIR: path.dirname(mirror), GSTACK_PROJECT_SLUG: 'some-slug' };
|
||||
const words = recordedOwnerWords(state, env);
|
||||
expect(ownerContext(state, words).verified).toEqual(new Set([1]));
|
||||
expect(fs.existsSync(home)).toBe(false);
|
||||
});
|
||||
|
||||
test("another repo's approvals record on a branch of the same name never verifies a quote", () => {
|
||||
const state = makeRun('ledger-other-repo', { plan: planText(), round2: round2(), round1: false });
|
||||
expect(spawnSync('git', ['init', '-q', '-b', 'main', path.dirname(state.plan_source)], { encoding: 'utf8', timeout: 30_000 }).status).toBe(0);
|
||||
const root = path.join(TMP, 'ledger-other-mirror');
|
||||
fs.mkdirSync(path.join(root, 'other-repo'), { recursive: true });
|
||||
fs.writeFileSync(path.join(root, 'other-repo', `${branchKey('main')}.jsonl`), JSON.stringify({ schema_version: 1, id: 'y', words: 'I want to remember what I promised people after every meeting' }) + '\n');
|
||||
const env = { HOME: path.join(TMP, 'ledger-other-home'), GSTACK_APPROVALS_DIR: root, GSTACK_PROJECT_SLUG: 'this-repo' };
|
||||
expect(ownerContext(state, recordedOwnerWords(state, env)).verified).toEqual(new Set());
|
||||
expect(ownerContext(state, recordedOwnerWords(state, { ...env, GSTACK_PROJECT_SLUG: 'other-repo' })).verified).toEqual(new Set([1]));
|
||||
});
|
||||
|
||||
test('without GSTACK_PROJECT_SLUG the repository slug cache entry finds the record, even under a project marker', () => {
|
||||
const state = makeRun(path.join('ledger-marker', 'run'), { plan: planText(), round2: round2(), round1: false });
|
||||
const repo = path.dirname(state.plan_source);
|
||||
expect(spawnSync('git', ['init', '-q', '-b', 'main', repo], { encoding: 'utf8', timeout: 30_000 }).status).toBe(0);
|
||||
const stateRoot = path.join(TMP, 'ledger-marker-state');
|
||||
const cache = slugCacheReadOrder(stateRoot, repo)[0];
|
||||
fs.mkdirSync(path.dirname(cache), { recursive: true });
|
||||
fs.writeFileSync(cache, 'v2:this-repo');
|
||||
const record = path.join(stateRoot, 'projects', 'this-repo', 'approvals', `${branchKey('main')}.jsonl`);
|
||||
fs.mkdirSync(path.dirname(record), { recursive: true });
|
||||
fs.writeFileSync(record, JSON.stringify({ schema_version: 1, id: 'z', words: 'I want to remember what I promised people after every meeting' }) + '\n');
|
||||
const env = { HOME: path.join(TMP, 'ledger-marker-home'), GSTACK_STATE_ROOT: stateRoot };
|
||||
expect(ownerContext(state, recordedOwnerWords(state, env)).verified).toEqual(new Set([1]));
|
||||
fs.writeFileSync(path.join(TMP, 'ledger-marker', '.gstack-project'), '');
|
||||
expect(ownerContext(state, recordedOwnerWords(state, env)).verified).toEqual(new Set([1]));
|
||||
});
|
||||
|
||||
test('with no GSTACK_PROJECT_SLUG and no slug cache entry, no record verifies a quote', () => {
|
||||
const state = makeRun('ledger-no-slug', { plan: planText(), round2: round2(), round1: false });
|
||||
expect(spawnSync('git', ['init', '-q', '-b', 'main', path.dirname(state.plan_source)], { encoding: 'utf8', timeout: 30_000 }).status).toBe(0);
|
||||
const stateRoot = path.join(TMP, 'ledger-no-slug-state');
|
||||
const record = path.join(stateRoot, 'projects', 'other-repo', 'approvals', `${branchKey('main')}.jsonl`);
|
||||
fs.mkdirSync(path.dirname(record), { recursive: true });
|
||||
fs.writeFileSync(record, JSON.stringify({ schema_version: 1, id: 'w', words: 'I want to remember what I promised people after every meeting' }) + '\n');
|
||||
const env = { HOME: path.join(TMP, 'ledger-no-slug-home'), GSTACK_STATE_ROOT: stateRoot };
|
||||
expect(ownerContext(state, recordedOwnerWords(state, env)).verified).toEqual(new Set());
|
||||
expect(ownerContext(state, recordedOwnerWords(state, { ...env, GSTACK_PROJECT_SLUG: 'other-repo' })).verified).toEqual(new Set([1]));
|
||||
});
|
||||
|
||||
test('quoted owner words are redacted from public copies', () => {
|
||||
const state = makeRun('redact', { plan: planText(), round2: round2(), round1: false });
|
||||
const words = ownerWords(state);
|
||||
|
||||
@@ -133,7 +133,7 @@ describe('B6: PLAN_REALITY flag beside the approval item', () => {
|
||||
const gate = JSON.parse(fs.readFileSync(path.join(out, 'gate.json'), 'utf8')) as GateList;
|
||||
const p1 = gate.items.find(i => i.id === 'p1')!;
|
||||
expect(p1.flags).toEqual([expect.stringMatching(/^PLAN_REALITY "minimum effect" has no number, arm list or rule body \(plan\.md:6: .*minimum effect.*\) \(report only\)$/)]);
|
||||
expect(renderGate(gate)).toMatch(/\n flagged: PLAN_REALITY "minimum effect"/);
|
||||
expect([1, 2].map(p => renderGate(gate, p)).join('\n')).toMatch(/\n flagged: PLAN_REALITY "minimum effect"/);
|
||||
expect(readRun(out).status).toBe('gate_pending');
|
||||
});
|
||||
});
|
||||
@@ -106,7 +106,7 @@ describe('next/submit loop', () => {
|
||||
expect(v.status, v.stdout + v.stderr).toBe(0);
|
||||
expect(JSON.parse(v.stdout)).toMatchObject({ valid: true, status: 'gate_pending' });
|
||||
const gate = JSON.parse(fs.readFileSync(path.join(out, 'gate.json'), 'utf8'));
|
||||
expect(gate.items.map((i: any) => i.id)).toEqual(['uc1', 'd1', 'd2', 'uc2', 'd3', 'd4', 'p1']);
|
||||
expect(gate.items.map((i: any) => i.id)).toEqual(['uc1', 'd1', 'd2', 'uc2', 'd3', 'd4', 'd5', 'p1']);
|
||||
expect(gate.items.filter((i: any) => i.finding).map((i: any) => i.title.split(':')[0])).toEqual(['Critical', 'High', 'Critical', 'High']);
|
||||
const decisions = fs.readFileSync(path.join(out, 'decisions.jsonl'), 'utf8').trim().split('\n').map(l => JSON.parse(l));
|
||||
expect(decisions.every((d: any) => d.status === 'pending' && d.gate_rev === 1)).toBe(true);
|
||||
|
||||
@@ -90,7 +90,7 @@ describe('submit records the shadow before GATE; the record stays private and ch
|
||||
test('one row per item with the frozen item, the rule pick, eligibility, class and hashes', () => {
|
||||
expect(shadowRows.map(r => [r.item, r.item_class, r.ceo_eligible, r.rule_pick, r.status, r.retrospective])).toEqual([
|
||||
['uc1', 'challenge2', true, 'a', 'recorded', false], ['d1', 'finding', false, null, 'recorded', false], ['d2', 'finding', false, null, 'recorded', false],
|
||||
['uc2', 'challenge2', false, 'a', 'recorded', false], ['d3', 'finding', false, null, 'recorded', false], ['d4', 'finding', false, null, 'recorded', false], ['p1', 'p1', true, 'a', 'recorded', false]]);
|
||||
['uc2', 'challenge2', false, 'a', 'recorded', false], ['d3', 'finding', false, null, 'recorded', false], ['d4', 'finding', false, null, 'recorded', false], ['d5', 'confirm_direction', false, 'a', 'recorded', false], ['p1', 'p1', true, 'a', 'recorded', false]]);
|
||||
expect(shadowRows[0].decision).toBe(`${readRun(out).run}-uc1`);
|
||||
expect(shadowRows[0].item_frozen.options.map((o: { key: string }) => o.key)).toEqual(['a', 'b']);
|
||||
expect(typeof shadowRows[0].audit).toBe('boolean');
|
||||
@@ -158,12 +158,12 @@ describe('submit records the shadow before GATE; the record stays private and ch
|
||||
expect(r.stderr).toContain('(SHADOW_BEFORE_ANSWER)');
|
||||
});
|
||||
test('"Do it" relayed as tokens records owner_mode rec, owner_named false, the rec_hash and the agent role', () => {
|
||||
ok(ap(['answer', '--out', out, '--gate-rev', '1', '--reply', 'uc1a d1a d2a uc2a d3a d4a p1b', '--by', 'agent:capy', '--owner-reply', 'Do it'], e), 'answer');
|
||||
ok(ap(['answer', '--out', out, '--gate-rev', '1', '--reply', 'uc1a d1a d2a uc2a d3a d4a d5a p1b', '--by', 'agent:capy', '--owner-reply', 'Do it'], e), 'answer');
|
||||
const d = rows(path.join(out, 'decisions.jsonl'));
|
||||
const gateRec = shadowRows[0].rec_hash;
|
||||
expect(d.map(r => [r.label, r.owner_mode, r.owner_named, r.answered_role, r.rec_hash === gateRec, r.answered_by])).toEqual([
|
||||
['UC1', 'rec', false, 'agent', true, 'agent:capy'], ['D1', 'rec', false, 'agent', true, 'agent:capy'], ['D2', 'rec', false, 'agent', true, 'agent:capy'],
|
||||
['UC2', 'rec', false, 'agent', true, 'agent:capy'], ['D3', 'rec', false, 'agent', true, 'agent:capy'], ['D4', 'rec', false, 'agent', true, 'agent:capy'], ['P1', 'rec', false, 'agent', true, 'agent:capy']]);
|
||||
['UC2', 'rec', false, 'agent', true, 'agent:capy'], ['D3', 'rec', false, 'agent', true, 'agent:capy'], ['D4', 'rec', false, 'agent', true, 'agent:capy'], ['D5', 'rec', false, 'agent', true, 'agent:capy'], ['P1', 'rec', false, 'agent', true, 'agent:capy']]);
|
||||
});
|
||||
test('the reader exports the answered revision with the owner provenance', () => {
|
||||
const r = exported(['--out', out], e);
|
||||
@@ -171,9 +171,9 @@ describe('submit records the shadow before GATE; the record stays private and ch
|
||||
const x = JSON.parse(r.stdout);
|
||||
expect(x.rows.map((row: any) => [row.item, row.status, row.shadow.pick, row.owner.chosen, row.owner.owner_mode, row.owner.rec_hash_at_answer === row.owner.rec_hash_at_record])).toEqual([
|
||||
['uc1', 'recorded', 'a', 'a', 'rec', true], ['d1', 'recorded', 'a', 'a', 'rec', true], ['d2', 'recorded', 'a', 'a', 'rec', true],
|
||||
['uc2', 'recorded', 'a', 'a', 'rec', true], ['d3', 'recorded', 'a', 'a', 'rec', true], ['d4', 'recorded', 'a', 'a', 'rec', true], ['p1', 'recorded', 'a', 'b', 'rec', true]]);
|
||||
expect(x.counts.status).toEqual({ recorded: 7 });
|
||||
expect(x.counts.ceo_eligible).toEqual({ true: 2, false: 5 });
|
||||
['uc2', 'recorded', 'a', 'a', 'rec', true], ['d3', 'recorded', 'a', 'a', 'rec', true], ['d4', 'recorded', 'a', 'a', 'rec', true], ['d5', 'recorded', 'a', 'a', 'rec', true], ['p1', 'recorded', 'a', 'b', 'rec', true]]);
|
||||
expect(x.counts.status).toEqual({ recorded: 8 });
|
||||
expect(x.counts.ceo_eligible).toEqual({ true: 2, false: 6 });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -207,7 +207,7 @@ describe('a failed shadow never blocks: the gate prints and answers; owner words
|
||||
const d = rows(path.join(out, 'decisions.jsonl'));
|
||||
expect(d.find(r => r.label === 'UC2')).toMatchObject({ owner_mode: 'unparsed', owner_named: false, owner_reply_parsed: false });
|
||||
const x = JSON.parse(exported(['--out', out], e).stdout);
|
||||
expect(x.rows.map((row: any) => [row.item, row.status, row.code])).toEqual([['uc1', 'failed', 'http'], ['d1', 'failed', 'http'], ['d2', 'failed', 'http'], ['uc2', 'failed', 'http'], ['d3', 'failed', 'http'], ['d4', 'failed', 'http'], ['p1', 'failed', 'http']]);
|
||||
expect(x.rows.map((row: any) => [row.item, row.status, row.code])).toEqual([['uc1', 'failed', 'http'], ['d1', 'failed', 'http'], ['d2', 'failed', 'http'], ['uc2', 'failed', 'http'], ['d3', 'failed', 'http'], ['d4', 'failed', 'http'], ['d5', 'failed', 'http'], ['p1', 'failed', 'http']]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -270,7 +270,7 @@ describe('a crash mid-call: the dispatched call settles unknown and answer recor
|
||||
});
|
||||
test('the reader labels those rows retrospective', () => {
|
||||
const x = JSON.parse(exported(['--out', out], e).stdout);
|
||||
expect(x.counts.status).toEqual({ retrospective: 7 });
|
||||
expect(x.counts.status).toEqual({ retrospective: 8 });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -300,7 +300,7 @@ describe('every gate writer records the shadow before it prints GATE', async ()
|
||||
ok(ap(['next', '--out', out], g.env), 'next eng');
|
||||
const r = ok(ap(['close', '--out', out, '--phase', 'eng', '--deferred', 'out of time', '--by', 'human'], g.env), 'close');
|
||||
expect(lineIndex(r.stdout, 'SHADOW: recorded')).toBeLessThan(lineIndex(r.stdout, 'GATE: gate_rev=1'));
|
||||
expect(rows(record(out)).map(x => [x.item, x.item_class, x.status])).toEqual([['uc1', 'challenge2', 'malformed'], ['d1', 'finding', 'recorded'], ['d2', 'finding', 'recorded'], ['d3', 'cap_deferred', 'recorded'], ['p1', 'p1', 'recorded']]);
|
||||
expect(rows(record(out)).map(x => [x.item, x.item_class, x.status])).toEqual([['uc1', 'challenge2', 'malformed'], ['d1', 'finding', 'recorded'], ['d2', 'finding', 'recorded'], ['d3', 'cap_deferred', 'recorded'], ['d4', 'confirm_direction', 'recorded'], ['p1', 'p1', 'recorded']]);
|
||||
ok(ap(['answer', '--out', out, '--gate-rev', '1', '--reply', 'rec', '--by', 'human', '--rec-hash', rows(record(out))[0].rec_hash], g.env), 'answer');
|
||||
closeOut = out;
|
||||
});
|
||||
@@ -399,7 +399,7 @@ describe('the reader over many runs', () => {
|
||||
fs.writeFileSync(list, [path.join(TMP, 'fail', 'run'), copy, copy].join('\n') + '\n');
|
||||
const r = exported(['--runs', list], baseEnv({ GSTACK_CAPY: '' }));
|
||||
expect(r.status).toBe(0);
|
||||
expect(JSON.parse(r.stdout).counts.status).toEqual({ failed: 14 });
|
||||
expect(JSON.parse(r.stdout).counts.status).toEqual({ failed: 16 });
|
||||
});
|
||||
test('missing, failed, malformed, retrospective and recorded rows; dedupe across a repeated entry; CEO_ONLY_NO_GATE', () => {
|
||||
const e = baseEnv({ GSTACK_CAPY: '' });
|
||||
@@ -416,11 +416,11 @@ describe('the reader over many runs', () => {
|
||||
expect(r.status).toBe(0);
|
||||
expect(r.stderr).toContain('(CEO_ONLY_NO_GATE)');
|
||||
const x = JSON.parse(r.stdout);
|
||||
expect(x.counts.status).toEqual({ missing: 7, failed: 7, retrospective: 7, malformed: 1, recorded: 11 });
|
||||
expect(x.rows.length).toBe(33);
|
||||
expect(x.counts.status).toEqual({ missing: 8, failed: 8, retrospective: 8, malformed: 1, recorded: 13 });
|
||||
expect(x.rows.length).toBe(38);
|
||||
expect(x.counts.item_class.cap_deferred).toBe(1);
|
||||
expect(x.rows.filter((row: any) => row.item_class === 'finding').every((row: any) => row.ceo_eligible === false)).toBe(true);
|
||||
const text = ap(['shadow', '--runs', list], e);
|
||||
expect(text.stdout).toContain('SHADOW_EXPORT: rows=33 refused=1');
|
||||
expect(text.stdout).toContain('SHADOW_EXPORT: rows=38 refused=1');
|
||||
});
|
||||
});
|
||||
@@ -120,7 +120,7 @@ describe('the reply contract: exactly one valid object per requested id', () =>
|
||||
const P1 = valid.findIndex(v => v.id === 'p1');
|
||||
const reply = (objs: unknown[]) => '```json\n' + JSON.stringify(objs) + '\n```';
|
||||
test('the page carries each item as rendered, with its findings, and p1 lists every finding the others name', () => {
|
||||
expect(page.items.map(i => i.item.id)).toEqual(['uc1', 'd1', 'd2', 'uc2', 'd3', 'd4', 'p1']);
|
||||
expect(page.items.map(i => i.item.id)).toEqual(['uc1', 'd1', 'd2', 'uc2', 'd3', 'd4', 'd5', 'p1']);
|
||||
expect(page.items[0]!.frozen).toMatchObject({ item_class: 'challenge2', kind: 'approval', recommended: 'a' });
|
||||
expect(page.items[0]!.findings.map(f => f.severity).every(Boolean)).toBe(true);
|
||||
expect(page.items[P1]!.findings.map(f => f.id).sort()).toEqual(page.items.filter((_, i) => i !== P1).flatMap(i => i.findings).map(f => f.id).sort());
|
||||
@@ -133,7 +133,7 @@ describe('the reply contract: exactly one valid object per requested id', () =>
|
||||
const p = buildPage({ ...readState(out), active_plan: planFile }, gateOf(out), decider());
|
||||
expect(p.prompt).toContain('fill {{QUESTIONS}} and {{IDS}} in {{PLAN}}');
|
||||
expect(p.prompt).toContain('"id": "uc1"');
|
||||
expect(p.prompt).toContain('uc1, d1, d2, uc2, d3, d4, p1');
|
||||
expect(p.prompt).toContain('uc1, d1, d2, uc2, d3, d4, d5, p1');
|
||||
expect(p.prompt.split('{{').length).toBe(4);
|
||||
});
|
||||
test('a good reply is clean', () => {
|
||||
@@ -208,7 +208,7 @@ describe('runShadow: retry, truncation, failures and the deadline', async () =>
|
||||
expect(outcome.status).toBe('recorded');
|
||||
expect(outcome.line).toMatch(/^SHADOW: recorded model=claude-opus-5-5 usd=0\.0140$/);
|
||||
expect(ids(out).map(r => [r.item, r.status, r.pick, r.rule_pick, r.prompt_sha256 === SHADOW_PROMPT_SHA256])).toEqual([['uc1', 'recorded', 'a', 'a', true], ['d1', 'recorded', 'a', null, true], ['d2', 'recorded', 'a', null, true],
|
||||
['uc2', 'recorded', 'a', 'a', true], ['d3', 'recorded', 'a', null, true], ['d4', 'recorded', 'a', null, true], ['p1', 'recorded', 'a', 'a', true]]);
|
||||
['uc2', 'recorded', 'a', 'a', true], ['d3', 'recorded', 'a', null, true], ['d4', 'recorded', 'a', null, true], ['d5', 'recorded', 'a', 'a', true], ['p1', 'recorded', 'a', 'a', true]]);
|
||||
expect(f.requests().map(r => r.body.max_tokens)).toEqual([4000]);
|
||||
expect(fs.existsSync(path.join(out, 'shadow', 'r1.prompt.md'))).toBe(true);
|
||||
expect(fs.readFileSync(path.join(out, 'shadow', 'shadow-r1-1.reply.txt'), 'utf8')).toContain('fake prediction');
|
||||
@@ -247,9 +247,9 @@ describe('runShadow: retry, truncation, failures and the deadline', async () =>
|
||||
expect(ids(fixed.out).every(r => r.status === 'recorded')).toBe(true);
|
||||
const still = await shadowed('bad-twice', [{ mode: 'bad_option' }, { mode: 'bad_option' }]);
|
||||
expect(still.f.requests().length).toBe(2);
|
||||
expect(ids(still.out).map(r => r.status)).toEqual(['malformed', 'recorded', 'recorded', 'recorded', 'recorded', 'recorded', 'recorded']);
|
||||
expect(ids(still.out).map(r => r.status)).toEqual(['malformed', 'recorded', 'recorded', 'recorded', 'recorded', 'recorded', 'recorded', 'recorded']);
|
||||
expect(ids(still.out)[0]).toMatchObject({ pick: null, audit: false, malformed: 'pick "z" is not an option of uc1' });
|
||||
expect(still.outcome.line).toContain('malformed=1/7');
|
||||
expect(still.outcome.line).toContain('malformed=1/8');
|
||||
});
|
||||
test('a truncated reply raises max_tokens once to 8,000 (no page retry); truncated twice fails truncated', async () => {
|
||||
const raised = await shadowed('truncated-once', [{ mode: 'truncated' }, { mode: 'valid' }]);
|
||||
@@ -274,7 +274,7 @@ describe('runShadow: retry, truncation, failures and the deadline', async () =>
|
||||
const r = await shadowed('deadline-retry', [{ mode: 'missing', delay_ms: 2500 }, { mode: 'valid', delay_ms: 3400 }], undefined, 3500);
|
||||
expect(r.ms).toBeLessThan(5900);
|
||||
expect(r.outcome.status).toBe('recorded');
|
||||
expect(ids(r.out).map(x => x.status)).toEqual(['recorded', 'recorded', 'recorded', 'recorded', 'recorded', 'recorded', 'malformed']);
|
||||
expect(ids(r.out).map(x => x.status)).toEqual(['recorded', 'recorded', 'recorded', 'recorded', 'recorded', 'recorded', 'recorded', 'malformed']);
|
||||
const slow = await shadowed('deadline-call', [{ mode: 'valid', delay_ms: 6000 }], undefined, 800);
|
||||
expect(slow.outcome.line).toBe('SHADOW: failed timeout');
|
||||
expect(slow.ms).toBeLessThan(5000);
|
||||
|
||||
Reference in new issue
Block a user