mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-03 23:37:53 +01:00
* feat(loadouts): Opus 5.5 lite loadout with explicit model detection (GSTA-161)
modules/loadouts.json gains the lite loadout (eleven coaching preamble
pieces dropped; upgrade-check and brain-sync kept) and a models map that
sends only claude-opus-5-5 (and its dated snapshots) to it. Detection is
explicit: Capy passes --model to the installer, Claude Code opts in with
./setup --loadout lite; anything unmapped, invalid or unreadable serves full.
The checkout always stays the full render.
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* docs(loadouts): Opus 5.5 lite, detection order, Capy SKILL_ROOT, setup --loadout; troubleshooting anchors; changelog fragment (GSTA-161)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* fix(loadouts): claude_loadout read through CONFIG_BIN; capy skill context ceiling for the --model line (GSTA-161)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* fix: pre-landing review fixes (prune keeps a reused loadout folder, GSTACK_LOADOUT_SKILL_ROOT, claude_loadout guard test) (GSTA-161)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* fix: pre-landing review fixes (Capy skill passes the model as GSTACK_MODEL so an older installer ignores it) (GSTA-161)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* fix: pre-landing review fixes (prune only owned, unused folders; full root from the canonical Claude install; fallback reason and source) (GSTA-161)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* docs: loadout source example, install and resolve commands, Capy launch-file note (GSTA-161)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* feat(autoplan): --ceo-only runs the CEO review alone with zero questions (GST-537)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* fix: pre-landing review fixes
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* test: classify autoplan-ceo-only in the Codex host metadata
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* fix: select autoplan-ceo-only on the skills it runs; say it needs Claude Code
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* test: run autoplan-ceo-only in the marathon lane
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* docs: changelog entry for /autoplan --ceo-only
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* test: count the autoplan-ceo-only paid file in the shard inventory
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* feat: gstack-autoplan next --ceo-only runs the CEO review alone and ends reviewed (GST-537)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* fix: budget and close recover a budget-stopped CEO-only run; reviewed is at rest for checkpoints
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* fix: CEO-only finish leaves owner-only cap items undecided, refuses marker phase flags and a plan at the report path, and replaces ceo-review.md atomically
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* fix: the CEO-only report-path guard compares real paths and refuses at the finish when the report file is the plan
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* docs: CEO_ONLY_CONFLICT covers a plan linked to the report path
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* test(browse): failing-first device and touch tests (GSTA-227 F3, #3058)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* fix: the report-path guard keeps the lexical check, skips safe hard links and compares bigint file ids
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* browse: per-session tabs for parallel agents; tab pins stop moving the shared active tab (#1961)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* browse: keep the server.ts tab-pin anchor comment for source guards
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* feat(browse): per-tab phone emulation and touch input ($B device, tap, swipe, pinch, touch; #3058, GSTA-227 F3)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* test: copy lib/result-codes-autoplan.ts into the fixtures that copy the result-code tables, and keep it out of derived closures
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* docs(qa, aside): touch pointer in /qa, Aside touch recipe verified on Aside 1.26, report Phones row (GSTA-227 F3, #3058)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* browse: drop the sessions paragraph from /browse to stay inside the parity cap
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* changelog: F3 phone emulation and touch entry
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* browse: hide the ps window in the session harness walk
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* docs: the decisions.jsonl status list names auto_kept
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* test(browse): session cap, LRU eviction, headed release and scoped auto-session rows
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* browse: sessions keep tab <id> picks, sidebar claude stays on the user's tab, pinned rebuild and state load rebind the scope
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* fix: pre-landing review fixes (touch cancel with no finger held is a no-op)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* fix: Step 11 review fixes (headed tab adoption, pinned viewport reset, CLI retry keeps --session, rebuild id remap, --json notice, evicted-session reset, device kept after annotated captures, render opts out of sessions, session: client ids reserved)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* fix: headed viewport auto keeps the user's active tab; render's closetab opts out of sessions
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* docs: sessions, phone emulation and touch in BROWSER.md; session: client ids reserved; TODOS #1961/#3058 shipped
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* test: copy lib/result-codes-browse.ts into the fixtures that copy the result-code modules
lib/result-codes.ts imports ./result-codes-browse, so fixtures with an explicit
lib list failed with 'Cannot find module'. pr-dependencies lists it with the other
reason-code modules as selection data. GSTA-227
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* feat: gstack-clean reclaims per-repo .gstack scratch and logs, dry run by default (#2627)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* browse: name the Aside touch recipe's coordinates <tap-x>/<tap-y> with a numeric grammar
The generated-bash lint (INV-3) rejected the free-text <x>/<y> placeholders. GSTA-227
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* fix(clean): leave hard-linked and unreadable logs alone; cut through a no-follow descriptor checked against the evaluated inode
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* test(clean): tests that expect tmp removal see a quiet ps, so a codex elsewhere on the host cannot flake them
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* fix(clean): an unlistable tmp/ skips its group; liveness asked once per group on apply; BROWSER.md says which logs reset at daemon start
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* fix(clean): write output synchronously so --json reaches a pipe whole; test fixes for root, wrapper git and the ps stub
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* fix(clean): every change rechecks the verified folder and the measured file; full writes; EPIPE ends output quietly
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* test(clean): the early-close test checks the bin's own exit status
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* docs: list gstack-clean among the standalone CLIs
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* test(clean): copied-install fixtures and the dependency list carry lib/result-codes-clean.ts
The remote pregate failed 11 cases in gstack-doctor, plan-tune-cathedral and
timeline-stop-hook-slug: each copies an explicit list of lib/result-codes-*.ts
files into a temp install, and lib/result-codes.ts now imports
./result-codes-clean. Same lists GSTA-168 extended for result-codes-eval.
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* chore: regenerate gstack-doctor-components.sh for the gstack-clean result codes
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* qa: fold the touch pointer into the Phases sentence to fit the parity size cap after the main merge
qa's union was 95988 bytes (ratio 1.142 > maxSizeRatio 1.142, train 75). Now 95975 (1.1419). GSTA-227
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* docs: #3058 troubleshooting 'Fixed in' lines name v1.92.10.0 after the main merge
GSTA-227
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* test(clean): classify gstack-tasks rows files (tasks-*) as kept, after main added the producer
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* docs: #3058 troubleshooting 'Fixed in' lines name v1.92.20.0 after the main merge
GSTA-227
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* fix: pre-landing review fixes (restore the lib/result-codes-plan.ts ratchet row the base merge dropped)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* test(clean): the producer scan ends a name at a colon, so prose after a path names the file (train 78)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* docs: #3058 troubleshooting 'Fixed in' lines name v1.92.30.0 after the main merge
GSTA-227
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* outside-voice-api: caller receipts, stop_reason, OpenAI max_completion_tokens
A caller can write its own egress receipt (sink, consent, payload class);
the default stays the outside-voice receipt, so existing callers send and
record the same bytes. The result carries the provider's stop reason, and
an OpenAI request gets max_completion_tokens only when the caller sets an
output limit (GST-537 PR 3).
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* autoplan gate: item classes and the owner's answer provenance
Gate items carry item_class, ruleKept() names the keep-direction pick, and
answered decision rows record owner_mode, owner_named, rec_hash and
answered_role parsed from the owner's own words (GST-537 PR 3).
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* autoplan: shadow-mode decider module, frozen prompt and spend accounting
The switch (off by default, fixed at init), the lease, one model call per
gate with a page retry and a truncation retry under one deadline, the
once-per-revision record with its journaled sha256, the 10% audit mark,
and shadow-labeled ledger rows that count against the cap, show as
shadow= on GATE_SPEND and stay out of the per-voice price (GST-537 PR 3).
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* gstack-autoplan: shadow every published gate, the shadow export, config and docs
Every driver command that prints a gate records the shadow first and
checkpoints after it; answer waits for a live lease and records a dead
one retrospectively; gstack-autoplan shadow exports answered revisions
for taste-replay R2 (SHADOW_BEFORE_ANSWER otherwise); gstack-config
autoplan_shadow / autoplan_shadow_model; gstack-artifact validate checks
the record's hash (GST-537 PR 3).
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* changelog: shadow mode for the CEO-only decider (GST-537 PR 3)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* autoplan shadow: /ship review fixes
- register test/fixtures/shadow-fake-model.ts in FREE_FIXTURES (free-fixtures inventory failed)
- withRunLock: count dead-owner reclaims apart from waitMs iterations, so a long wait can still reclaim
- buildPage: fill the decider template in one pass, so slot names inside the plan stay plan text
- revise regate: the shared publish step prints the GATE and ESCALATE lines once
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* autoplan shadow: reply files never follow a link; the retry reserves against the current cap
- a model reply is written by temp file and rename, so a link at the reply path is replaced, not written through
- each shadow reservation reads the run's current spend cap, so a budget change during the call is not undone
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* outside-voice-api: write the egress receipt under the caller's env
callModel resolved the receipt's state root from process.env even when the caller passed env, so the shadow
tests wrote to the real ~/.gstack (the free-suite home-containment guard caught it). The receipt now uses
req.env; the shadow tests give the fake model a private HOME, GSTACK_HOME and GSTACK_STATE_ROOT, and the receipt
test no longer sets process.env to pass.
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* fix(autoplan): CEO-only runs leave main's open Critical/High finding items to the owner (no decision row)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* fix(autoplan): shadow pages carry main's finding items; a finding item is never ceo_eligible
After merging #3767 (main 90a170c), open Critical/High findings are gate items
(item_class finding). The shadow records one row per gate item, so its pages now
include them. A CEO-only run leaves those items to the owner, so their rows are
ceo_eligible false (rule_pick was already null). The shadow fixtures follow
main's gate (items, counts, "Do it" tokens); the two merge adapters follow
main's doNext(out, rest, price) and its per-voice reservation (--estimate-usd).
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* docs: #3058 troubleshooting 'Fixed in' lines name v1.92.61.0 after the main merge
GSTA-227
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* fix(investigate): drop the five-phase step procedure (GST-938)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* changelog: GST-938 fragment wording
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* fix(investigate): keep the web-search sanitize guard; update BROWSER.md and docs/skills.md (review pass 1)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* changelog: GST-938 wording
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* test(investigate): pin the web-search guard; changelog size (adversarial review)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* test(clean): classify restamp-staging.json, the restamp producer main gained after #3779's base (wave browse)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* fix: pre-landing review fixes (wave browse): scoped newtab keeps a concurrent tab switch; shadow validate catches a deleted record; a failed shadow publish is journaled
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* fix: a failed shadow publish journals under the run lock and never throws (wave browse review pass 2)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* docs(unattended): shadow publish failures and the missing-record validate check (wave browse docs audit)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* changelog: wave browse integrator fixes fragment
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* fix: shadow reader dedupes by run folder, so two runs started in the same second both count (wave browse CI free-suite 16)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* chore: regenerate doctor result anchors in generator order (wave browse rebuild without #3818)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* changelog: name the wave fragment after the rebuilt wave
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
---------
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
334 lines
22 KiB
TypeScript
334 lines
22 KiB
TypeScript
/**
|
||
* GST-537 PR 3, shadow mode for the CEO-only decider (lib/autoplan-shadow.ts):
|
||
* the frozen decider prompt, the switch, the reply contract, the 10% audit
|
||
* mark, the call (truncation, the OpenAI output bound, the 90 s deadline, the
|
||
* shadow's own egress receipt), and the gate-cost exclusion of shadow spend.
|
||
* Model calls go to test/fixtures/shadow-fake-model.ts through
|
||
* ANTHROPIC_BASE_URL and OPENAI_BASE_URL: never a real network call.
|
||
*/
|
||
import { afterAll, describe, expect, test } from 'bun:test';
|
||
import * as crypto from 'node:crypto';
|
||
import * as fs from 'node:fs';
|
||
import * as os from 'node:os';
|
||
import * as path from 'node:path';
|
||
import { SHADOW_PROMPT_SHA256, SHADOW_PROMPT_VERSION, auditMark, buildPage, claimShadow, parsePage, recordFile, runShadow, shadowPromptFile, shadowSwitch } from '../lib/autoplan-shadow';
|
||
import { callModel } from '../lib/outside-voice-api';
|
||
import { gateCost, renderGateCost } from '../lib/autoplan-gate-cost';
|
||
import { readJournal, readState } from '../lib/autoplan-state';
|
||
import { reserve, settle, readLedger } from '../lib/spend-ledger';
|
||
import type { GateList } from '../lib/gate-list';
|
||
import { ROOT, readRun, runPhase, startSmallRun } from './helpers/autoplan-run-fixture';
|
||
|
||
const TMP = fs.mkdtempSync(path.join(os.tmpdir(), 'ap-shadow-'));
|
||
const FAKE = path.join(ROOT, 'test', 'fixtures', 'shadow-fake-model.ts');
|
||
const servers: Array<{ kill: () => void }> = [];
|
||
afterAll(() => { for (const s of servers) s.kill(); fs.rmSync(TMP, { recursive: true, force: true }); });
|
||
const rows = (file: string) => (fs.existsSync(file) ? fs.readFileSync(file, 'utf8').split('\n').filter(Boolean).map(l => JSON.parse(l)) : []);
|
||
|
||
/** A fake model server in its own process, answering step n of its script for the n-th request. */
|
||
async function fake(name: string, script: Array<Record<string, unknown>>) {
|
||
const dir = path.join(TMP, `fake-${name}`);
|
||
fs.mkdirSync(dir, { recursive: true });
|
||
fs.writeFileSync(path.join(dir, 'script.json'), JSON.stringify(script));
|
||
const proc = Bun.spawn(['bun', FAKE, dir], { stdout: 'ignore', stderr: 'inherit' });
|
||
servers.push(proc);
|
||
for (let i = 0; i < 200 && !fs.existsSync(path.join(dir, 'port')); i++) await Bun.sleep(25);
|
||
const port = Number(fs.readFileSync(path.join(dir, 'port'), 'utf8'));
|
||
const state = path.join(dir, 'state');
|
||
fs.mkdirSync(state, { recursive: true });
|
||
const env = { HOME: state, GSTACK_HOME: state, GSTACK_STATE_ROOT: state, ANTHROPIC_API_KEY: 'sk-fake-anthropic', OPENAI_API_KEY: 'sk-fake-openai', ANTHROPIC_BASE_URL: `http://127.0.0.1:${port}`, OPENAI_BASE_URL: `http://127.0.0.1:${port}/v1` };
|
||
return { dir, env, requests: () => rows(path.join(dir, 'requests.jsonl')), receipts: () => rows(path.join(state, 'security', 'egress.jsonl')) };
|
||
}
|
||
|
||
// One run at its gate (uc1 from CEO, uc2 from Eng, p1), shadow off; each case copies it and turns shadow on in run.json.
|
||
const BASE = startSmallRun(path.join(TMP, 'base'));
|
||
runPhase(BASE, 'ceo');
|
||
runPhase(BASE, 'eng');
|
||
let copies = 0;
|
||
function gateRun(model = 'claude-opus-5-5'): string {
|
||
const out = path.join(TMP, `run-${++copies}`);
|
||
fs.cpSync(BASE, out, { recursive: true });
|
||
const run = readRun(out);
|
||
run.shadow = { enabled: true, model, source: 'env' };
|
||
fs.writeFileSync(path.join(out, 'run.json'), JSON.stringify(run, null, 2) + '\n');
|
||
return out;
|
||
}
|
||
const gateOf = (out: string) => JSON.parse(fs.readFileSync(path.join(out, 'gate.json'), 'utf8')) as GateList;
|
||
const decider = () => fs.readFileSync(shadowPromptFile(), 'utf8');
|
||
|
||
describe('the decider prompt is frozen and predicts; it imports no mandate', () => {
|
||
test('its sha256 is pinned beside its version, and every row stores it', () => {
|
||
expect(SHADOW_PROMPT_VERSION).toBe(1);
|
||
expect(crypto.createHash('sha256').update(fs.readFileSync(shadowPromptFile())).digest('hex')).toBe(SHADOW_PROMPT_SHA256);
|
||
});
|
||
test('no take-the-recommendation or keep-the-direction rule; the recommendation is a labeled input', () => {
|
||
const text = decider();
|
||
expect(text).not.toMatch(/\(recommended\)/i);
|
||
expect(text).not.toMatch(/keep (the|your|its) (plan's )?direction/i);
|
||
expect(text).not.toMatch(/sections\/ceo-only\.md/);
|
||
expect(text).toContain("The reviewers' recommendation is one input among several");
|
||
for (const slot of ['{{PLAN}}', '{{QUESTIONS}}', '{{IDS}}']) expect(text.split(slot).length).toBe(2);
|
||
expect(text.trimEnd().endsWith('```')).toBe(true);
|
||
});
|
||
test('the CEO-only driver module does not name the CEO-only skill section', () => {
|
||
expect(fs.readFileSync(path.join(ROOT, 'lib', 'autoplan-ceo-only.ts'), 'utf8')).not.toContain('sections/ceo-only.md');
|
||
});
|
||
});
|
||
|
||
describe('the switch is off by default and fixed at init', () => {
|
||
const state = fs.mkdtempSync(path.join(TMP, 'cfg-'));
|
||
const keys = { GSTACK_STATE_ROOT: state, ANTHROPIC_API_KEY: 'sk-fake', OPENAI_API_KEY: '' };
|
||
const on = { outsideEnabled: true, ceoOnly: false };
|
||
test('off by default, with no line', () => {
|
||
expect(shadowSwitch(on, keys)).toEqual({ shadow: { enabled: false, reason: 'off by default', source: 'default' } });
|
||
});
|
||
test('config on turns it on with the default model; the env var wins over config', () => {
|
||
fs.writeFileSync(path.join(state, 'config.yaml'), 'autoplan_shadow: on\n');
|
||
const r = shadowSwitch(on, keys);
|
||
expect(r.shadow).toEqual({ enabled: true, model: 'claude-opus-5-5', source: 'config' });
|
||
expect(r.line).toStartWith('SHADOW: on model=claude-opus-5-5');
|
||
expect(shadowSwitch(on, { ...keys, GSTACK_AUTOPLAN_SHADOW: '0' }).shadow).toMatchObject({ enabled: false, source: 'env' });
|
||
fs.rmSync(path.join(state, 'config.yaml'));
|
||
expect(shadowSwitch(on, { ...keys, GSTACK_AUTOPLAN_SHADOW: '1' }).shadow).toMatchObject({ enabled: true, source: 'env' });
|
||
});
|
||
test('no key, --no-outside, a CEO-only run, an unpriced model or a bad value give off with the reason', () => {
|
||
const env = { ...keys, GSTACK_AUTOPLAN_SHADOW: 'on' };
|
||
expect(shadowSwitch(on, { ...env, ANTHROPIC_API_KEY: '' }).line).toBe('SHADOW: off (no API key for claude-opus-5-5 (ANTHROPIC_API_KEY))');
|
||
expect(shadowSwitch({ ...on, outsideEnabled: false }, env).line).toBe('SHADOW: off (--no-outside)');
|
||
expect(shadowSwitch({ ...on, ceoOnly: true }, env).line).toBe('SHADOW: off (a CEO-only run writes no gate)');
|
||
fs.writeFileSync(path.join(state, 'config.yaml'), 'autoplan_shadow_model: claude-unpriced-9\n');
|
||
expect(shadowSwitch(on, env).line).toBe('SHADOW: off (claude-unpriced-9 has no price in lib/pricing.ts, so --spend-cap cannot bound it)');
|
||
fs.rmSync(path.join(state, 'config.yaml'));
|
||
expect(shadowSwitch(on, { ...keys, GSTACK_AUTOPLAN_SHADOW: 'yes' }).line).toBe('SHADOW: off (GSTACK_AUTOPLAN_SHADOW=yes is not on or off)');
|
||
});
|
||
test('a run.json without the field reads off: nothing is claimed', () => {
|
||
const out = path.join(TMP, 'no-field');
|
||
fs.cpSync(BASE, out, { recursive: true });
|
||
const run = readRun(out);
|
||
expect(run.shadow).toEqual({ enabled: false, reason: 'off by default', source: 'default' });
|
||
delete run.shadow;
|
||
fs.writeFileSync(path.join(out, 'run.json'), JSON.stringify(run, null, 2) + '\n');
|
||
expect(claimShadow(out)).toEqual({});
|
||
expect(fs.existsSync(path.join(out, 'shadow'))).toBe(false);
|
||
});
|
||
});
|
||
|
||
describe('the reply contract: exactly one valid object per requested id', () => {
|
||
const out = gateRun();
|
||
const page = buildPage(readState(out), gateOf(out), decider());
|
||
const valid = page.items.map(i => ({ id: i.item.id, action: 'decide', pick: 'a', confidence: 0.6, cites: i.findings.slice(0, 1).map(f => f.id), reason: 'r' }));
|
||
const P1 = valid.findIndex(v => v.id === 'p1');
|
||
const reply = (objs: unknown[]) => '```json\n' + JSON.stringify(objs) + '\n```';
|
||
test('the page carries each item as rendered, with its findings, and p1 lists every finding the others name', () => {
|
||
expect(page.items.map(i => i.item.id)).toEqual(['uc1', 'd1', 'd2', 'uc2', 'd3', 'd4', 'p1']);
|
||
expect(page.items[0]!.frozen).toMatchObject({ item_class: 'challenge2', kind: 'approval', recommended: 'a' });
|
||
expect(page.items[0]!.findings.map(f => f.severity).every(Boolean)).toBe(true);
|
||
expect(page.items[P1]!.findings.map(f => f.id).sort()).toEqual(page.items.filter((_, i) => i !== P1).flatMap(i => i.findings).map(f => f.id).sort());
|
||
expect(page.prompt).toContain('"id": "uc1"');
|
||
expect(page.prompt).not.toContain('{{');
|
||
});
|
||
test('slot names inside the plan text stay plan text; the template slots are still filled', () => {
|
||
const planFile = path.join(TMP, 'slot-plan.md');
|
||
fs.writeFileSync(planFile, '## Implementation plan\n\nfill {{QUESTIONS}} and {{IDS}} in {{PLAN}}\n');
|
||
const p = buildPage({ ...readState(out), active_plan: planFile }, gateOf(out), decider());
|
||
expect(p.prompt).toContain('fill {{QUESTIONS}} and {{IDS}} in {{PLAN}}');
|
||
expect(p.prompt).toContain('"id": "uc1"');
|
||
expect(p.prompt).toContain('uc1, d1, d2, uc2, d3, d4, p1');
|
||
expect(p.prompt.split('{{').length).toBe(4);
|
||
});
|
||
test('a good reply is clean', () => {
|
||
expect(parsePage(reply(valid), page.items).clean).toBe(true);
|
||
});
|
||
const cases: Array<[string, unknown[] | string, string]> = [
|
||
['missing', valid.slice(0, 2), 'missing'],
|
||
['duplicate', [...valid, valid[P1]], 'duplicate'],
|
||
['unknown option', valid.map((v, i) => (i === P1 ? { ...v, pick: 'z' } : v)), 'pick "z" is not an option of p1'],
|
||
['foreign cite', valid.map((v, i) => (i === P1 ? { ...v, cites: ['nope'] } : v)), 'cites ["nope"] name a finding outside p1\'s origin'],
|
||
['confidence above 1', valid.map((v, i) => (i === P1 ? { ...v, confidence: 1.2 } : v)), 'confidence 1.2 is not a number from 0 to 1'],
|
||
['bad action', valid.map((v, i) => (i === P1 ? { ...v, action: 'approve' } : v)), 'action "approve" is not decide, send_back or hold'],
|
||
['long reason', valid.map((v, i) => (i === P1 ? { ...v, reason: 'x'.repeat(401) } : v)), 'reason is not a string of at most 400 characters'],
|
||
['no JSON', 'I would approve.', 'no JSON array in the reply'],
|
||
];
|
||
for (const [name, objs, why] of cases) test(`${name} marks that item malformed`, () => {
|
||
const p = parsePage(typeof objs === 'string' ? objs : reply(objs), page.items);
|
||
expect(p.clean).toBe(false);
|
||
expect(p.byId.get('p1')).toEqual({ ok: false, why });
|
||
});
|
||
test('a foreign id leaves the items valid but the page unclean, so it is asked again', () => {
|
||
const p = parsePage(reply([...valid, { ...valid[0], id: 'zz9' }]), page.items);
|
||
expect(p.foreign).toEqual(['zz9']);
|
||
expect([...p.byId.values()].every(v => v.ok)).toBe(true);
|
||
expect(p.clean).toBe(false);
|
||
});
|
||
});
|
||
|
||
describe('the 10% audit mark', () => {
|
||
test('on a fixed 2,000-row corpus the marked share is what the hash gives', () => {
|
||
const marked = Array.from({ length: 2000 }, (_, i) => auditMark('fixed-salt-0001', `run-fixture-r${1 + (i % 3)}-d${i}`)).filter(Boolean).length;
|
||
expect(marked).toBe(Array.from({ length: 2000 }, (_, i) => BigInt('0x' + crypto.createHash('sha256').update(`fixed-salt-0001run-fixture-r${1 + (i % 3)}-d${i}`).digest('hex')) % 1000n < 100n).filter(Boolean).length);
|
||
expect(marked / 2000).toBeGreaterThanOrEqual(0.08);
|
||
expect(marked / 2000).toBeLessThanOrEqual(0.12);
|
||
});
|
||
});
|
||
|
||
describe('callModel: the shadow receipt, stop_reason and the OpenAI output bound (existing callers unchanged)', async () => {
|
||
const f = await fake('transport', [{ mode: 'valid' }, { mode: 'truncated' }, { mode: 'valid' }, { mode: 'truncated' }]);
|
||
test('a receipt override writes exactly one receipt with its sink and consent; the default stays the outside-voice receipt', async () => {
|
||
await callModel({ model: 'claude-opus-5-5', prompt: 'p', timeoutMs: 5000, env: f.env, maxTokens: 10, receipt: { sink: 'autoplan-shadow', consent: 'autoplan_shadow on', payloadClass: 'plan-and-gate' } });
|
||
expect(f.receipts().map(r => [r.sink, r.consent, r.payload_class ?? r.payloadClass])).toEqual([['autoplan-shadow', 'autoplan_shadow on', 'plan-and-gate']]);
|
||
const cut = await callModel({ model: 'claude-opus-5-5', prompt: 'p', timeoutMs: 5000, env: f.env });
|
||
expect(cut.stop_reason).toBe('max_tokens');
|
||
expect(f.receipts().at(-1)!.sink).toBe('outside-voice-api');
|
||
expect(f.receipts().length).toBe(2);
|
||
});
|
||
test('OpenAI gets max_completion_tokens only when the caller set an output limit, and its finish_reason comes back', async () => {
|
||
const plain = await callModel({ model: 'gpt-6.1-sol', prompt: 'p', timeoutMs: 5000, env: f.env });
|
||
const bounded = await callModel({ model: 'gpt-6.1-sol', prompt: 'p', timeoutMs: 5000, env: f.env, maxTokens: 4000 });
|
||
const [a, b] = f.requests().slice(2).map(r => r.body);
|
||
expect(Object.keys(a)).toEqual(['model', 'messages']);
|
||
expect(b.max_completion_tokens).toBe(4000);
|
||
expect(plain.stop_reason).toBe('stop');
|
||
expect(bounded.stop_reason).toBe('length');
|
||
});
|
||
});
|
||
|
||
describe('runShadow: retry, truncation, failures and the deadline', async () => {
|
||
const ids = (out: string) => rows(recordFile(out, 1));
|
||
const shadowed = async (name: string, script: Array<Record<string, unknown>>, model?: string, deadlineMs?: number) => {
|
||
const f = await fake(name, script);
|
||
const out = gateRun(model);
|
||
const { claim } = claimShadow(out);
|
||
expect(claim).toEqual({ gate_rev: 1, retrospective: false });
|
||
const started = Date.now();
|
||
const outcome = await runShadow(out, claim!, { env: { ...process.env, ...f.env }, deadlineMs });
|
||
return { f, out, outcome, ms: Date.now() - started };
|
||
};
|
||
test('a good reply records every item, keeps the prompt and the raw reply, and settles one shadow-labeled call', async () => {
|
||
const { f, out, outcome } = await shadowed('good', [{ mode: 'valid' }]);
|
||
expect(outcome.status).toBe('recorded');
|
||
expect(outcome.line).toMatch(/^SHADOW: recorded model=claude-opus-5-5 usd=0\.0140$/);
|
||
expect(ids(out).map(r => [r.item, r.status, r.pick, r.rule_pick, r.prompt_sha256 === SHADOW_PROMPT_SHA256])).toEqual([['uc1', 'recorded', 'a', 'a', true], ['d1', 'recorded', 'a', null, true], ['d2', 'recorded', 'a', null, true],
|
||
['uc2', 'recorded', 'a', 'a', true], ['d3', 'recorded', 'a', null, true], ['d4', 'recorded', 'a', null, true], ['p1', 'recorded', 'a', 'a', true]]);
|
||
expect(f.requests().map(r => r.body.max_tokens)).toEqual([4000]);
|
||
expect(fs.existsSync(path.join(out, 'shadow', 'r1.prompt.md'))).toBe(true);
|
||
expect(fs.readFileSync(path.join(out, 'shadow', 'shadow-r1-1.reply.txt'), 'utf8')).toContain('fake prediction');
|
||
expect(readLedger(path.join(out, 'spend.json')).settled.filter(s => s.label === 'shadow').map(s => [s.attempt, s.usd])).toEqual([['shadow-r1-1', 0.014]]);
|
||
expect(fs.existsSync(path.join(out, 'shadow', 'r1.pending'))).toBe(false);
|
||
});
|
||
test('a link at a reply path is replaced, never written through', async () => {
|
||
const f = await fake('linked-reply', [{ mode: 'valid' }]);
|
||
const out = gateRun();
|
||
const { claim } = claimShadow(out);
|
||
const sentinel = path.join(TMP, 'sentinel.txt');
|
||
fs.writeFileSync(sentinel, 'keep');
|
||
const reply = path.join(out, 'shadow', 'shadow-r1-1.reply.txt');
|
||
fs.symlinkSync(sentinel, reply);
|
||
expect((await runShadow(out, claim!, { env: { ...process.env, ...f.env } })).status).toBe('recorded');
|
||
expect(fs.readFileSync(sentinel, 'utf8')).toBe('keep');
|
||
expect(fs.lstatSync(reply).isSymbolicLink()).toBe(false);
|
||
expect(fs.readFileSync(reply, 'utf8')).toContain('fake prediction');
|
||
});
|
||
test('a cap changed during the call admits the retry against the new cap and is not written back over', async () => {
|
||
const f = await fake('cap-moved', [{ mode: 'truncated', delay_ms: 400 }, { mode: 'valid' }]);
|
||
const out = gateRun();
|
||
const { claim } = claimShadow(out);
|
||
const pending = runShadow(out, claim!, { env: { ...process.env, ...f.env } });
|
||
await Bun.sleep(150);
|
||
const run = readRun(out);
|
||
run.spend_cap_usd = 2;
|
||
fs.writeFileSync(path.join(out, 'run.json'), JSON.stringify(run, null, 2) + '\n');
|
||
expect((await pending).status).toBe('recorded');
|
||
expect(f.requests().length).toBe(2);
|
||
expect(readLedger(path.join(out, 'spend.json')).cap_usd).toBe(2);
|
||
});
|
||
test('a malformed page is asked once more; items valid in either reply record, the rest record malformed', async () => {
|
||
const fixed = await shadowed('missing-then-valid', [{ mode: 'missing' }, { mode: 'valid' }]);
|
||
expect(fixed.f.requests().length).toBe(2);
|
||
expect(ids(fixed.out).every(r => r.status === 'recorded')).toBe(true);
|
||
const still = await shadowed('bad-twice', [{ mode: 'bad_option' }, { mode: 'bad_option' }]);
|
||
expect(still.f.requests().length).toBe(2);
|
||
expect(ids(still.out).map(r => r.status)).toEqual(['malformed', 'recorded', 'recorded', 'recorded', 'recorded', 'recorded', 'recorded']);
|
||
expect(ids(still.out)[0]).toMatchObject({ pick: null, audit: false, malformed: 'pick "z" is not an option of uc1' });
|
||
expect(still.outcome.line).toContain('malformed=1/7');
|
||
});
|
||
test('a truncated reply raises max_tokens once to 8,000 (no page retry); truncated twice fails truncated', async () => {
|
||
const raised = await shadowed('truncated-once', [{ mode: 'truncated' }, { mode: 'valid' }]);
|
||
expect(raised.f.requests().map(r => r.body.max_tokens)).toEqual([4000, 8000]);
|
||
expect(raised.outcome.status).toBe('recorded');
|
||
const twice = await shadowed('truncated-twice', [{ mode: 'truncated' }]);
|
||
expect(twice.f.requests().length).toBe(2);
|
||
expect(twice.outcome.line).toBe('SHADOW: failed truncated');
|
||
expect(fs.existsSync(recordFile(twice.out, 1))).toBe(false);
|
||
});
|
||
test('an OpenAI shadow model sends max_completion_tokens and treats finish_reason length as truncated', async () => {
|
||
const r = await shadowed('openai', [{ mode: 'truncated' }, { mode: 'valid' }], 'gpt-6.1-sol');
|
||
expect(r.f.requests().map(x => [x.path, x.body.max_completion_tokens])).toEqual([['/v1/chat/completions', 4000], ['/v1/chat/completions', 8000]]);
|
||
expect(r.outcome.status).toBe('recorded');
|
||
});
|
||
test('a 500 fails http and settles the call at 0; the journal names the code', async () => {
|
||
const r = await shadowed('http500', [{ mode: 'http500' }]);
|
||
expect(r.outcome.line).toBe('SHADOW: failed http');
|
||
expect(readLedger(path.join(r.out, 'spend.json')).settled.filter(s => s.label === 'shadow').map(s => s.usd)).toEqual([0]);
|
||
});
|
||
test('one deadline covers the call and its retry: a slow malformed reply leaves the retry no time', async () => {
|
||
const r = await shadowed('deadline-retry', [{ mode: 'missing', delay_ms: 1200 }, { mode: 'valid', delay_ms: 1200 }], undefined, 1500);
|
||
expect(r.ms).toBeLessThan(2400);
|
||
expect(r.outcome.status).toBe('recorded');
|
||
expect(ids(r.out).map(x => x.status)).toEqual(['recorded', 'recorded', 'recorded', 'recorded', 'recorded', 'recorded', 'malformed']);
|
||
const slow = await shadowed('deadline-call', [{ mode: 'valid', delay_ms: 3000 }], undefined, 800);
|
||
expect(slow.outcome.line).toBe('SHADOW: failed timeout');
|
||
expect(slow.ms).toBeLessThan(2500);
|
||
expect(readLedger(path.join(slow.out, 'spend.json')).settled.filter(s => s.label === 'shadow').map(s => s.usd)).toEqual(['unknown']);
|
||
});
|
||
test('a publish that throws journals shadow_failed and drops the lease, so a later claim pays no second call', async () => {
|
||
const f = await fake('publish-throws', [{ mode: 'valid' }]);
|
||
const out = gateRun();
|
||
const { claim } = claimShadow(out);
|
||
fs.mkdirSync(path.join(out, 'shadow', 'ceo-only.r1.jsonl.stuck.tmp'), { recursive: true });
|
||
const outcome = await runShadow(out, claim!, { env: { ...process.env, ...f.env } });
|
||
expect(outcome).toMatchObject({ status: 'failed', code: 'internal' });
|
||
expect(readJournal(out).filter(r => r.event === 'shadow_failed' && r.gate_rev === 1).map(r => r.code)).toEqual(['internal']);
|
||
expect(fs.existsSync(path.join(out, 'shadow', 'r1.pending'))).toBe(false);
|
||
expect(claimShadow(out)).toEqual({});
|
||
expect(f.requests().length).toBe(1);
|
||
});
|
||
test('runShadow never throws when the journal cannot be appended', async () => {
|
||
const f = await fake('journal-readonly', [{ mode: 'valid' }]);
|
||
const out = gateRun();
|
||
const { claim } = claimShadow(out);
|
||
fs.chmodSync(path.join(out, 'attempts.jsonl'), 0o444);
|
||
try {
|
||
expect(await runShadow(out, claim!, { env: { ...process.env, ...f.env } })).toMatchObject({ status: 'failed' });
|
||
} finally { fs.chmodSync(path.join(out, 'attempts.jsonl'), 0o644); }
|
||
});
|
||
test('a record already published makes a later claim a no-op: no second call is paid', async () => {
|
||
const { f, out } = await shadowed('noop', [{ mode: 'valid' }]);
|
||
const before = fs.readFileSync(recordFile(out, 1));
|
||
expect(claimShadow(out)).toEqual({});
|
||
expect(f.requests().length).toBe(1);
|
||
expect(fs.readFileSync(recordFile(out, 1)).equals(before)).toBe(true);
|
||
});
|
||
});
|
||
|
||
describe('gate cost: shadow spend counts against the cap and shows in GATE_SPEND, never in the per-voice price', () => {
|
||
test('a $5 voice settle followed by a shadow settle keeps GATE_REC_COST at x 5.00', () => {
|
||
const out = path.join(TMP, 'cost');
|
||
fs.mkdirSync(out, { recursive: true });
|
||
const file = path.join(out, 'spend.json');
|
||
reserve(file, 'r-a1', 5, { cap: 5.2, label: 'ceo/native' }); settle(file, 'r-a1', 'unknown');
|
||
reserve(file, 'shadow-r1-1', 0.15, { label: 'shadow' }); settle(file, 'shadow-r1-1', 0.014);
|
||
const list: GateList = { schema_version: 1, run: 'r', gate_rev: 1, items: [
|
||
{ id: 'd1', title: 't', kind: 'auto', recommended: 'b', phase: 'ceo', options: [{ key: 'a', text: 'keep', changes_plan: false }, { key: 'b', text: 'change', changes_plan: true }] },
|
||
{ id: 'p1', title: 'Plan approval', kind: 'approval', recommended: 'b', options: [{ key: 'a', text: 'x' }, { key: 'b', text: 'y' }] }] };
|
||
const c = gateCost(out, list, { phase_order: ['ceo', 'eng'], outside_enabled: true, spend_cap_usd: 5.2 });
|
||
expect(c.per_voice_usd).toBe(5);
|
||
expect(c.attempts).toBe(1);
|
||
expect(c.shadow_usd).toBe(0.014);
|
||
expect(c.summary.worst_case_usd).toBe(5.014);
|
||
const [spend, rec] = renderGateCost(c);
|
||
expect(spend).toContain('attempts=1 shadow=0.0140');
|
||
expect(rec).toContain('× 5.00');
|
||
expect(rec).toContain('(over the cap: raise it with gstack-autoplan budget before answering rec)');
|
||
});
|
||
});
|