mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-31 14:06:50 +01:00
feat(ship): /ship and /autoplan size their review to the diff (GSTA-131) (#3527)
* wip(GSTA-131): review depth, fix-check transitions, dispatch counters, carry pregate, autoplan round policy Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * wip(GSTA-131): reviewers stay at full effort; audits light Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * feat(ship): Step 9 dispatches what gstack-review-plan plans; fix-check after below-floor fixes; cap two full passes plus one fix-check Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test: review-dispatch-compare has a free lane; review-depth fixtures registered free; resumed max_rounds case off the old default Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(GSTA-131): ship review pass-1 repairs: fix-check binds under its own skill, Step 11.5 accepts a fix-check or a planned adversarial skip, delta carry with a pregate, carry --pregate pinned to the checkout, quiet-after-cap revise regates, skill-dir markdown and rule docs classed code, gate signals from tests class up, plan fails closed on git errors, per-class compare ratios Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(GSTA-131): ship review pass-2 repairs: Step 11 follows the last full-pass plan, Important Rules restored, ship goldens regenerated, true per-class medians, review-policy transitions and once-per-invocation voices, docs/** assets stay docs-class, CLI and fix-check binding tests Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs(changelog): GSTA-131 review depth, fix-check, delta /ship and autoplan rounds Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs: ship-policy cap and counters, review-policy /review cycle cap, unattended passes (docs audit) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(review): 400-word findings cap fits the /review size pins (five host pins lowered by 2); CHANGELOG max_rounds wording Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs(changelog): gate integrity names the size-ledger entry and the lowered /review pins Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test: the round-budget case asserts the default through ROUNDS output only (pregate literals heuristic) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test: curate ship-receipt-carry-pregate out of the Windows lane like its --pregate siblings Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs(changelog): end the Windows curation sentence instead of a mid-sentence colon Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(ship): one decision each for the review-loop exits, the planned adversarial skip and Step 16's TODOS rerun (screen judge repair) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(ship): fix-check repairs: pin the renamed Step 11 heading, keep the below-floor exit at the cap, name the plan run a repeat reuses, print the native skip on outside-only plans Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
44f19d0106
commit
148d28ea81
68 files changed
+2069
-266
No files matched your search
@@ -80,6 +80,7 @@ export const KNOWN_WINDOWS_INCOMPATIBLE: Array<{ file: string; reason: string }>
|
||||
// test/helpers/pregate-fixture.ts runBin.
|
||||
{ file: 'test/stamp-templates.test.ts', reason: 'spawns bin/gstack-tree-receipt through test/helpers/restamp-fixture.ts runBin (shebang via CreateProcess)' },
|
||||
{ file: 'test/ship-issue-intent.test.ts', reason: 'spawns bin/gstack-issue-links and bin/gstack-ship-receipt through test/helpers/pregate-fixture.ts runBin (shebang via CreateProcess) with a `gh` shell shim on PATH' },
|
||||
{ file: 'test/ship-receipt-carry-pregate.test.ts', reason: 'passes a gstack-pregate result to gstack-ship-receipt write and carry; on Windows the fixture\'s Linux lane is a requires-remote obligation, so write refuses the pregate as uncleared and exits 1 (windows-free-shard run 40611810409), like the other --pregate receipt tests' },
|
||||
// PR E2: lane ownership and evidence verification run bin/gstack-lane-check and bin/gstack-evidence
|
||||
// through test/helpers/restamp-fixture.ts runBin (shebang via CreateProcess); the lib paths are
|
||||
// exercised in the same file, so there is no Windows-safe split to keep.
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
/**
|
||||
* Fix-check prompt (docs/review-policy.md#transitions): the one reviewer /ship dispatches after a
|
||||
* step whose fixes do not earn a full pass. It reads the fixes, never the whole diff. The parent
|
||||
* writes every input in literally: a reviewer never sees the parent's shell.
|
||||
*/
|
||||
export interface FixCheckInput { fixList: string; touchedHunks: string; callers: string; testResults: string }
|
||||
|
||||
export const FIX_CHECK_INSTRUCTIONS = `You are the fix-check reviewer for a pre-landing review. An earlier review pass found
|
||||
findings and the author fixed them. You check only those fixes: the fix list, the hunks the fixes
|
||||
touched, the callers of every symbol the fixes changed, and the test results after the fixes.
|
||||
|
||||
Find any defect a fix introduced or left in place: a caller the change breaks, a guard that now
|
||||
fails open (accepts what it refused) or fails closed (refuses what it accepted), a regression of
|
||||
behaviour the code had before the fix, an error or unreadable state now read as success, a test
|
||||
made weaker. Name the input or call site that reaches each defect.
|
||||
|
||||
Rate each finding against the blocking floor: CRITICAL only when it would fail a test, break
|
||||
production, or lose data or secrets; everything else is INFORMATIONAL. Output one JSON object per
|
||||
line, {"severity":"CRITICAL|INFORMATIONAL","path":"<file>","line":<n>,"summary":"<what breaks, for which input>"},
|
||||
or the two words NO FINDINGS when the fixes are sound.`;
|
||||
|
||||
export function renderFixCheckPrompt(i: FixCheckInput): string {
|
||||
return [FIX_CHECK_INSTRUCTIONS, '', '## Fix list', i.fixList.trim(), '', '## Touched hunks', '```diff', i.touchedHunks.trim(), '```', '',
|
||||
'## Callers of changed symbols', i.callers.trim() || '(none found)', '', '## Test results after the fixes', i.testResults.trim()].join('\n') + '\n';
|
||||
}
|
||||
@@ -116,7 +116,7 @@ If A: revise the premise and note the revision. If B: proceed (and note that the
|
||||
// ─── Adversarial Review (always-on) ──────────────────────────────────
|
||||
|
||||
function adversarialNativePass(ctx: TemplateContext, isShip: boolean): string {
|
||||
return `### ${outsideVoiceFor(ctx).nativeLabel} adversarial subagent (always runs)
|
||||
return `### ${outsideVoiceFor(ctx).nativeLabel} adversarial subagent (${isShip ? 'on \`DISPATCH: adversarial-native\`' : 'always runs'})
|
||||
|
||||
Before dispatch, run \`~/.claude/skills/gstack/bin/gstack-review-log --start adversarial-review\`
|
||||
and save the returned token for this native attempt. Do the same before each outside
|
||||
@@ -235,9 +235,9 @@ export function generateAdversarialStep(ctx: TemplateContext): string {
|
||||
const isShip = ctx.skillName === 'ship';
|
||||
const stepNum = isShip ? '11' : '4.8';
|
||||
|
||||
return `## Step ${stepNum}: Adversarial review (always-on)
|
||||
return `## Step ${stepNum}: Adversarial review${isShip ? '' : ' (always-on)'}
|
||||
|
||||
Every diff gets the ${outsideVoiceFor(ctx).nativeLabel} adversarial pass. Add ${outsideVoiceFor(ctx).label} when its preflight is ready; unavailable or disabled outside coverage stays explicit.
|
||||
${isShip ? 'Unavailable' : `Every diff gets the ${outsideVoiceFor(ctx).nativeLabel} adversarial pass. Add ${outsideVoiceFor(ctx).label} when its preflight is ready; unavailable`} or disabled outside coverage stays explicit.
|
||||
|
||||
**Detect diff size:**
|
||||
|
||||
@@ -252,8 +252,8 @@ echo "DIFF_SIZE: $DIFF_TOTAL"
|
||||
${outsideVoicePreflight(ctx, { disabledBehavior: 'codex-only', nativeReview: true })}
|
||||
|
||||
\`CODEX_MODE: disabled\` means skip the ${outsideVoiceFor(ctx).label} passes ONLY.
|
||||
\`ready\` and \`unverified\` run them; every other mode skips them with the printed reason.
|
||||
The ${outsideVoiceFor(ctx).nativeLabel} adversarial subagent always runs.
|
||||
\`ready\` and \`unverified\` run them; every other mode skips them with the printed reason.${isShip ? '' : `
|
||||
The ${outsideVoiceFor(ctx).nativeLabel} adversarial subagent always runs.`}
|
||||
|
||||
**User override:** If the user explicitly requested "full review", "structured review", or "P1 gate", also run the ${outsideVoiceFor(ctx).label} structured review regardless of diff size (still requires \`CODEX_MODE: ready\` (or \`unverified\`)).
|
||||
|
||||
@@ -274,7 +274,7 @@ ${outsideVoiceProvenance(ctx, 'adversarial')}
|
||||
After all passes complete, synthesize findings across all sources:
|
||||
|
||||
\`\`\`
|
||||
ADVERSARIAL REVIEW SYNTHESIS (always-on, N lines):
|
||||
ADVERSARIAL REVIEW SYNTHESIS (${isShip ? '' : 'always-on, '}N lines):
|
||||
════════════════════════════════════════════════════════════
|
||||
High confidence (found by multiple sources): [findings agreed on by >1 pass]
|
||||
Unique to the parent checklist/specialists: [from earlier steps]
|
||||
@@ -309,7 +309,7 @@ in order before leaving Step 11:
|
||||
Step 9 completes full review before fixes; any further repair inserts its checks
|
||||
ahead of the remaining items. These fresh reviews after code edits are not recovery retries.
|
||||
Returning here never resets Step 9's three-cycle fix limit.
|
||||
3. **Native complete with no queued fixes:** Finish the memory updates below,
|
||||
3. **Native complete with no queued fixes, or not dispatched:** Finish the memory updates below,
|
||||
then continue to Step 11.5. Never jump directly to release preparation.` : 'The native pass is required for Step 5.8 completion. Optional outside failures remain separately recorded, not completed by native coverage. Return all findings and structured-review decisions to Step 5; the parent owns fixes and the full rerun.'}
|
||||
|
||||
---`;
|
||||
|
||||
@@ -141,7 +141,7 @@ Every other specialist, including Testing, reads the full diff with fixtures.
|
||||
"You are a specialist code reviewer. Read the checklist at {checklist path}, then run
|
||||
\`${diffBase(ctx)} && git diff "$DIFF_BASE"\` to get the ${ctx.skillName === 'ship' ? 'diff (`<diff-base>`: write the sha in before dispatch, the merge base on pass 1 and `RANGE_BASE` from pass 2 on; never a shell variable)' : 'full diff'}. Apply the checklist against the diff.
|
||||
|
||||
\`NO FINDINGS\` alone is the expected, complete result for a clean diff; never invent an item to show work. Otherwise output one JSON object per finding, on its own line:
|
||||
\`NO FINDINGS\` alone is the expected, complete result for a clean diff; never invent an item. Otherwise output one JSON object per finding per line, 400 words at most:
|
||||
{\\"severity\\":\\"CRITICAL|INFORMATIONAL\\",\\"confidence\\":N,\\"path\\":\\"file\\",\\"line\\":N,\\"category\\":\\"category\\",\\"summary\\":\\"description\\",\\"fix\\":\\"recommended fix\\",\\"fingerprint\\":\\"path:line:category\\",\\"specialist\\":\\"name\\"}
|
||||
|
||||
Required fields: severity, confidence, path, category, summary, specialist.
|
||||
|
||||
@@ -65,9 +65,7 @@ ${ctx.skillName === 'ship' ? 'This verdict never skips Step 9 or its finding, ap
|
||||
Other rows provide context, not a substitute for Eng Review:
|
||||
- Recommend CEO Review for product/business or scope decisions, not routine fixes or cleanup.
|
||||
- Recommend Design Review for UI/UX work, not backend, infrastructure or prompt-only work.
|
||||
- Adversarial review always includes a native pass. Available, enabled outside
|
||||
challenges supplement it; diffs of 200+ lines also get the structured P1 gate.
|
||||
- Outside Voice is the default-on plan review after CEO/Eng review. \`codex_reviews\`
|
||||
${ctx.skillName === 'ship' ? '- Step 9\'s review plan decides the adversarial passes (Step 11).\n' : '- Adversarial review always includes a native pass. Available, enabled outside\n challenges supplement it; diffs of 200+ lines also get the structured P1 gate.\n'}- Outside Voice is the default-on plan review after CEO/Eng review. \`codex_reviews\`
|
||||
disables that extra step. Provider failure uses native fallback and records
|
||||
missing outside coverage; this dashboard row never gates shipping.
|
||||
|
||||
|
||||
Executable
+175
@@ -0,0 +1,175 @@
|
||||
#!/usr/bin/env -S bun --no-env-file --config=/dev/null
|
||||
/**
|
||||
* review-dispatch-compare — how many reviewers a /ship run and an /autoplan run dispatch under the
|
||||
* legacy policy and under docs/review-policy.md#review-depth, on fixed fixture diffs, with an input
|
||||
* plus output token estimate per run. The proof-of-saving table in the GSTA-131 PR comes from here.
|
||||
*
|
||||
* bun scripts/review-dispatch-compare.ts [--fixture <file>]... [--tokens <measured.json>] [--json]
|
||||
* bun scripts/review-dispatch-compare.ts --capture <sha> --name <name> --out <file>
|
||||
*
|
||||
* --capture writes a fixture from one commit (its diff against its first parent) through
|
||||
* gstack-review-plan --json. --tokens takes the live per-dispatch output-token medians by role and
|
||||
* effort ({ "specialist": { "full": n, "light": n }, ... }); without it the script uses the shipped
|
||||
* defaults below and says so. Counts are exact for the modelled scenarios; tokens are estimates.
|
||||
*/
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { runScriptSync } from '../lib/script-invocation';
|
||||
import { planReview, specialistsFor, type DiffClass, type Dispatch, type History, type NumbersResult } from '../lib/review-depth';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const BYTES_PER_TOKEN = 4;
|
||||
const FIXED_AUDITS = ['coverage-audit', 'plan-completion', 'docs-audit'];
|
||||
const RANGE_FRACTION = 0.15;
|
||||
const FIX_CHECK_FRACTION = 0.1;
|
||||
/** Output plus reasoning tokens per dispatch when no live measurement is given (labelled DEFAULT in the output). */
|
||||
const DEFAULT_OUT: Record<string, { full: number; light: number }> = {
|
||||
specialist: { full: 6000, light: 3000 }, 'red-team': { full: 8000, light: 4000 }, 'adversarial-native': { full: 8000, light: 4000 },
|
||||
'outside-voice': { full: 6000, light: 6000 }, 'fix-check': { full: 4000, light: 2000 }, audit: { full: 5000, light: 2500 }, 'autoplan-voice': { full: 9000, light: 4500 },
|
||||
};
|
||||
|
||||
export interface Fixture { name: string; commit: string; paths: string[]; deleted: string[]; stamp_only?: string[]; scope: Record<string, string>; gate_tagged: number | null; diff_lines: number; diff_bytes: number; numbers: NumbersResult }
|
||||
/** clean: pass 1 finds nothing; nits: pass 1 fixes below-floor findings; blocker: pass 1 fixes a blocker; adversarial-fix: Step 11 finds a should that is fixed; fix-regression: a pass-1 fix introduces a blocker the next step finds. */
|
||||
type Scenario = 'clean' | 'nits' | 'blocker' | 'adversarial-fix' | 'fix-regression';
|
||||
const SCENARIOS: Scenario[] = ['clean', 'nits', 'blocker', 'adversarial-fix', 'fix-regression'];
|
||||
interface Call { role: string; name?: string; effort: 'full' | 'light' | 'host'; diffBytes: number }
|
||||
|
||||
const fileBytes = (rel: string) => { try { return fs.statSync(path.join(ROOT, rel)).size; } catch { return 0; } };
|
||||
function promptBytes(role: string, name: string): number {
|
||||
if (role === 'specialist') return fileBytes(`review/specialists/${name}.md`) + fileBytes('review/checklist.md') / 4;
|
||||
if (role === 'red-team') return fileBytes('review/specialists/red-team.md');
|
||||
if (role === 'adversarial-native' || role === 'outside-voice') return fileBytes('ship/sections/adversarial.md') / 2;
|
||||
if (role === 'fix-check') return fileBytes('review/checklist.md') / 2;
|
||||
return 20_000;
|
||||
}
|
||||
|
||||
function legacyShip(f: Fixture, sc: Scenario): Call[] {
|
||||
const calls: Call[] = [];
|
||||
const passes = sc === 'clean' ? 1 : sc === 'fix-regression' ? 3 : 2;
|
||||
const step11Runs = sc === 'adversarial-fix' ? 2 : 1;
|
||||
const legacySpecialists = specialistsFor('code', { ...base(f), history: { fullPasses: 0, last: 'none' } });
|
||||
for (let p = 1; p <= passes; p++) {
|
||||
const diffBytes = p === 1 ? f.diff_bytes : f.diff_bytes * RANGE_FRACTION;
|
||||
for (const name of legacySpecialists) calls.push({ role: 'specialist', name, effort: 'full', diffBytes });
|
||||
const critical = (sc === 'blocker' && p === 1) || (sc === 'fix-regression' && p === 2);
|
||||
if (f.diff_lines > 200 || critical) calls.push({ role: 'red-team', effort: 'full', diffBytes });
|
||||
}
|
||||
for (let s = 0; s < step11Runs; s++) calls.push({ role: 'adversarial-native', effort: 'full', diffBytes: f.diff_bytes }, { role: 'outside-voice', effort: 'host', diffBytes: f.diff_bytes });
|
||||
for (const a of FIXED_AUDITS) calls.push({ role: 'audit', effort: 'full', diffBytes: f.diff_bytes / 2 });
|
||||
return calls;
|
||||
}
|
||||
|
||||
function base(f: Fixture) {
|
||||
return { paths: f.paths, deleted: f.deleted, stampOnly: f.stamp_only ?? [], scope: f.scope, gateTagged: f.gate_tagged, diffLines: f.diff_lines, numbers: f.numbers };
|
||||
}
|
||||
|
||||
/** The new policy, step by step through lib/review-depth's own transition table. */
|
||||
function newShip(f: Fixture, sc: Scenario): { calls: Call[]; ending: string } {
|
||||
const calls: Call[] = [];
|
||||
const outcomes: Record<Scenario, Array<Partial<History>>> = {
|
||||
clean: [{}],
|
||||
nits: [{ fixedBelow: true }, {}],
|
||||
blocker: [{ fixedBlocking: true }, {}],
|
||||
'adversarial-fix': [{ fixedBelow: true }, {}],
|
||||
'fix-regression': [{ fixedBelow: true }, { blockingOpen: true }],
|
||||
};
|
||||
let h: History = { fullPasses: 0, last: 'none' };
|
||||
let ending = 'done';
|
||||
let adversarialRan = false;
|
||||
for (let i = 0; i < 6; i++) {
|
||||
const plan = planReview({ ...base(f), history: h, criticalFound: sc === 'blocker' && i === 0, adversarialRan });
|
||||
if (plan.next.step === 'done' || plan.next.step === 'stop') { ending = plan.next.step; break; }
|
||||
const diffBytes = plan.next.step === 'fix-check' ? f.diff_bytes * FIX_CHECK_FRACTION : h.fullPasses === 0 ? f.diff_bytes : f.diff_bytes * RANGE_FRACTION;
|
||||
for (const d of plan.dispatches) calls.push({ role: d.role, name: d.name, effort: d.effort, diffBytes: d.role === 'outside-voice' ? f.diff_bytes : diffBytes });
|
||||
adversarialRan ||= plan.dispatches.some(d => d.role === 'adversarial-native');
|
||||
const o = outcomes[sc][Math.min(i, outcomes[sc].length - 1)] ?? {};
|
||||
h = { fullPasses: h.fullPasses + (plan.next.step === 'full' ? 1 : 0), last: plan.next.step, ...o };
|
||||
}
|
||||
for (const a of FIXED_AUDITS) calls.push({ role: 'audit', effort: 'light', diffBytes: f.diff_bytes / 2 });
|
||||
return { calls, ending };
|
||||
}
|
||||
|
||||
function tokens(calls: Call[], out: typeof DEFAULT_OUT): number {
|
||||
return Math.round(calls.reduce((n, c) => {
|
||||
const o = out[c.role] ?? out.audit!;
|
||||
return n + (promptBytes(c.role, c.name ?? '') + c.diffBytes) / BYTES_PER_TOKEN + (c.effort === 'light' ? o.light : o.full);
|
||||
}, 0));
|
||||
}
|
||||
|
||||
/** /autoplan: phases x voices x rounds. Legacy reopens every touched phase on any accepted change, up to 3 rounds. */
|
||||
function autoplanRuns(docsOnly: boolean): Array<{ scenario: string; legacy: number; next: number }> {
|
||||
const phases = ['ceo', 'eng'];
|
||||
const legacyRounds = { 'no change': 1, 'medium-only change': 2, 'high change': 2, 'high twice': 3 } as Record<string, number>;
|
||||
const newRounds = { 'no change': 1, 'medium-only change': 1, 'high change': 2, 'high twice': 2 } as Record<string, number>;
|
||||
return Object.keys(legacyRounds).map(s => ({
|
||||
scenario: s,
|
||||
legacy: phases.length * 2 * legacyRounds[s]!,
|
||||
next: (docsOnly ? 3 : phases.length * 2) * newRounds[s]!,
|
||||
}));
|
||||
}
|
||||
|
||||
function capture(sha: string, name: string, out: string): number {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'review-compare-'));
|
||||
const captureBase = `review-compare-base-${process.pid}-${sha.slice(0, 8)}`;
|
||||
const git = (args: string[], cwd = ROOT) => spawnSync('git', args, { cwd, encoding: 'utf8', timeout: 120_000, maxBuffer: 64 * 1024 * 1024 });
|
||||
try {
|
||||
if (git(['worktree', 'add', '-q', '--detach', dir, sha]).status !== 0) throw new Error(`cannot check out ${sha}`);
|
||||
git(['branch', '-f', captureBase, `${sha}^`], dir);
|
||||
const r = runScriptSync(path.join(ROOT, 'bin', 'gstack-review-plan'), ['--base', captureBase, '--json'], { cwd: dir, encoding: 'utf8', timeout: 300_000, maxBuffer: 64 * 1024 * 1024 });
|
||||
if (r.status !== 0) throw new Error(`gstack-review-plan failed: ${r.stderr}`);
|
||||
const plan = JSON.parse(r.stdout);
|
||||
const diffBytes = Buffer.byteLength(git(['diff', `${sha}^`, sha], dir).stdout ?? '');
|
||||
const fixture: Fixture = { name, commit: sha, paths: plan.paths, deleted: plan.deleted, stamp_only: plan.stamp_only, scope: plan.scope, gate_tagged: plan.gate_tagged, diff_lines: plan.diff_lines, diff_bytes: diffBytes, numbers: plan.numbers };
|
||||
fs.mkdirSync(path.dirname(out), { recursive: true });
|
||||
fs.writeFileSync(out, JSON.stringify(fixture, null, 2) + '\n');
|
||||
process.stdout.write(`CAPTURED: ${name} ${sha} class=${plan.class} paths=${plan.paths.length} lines=${plan.diff_lines} -> ${out}\n`);
|
||||
return 0;
|
||||
} finally {
|
||||
git(['worktree', 'remove', '--force', dir]);
|
||||
git(['branch', '-D', captureBase]);
|
||||
}
|
||||
}
|
||||
|
||||
export function compare(fixtures: Fixture[], out: typeof DEFAULT_OUT) {
|
||||
const rows: Array<Record<string, string | number>> = [];
|
||||
for (const f of fixtures) {
|
||||
const cls: DiffClass = planReview({ ...base(f), history: { fullPasses: 0, last: 'none' } }).class;
|
||||
for (const sc of SCENARIOS) {
|
||||
const legacy = legacyShip(f, sc);
|
||||
const next = newShip(f, sc);
|
||||
const lt = tokens(legacy, out); const nt = tokens(next.calls, out);
|
||||
rows.push({ fixture: f.name, class: cls, scenario: sc, legacy_dispatches: legacy.length, new_dispatches: next.calls.length, legacy_tokens: lt, new_tokens: nt, ratio: Math.round((nt / lt) * 100) / 100, ending: next.ending });
|
||||
}
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
function main(argv: string[]): number {
|
||||
const flag = (n: string) => { const i = argv.indexOf(n); return i >= 0 ? argv[i + 1] : undefined; };
|
||||
const cap = flag('--capture');
|
||||
if (cap) return capture(cap, flag('--name') ?? cap, flag('--out') ?? path.join(ROOT, 'test/fixtures/review-depth', `${flag('--name') ?? cap}.json`));
|
||||
const files = argv.flatMap((a, i) => (a === '--fixture' ? [argv[i + 1]!] : []));
|
||||
const fixtureFiles = files.length ? files : fs.readdirSync(path.join(ROOT, 'test/fixtures/review-depth')).filter(f => f.endsWith('.json')).map(f => path.join(ROOT, 'test/fixtures/review-depth', f));
|
||||
const fixtures = fixtureFiles.map(f => JSON.parse(fs.readFileSync(f, 'utf8')) as Fixture);
|
||||
const tokFile = flag('--tokens');
|
||||
const out = tokFile ? { ...DEFAULT_OUT, ...JSON.parse(fs.readFileSync(tokFile, 'utf8')) } : DEFAULT_OUT;
|
||||
const rows = compare(fixtures, out);
|
||||
const ap = fixtures.map(f => ({ fixture: f.name, runs: autoplanRuns(planReview({ ...base(f), history: { fullPasses: 0, last: 'none' } }).class === 'docs') }));
|
||||
if (argv.includes('--json')) { process.stdout.write(JSON.stringify({ tokens_source: tokFile ?? 'DEFAULT', ship: rows, autoplan: ap }) + '\n'); return 0; }
|
||||
const lines = [`TOKENS: ${tokFile ? `measured (${tokFile})` : 'DEFAULT estimates (pass --tokens for measured output)'}`, '', '| fixture | class | scenario | /ship dispatches legacy → new | est. tokens legacy → new | ratio | ending |', '|---|---|---|---|---|---|---|'];
|
||||
for (const r of rows) lines.push(`| ${r.fixture} | ${r.class} | ${r.scenario} | ${r.legacy_dispatches} → ${r.new_dispatches} | ${r.legacy_tokens} → ${r.new_tokens} | ${r.ratio} | ${r.ending} |`);
|
||||
lines.push('', '| fixture | /autoplan scenario | voice dispatches legacy → new |', '|---|---|---|');
|
||||
for (const a of ap) for (const r of a.runs) lines.push(`| ${a.fixture} | ${r.scenario} | ${r.legacy} → ${r.next} |`);
|
||||
const median = (v: number[]) => { const m = Math.floor(v.length / 2); return v.length % 2 ? v[m]! : Math.round(((v[m - 1]! + v[m]!) / 2) * 100) / 100; };
|
||||
const ratios = rows.map(r => Number(r.ratio)).sort((a, b) => a - b);
|
||||
lines.push('', `SHIP_TOKEN_RATIO: median=${median(ratios)} min=${ratios[0]} max=${ratios[ratios.length - 1]} (n=${ratios.length})`);
|
||||
const byClass = new Map<string, number[]>();
|
||||
for (const r of rows) byClass.set(String(r.class), [...(byClass.get(String(r.class)) ?? []), Number(r.ratio)].sort((x, y) => x - y));
|
||||
lines.push(`SHIP_TOKEN_RATIO_BY_CLASS: ${[...byClass].map(([c, v]) => `${c} median=${median(v)} (n=${v.length})`).join('; ')}`);
|
||||
process.stdout.write(lines.join('\n') + '\n');
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (import.meta.main) process.exit(main(process.argv.slice(2)));
|
||||
Reference in new issue
Block a user