mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-09 06:28:59 +02:00
fix: retire decided A/B experiments; vendor the pre-cut fixture; ban raw-SHA fixtures
Three one-shot decision experiments kept re-running weekly as N=1
stochastic comparisons — flaky by construction with near-zero remaining
information: skill-e2e-auq-repetition-cut-ab (its own header: gate "passed
pre-landing, approved 2026-08-25"), skill-e2e-preamble-script-ab ("demoted
post-Phase-3"), and opus-47's fanout arm-vs-arm (parA >= parB across two
SINGLE stochastic runs — a coin flip). Deleted, with their selection keys;
the SDK overlay-harness stays as the maintained instrument for the next
experiment, and opus-47 keeps its routing-precision cases.
verboseSkill() now reads the VENDORED test/fixtures/auq-pre-cut-...-SKILL.md
instead of `git show ab66193e^:...` — a branch-local ref that dies on
branch prune and already failed on shallow clones. New free tripwire
(test/git-ref-fixture-tripwire.test.ts) bans the raw-SHA fixture class
outright: quoted SHA:path rev-specs and gitRef-style hex defaults in the
test trees fail the suite with the vendor-instead instruction.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
co-authored by
Claude Fable 5
Claude Fable 5
parent
99b5fa2e46
commit
d6df2af7dd
@@ -0,0 +1,64 @@
|
||||
/**
|
||||
* Git-ref fixture tripwire: no test or helper may pin repo content to a raw
|
||||
* commit SHA (the `git show <sha>:path` fixture pattern).
|
||||
*
|
||||
* The class: test/helpers/auq-sdk-capture.ts defaulted verboseSkill() to
|
||||
* `git show ab66193e^:plan-ceo-review/SKILL.md` — a BRANCH-LOCAL ref. That
|
||||
* fixture dies the day the branch is pruned, and already failed on shallow
|
||||
* clones (CI executors fetch-depth-0 exists precisely because self-derived
|
||||
* selection crashed on shallow checkouts). The v1.75 precedent is to VENDOR
|
||||
* the frozen content under test/fixtures/ instead — content-addressed by the
|
||||
* repo itself, immune to ref pruning and clone depth.
|
||||
*
|
||||
* Scans test trees + helpers for two shapes:
|
||||
* - a quoted `<hex>{7,40}[^]?:` rev-path (the `git show SHA:path` form)
|
||||
* - a gitRef-style default parameter carrying a raw hex SHA
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const SCAN_ROOTS = ['test', 'browse/test', 'design/test', 'make-pdf/test'];
|
||||
const SELF = path.join('test', 'git-ref-fixture-tripwire.test.ts');
|
||||
|
||||
// Quoted `SHA:` rev-path (7-40 hex chars, optional ^/~ suffix, then colon) —
|
||||
// requires >= 2 digits among the hex so ordinary words ('deadbeef' aside)
|
||||
// and pure-alpha identifiers don't false-positive.
|
||||
const REV_PATH = /['"`]([0-9a-f]{7,40})[\^~]?:/g;
|
||||
const GIT_REF_DEFAULT = /gitRef\s*=\s*['"`][0-9a-f]{7,40}/;
|
||||
|
||||
const looksLikeSha = (s: string): boolean => /[0-9]/.test(s) && /[a-f]/.test(s);
|
||||
|
||||
describe('git-ref fixture tripwire', () => {
|
||||
test('no raw-SHA fixture refs in the test trees (vendor the content instead)', () => {
|
||||
const hits: string[] = [];
|
||||
for (const root of SCAN_ROOTS) {
|
||||
const abs = path.join(ROOT, root);
|
||||
if (!fs.existsSync(abs)) continue;
|
||||
const stack = [abs];
|
||||
while (stack.length > 0) {
|
||||
const dir = stack.pop()!;
|
||||
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
||||
const full = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) { stack.push(full); continue; }
|
||||
if (!/\.(?:[cm]?[jt]s|tsx)$/.test(entry.name)) continue;
|
||||
const rel = path.relative(ROOT, full);
|
||||
if (rel === SELF) continue;
|
||||
const src = fs.readFileSync(full, 'utf-8');
|
||||
src.split('\n').forEach((line, i) => {
|
||||
for (const m of line.matchAll(REV_PATH)) {
|
||||
if (looksLikeSha(m[1])) hits.push(`${rel}:${i + 1} ${line.trim().slice(0, 100)}`);
|
||||
}
|
||||
if (GIT_REF_DEFAULT.test(line)) hits.push(`${rel}:${i + 1} ${line.trim().slice(0, 100)}`);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
expect(
|
||||
hits,
|
||||
`raw-SHA fixture reference(s) — these die on branch prune and fail on shallow clones. `
|
||||
+ `Vendor the frozen content under test/fixtures/ instead (v1.75 precedent):\n ${hits.join('\n ')}`,
|
||||
).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -281,9 +281,17 @@ export function carvedSkill(): { skillMd: string; sectionsFrom: string | null }
|
||||
};
|
||||
}
|
||||
|
||||
/** Read the pre-carve verbose monolith plan-ceo SKILL.md from git. */
|
||||
export function verboseSkill(gitRef = 'ab66193e^'): string {
|
||||
return execGit(['show', `${gitRef}:plan-ceo-review/SKILL.md`]);
|
||||
/** Read the pre-carve verbose monolith plan-ceo SKILL.md.
|
||||
* VENDORED fixture (v1.75 precedent), not a git ref: the old default
|
||||
* `git show ab66193e^:...` pinned a BRANCH-LOCAL commit — it dies the day
|
||||
* that branch is pruned and already fails on shallow clones. The fixture
|
||||
* is the frozen pre-cut render; test/git-ref-fixture-tripwire.test.ts
|
||||
* keeps this class from coming back. */
|
||||
export function verboseSkill(): string {
|
||||
return fs.readFileSync(
|
||||
path.join(ROOT, 'test', 'fixtures', 'auq-pre-cut-plan-ceo-review-SKILL.md'),
|
||||
'utf-8',
|
||||
);
|
||||
}
|
||||
|
||||
function execGit(args: string[]): string {
|
||||
|
||||
@@ -129,9 +129,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
// Real-PTY E2E batch (#6 new tests on the harness).
|
||||
// Each one tests behavior the SDK harness can't observe (rendered TTY,
|
||||
// numbered-option lists, multi-phase ordering, idempotency state echo).
|
||||
'preamble-script-ab': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'scripts/resolvers/preamble/generate-brain-sync-block.ts', 'scripts/resolvers/preamble.ts', 'plan-ceo-review/**', 'test/helpers/auq-sdk-capture.ts', 'test/skill-e2e-preamble-script-ab.test.ts'],
|
||||
'auq-format-gate': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/helpers/llm-judge.ts', 'test/skill-e2e-ask-user-question-format-compliance.test.ts'],
|
||||
'auq-repetition-cut-ab': ['scripts/resolvers/preamble/generate-ask-user-format.ts', 'plan-ceo-review/**', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/llm-judge.ts', 'test/fixtures/auq-pre-cut-plan-ceo-review-SKILL.md', 'test/skill-e2e-auq-repetition-cut-ab.test.ts'],
|
||||
'plan-ceo-mode-routing': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-ceo-mode-routing.test.ts'],
|
||||
'plan-design-with-ui-scope': ['plan-design-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-design-with-ui.test.ts'],
|
||||
'ship-idempotency-pty': ['ship/**', 'bin/gstack-next-version', 'bin/gstack-version-bump', 'scripts/resolvers/sections.ts', 'lib/worktree.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-ship-idempotency.test.ts'],
|
||||
@@ -569,8 +567,6 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic'> = {
|
||||
// Real-PTY E2E batch — tier classification:
|
||||
// gate: cheap, deterministic, run on every PR
|
||||
// periodic: long-running or expensive (>$3/run), run weekly
|
||||
'preamble-script-ab': 'periodic', // Phase 1-3 A/B: script vs inline preamble; demoted post-Phase-3 (OV7)
|
||||
'auq-repetition-cut-ab': 'periodic', // AUQ repetition-cut NOT-WORSE gate (passed pre-landing). Periodic runs force EVALS_ALL, so the dep list cannot auto-trigger it — an AUQ format edit carries a MANUAL re-run obligation (bun test test/skill-e2e-auq-repetition-cut-ab.test.ts with EVALS=1 EVALS_TIER=periodic)
|
||||
'auq-format-gate': 'gate', // ~$0.50/run, SDK capture, single skill probe
|
||||
'plan-ceo-mode-routing': 'periodic', // ~$3/run, deep navigation through 8-12 prior AskUserQuestions
|
||||
'plan-design-with-ui-scope': 'gate', // ~$0.80/run
|
||||
|
||||
@@ -1,124 +0,0 @@
|
||||
/**
|
||||
* AUQ no-degradation A/B: pre-cut vs post-cut AskUserQuestion Format — periodic,
|
||||
* paid, SDK capture.
|
||||
*
|
||||
* The AskUserQuestion Format preamble section stated several of its rules more
|
||||
* than once (the completeness rule three times, the auto-decide marker twice,
|
||||
* the tool-not-prose rule three times). The repetition cut removes the
|
||||
* duplicate statements while keeping every floor and all 14 format pins
|
||||
* (Layer 0, auq-format-always-loaded.test.ts, proves presence deterministically).
|
||||
*
|
||||
* The risk under test: repetition may be load-bearing for RUNTIME compliance —
|
||||
* a model may follow rules better because they repeat. This A/B is the gate
|
||||
* that decision rested on (approved 2026-08-25, option A: "the gate outranks
|
||||
* the approval"): identical prompt, two renders, and the post-cut AUQ must be
|
||||
* NOT WORSE than the pre-cut AUQ on format elements and recommendation
|
||||
* substance. Same harness and bar as skill-e2e-auq-verbose-vs-carved-ab.
|
||||
*
|
||||
* - PRE : the pre-cut plan-ceo-review/SKILL.md render, vendored at
|
||||
* test/fixtures/auq-pre-cut-plan-ceo-review-SKILL.md (captured
|
||||
* from branch commit 3263fffe, the last commit before the cut —
|
||||
* vendored because that SHA is branch-local and unreachable from
|
||||
* fresh clones after the squash-merge), with the current
|
||||
* sections/ (the cut touched only the preamble skeleton).
|
||||
* - POST : this worktree's render.
|
||||
*/
|
||||
import { test } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import {
|
||||
setupPlanCeoDir,
|
||||
captureModeSelectionAuq,
|
||||
scoreAuqFormat,
|
||||
carvedSkill,
|
||||
} from './helpers/auq-sdk-capture';
|
||||
import { judgeRecommendation } from './helpers/llm-judge';
|
||||
|
||||
const describeE2E = describeE2ETier('periodic');
|
||||
const runId = `auq-cut-ab-${process.env.EVALS_RUN_ID ?? 'local'}`;
|
||||
const PRE_CUT_FIXTURE = path.join(import.meta.dir, 'fixtures', 'auq-pre-cut-plan-ceo-review-SKILL.md');
|
||||
|
||||
async function grade(label: string, dir: string) {
|
||||
const text = await captureModeSelectionAuq({ planDir: dir, testName: `auq-cut-ab-${label}`, runId });
|
||||
const fmt = scoreAuqFormat(text);
|
||||
// null = judge unavailable. Never coerced to 0: a transient judge failure
|
||||
// on one side must read as INCONCLUSIVE, not as a fabricated degradation
|
||||
// (POST-side failure) or a masked regression (PRE-side failure) — same
|
||||
// taxonomy as armJudge's judge_error cells.
|
||||
let substance: number | null = null;
|
||||
if (text.trim()) {
|
||||
try {
|
||||
const r = await judgeRecommendation(text);
|
||||
substance = r.reason_substance;
|
||||
} catch { /* judge unavailable — recorded as null */ }
|
||||
}
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(
|
||||
`[AUQ-CUT-AB ${label}] captured=${text.length}B format=${fmt.present}/${fmt.total} ` +
|
||||
`missing=[${fmt.missing.join(',')}] substance=${substance ?? 'inconclusive'}`,
|
||||
);
|
||||
return { text, fmt, substance };
|
||||
}
|
||||
|
||||
describeE2E('AUQ no-degradation: repetition cut (periodic)', () => {
|
||||
test(
|
||||
'post-cut AskUserQuestion Format render is not worse than pre-cut on the same prompt',
|
||||
async () => {
|
||||
const post = carvedSkill();
|
||||
const postDir = setupPlanCeoDir({
|
||||
skillMd: post.skillMd,
|
||||
sectionsFrom: post.sectionsFrom,
|
||||
tmpPrefix: 'auq-cut-ab-post-',
|
||||
});
|
||||
const preDir = setupPlanCeoDir({
|
||||
skillMd: fs.readFileSync(PRE_CUT_FIXTURE, 'utf-8'),
|
||||
sectionsFrom: post.sectionsFrom,
|
||||
tmpPrefix: 'auq-cut-ab-pre-',
|
||||
});
|
||||
|
||||
let p, q;
|
||||
try {
|
||||
q = await grade('POST', postDir);
|
||||
p = await grade('PRE', preDir);
|
||||
} finally {
|
||||
fs.rmSync(postDir, { recursive: true, force: true });
|
||||
fs.rmSync(preDir, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
const summary = [
|
||||
`POST: format ${q.fmt.present}/${q.fmt.total}, substance ${q.substance ?? 'inconclusive'}`,
|
||||
`PRE : format ${p.fmt.present}/${p.fmt.total}, substance ${p.substance ?? 'inconclusive'}`,
|
||||
].join('\n');
|
||||
|
||||
if (!q.text.trim() || !p.text.trim()) {
|
||||
throw new Error(
|
||||
`A/B inconclusive — a side produced no AUQ capture:\n${summary}\n` +
|
||||
`--- post ---\n${q.text.slice(0, 2000)}\n--- pre ---\n${p.text.slice(0, 2000)}`,
|
||||
);
|
||||
}
|
||||
|
||||
const formatRegressed = q.fmt.present < p.fmt.present;
|
||||
// Substance compares only when BOTH judge calls succeeded; a null on
|
||||
// either side logs as inconclusive and the format comparison still gates.
|
||||
const substanceComparable = q.substance !== null && p.substance !== null;
|
||||
if (!substanceComparable) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('[AUQ-CUT-AB] substance inconclusive (judge error on at least one side) — format elements still compared.');
|
||||
}
|
||||
const substanceRegressed = substanceComparable && q.substance! < p.substance! - 1; // 1-pt judge tolerance
|
||||
if (formatRegressed || substanceRegressed) {
|
||||
throw new Error(
|
||||
`AUQ DEGRADATION from the repetition cut — the gate outranks the approval; revert the cut:\n${summary}` +
|
||||
(formatRegressed ? `\n -> post-cut dropped: [${q.fmt.missing.join(',')}]` : '') +
|
||||
(substanceRegressed ? `\n -> post-cut substance regressed >1 pt` : '') +
|
||||
`\n--- post AUQ ---\n${q.text}\n--- pre AUQ ---\n${p.text}`,
|
||||
);
|
||||
}
|
||||
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('[AUQ-CUT-AB] NO DEGRADATION:\n' + summary);
|
||||
},
|
||||
600_000,
|
||||
);
|
||||
});
|
||||
@@ -1,17 +1,13 @@
|
||||
/**
|
||||
* Opus 4.7 behavior evals.
|
||||
*
|
||||
* Two cases, both pinned to claude-opus-4-7:
|
||||
* One case, pinned to claude-opus-4-7:
|
||||
*
|
||||
* 1. Fanout rate — the "Fan out explicitly" overlay nudge should make 4.7
|
||||
* spawn parallel tool calls when the prompt has independent sub-problems.
|
||||
* A/B: SKILL.md regenerated with `--model opus-4-7` (overlay ON) vs
|
||||
* default `--model claude` (overlay OFF). Assert A ≥ B on parallel-call
|
||||
* count in the first assistant turn.
|
||||
*
|
||||
* 2. Routing precision — the new "when in doubt, invoke the skill" policy
|
||||
* should route ambiguous dev prompts to the right skill WITHOUT routing
|
||||
* casual/non-dev prompts. A handful of positive and negative controls.
|
||||
* Routing precision — the "when in doubt, invoke the skill" policy should
|
||||
* route ambiguous dev prompts to the right skill WITHOUT routing
|
||||
* casual/non-dev prompts. A handful of positive and negative controls.
|
||||
* (The fanout A/B retired 2026-08 — single-run parallel-call comparison was
|
||||
* a coin flip; the SDK overlay-harness is the maintained instrument.)
|
||||
*
|
||||
* Both cases require a running Anthropic API key. Gated behind EVALS=1.
|
||||
* Classify as `periodic` in touchfiles — behavior measurement, not gate.
|
||||
@@ -177,91 +173,12 @@ describeE2E('Opus 4.7 overlay behavior evals', () => {
|
||||
// concurrent shards.
|
||||
});
|
||||
|
||||
test(
|
||||
'fanout: overlay ON emits >= parallel calls vs overlay OFF on 3-file investigate task',
|
||||
async () => {
|
||||
const armA = mkEvalRoot('on', true);
|
||||
const armB = mkEvalRoot('off', false);
|
||||
// (fanout A/B retired, 2026-08 audit: it compared parallel-call counts of
|
||||
// two SINGLE stochastic runs — parA >= parB is a coin flip with near-zero
|
||||
// remaining information; the overlay-fanout question is answered and the
|
||||
// SDK overlay-harness (test/skill-e2e-overlay-harness.test.ts) is the
|
||||
// maintained instrument for the next experiment.)
|
||||
|
||||
// Populate three tiny independent files in each arm. The prompt asks
|
||||
// the agent to read all three and report. Opus 4.7 (without nudge)
|
||||
// tends to serialize; with the nudge it should parallelize.
|
||||
for (const dir of [armA, armB]) {
|
||||
fs.writeFileSync(path.join(dir, 'alpha.txt'), 'alpha content: 1\n');
|
||||
fs.writeFileSync(path.join(dir, 'beta.txt'), 'beta content: 2\n');
|
||||
fs.writeFileSync(path.join(dir, 'gamma.txt'), 'gamma content: 3\n');
|
||||
}
|
||||
|
||||
const prompt =
|
||||
"Read alpha.txt, beta.txt, and gamma.txt in this directory and report what's inside each. These three reads are independent.";
|
||||
|
||||
try {
|
||||
const [resA, resB] = await Promise.all([
|
||||
runSkillTest({
|
||||
prompt,
|
||||
workingDirectory: armA,
|
||||
maxTurns: 5,
|
||||
allowedTools: ['Read', 'Bash', 'Glob', 'Grep'],
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'fanout-arm-overlay-on',
|
||||
runId,
|
||||
model: OPUS_47,
|
||||
}),
|
||||
runSkillTest({
|
||||
prompt,
|
||||
workingDirectory: armB,
|
||||
maxTurns: 5,
|
||||
allowedTools: ['Read', 'Bash', 'Glob', 'Grep'],
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'fanout-arm-overlay-off',
|
||||
runId,
|
||||
model: OPUS_47,
|
||||
}),
|
||||
]);
|
||||
|
||||
const parA = firstTurnParallelism(resA.transcript);
|
||||
const parB = firstTurnParallelism(resB.transcript);
|
||||
|
||||
console.log(
|
||||
`[opus-4-7 fanout] arm A (overlay ON): ${parA} parallel tool calls in first turn; ` +
|
||||
`arm B (overlay OFF): ${parB}`,
|
||||
);
|
||||
console.log(` cost A=$${resA.costEstimate.estimatedCost.toFixed(2)} B=$${resB.costEstimate.estimatedCost.toFixed(2)}`);
|
||||
|
||||
evalCollector?.addTest({
|
||||
name: 'fanout-arm-overlay-on',
|
||||
suite: 'Opus 4.7 overlay',
|
||||
tier: 'e2e',
|
||||
passed: parA >= parB,
|
||||
duration_ms: resA.duration,
|
||||
cost_usd: resA.costEstimate.estimatedCost,
|
||||
transcript: resA.transcript,
|
||||
output: `parallel=${parA}`,
|
||||
turns_used: resA.costEstimate.turnsUsed,
|
||||
exit_reason: resA.exitReason,
|
||||
});
|
||||
evalCollector?.addTest({
|
||||
name: 'fanout-arm-overlay-off',
|
||||
suite: 'Opus 4.7 overlay',
|
||||
tier: 'e2e',
|
||||
passed: true, // baseline arm, recorded for comparison
|
||||
duration_ms: resB.duration,
|
||||
cost_usd: resB.costEstimate.estimatedCost,
|
||||
transcript: resB.transcript,
|
||||
output: `parallel=${parB}`,
|
||||
turns_used: resB.costEstimate.turnsUsed,
|
||||
exit_reason: resB.exitReason,
|
||||
});
|
||||
|
||||
// Main assertion: overlay arm is at least as parallel as baseline.
|
||||
expect(parA, `overlay arm emitted ${parA} parallel calls, baseline ${parB}`).toBeGreaterThanOrEqual(parB);
|
||||
} finally {
|
||||
fs.rmSync(armA, { recursive: true, force: true });
|
||||
fs.rmSync(armB, { recursive: true, force: true });
|
||||
}
|
||||
},
|
||||
CAPTURE_MS,
|
||||
);
|
||||
|
||||
test(
|
||||
'routing precision: positives route, negatives do not',
|
||||
|
||||
@@ -1,109 +0,0 @@
|
||||
/**
|
||||
* Preamble script-consolidation A/B: inline-bash render vs script render —
|
||||
* gate during token-reduction Phases 1-3 (demote to periodic after Phase 3,
|
||||
* plan OV7), paid, SDK capture.
|
||||
*
|
||||
* Phase 1 moved ~13KB of inline preamble bash per skill into
|
||||
* bin/gstack-skill-start. Layer 0 (test/gstack-skill-start.test.ts) proves the
|
||||
* script emits the same STATUS lines deterministically; THIS proves the model
|
||||
* driven by the slim render still runs the preamble and produces an
|
||||
* equal-quality decision brief on the same prompt.
|
||||
*
|
||||
* Arms (precedent: skill-e2e-auq-verbose-vs-carved-ab.test.ts):
|
||||
* - INLINE : pre-Phase-1 plan-ceo-review/SKILL.md read from git
|
||||
* (29785978 = the v1.69.1.0 bump, the last inline-bash render).
|
||||
* - SCRIPT : this worktree's render, with the fence's install-root bin path
|
||||
* rewritten to THIS WORKTREE's bin/ (plan EOV2: hermetic evals
|
||||
* resolve $HOME/.claude/skills/gstack/bin to the operator
|
||||
* install, which would silently exercise the degraded path;
|
||||
* the rewrite makes the branch's script the subject under test).
|
||||
*
|
||||
* Both arms pin GSTACK_HOME to the fixture dir (EOV7: onboarding state is
|
||||
* hermetic now that the script honors GSTACK_HOME).
|
||||
*/
|
||||
import { test, expect } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { execSync } from 'node:child_process';
|
||||
import {
|
||||
setupPlanCeoDir,
|
||||
captureModeSelectionAuq,
|
||||
scoreAuqFormat,
|
||||
carvedSkill,
|
||||
} from './helpers/auq-sdk-capture';
|
||||
|
||||
const describeE2E = describeE2ETier('periodic');
|
||||
const runId = `preamble-ab-${process.env.EVALS_RUN_ID ?? 'local'}`;
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const INLINE_REF = '29785978'; // last pre-Phase-1 commit (v1.69.1.0 bump)
|
||||
|
||||
function inlineSkill(): string {
|
||||
return execSync(`git show ${INLINE_REF}:plan-ceo-review/SKILL.md`, {
|
||||
// LIVE-REPO CWD: git show needs this repo's history to read the
|
||||
// pre-Phase-1 SKILL.md render at INLINE_REF.
|
||||
cwd: ROOT,
|
||||
encoding: 'utf-8',
|
||||
maxBuffer: 8 * 1024 * 1024,
|
||||
timeout: 30_000,
|
||||
});
|
||||
}
|
||||
|
||||
/** EOV2 redirection: point the fence at the worktree's bin. */
|
||||
function scriptSkillWorktreeBin(): string {
|
||||
const current = carvedSkill();
|
||||
const rewritten = current.skillMd.replaceAll(
|
||||
'$HOME/.claude/skills/gstack/bin/gstack-skill-start',
|
||||
path.join(ROOT, 'bin', 'gstack-skill-start'),
|
||||
);
|
||||
if (!rewritten.includes(path.join(ROOT, 'bin', 'gstack-skill-start'))) {
|
||||
throw new Error('binDir rewrite matched nothing — fence shape changed; update the A/B redirection');
|
||||
}
|
||||
return rewritten;
|
||||
}
|
||||
|
||||
async function grade(label: string, dir: string) {
|
||||
const text = await captureModeSelectionAuq({ planDir: dir, testName: `preamble-ab-${label}`, runId });
|
||||
const fmt = scoreAuqFormat(text);
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(`[PREAMBLE-AB ${label}] captured=${text.length}B format=${fmt.present}/${fmt.total} missing=[${fmt.missing.join(',')}]`);
|
||||
return { text, fmt };
|
||||
}
|
||||
|
||||
describeE2E('Preamble consolidation no-degradation: inline bash vs script (gate)', () => {
|
||||
test(
|
||||
'script-render plan-ceo-review AUQ is not worse than the inline-bash render on the same prompt',
|
||||
async () => {
|
||||
const sections = carvedSkill().sectionsFrom;
|
||||
const scriptDir = setupPlanCeoDir({
|
||||
skillMd: scriptSkillWorktreeBin(),
|
||||
sectionsFrom: sections,
|
||||
tmpPrefix: 'preamble-ab-script-',
|
||||
});
|
||||
const inlineDir = setupPlanCeoDir({
|
||||
skillMd: inlineSkill(),
|
||||
sectionsFrom: sections,
|
||||
tmpPrefix: 'preamble-ab-inline-',
|
||||
});
|
||||
|
||||
let s, i;
|
||||
try {
|
||||
s = await grade('SCRIPT', scriptDir);
|
||||
i = await grade('INLINE', inlineDir);
|
||||
} finally {
|
||||
fs.rmSync(scriptDir, { recursive: true, force: true });
|
||||
fs.rmSync(inlineDir, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
// Both arms must produce a capture at all (an empty script-arm capture
|
||||
// means the preamble derailed the workflow — exactly the regression this
|
||||
// guards against).
|
||||
expect(s.text.length).toBeGreaterThan(100);
|
||||
expect(i.text.length).toBeGreaterThan(100);
|
||||
// Relative parity: the script render is NOT WORSE on decision-brief
|
||||
// format elements (absolute compliance is auq-format-gate's job).
|
||||
expect(s.fmt.present).toBeGreaterThanOrEqual(i.fmt.present);
|
||||
},
|
||||
20 * 60 * 1000,
|
||||
);
|
||||
});
|
||||
@@ -109,12 +109,8 @@ describe('selectTests', () => {
|
||||
expect(result.selected).toContain('plan-ceo-split-overflow');
|
||||
// v2 plan Phase B carve: the section-loading E2E depends on plan-ceo-review/**.
|
||||
expect(result.selected).toContain('plan-ceo-section-loading');
|
||||
// Token-reduction Phase 1: the preamble script A/B also keys on plan-ceo-review/**.
|
||||
expect(result.selected).toContain('preamble-script-ab');
|
||||
// AUQ repetition-cut NOT-WORSE gate drives plan-ceo-review, so it keys on it too.
|
||||
expect(result.selected).toContain('auq-repetition-cut-ab');
|
||||
expect(result.selected.length).toBe(23);
|
||||
expect(result.skipped.length).toBe(Object.keys(E2E_TOUCHFILES).length - 23);
|
||||
expect(result.selected.length).toBe(21);
|
||||
expect(result.skipped.length).toBe(Object.keys(E2E_TOUCHFILES).length - 21);
|
||||
});
|
||||
|
||||
test('global touchfile triggers ALL tests', () => {
|
||||
|
||||
Reference in New Issue
Block a user