diff --git a/test/helpers/ship-measure-seeded-fixture.ts b/test/helpers/ship-measure-seeded-fixture.ts new file mode 100644 index 000000000..b9703bec8 --- /dev/null +++ b/test/helpers/ship-measure-seeded-fixture.ts @@ -0,0 +1,165 @@ +/** + * Fixture project for the seeded-flake measure-loop case (A6, CEO-3, ENG-13). + * + * A tiny repo with no remote whose CLAUDE.md documents a free test command + * (`./test.sh`, passes instantly), a single-case eval command + * (`./evals.sh case `) and a gate command (`./evals.sh gate`), all local + * stubs that cost nothing. The rule case `queue-priorities` reads a task list + * that storage returns in seed-dependent order and compares it to the sorted + * expectation. Each `case` invocation claims the lowest free trial slot with + * an atomic `mkdir` (slots reset per ship-measure round, keyed on the round + * directory), and the slot is the seed: while src/priorities.js leaves the + * list unsorted, slots whose number mod 10 is 2, 5 or 9 fail (3 of 10). The + * failure prints `failure_cause` and `failure_detail` naming the line. The + * deterministic fix sorts the list; an unrelated edit still measures 7 of 10. + * The gate runs every case once at a seed that fails while unsorted, so the + * first gate run is red. Every invocation appends one line to + * SEEDED_FLAKE_LOG, which the case reads. + */ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { spawnSync } from 'node:child_process'; + +export const SEEDED_CASE = 'queue-priorities'; +export const STABLE_CASE = 'queue-empty'; +/** The line of src/priorities.js that returns the list unsorted. */ +export const SEEDED_LINE = 6; + +const PRIORITIES = `const { loadTasks } = require('./tasks'); + +// Dashboard order: highest priority first. +function priorities(seed) { + const tasks = loadTasks(seed); + return tasks.map(task => task.priority); +} + +// Label shown above the list. +const TITLE = 'Queue'; + +module.exports = { priorities, TITLE }; +`; + +const TASKS = `// Storage returns tasks in no guaranteed order; the seed models that order. +const STORED = [9, 7, 5, 3, 1].map(priority => ({ priority })); + +function loadTasks(seed) { + if ([2, 5, 9].includes(seed % 10)) return [...STORED.slice(2), ...STORED.slice(0, 2)]; + return [...STORED]; +} + +module.exports = { loadTasks }; +`; + +const CHECK = `// Eval case ${SEEDED_CASE}: the dashboard list is sorted, highest first. +const { priorities } = require('../src/priorities'); +const seed = Number(process.argv[2] || 0); +const sortedDesc = list => [...list].sort((a, b) => b - a); +const got = priorities(seed); +const want = sortedDesc(got); +console.log('fixture_sorted: ' + (JSON.stringify(priorities(2)) === JSON.stringify(sortedDesc(priorities(2))) ? 1 : 0)); +console.log('cost_usd: 0'); +if (JSON.stringify(got) !== JSON.stringify(want)) { + console.log('failure_cause: assertion'); + console.log('failure_detail: src/priorities.js:${SEEDED_LINE} returns loadTasks(seed) in storage order, which seed ' + seed + ' shuffles; expected ' + JSON.stringify(want) + ' received ' + JSON.stringify(got)); + process.exit(1); +} +console.log('PASS ${SEEDED_CASE} (seed ' + seed + ')'); +`; + +const EVALS = `#!/usr/bin/env bash +# Local eval stub. ./evals.sh case runs one trial of one case; +# ./evals.sh gate runs every case once (the full gate). +set -u +here=$(cd "$(dirname "$0")" && pwd) +log="\${SEEDED_FLAKE_LOG:-$here/.evals/runs.log}" +mkdir -p "$here/.evals" "$(dirname "$log")" +run() { # case-id seed + if [ "$1" = ${STABLE_CASE} ]; then echo "cost_usd: 0"; echo "PASS ${STABLE_CASE}"; echo "fixture_sorted: -"; return 0; fi + node "$here/test/priorities.check.js" "$2" +} +sorted_of() { sed -n 's/^fixture_sorted: //p' <<<"$1" | head -1; } +case "\${1:-}" in + case) + id="\${2:-}" + case "$id" in ${SEEDED_CASE}|${STABLE_CASE}) ;; *) echo "unknown case: $id (cases: ${SEEDED_CASE}, ${STABLE_CASE})" >&2; exit 2 ;; esac + if [ -n "\${GSTACK_SHIP_MEASURE_DIR:-}" ]; then slots="$(dirname "$GSTACK_SHIP_MEASURE_DIR")/slots"; else slots="$here/.evals/slots"; fi + mkdir -p "$slots" + n=1; until mkdir "$slots/$n" 2>/dev/null; do n=$((n + 1)); done + out=$(run "$id" "$n" 2>&1); rc=$? + printf '%s\\n' "$out" + echo "case id=$id slot=$n sorted=$(sorted_of "$out") result=$([ $rc = 0 ] && echo pass || echo fail) measure_dir=\${GSTACK_SHIP_MEASURE_DIR:--}" >>"$log" + exit $rc ;; + gate) + rc=0; sorted=- + for id in ${SEEDED_CASE} ${STABLE_CASE}; do + out=$(run "$id" 5 2>&1); r=$? + [ "$id" = ${SEEDED_CASE} ] && sorted=$(sorted_of "$out") + echo "[$id] $([ $r = 0 ] && echo PASS || echo FAIL)" + [ $r = 0 ] || { printf '%s\\n' "$out" | sed 's/^/ /'; rc=1; } + done + echo "gate: $([ $rc = 0 ] && echo green || echo red)" + echo "gate sorted=$sorted result=$([ $rc = 0 ] && echo pass || echo fail)" >>"$log" + exit $rc ;; + *) echo "usage: ./evals.sh case | ./evals.sh gate" >&2; exit 2 ;; +esac +`; + +const CLAUDE_MD = `# queue-dash + +Dashboard that lists queued tasks, highest priority first. + +## Commands + +- Tests (free, fast): \`./test.sh\` +- Evals (local stubs in this repo; each run prints cost_usd): + - One case alone: \`./evals.sh case \` + - Full gate: \`./evals.sh gate\` + +Eval cases: \`${SEEDED_CASE}\` (kind: rule), \`${STABLE_CASE}\` (kind: rule). +`; + +export interface SeededFixture { repo: string; log: string } + +export interface FixtureRun { kind: 'case' | 'gate'; id?: string; slot?: number; sorted: string; result: 'pass' | 'fail'; measureDir?: string } + +/** Create the fixture repo at /queue-dash, committed and clean. */ +export function createSeededFixture(root: string, log = path.join(root, 'runs.log')): SeededFixture { + const repo = path.join(root, 'queue-dash'); + for (const dir of ['src', 'test']) fs.mkdirSync(path.join(repo, dir), { recursive: true }); + fs.writeFileSync(path.join(repo, 'src/priorities.js'), PRIORITIES.replace(/\n\/\/ Label shown above the list\.\nconst TITLE = 'Queue';\n/, '').replace('{ priorities, TITLE }', '{ priorities }')); + fs.writeFileSync(path.join(repo, 'src/tasks.js'), TASKS); + fs.writeFileSync(path.join(repo, 'test/priorities.check.js'), CHECK); + fs.writeFileSync(path.join(repo, 'evals.sh'), EVALS, { mode: 0o755 }); + fs.writeFileSync(path.join(repo, 'test.sh'), '#!/usr/bin/env bash\necho "unit: 3 passed"\n', { mode: 0o755 }); + fs.writeFileSync(path.join(repo, 'CLAUDE.md'), CLAUDE_MD); + fs.writeFileSync(path.join(repo, 'VERSION'), '0.3.1\n'); + const git = (...args: string[]) => { + const r = spawnSync('git', ['-c', 'user.email=test@example.invalid', '-c', 'user.name=Test', ...args], { cwd: repo, encoding: 'utf8', timeout: 10_000 }); + if (r.status !== 0) throw new Error(`git ${args.join(' ')}: ${r.stderr}`); + }; + git('init', '-q', '-b', 'main'); + fs.appendFileSync(path.join(repo, '.git/info/exclude'), '.evals/\n.context/\n'); + git('add', '.'); + git('commit', '-q', '-m', 'queue dashboard'); + git('checkout', '-q', '-b', 'feature/queue-title'); + fs.writeFileSync(path.join(repo, 'src/priorities.js'), PRIORITIES); + git('commit', '-q', '-am', 'show a title above the queue'); + return { repo, log }; +} + +export function readFixtureRuns(log: string): FixtureRun[] { + if (!fs.existsSync(log)) return []; + return fs.readFileSync(log, 'utf8').split('\n').filter(Boolean).map(line => { + const field = (name: string) => new RegExp(`(?:^| )${name}=(\\S+)`).exec(line)?.[1]; + const kind = line.startsWith('gate ') ? 'gate' : 'case'; + return { + kind, sorted: field('sorted') ?? '-', result: field('result') === 'pass' ? 'pass' : 'fail', + ...(kind === 'case' ? { id: field('id'), slot: Number(field('slot')), measureDir: field('measure_dir') } : {}), + } as FixtureRun; + }); +} + +/** The repaired source: the deterministic fix sorts the list. */ +export const SORTED_FIX = PRIORITIES.replace('return tasks.map(task => task.priority);', 'return tasks.map(task => task.priority).sort((a, b) => b - a);'); +/** A negative-control "fix" that edits an unrelated line. */ +export const UNRELATED_EDIT = PRIORITIES.replace("const TITLE = 'Queue';", "const TITLE = 'Task queue';"); diff --git a/test/helpers/touchfiles-data.ts b/test/helpers/touchfiles-data.ts index b9c90fc53..92708be65 100644 --- a/test/helpers/touchfiles-data.ts +++ b/test/helpers/touchfiles-data.ts @@ -389,6 +389,7 @@ export const E2E_TOUCHFILES: Record = { 'tpa-apple-ban': [ 'scripts/resolvers/third-party-actions.ts', 'ship/SKILL.md.tmpl', 'ship/sections/apple-release.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-third-party-actions.test.ts', 'test/helpers/third-party-actions.ts', 'lib/eval-model.ts' ], + 'ship-measure-seeded-flake': ['ship/**', 'scripts/ship-measure.ts', 'scripts/test-free-shards.ts', 'scripts/lib/shard-engine.ts', 'scripts/lib/paid-select.ts', 'scripts/lib/free-home-guard.ts', 'bin/gstack-config', 'bin/gstack-state-root.sh', 'lib/state-root.ts', 'scripts/resolvers/sections.ts', 'scripts/resolvers/testing.ts', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-ship-measure-loop.test.ts', 'test/helpers/ship-measure-seeded-fixture.ts', 'test/helpers/agent-sdk-runner.ts', 'test/helpers/skill-fixture.ts'], 'ship-section-loading': [ 'ship/**', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-ship-section-loading.test.ts', 'scripts/resolvers/testing.ts', 'test/helpers/agent-sdk-runner.ts', 'test/helpers/auq-native-capture.ts', 'test/helpers/auto-decision-state.ts', 'test/helpers/autoplan-artifact-digest.ts', 'test/helpers/autoplan-artifact-permission.ts', 'test/helpers/autoplan-artifact-recorder.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty/**', 'test/helpers/dx-selected-navigation.ts', 'test/helpers/e2e-gate.ts', 'test/helpers/eng-cache-writer-decision.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/helpers/llm-judge.ts', 'test/helpers/native-auto-decide.ts', 'test/helpers/owned-claude-transcript.ts', 'test/helpers/plan-count-artifacts.ts', 'test/helpers/plan-count-file-permission.ts', 'test/helpers/plan-count-fixture.ts', 'test/helpers/plan-count-pending-exit.ts', 'test/helpers/plan-count-pending-question.ts', 'test/helpers/plan-count-transcript.ts', 'test/helpers/plan-floor-review.ts', 'test/helpers/plan-floor-target.ts', 'test/helpers/plan-scope-selection.ts', 'test/helpers/plan-seed-submission.ts', 'test/helpers/plan-skill-question-events.ts', 'test/helpers/plan-skill-question-hook-scope.ts', 'test/helpers/plan-skill-questions.ts', 'test/helpers/pty-screen.ts', 'test/helpers/pty-trust-dialog.ts', 'test/helpers/skill-census.ts'], 'plan-ceo-section-loading': [ @@ -1262,6 +1263,7 @@ export const E2E_TIERS: Record = { 'tpa-absent-darwin': 'gate', // consent/credential safety guardrail; deterministic shims + grep asserts 'tpa-apple-ban': 'gate', // consent/credential safety guardrail; deterministic shims + grep asserts + 'ship-measure-seeded-flake': 'periodic', // free stub evals; real /ship measure loop (A6) 'ship-section-loading': 'periodic', // ~$3/run, real /ship; asserts section reads 'plan-ceo-section-loading': 'periodic', // ~$3-5/run, real /plan-ceo-review; asserts section read 'carve-section-loading-browse': 'periodic', @@ -1714,6 +1716,7 @@ export const E2E_KINDS: Record = { 'tpa-broken': 'rule', 'tpa-absent-darwin': 'rule', 'tpa-apple-ban': 'rule', + 'ship-measure-seeded-flake': 'behavior', 'ship-section-loading': 'rule', 'plan-ceo-section-loading': 'rule', 'carve-section-loading-browse': 'rule', @@ -1893,6 +1896,8 @@ export const E2E_KINDS: Record = { * E2E_KINDS; values are non-empty. */ export const BEHAVIOR_WHY: Record = { + 'ship-measure-seeded-flake': + "How the live model words its classification and fix can vary run to run; measuring before fixing, re-measuring at target and running the gate once after stay assertions, and answering only the spend question is a contract.", 'shared-libs-opportunity-judgment': "Whether a candidate extraction is worth recommending is a judgment call; the read-only invariant stays a contract.", 'context-restore-provenance-order': diff --git a/test/ship-measure-seeded-fixture.test.ts b/test/ship-measure-seeded-fixture.test.ts new file mode 100644 index 000000000..47afd99ef --- /dev/null +++ b/test/ship-measure-seeded-fixture.test.ts @@ -0,0 +1,63 @@ +/** + * Free checks of the seeded-flake fixture the paid case drives (CEO-3, + * ENG-13), measured through the real scripts/ship-measure.ts CLI: unsorted it + * measures 7 of 10 with failures that name the line, parallel trials claim + * unique slots that reset per round, an unrelated edit still measures 7 of 10 + * and leaves the gate red, and only the sorting fix measures 10 of 10 and turns + * the gate green. + */ +import { afterAll, describe, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { SEEDED_CASE, SEEDED_LINE, SORTED_FIX, UNRELATED_EDIT, createSeededFixture, readFixtureRuns } from './helpers/ship-measure-seeded-fixture'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const root = fs.mkdtempSync(path.join(os.tmpdir(), 'seeded-flake-')); +afterAll(() => fs.rmSync(root, { recursive: true, force: true })); + +describe('seeded-flake fixture (free)', () => { + test('measures 7/10 unsorted, 7/10 after an unrelated edit, 10/10 after the sort; the gate follows', () => { + const { repo, log } = createSeededFixture(root); + const state = path.join(root, 'state'); + const env = { ...process.env, GSTACK_HOME: state, GSTACK_STATE_ROOT: state, SEEDED_FLAKE_LOG: log }; + const measure = (round: string) => spawnSync(process.execPath, ['run', path.join(ROOT, 'scripts/ship-measure.ts'), 'measure', '--case', SEEDED_CASE, + '--round', round, '--command', './evals.sh case {case}', '--cost-per-trial', '0', '--jobs', '10'], { cwd: repo, env, encoding: 'utf8', timeout: 120_000 }); + const gate = () => spawnSync('./evals.sh', ['gate'], { cwd: repo, env, encoding: 'utf8', timeout: 30_000 }); + const roundDir = (round: string) => path.join(repo, '.context/ship-measure', SEEDED_CASE, round); + + expect(spawnSync('git', ['status', '--porcelain'], { cwd: repo, encoding: 'utf8', timeout: 10_000 }).stdout).toBe(''); + expect(gate().status).toBe(1); + + const baseline = measure('baseline'); + expect(baseline.stdout).toContain(`DIAGNOSTIC ${SEEDED_CASE} baseline: observed 7/10 trials`); + expect(baseline.status).toBe(1); + const failed = fs.readdirSync(roundDir('baseline')).filter(t => /^t\d+$/.test(t)) + .map(t => JSON.parse(fs.readFileSync(path.join(roundDir('baseline'), t, 'trial.json'), 'utf8'))).filter(t => !t.passed); + expect(failed).toHaveLength(3); + for (const trial of failed) { + expect(trial.failureCause).toBe('assertion'); + expect(trial.failureDetail).toContain(`src/priorities.js:${SEEDED_LINE} returns loadTasks(seed) in storage order`); + } + const slotsOf = (dir: string) => readFixtureRuns(log).filter(r => r.kind === 'case' && r.measureDir?.startsWith(dir)).map(r => r.slot).sort((a, b) => a! - b!); + expect(slotsOf(roundDir('baseline'))).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + expect(readFixtureRuns(log).filter(r => r.kind === 'case' && r.result === 'fail').map(r => r.slot).sort()).toEqual([2, 5, 9]); + + fs.writeFileSync(path.join(repo, 'src/priorities.js'), UNRELATED_EDIT); + const control = measure('round-1'); + expect(control.stdout).toContain(`DIAGNOSTIC ${SEEDED_CASE} round-1: observed 7/10 trials`); + expect(control.status).toBe(1); + expect(slotsOf(roundDir('round-1'))).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + expect(gate().status).toBe(1); + + fs.writeFileSync(path.join(repo, 'src/priorities.js'), SORTED_FIX); + const fixed = measure('round-2'); + expect(fixed.stdout).toContain(`DIAGNOSTIC ${SEEDED_CASE} round-2: observed 10/10 trials`); + expect(fixed.status).toBe(0); + expect(gate().status).toBe(0); + const runs = readFixtureRuns(log); + expect(runs.filter(r => r.kind === 'gate').map(r => `${r.sorted}:${r.result}`)).toEqual(['0:fail', '0:fail', '1:pass']); + expect(runs.filter(r => r.kind === 'case' && r.measureDir?.startsWith(roundDir('round-2'))).every(r => r.sorted === '1' && r.result === 'pass')).toBe(true); + }); +}); diff --git a/test/skill-e2e-ship-measure-loop.test.ts b/test/skill-e2e-ship-measure-loop.test.ts new file mode 100644 index 000000000..159f3a363 --- /dev/null +++ b/test/skill-e2e-ship-measure-loop.test.ts @@ -0,0 +1,141 @@ +/** + * Paid E2E (periodic, behavior): /ship's measure-then-fix loop on a seeded + * flaky eval (A6; CEO-3, ENG-13). The fixture (test/helpers/ + * ship-measure-seeded-fixture.ts) is a repo with no remote whose CLAUDE.md + * documents a single-case eval command and a gate command, both free local + * stubs. Steps 1-5 are recorded as done and the first gate run is red on + * queue-priorities, which fails trial slots 2, 5 and 9 (3 of 10) until + * src/priorities.js sorts its list. The session gets the generated Step 6 and + * the measure section, and must measure the case alone (at least 10 trials, + * unsorted), fix the named line, re-measure at least 10 trials against the + * sorted file with every trial passing, and only then run the gate command + * once. It never gets as far as version, push or PR steps. + * + * Declared actor interface: the only question it may answer is the spend + * question (cost, budget or estimate), which it approves. Any other question + * gets its first option so the session can continue, and is a contract + * violation. + */ +import { afterAll, expect } from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { passThroughNonAskUserQuestion, resolveClaudeBinary, runAgentSdkTest, toSkillTestResult } from './helpers/agent-sdk-runner'; +import { expectContract } from './helpers/eval-store'; +import { ROOT, runId, describeIfSelected, testConcurrentIfSelected, logCost, recordE2E, createEvalCollector, finalizeEvalCollector } from './helpers/e2e-helpers'; +import { CAPTURE_LONG_MS, SHARD_RESERVE_MS } from './helpers/eval-budgets'; +import { sliceBetween } from './helpers/skill-fixture'; +import { SEEDED_CASE, createSeededFixture, readFixtureRuns } from './helpers/ship-measure-seeded-fixture'; + +const CASE = 'ship-measure-seeded-flake'; +const collector = createEvalCollector('e2e-ship-measure-loop'); +const SPEND_QUESTION = /spend|cost|budget|estimate|\$/i; + +describeIfSelected('/ship measure-then-fix loop on a seeded flaky eval', ['ship-measure-seeded-flake'], () => { + const dirs: string[] = []; + afterAll(() => { + for (const dir of dirs) fs.rmSync(dir, { recursive: true, force: true }); + finalizeEvalCollector(collector); + }); + + testConcurrentIfSelected('ship-measure-seeded-flake', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ship-measure-loop-')); + dirs.push(root); + const { repo, log } = createSeededFixture(root); + const state = path.join(root, 'state'); + const bin = path.join(root, 'bin'); + fs.mkdirSync(state, { recursive: true }); + fs.mkdirSync(bin, { recursive: true }); + fs.writeFileSync(path.join(bin, 'gh'), `#!/usr/bin/env bash\necho "$*" >> "${root}/gh-calls"\necho "gh: not available in this fixture" >&2\nexit 1\n`, { mode: 0o755 }); + const env = { GSTACK_HOME: state, GSTACK_STATE_ROOT: state, SEEDED_FLAKE_LOG: log, PATH: `${bin}${path.delimiter}${process.env.PATH ?? ''}` }; + + // The gate already ran once and was red; that run is setup, not the session's. + const red = spawnSync('./evals.sh', ['gate'], { cwd: repo, encoding: 'utf8', timeout: 30_000, env: { ...process.env, SEEDED_FLAKE_LOG: path.join(root, 'setup.log') } }); + if (red.status !== 1) throw new Error(`fixture gate should start red: ${red.stdout}${red.stderr}`); + + const local = (text: string) => text.replaceAll('~/.claude/skills/gstack/', `${ROOT}/`); + const tests = fs.readFileSync(path.join(ROOT, 'ship/sections/tests.md'), 'utf8'); + fs.writeFileSync(path.join(root, 'ship-step6.md'), local(sliceBetween(tests, '## Step 6: Eval Suites', '\n---\n'))); + fs.writeFileSync(path.join(root, 'ship-measure.md'), local(fs.readFileSync(path.join(ROOT, 'ship/sections/measure.md'), 'utf8'))); + fs.writeFileSync(path.join(root, 'ship-state.md'), `# /ship state for feature/queue-title (Steps 1-5 complete) + +- Base branch: main. No remote is configured; this run never pushes or opens a PR. +- Step 5: \`./test.sh\` passed (unit: 3 passed). +- Step 6 selection: CLAUDE.md documents the gate command \`./evals.sh gate\`; it covers every eval case. +- Step 6 item 2: the gate ran once and was red: + +\`\`\` +${red.stdout.trim()} +\`\`\` +`); + + const unexpected: string[] = []; + const binary = resolveClaudeBinary(); + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(new Error(`${CASE} exceeded ${CAPTURE_LONG_MS}ms`)), CAPTURE_LONG_MS); + let result; + try { + result = await runAgentSdkTest({ + systemPrompt: { type: 'preset', preset: 'claude_code' }, + userPrompt: `You are running the /ship skill in this repository (${repo}, branch feature/queue-title). Steps 1-5 are complete; their results are in ${path.join(root, 'ship-state.md')}. Read ${path.join(root, 'ship-step6.md')} and continue Step 6 at item 3, "Check results", with the red gate result recorded there; it sends you to ${path.join(root, 'ship-measure.md')}, which you follow. Stop after Step 6 ends, either with the full gate command passing or at the loop's named-red stop, and report what you measured and changed. Do not bump VERSION, edit CHANGELOG, push or open a PR.`, + workingDirectory: repo, + env, + maxTurns: 60, + maxRetries: 0, + allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'], + signal: controller.signal, + sessionBudgetMs: CAPTURE_LONG_MS, + testName: 'ship-measure-seeded-flake', + runId, + ...(binary ? { pathToClaudeCodeExecutable: binary } : {}), + canUseTool: async (toolName, input) => { + if (toolName !== 'AskUserQuestion') return passThroughNonAskUserQuestion(toolName, input); + const questions = input.questions as Array<{ question: string; options: Array<{ label: string }> }>; + const answers: Record = {}; + for (const q of questions) { + if (!SPEND_QUESTION.test(q.question)) unexpected.push(q.question); + answers[q.question] = (q.options.find(o => /approve|yes|run|proceed|continue/i.test(o.label)) ?? q.options[0]!).label; + } + return { behavior: 'allow', updatedInput: { questions, answers } }; + }, + }); + } finally { + clearTimeout(timer); + } + const skillResult = toSkillTestResult(result); + logCost('/ship measure loop', skillResult); + + const runs = readFixtureRuns(log); + const seeded = runs.map((run, index) => ({ ...run, index })).filter(run => run.kind === 'case' && run.id === SEEDED_CASE); + const gates = runs.map((run, index) => ({ ...run, index })).filter(run => run.kind === 'gate'); + const firstSorted = seeded.find(run => run.sorted === '1')?.index ?? Infinity; + const lastUnsorted = Math.max(-1, ...seeded.filter(run => run.sorted !== '1').map(run => run.index)); + const baseline = seeded.filter(run => run.index < firstSorted); + const closing = seeded.filter(run => run.index > lastUnsorted); + const tenthClosing = closing[9]?.index ?? Infinity; + const changed = (file: string) => spawnSync('git', ['diff', '--quiet', 'HEAD', '--', file], { cwd: repo, timeout: 10_000 }).status !== 0; + const fixedEverySeed = spawnSync('node', ['-e', `const {priorities}=require('./src/priorities');for(let s=0;s<20;s++){const g=priorities(s);if(JSON.stringify(g)!==JSON.stringify([...g].sort((a,b)=>b-a)))process.exit(1)}`], + { cwd: repo, timeout: 10_000 }).status === 0; + const checks = { + measuredAloneBeforeFixing: baseline.length >= 10, + fixedTheNamedLine: changed('src/priorities.js') && fixedEverySeed, + evalUntouched: !changed('src/tasks.js') && !changed('test/priorities.check.js') && !changed('evals.sh'), + closingMeasurementAtTarget: closing.length >= 10 && closing.slice(0, 10).every(run => run.result === 'pass'), + gateOnceAfterRemeasure: gates.length === 1 && gates[0]!.result === 'pass' && gates[0]!.index > tenthClosing, + noRelease: !fs.existsSync(path.join(root, 'gh-calls')) && fs.readFileSync(path.join(repo, 'VERSION'), 'utf8') === '0.3.1\n', + }; + const passed = result.exitReason === 'success' && Object.values(checks).every(Boolean) && unexpected.length === 0; + console.log(`${CASE}: ${JSON.stringify({ checks, baseline: baseline.length, closing: closing.length, gates: gates.map(g => `${g.sorted}:${g.result}`), unexpected })}`); + recordE2E(collector, CASE, '/ship measure loop', skillResult, { + passed, + ...(passed ? {} : { error: `checks failed: ${JSON.stringify({ exit: result.exitReason, checks, unexpected })}`.slice(0, 300) }), + }); + expectContract(unexpected.length === 0, `asked a question outside the declared spend question: ${unexpected.join(' | ').slice(0, 200)}`, { collector, name: CASE }); + expect(result.exitReason).toBe('success'); + expect(checks).toEqual({ + measuredAloneBeforeFixing: true, fixedTheNamedLine: true, evalUntouched: true, + closingMeasurementAtTarget: true, gateOnceAfterRemeasure: true, noRelease: true, + }); + }, CAPTURE_LONG_MS + SHARD_RESERVE_MS); +});