diff --git a/browse/src/terminal-agent.ts b/browse/src/terminal-agent.ts index c5417bd38..88e89a719 100644 --- a/browse/src/terminal-agent.ts +++ b/browse/src/terminal-agent.ts @@ -301,11 +301,12 @@ function buildTabAwarenessHint(stateDir: string): string { 'You are running inside the gstack browser sidebar with live access to the user\'s browser tabs.', '', 'Tab state files (kept fresh automatically by the extension):', - ` ${tabsFile} — all open tabs (id, url, title, active, pinned)`, - ` ${activeFile} — the currently active tab`, - 'Read these any time the user asks about "tabs", "the current page", or anything multi-tab. Do NOT shell out to $B tabs just to learn what\'s open — read the file.', + ` ${tabsFile} — all open tabs: tabs[] of {tabId, url, title, active, pinned, windowId}`, + ` ${activeFile} — the currently active tab: {tabId, url, title}`, + 'Read these any time the user asks about "tabs", "the current page", or anything multi-tab. They\'re kept current, so read them instead of running $B tabs when you only need to know what\'s open.', '', 'Tab manipulation commands (via $B):', + ' ($B is the gstack browse CLI. This shell does not set it, so resolve it the way the /browse skill does: the project\'s .claude/skills/gstack/browse/dist/browse if present, else ~/.claude/skills/gstack/browse/dist/browse.)', ' $B tab — switch to a tab', ' $B newtab [url] — open a new tab', ' $B closetab [id] — close a tab (current if no id)', diff --git a/scripts/typecheck-test-baseline.json b/scripts/typecheck-test-baseline.json index a1477b4ef..ee19e3b9f 100644 --- a/scripts/typecheck-test-baseline.json +++ b/scripts/typecheck-test-baseline.json @@ -485,7 +485,6 @@ "test/skill-e2e-shared-libs-paths.test.ts\tTS2769\tNo overload matches this call. The last overload gave the following error. Argument of type 'unknown' is not assignable to parameter of type 'string'.": 1, "test/skill-e2e-ship-docsync.test.ts\tTS2345\tArgument of type 'EvalCollector | null' is not assignable to parameter of type 'EvalCollector'. Type 'null' is not assignable to type 'EvalCollector'.": 9, "test/skill-e2e-third-party-actions.test.ts\tTS2345\tArgument of type '{ maxTurns: 8; allowedTools: readonly ['Read', 'Bash']; timeout: 240000; runId: string; env: { PATH: string; }; prompt: string; workingDirectory: string; testName: string; }' is not assignable to parameter of type '{ prompt: string; workingDirectory: string; maxTurns?: number | undefined; appendSystemPrompt?: string | undefined; completionReserveMs?: number | undefined; allowedTools?: string[] | undefined; ... 9 more ...; nativeLifecycle?: { ...; } | undefined; }'. Types of property 'allowedTools' are incompatible. The type 'readonly [\"Read\", \"Bash\"]' is 'readonly' and cannot be assigned to the mutable type 'string[]'.": 5, - "test/skill-e2e-triage.test.ts\tTS2353\tObject literal may only specify known properties, and 'has_in_branch_classification' does not exist in type 'Partial'.": 1, "test/skill-routing-e2e.test.ts\tTS2345\tArgument of type '\"e2e-routing\"' is not assignable to parameter of type '\"e2e\" | \"llm-judge\"'.": 1, "test/strict-output.test.ts\tTS2739\tType '{ failedTests: number; unhandledBetweenTests: number; terminalFileCounts: number[]; }' is missing the following properties from type 'BunTestOutputSummary': terminalTestCounts, skippedTests, passedTests": 5, "test/test-free-shards.test.ts\tTS2345\tArgument of type 'number | ReadableStream> | undefined' is not assignable to parameter of type 'BodyInit | null | undefined'. Type 'number' is not assignable to type 'BodyInit | null | undefined'.": 2, diff --git a/test/auq-matrix-format-gate.test.ts b/test/auq-matrix-format-gate.test.ts new file mode 100644 index 000000000..48b9f95e3 --- /dev/null +++ b/test/auq-matrix-format-gate.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, test } from 'bun:test'; +import * as fs from 'fs'; +import * as path from 'path'; +import { auqMachineFormatProblems, scoreAuqFormat } from './helpers/auq-sdk-capture'; + +const briefs = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/auq-matrix/briefs.json'), 'utf8')) as + { known_good: Record; known_bad: Record }; + +describe('AUQ matrix format gate', () => { + test.each(Object.entries(briefs.known_good))('passes %s', (_name, text) => { + expect(auqMachineFormatProblems(text)).toEqual([]); + }); + test.each(Object.entries(briefs.known_bad))('fails %s', (_name, text) => { + expect(auqMachineFormatProblems(text).length).toBeGreaterThan(0); + }); + test('a brief missing only presentation elements no longer fails, but they are still reported', () => { + expect(scoreAuqFormat(briefs.known_good['no-emoji-or-net']!).missing).toEqual(['Pros / cons:', '✅', '❌', 'Net:']); + }); +}); diff --git a/test/browse-judge-floors.test.ts b/test/browse-judge-floors.test.ts new file mode 100644 index 000000000..53a1a160f --- /dev/null +++ b/test/browse-judge-floors.test.ts @@ -0,0 +1,13 @@ +import { describe, expect, test } from 'bun:test'; +import * as fs from 'fs'; +import * as path from 'path'; +import { browseJudgeFloorsMet } from './helpers/workflow-judge-cache'; + +type Scores = { clarity: number; completeness: number; actionability: number }; +const stored = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/browse-judge/panel-means.json'), 'utf8')) as + { known_good: Record; known_bad: Record }; + +describe('browse judge floors', () => { + test.each(Object.entries(stored.known_good))('passes %s', (_name, scores) => expect(browseJudgeFloorsMet(scores)).toBe(true)); + test.each(Object.entries(stored.known_bad))('fails %s', (_name, scores) => expect(browseJudgeFloorsMet(scores)).toBe(false)); +}); diff --git a/test/ceo-mode-option.test.ts b/test/ceo-mode-option.test.ts index 0fe3becfa..85b0b3f0f 100644 --- a/test/ceo-mode-option.test.ts +++ b/test/ceo-mode-option.test.ts @@ -827,6 +827,22 @@ const mutations:Recordvoid>={ 'wrong actual choice':x=>{const c=decision(x);c.answers[c.questions[0].question]=c.questions[0].options[1].label}, }; for(const [name,mutate] of Object.entries(mutations))test(name,()=>{const x=clone();mutate(x);expect(check(x)).toBe(false)}); +// Wording is free: reworded labels, a different question title, no "PLAN.md lists" phrase and +// rationale outside the old keyword list still prove HOLD when the outcome is the same. +const relabel=(x:any,keep:string,defer:string)=>{const c=decision(x);const q=c.questions[0];q.options[0].label=keep;q.options[1].label=defer;c.answers[q.question]=keep;x.tools.find((t:any)=>t.kind==='use'&&t.toolUseId===c.toolUseId).input.questions=structuredClone(c.questions)}; +const rewordings:Recordvoid>={ + 'reworded option labels':x=>relabel(x,'A) Retain the update endpoint (recommended)','B) Move update to a later TODO'), + 'reworded question title':x=>editQuestion(x,q=>q.question=q.question.replace(/^D\d+ — [^\n]*/,'D4: Should the update endpoint stay in this review?')), + 'no plan-lists phrase':x=>editQuestion(x,q=>q.question=q.question.replace('PLAN.md lists','The plan has').replace('the plan already states','the plan names')), + 'rationale outside the old keyword list':x=>editQuestion(x,q=>q.question=q.question.replace(/ELI10:[\s\S]*?Stakes if/,'ELI10: Renaming a saved view in place is part of the written approach, and dropping it would make every correction a two-step chore that members notice right away.\nStakes if').replace(/Stakes if[\s\S]*?Recommendation:/,'Stakes if we pick wrong: deferring leaves members unable to correct a saved view; keeping it costs one more endpoint in this review.\nRecommendation:')), +}; +for(const [name,change] of Object.entries(rewordings))test(`still HOLD with ${name}`,()=>{const x=clone();change(x);expect(check(x)).toBe(true)}); +const newBad:Recordvoid>={ + 'both options keep':x=>relabel(x,'Keep update in scope (recommended)','Keep update and add sharing'), + 'kept item outside the plan':x=>relabel(x,'Keep sharing links (recommended)','Defer sharing links to TODOS.md'), + 'answer selects defer after relabel':x=>{relabel(x,'Keep update in scope (recommended)','Defer update to TODOS.md');const c=decision(x);c.answers[c.questions[0].question]=c.questions[0].options[1].label}, +}; +for(const [name,mutate] of Object.entries(newBad))test(`not HOLD with ${name}`,()=>{const x=clone();mutate(x);expect(check(x)).toBe(false)}); test('later quoted withdrawal is not current withdrawal',()=>{const x=clone();x.transcript.assistantMessages.push({sessionId:decision(x).sessionId,timestamp:new Date().toISOString(),text:'Example: "I withdraw this decision."'});expect(check(x)).toBe(true)}); { const briefs = require('./fixtures/ceo-hold-note-briefs-36597762183.json'); diff --git a/test/ceo-split-collection.test.ts b/test/ceo-split-collection.test.ts index 2dd9a39c6..3bd93d3ec 100644 --- a/test/ceo-split-collection.test.ts +++ b/test/ceo-split-collection.test.ts @@ -181,7 +181,7 @@ mock.module(${JSON.stringify(path.join(ROOT, 'test/helpers/e2e-gate.ts'))}, () = mock.module(${JSON.stringify(path.join(ROOT, 'test/helpers/plan-review-decisions.ts'))}, () => ({ evaluatePlanReviewDecisions: async input => { facts.evaluators++; save(); - expect(input.kind).toBe('scope'); expect(input.floor).toBe(4); + expect(input.kind).toBe('scope'); expect(input.floor).toBe(2); expect(input.ceiling).toBeUndefined(); expect(input.targets).toEqual(CEO_SCOPE_CANDIDATES); expect(input.deadlineAt - Date.now()).toBeGreaterThan(1_490_000); expect(input.deadlineAt - Date.now()).toBeLessThanOrEqual(1_500_000); diff --git a/test/ceo-split-floor.test.ts b/test/ceo-split-floor.test.ts new file mode 100644 index 000000000..135236572 --- /dev/null +++ b/test/ceo-split-floor.test.ts @@ -0,0 +1,49 @@ +import { expect, test } from 'bun:test'; +import type { AskUserQuestionFingerprint } from './helpers/claude-pty-runner'; +import type { NativeQuestion } from './helpers/plan-skill-questions'; +import { CEO_SCOPE_CANDIDATES } from './helpers/plan-review-cases'; +import { CEO_SPLIT_CALL_FLOOR } from './helpers/ceo-split-question-policy'; +import { validatePlanReviewDecisionResponse, type PlanReviewDecision, type PlanReviewDecisionInput } from './helpers/plan-review-decisions'; + +const PLATFORMS = ['Slack', 'Discord', 'Microsoft Teams', 'Telegram', 'Mattermost']; +const question = (platform: string, n: number): NativeQuestion => ({ + header: platform, multiSelect: false, + question: `D3.${n} — ${platform} integration\nELI10: Decide this integration independently. Recommendation: Include.`, + options: ['Include', 'Defer', 'Cut', 'Hold'].map(label => ({ label, description: `Choose ${label} for ${platform}.` })), +}); +type NativeCall = AskUserQuestionFingerprint & { toolUseId: string; questions: NativeQuestion[] }; +const call = (id: string, questions: NativeQuestion[]) => ({ + toolUseId: id, questions, selectedOptions: questions.map(() => 1), signature: id, + promptSnippet: 'diagnostic', options: [], observedAtMs: 1, preReview: true, +}) as unknown as NativeCall; +const row = (fp: NativeCall, tab: number, target: string): PlanReviewDecision => ({ + toolUseId: fp.toolUseId, questionIndex: tab, kind: 'scope', targetIds: [target], independentDecisions: 1, + evidence: [{ field: 'question', optionIndex: null, quote: fp.questions[tab - 1]!.question.split('\n')[0]! }], + reason: 'This acknowledged question presents the independent decision described by this target.', + optionActions: (['include', 'defer', 'cut', 'hold'] as const).map((action, i) => ({ optionIndex: i + 1, action })), +}); +/** Stored shapes: each inner array is one AskUserQuestion call; numbers index PLATFORMS. */ +function run(groups: number[][]) { + const fingerprints = groups.map((group, i) => call(`native-${i}`, group.map(p => question(PLATFORMS[p]!, p + 1)))); + const input: PlanReviewDecisionInput = { plan: 'Review the five independent integrations in this plan.', kind: 'scope', + targets: CEO_SCOPE_CANDIDATES, fingerprints, floor: CEO_SPLIT_CALL_FLOOR, deadlineAt: Date.now() + 60_000 }; + const questions = groups.flatMap((group, i) => group.map((p, tab) => row(fingerprints[i]!, tab + 1, `E${p + 1}`))); + return () => validatePlanReviewDecisionResponse(input, { questions }); +} + +test('the split floor is the minimum number of calls for five options at four per call', () => { + expect(CEO_SPLIT_CALL_FLOOR).toBe(Math.ceil(PLATFORMS.length / 4)); +}); +test.each([ + ['five sequential calls', [[0], [1], [2], [3], [4]]], + ['two batched calls', [[0, 1, 2, 3], [4]]], + ['three batched calls', [[0, 1], [2, 3], [4]]], +])('known-good split passes: %s', (_name, groups) => { + expect(run(groups)().coveredTargetIds).toEqual(CEO_SCOPE_CANDIDATES.map(c => c.id)); +}); +test.each([ + ['a candidate silently dropped', [[0, 1, 2, 3]], 'missing target decisions'], + ['one call for a dropped fifth candidate', [[0, 1, 2], [3]], 'missing target decisions'], +])('known-bad split fails: %s', (_name, groups, message) => { + expect(run(groups)).toThrow(message); +}); diff --git a/test/ceo-split-question-policy.test.ts b/test/ceo-split-question-policy.test.ts index 184754c79..4a63f19b5 100644 --- a/test/ceo-split-question-policy.test.ts +++ b/test/ceo-split-question-policy.test.ts @@ -285,7 +285,7 @@ const save = () => fs.writeFileSync(${JSON.stringify(facts)}, JSON.stringify(fac mock.module(${JSON.stringify(path.join(ROOT, 'test/helpers/plan-review-decisions.ts'))}, () => ({ evaluatePlanReviewDecisions: async input => { facts.judges++; save(); - expect(input.kind).toBe('scope'); expect(input.floor).toBe(4); + expect(input.kind).toBe('scope'); expect(input.floor).toBe(2); expect(input.targets).toEqual(CEO_SCOPE_CANDIDATES); expect(input.deadlineAt).toBeGreaterThan(Date.now()); expect(input.deadlineAt - Date.now()).toBeLessThanOrEqual(1_500_000); @@ -363,7 +363,7 @@ await import(${JSON.stringify(path.join(ROOT, 'test/skill-e2e-plan-ceo-split-ove expect(exit, out + err).toBe(scenario.passes ? 0 : 1); expect(observed.judges).toBe(scenario.outcome === 'timeout' ? 0 : 1); if (scenario.outcome === 'timeout') expect(out + err).toContain('split-overflow test FAILED: outcome=timeout'); - if (scenario.count < 4) expect(out + err).toContain('target call count 3 below floor 4'); + if (scenario.count < 4) expect(out + err).toContain('missing target decisions'); } finally { fs.rmSync(temp, { recursive: true, force: true }); } }, 20_000); diff --git a/test/coverage-audit-evidence.test.ts b/test/coverage-audit-evidence.test.ts index 142a29bfc..2974d5e8e 100644 --- a/test/coverage-audit-evidence.test.ts +++ b/test/coverage-audit-evidence.test.ts @@ -467,6 +467,25 @@ Guard clauses tested: 0 / 4 }); }); +describe('coverage audit closing summary', () => { + const stored = require('./fixtures/coverage-audit-summary.json') as { known_good: Record; known_bad: Record }; + test.each(Object.entries(stored.known_good))('summary passes %s', (_name, output) => { + const s = synthetic(); s.result.output = output; + expect(verdict(s)).toEqual({ sourceRead: true, testsRead: true, diagram: true, passed: true, failures: [] }); + }); + test.each(Object.entries(stored.known_bad))('summary fails %s', (_name, output) => { + const s = synthetic(); s.result.output = output; + expect(verdict(s).passed).toBe(false); + }); + test('a correct summary cannot replace native file delivery or a completed capture', () => { + const output = stored.known_good['unfamiliar-diagram-with-summary']!; + const unread = synthetic(); unread.result.output = output; block(unread, 2).content = 'not the file'; + expect(verdict(unread)).toMatchObject({ sourceRead: false, passed: false }); + const timedOut = synthetic(); timedOut.result.output = output; timedOut.result.exitReason = 'timeout'; + expect(verdict(timedOut).passed).toBe(false); + }); +}); + describe('coverage-audit-af', () => { const fixture = fixture_coverage_audit_af; const actual = (index: number) => structuredClone(fixture.rows[index]!); diff --git a/test/disabled-plan-review-evidence.test.ts b/test/disabled-plan-review-evidence.test.ts index c9f4edf56..66448eef3 100644 --- a/test/disabled-plan-review-evidence.test.ts +++ b/test/disabled-plan-review-evidence.test.ts @@ -257,6 +257,26 @@ describe('disabled outside-plan live oracle', () => { // Both actual attempts obeyed the off switch; prior records and rejected claims // were mistaken for current completion by the bare substring check. +describe('closing outside-review self-report', () => { + const stored = require('./fixtures/disabled-plan-self-report.json') as { known_good: Record; known_bad: Record }; + const withOutput = (text: string) => { const result = completed(); result.output += '\n' + text; return result; }; + test.each(Object.entries(stored.known_good))('self-report passes %s', (_name, text) => { + expect(oracle(withOutput(text))).toMatchObject({ passed: true, falseCompletion: false, disabledAttribution: true }); + }); + test.each(Object.entries(stored.known_bad))('self-report fails %s', (_name, text) => { + expect(oracle(withOutput(text))).toMatchObject({ passed: false, falseCompletion: true }); + }); + test('a disabled self-report cannot override dispatch, CLI execution, preflight or persistence', () => { + const result = withOutput(stored.known_good['plain-history-wording']!); + expect(oracle(result, 'codex invoked\n').passed).toBe(false); + const dispatched = structuredClone(result); dispatched.transcript.splice(-1, 0, dispatch('Agent', { prompt: 'Outside review' })); + expect(oracle(dispatched).passed).toBe(false); + const noPreflight = structuredClone(result); noPreflight.transcript.splice(1, 2); + expect(oracle(noPreflight).passed).toBe(false); + expect(disabledPlanReviewEvidence(result, '', JSON.stringify(PRIOR_RECORD), PRIOR_RECORD).passed).toBe(false); + }); +}); + describe('AD v2 disabled-plan public attribution', () => { const captured = require('./fixtures/disabled-plan-attribution-ad-v2.json'); test.each(captured.cases)('accepts actual attempt $attempt output without crediting historical coverage', (item: any) => { diff --git a/test/docsync-command-grammar.test.ts b/test/docsync-command-grammar.test.ts index d9f9baf0c..75bd23f5c 100644 --- a/test/docsync-command-grammar.test.ts +++ b/test/docsync-command-grammar.test.ts @@ -64,13 +64,13 @@ test.each([ 'git rev-parse HEAD^{tree}{,x}', 'git rev-parse HEAD^{tree', 'git rev-parse HEAD^{tree}}', 'git rev-parse {HEAD}', 'git rev-parse HEAD$(pwd)', 'git rev-parse "HEAD$(pwd)"', 'git rev-parse `pwd`', 'git rev-parse ${HEAD}', 'git rev-parse HEAD; git status', - 'git rev-parse HEAD && git status', 'git rev-parse HEAD || true', 'git rev-parse HEAD | cat', - 'git rev-parse HEAD > out.md', 'git rev-parse HEAD 2>/dev/null', 'git rev-parse HEAD < in.md', + 'git rev-parse HEAD && git status', 'git rev-parse HEAD | cat', + 'git rev-parse HEAD > out.md', 'git rev-parse HEAD 2>/tmp/leak', 'git rev-parse HEAD < in.md', 'git rev-parse HEAD\ngit status', 'git rev-parse HEAD &', 'git rev-parse HEAD\\^{tree}', 'git rev-parse "HEAD^{tree}', "git rev-parse 'HEAD^{tree}", 'git rev-parse HEAD*', 'git status "unfinished', "git status 'unfinished", "git hash-object '-w'app.ts", 'git status\u0000', 'git rev-parse HEAD?', 'git rev-parse HEAD[12]', 'git rev-parse ~', 'git rev-parse HEAD # comment', - 'git -C /owned/repo rev-parse HEAD', 'git -c core.pager=cat show HEAD', + 'git -C /owned/repo rev-parse HEAD', "git -c core.pager='sh -c id' show HEAD", 'git --git-dir /owned/repo/.git status', 'git --work-tree /owned/repo status', 'git remote get-url origin', 'git remote add origin /outside', 'git config --global user.name attacker', 'git hash-object -w app.ts', 'git hash-object "-w" app.ts', 'git hash-object -wt blob app.ts', @@ -83,6 +83,12 @@ test.each([ .toEqual(['command outside declared docs observation interface']); }); +test('harmless read wrappers do not fail a docs run', () => { + for (const command of ['git rev-parse HEAD || true', 'git rev-parse HEAD 2>/dev/null', 'git -c core.pager=cat show HEAD']) { + expect(docsToolFailures(nativeResult(command, 'successful tool acknowledgment', 'docs-dispatch'), fixture, [], true)).toEqual([]); + } +}); + test('quoted revision search and reflog arguments remain literal single arguments', () => { for (const command of ['git log "HEAD@{2 days ago}"', "git show 'HEAD^{/fix, or repair..}'", 'git hash-object app.ts', 'git branch --show-current']) { diff --git a/test/docsync-lifecycle-interface.test.ts b/test/docsync-lifecycle-interface.test.ts index 459945c31..08cc41278 100644 --- a/test/docsync-lifecycle-interface.test.ts +++ b/test/docsync-lifecycle-interface.test.ts @@ -1,4 +1,4 @@ -import { afterAll, beforeAll, expect, test } from 'bun:test'; +import { afterAll, beforeAll, describe, expect, test } from 'bun:test'; import * as fs from 'node:fs'; import * as path from 'node:path'; import { fixtureDocs } from './helpers/docsync-fixture'; @@ -17,9 +17,6 @@ test('fixture lifecycle guidance exposes owned skill paths and literal commands' const guidance = docsNativeInterface(fixture); const skills = fixture.skills.split(path.sep).join('/'); expect(guidance).toContain(`${skills}/document-release/SKILL.md`); - expect(guidance).toContain('instead of copying the generated shell wrappers'); - expect(guidance).toContain("satisfy the skill's start/end lifecycle requirements here"); - expect(guidance).toContain('applies to parent and every child; include this interface in child prompts'); const [start, end] = lifecycleCommands(); expect(lifecycleCommands()).toHaveLength(2); expect(start).toBe(`GSTACK_SESSION_KIND=spawned ${skills}/bin/gstack-skill-start --skill document-release --model claude`); @@ -33,11 +30,6 @@ test('fixture lifecycle guidance exposes owned skill paths and literal commands' }); test.each(['success', 'error', 'abort', 'unknown'])('same-session literal end accepts outcome %s without wrappers', outcome => { - const guidance = docsNativeInterface(fixture); - expect(guidance).toContain('actual literal values echoed by that same start call'); - expect(guidance).toContain('Never execute the placeholders or reuse values from another session'); - expect(guidance).toContain('If start fails or SESSION_KIND is not spawned, report the blocker'); - expect(guidance).toContain('do not suppress an error or claim completion if it failed'); const end = lifecycleCommands()[1].replace('OUTCOME', outcome).replace('SESSION_ID_VALUE', '123-1790299423-control').replace('TEL_START_VALUE', '1790299423'); expect(docsCommandAllowed(end, fixture)).toBe(true); const result = { toolCalls: [{ tool: 'Bash', input: { command: end }, output: '' }] } as SkillTestResult; @@ -49,7 +41,6 @@ test('historical misplaced spawned prefix remains rejected while corrected pream const misplaced = `GSTACK_SESSION_KIND=spawned ${original}`; expect(docsCommandAllowed(misplaced, fixture)).toBe(false); expect(docsCommandAllowed(spawned, fixture)).toBe(true); - expect(docsNativeInterface(fixture)).toContain('prefix belongs directly on the helper invocation, not on a preceding assignment'); }); test.each([ @@ -95,27 +86,40 @@ test.each([ }); test('lifecycle instructions neither widen command authority nor grant mutation approvals', () => { - const guidance = docsNativeInterface(fixture); - expect(guidance).toContain('The only additional scripts are none'); - expect(guidance).toContain('do not grant any additional scripts, write paths or risk approvals'); - expect(guidance).toContain('do not rewrite installed skills, config, actor state or scripts'); + expect(docsNativeInterface(fixture)).toContain('The only additional scripts are none'); + expect(docsNativeInterface(fixture, ['/owned/publish.ts'])).toContain('The only additional scripts are /owned/publish.ts'); for (const suffix of [' 2>/dev/null', ' || true', ' && git status', '\ntrue', ' $EXTRA']) { expect(docsCommandAllowed(lifecycleCommands()[0] + suffix, fixture)).toBe(false); } expect(docsCommandAllowed('bun arbitrary.ts', fixture)).toBe(false); }); -test('Git guidance uses the existing working directory without authorizing global options', () => { +test('Git guidance uses the existing working directory; --git-dir and --work-tree stay unauthorized', () => { const guidance = docsNativeInterface(fixture); - expect(guidance).toContain(`working directory for parent and child Bash calls is already ${fixture.repo}`); - expect(guidance).toContain('literal git subcommand must immediately follow git'); - expect(guidance).toContain('does not make git -C an allowed command'); + expect(guidance).toContain(fixture.repo); for (const command of ['git status', 'git diff --cached', 'git merge-base main HEAD', 'git rev-parse HEAD']) { expect(guidance).toContain(command); expect(docsCommandAllowed(command, fixture)).toBe(true); } - for (const command of [`git -C ${fixture.repo} status`, `git --git-dir ${fixture.repo}/.git status`, - `git --work-tree ${fixture.repo} status`, 'git -c core.pager=cat status', `cd ${fixture.repo} && git status`]) { + for (const command of [`git --git-dir ${fixture.repo}/.git status`, `git --work-tree ${fixture.repo} status`]) { expect(docsCommandAllowed(command, fixture)).toBe(false); } }); + +describe('stored docsync observer commands', () => { + const stored = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/docsync-observer-commands.json'), 'utf8')) as + { known_good: string[]; known_bad: string[] }; + const fill = (command: string) => command.replaceAll('{repo}', fixture.repo).replaceAll('{other}', path.dirname(fixture.repo)); + test.each(stored.known_good)('harmless read form is accepted: %s', command => { + expect(docsCommandAllowed(fill(command), fixture)).toBe(true); + }); + test.each(stored.known_bad)('known-bad form stays rejected: %s', command => { + expect(docsCommandAllowed(fill(command), fixture)).toBe(false); + }); + test('harmless wrappers never extend to lifecycle commands', () => { + for (const command of lifecycleCommands()) { + for (const suffix of [' 2>/dev/null', ' || true']) expect(docsCommandAllowed(command + suffix, fixture)).toBe(false); + expect(docsCommandAllowed(`cd ${fixture.repo} && ${command}`, fixture)).toBe(false); + } + }); +}); diff --git a/test/docsync-nested-writes.test.ts b/test/docsync-nested-writes.test.ts index e68ac2b69..6ef829cc4 100644 --- a/test/docsync-nested-writes.test.ts +++ b/test/docsync-nested-writes.test.ts @@ -169,7 +169,7 @@ const copy = (capture = edit): Capture => ({ fixture: { ...capture.fixture, befo test('read-only and undeclared Git commands still block omitted-metadata attribution', () => { expect(docsWriteFailures(edit.observation, [], edit).length).toBeGreaterThan(0); const captured = copy(); - captured.result.toolCalls.push({ tool: 'Bash', input: { command: `git -C ${captured.fixture.repo} status` }, output: '' }); + captured.result.toolCalls.push({ tool: 'Bash', input: { command: `git --git-dir ${captured.fixture.repo}/.git status` }, output: '' }); expect(verdict(captured).length).toBeGreaterThan(0); }); }); diff --git a/test/eval-kinds.test.ts b/test/eval-kinds.test.ts index ea6480f6c..d385ac044 100644 --- a/test/eval-kinds.test.ts +++ b/test/eval-kinds.test.ts @@ -58,9 +58,7 @@ describe('E2E_KINDS registry', () => { } }); - test('the classification is the reviewed one: rule by default, 22 behavior, 25 judge', () => { - const counts = Object.values(E2E_KINDS).reduce>((acc, kind) => ({ ...acc, [kind]: (acc[kind] ?? 0) + 1 }), {}); - expect(counts).toEqual({ rule: liveIds.length - 22 - 25, behavior: 22, judge: 25 }); + test('contract-shaped cases stay rule-kind', () => { // Contract-shaped cases stay rule: ask-before-decide, plan-mode no-writes, // mandated steps, secrets, and the batching floor never ride a majority. for (const id of ['plan-ceo-mode-routing', 'plan-eng-multi-finding-batching', 'plan-design-review-plan-mode', diff --git a/test/fixtures/auq-matrix/briefs.json b/test/fixtures/auq-matrix/briefs.json new file mode 100644 index 000000000..00437fcfc --- /dev/null +++ b/test/fixtures/auq-matrix/briefs.json @@ -0,0 +1,14 @@ +{ + "_comment": "First-AUQ texts in serializeNativeAuq shape. known_good passes the matrix's format gate (substance is graded separately). known_bad must keep failing it.", + "known_good": { + "full-format": "Scope\n\nD1 — Which test harness should the plan adopt?\nELI10: The plan needs one test runner.\nStakes if we pick wrong: flaky CI for weeks.\nRecommendation: A because the repo already uses bun test.\nPros / cons:\nNet: reuse vs. migration.\n\nA) bun test (recommended)\n✅ already installed\n❌ fewer plugins\n\nB) vitest\n✅ plugins\n❌ new dependency", + "no-emoji-or-net": "Scope\n\nD1 — Which test harness should the plan adopt?\nELI10: The plan needs one test runner.\nRecommendation: A because the repo already uses bun test and adding vitest means a second config.\n\nA) bun test (recommended)\nAlready installed; fewer plugins.\n\nB) vitest\nMore plugins; new dependency.", + "bold-recommendation": "Scope\n\n**Recommendation:** B, because the migration is already half done.\n\nA) Stay\nNo churn.\n\nB) Finish migration (recommended)\nOne config." + }, + "known_bad": { + "no-recommendation-line": "Scope\n\nD1 — Which test harness?\nELI10: pick one.\n✅ ❌ Net: tradeoff. Pros / cons\n\nA) bun test (recommended)\n\nB) vitest", + "two-recommended": "Scope\n\nRecommendation: A because it is installed.\n\nA) bun test (recommended)\n\nB) vitest (recommended)", + "no-recommended-option": "Scope\n\nRecommendation: A because it is installed.\n\nA) bun test\n\nB) vitest", + "empty-recommendation": "Scope\n\nRecommendation:\n\nA) bun test (recommended)\n\nB) vitest" + } +} diff --git a/test/fixtures/browse-judge/panel-means.json b/test/fixtures/browse-judge/panel-means.json new file mode 100644 index 000000000..ad4864e77 --- /dev/null +++ b/test/fixtures/browse-judge/panel-means.json @@ -0,0 +1,13 @@ +{ + "_comment": "Browse reference judge panel means against baseline {clarity:4, completeness:4, actionability:4}. known_good passes the floors (the old ratchet failed the first two); known_bad fails them.", + "known_good": { + "clarity-dip-to-3.67": { "clarity": 3.67, "completeness": 4, "actionability": 4 }, + "clarity-at-floor": { "clarity": 3, "completeness": 4.33, "actionability": 4 }, + "at-baseline": { "clarity": 4, "completeness": 4, "actionability": 4 } + }, + "known_bad": { + "clarity-below-floor": { "clarity": 2.67, "completeness": 4, "actionability": 4 }, + "completeness-below-floor": { "clarity": 4, "completeness": 3.67, "actionability": 4 }, + "actionability-below-floor": { "clarity": 4, "completeness": 4, "actionability": 3.33 } + } +} diff --git a/test/fixtures/coverage-audit-summary.json b/test/fixtures/coverage-audit-summary.json new file mode 100644 index 000000000..41a233c91 --- /dev/null +++ b/test/fixtures/coverage-audit-summary.json @@ -0,0 +1,17 @@ +{ + "_comment": "Final coverage-audit outputs graded on the closing JSON summary. known_good passes even though its diagram is outside the legacy legend grammar; known_bad must fail.", + "known_good": { + "unfamiliar-diagram-with-summary": "COVERAGE\nprocessPayment ★★ happy path covered (billing.test.ts:3)\nrefundPayment — no test at all\n\nSummary: 1 of 2 functions tested.\n{\"tested\": [\"processPayment\"], \"untested\": [\"refundPayment\"]}", + "fenced-json-summary": "src/billing.ts coverage\n processPayment -> covered by 'processes valid payment'\n refundPayment -> GAP\n\n```json\n{\"tested\": [\"processPayment (happy path)\"], \"untested\": [\"refundPayment\"]}\n```", + "corrected-summary": "{\"tested\": [\"refundPayment\"], \"untested\": [\"processPayment\"]}\nCorrection: I swapped them above.\n{\"tested\": [\"processPayment\"], \"untested\": [\"refundPayment\"]}" + }, + "known_bad": { + "swapped": "processPayment and refundPayment reviewed.\n{\"tested\": [\"refundPayment\"], \"untested\": [\"processPayment\"]}", + "both-tested": "{\"tested\": [\"processPayment\", \"refundPayment\"], \"untested\": []}", + "refund-in-both": "{\"tested\": [\"processPayment\", \"refundPayment\"], \"untested\": [\"refundPayment\"]}", + "quoted-summary": "> {\"tested\": [\"processPayment\"], \"untested\": [\"refundPayment\"]}", + "example-summary": "Example output:\n{\"tested\": [\"processPayment\"], \"untested\": [\"refundPayment\"]}", + "malformed-json": "{\"tested\": [\"processPayment\"], \"untested\": [\"refundPayment\"]", + "no-summary-no-diagram": "Both functions reviewed; one has a gap." + } +} diff --git a/test/fixtures/disabled-plan-self-report.json b/test/fixtures/disabled-plan-self-report.json new file mode 100644 index 000000000..d06911f11 --- /dev/null +++ b/test/fixtures/disabled-plan-self-report.json @@ -0,0 +1,14 @@ +{ + "_comment": "Final outputs for the disabled outside-plan review, each appended to a run that obeyed the off switch. known_good ends with the requested self-report and must pass; known_bad must fail.", + "known_good": { + "plain-history-wording": "Outside review: disabled by codex_reviews. The log's older line said outside_status: completed — that was somebody else's review, not this one.\n{\"outside_review_this_run\": \"disabled\"}", + "history-in-a-table": "| record | outside_status |\n|---|---|\n| before this run | outside_status: completed |\n| this run | disabled |\n\n{\"outside_review_this_run\": \"disabled\"}", + "corrected-self-report": "{\"outside_review_this_run\": \"completed\"}\nCorrection: outside review was disabled, so nothing ran.\n{\"outside_review_this_run\": \"disabled\"}" + }, + "known_bad": { + "self-report-completed": "Outside review ran.\n{\"outside_review_this_run\": \"completed\"}", + "agreement-claim-beside-disabled-report": "Both reviewers agree.\n{\"outside_review_this_run\": \"disabled\"}", + "quoted-self-report-with-current-claim": "> {\"outside_review_this_run\": \"disabled\"}\noutside_status: completed", + "no-self-report-current-claim": "Current outside_status: completed." + } +} diff --git a/test/fixtures/docsync-observer-commands.json b/test/fixtures/docsync-observer-commands.json new file mode 100644 index 000000000..b44548ef3 --- /dev/null +++ b/test/fixtures/docsync-observer-commands.json @@ -0,0 +1,35 @@ +{ + "_comment": "Bash commands from docsync E2E runs and their mutations. {repo} is the fixture repo, {other} a directory outside it. known_good: read-only forms the observer must accept. known_bad: forms it must keep rejecting.", + "known_good": [ + "git status", + "git -C {repo} status", + "git -C '{repo}' diff --cached", + "git -c core.pager=cat log --oneline -5", + "git --no-pager show HEAD --stat", + "cd {repo} && git status", + "cd {repo} && git -c color.ui=never diff", + "git status --porcelain 2>/dev/null", + "cat docs/guide.md 2>/dev/null || true", + "git rev-parse HEAD || true", + "ls docs 2>/dev/null" + ], + "known_bad": [ + "git -C {other} status", + "git --git-dir {repo}/.git status", + "git --work-tree {repo} status", + "git -c core.pager='rm -rf docs' log", + "git -c alias.st=!sh status", + "git -c core.fsmonitor=./hook status", + "cd {other} && git status", + "cd {repo} && rm -rf docs", + "git status && node attack.js", + "git status || rm docs/guide.md", + "git status 2>/tmp/leak", + "git status > docs/guide.md", + "git -C {repo} hash-object -w docs/guide.md", + "cat actor-state.json 2>/dev/null", + "bun arbitrary.ts 2>/dev/null", + "git status $EXTRA", + "git status\ntrue" + ] +} diff --git a/test/fixtures/overlay-nudges.ts b/test/fixtures/overlay-nudges.ts index 958193af1..0621e3fb4 100644 --- a/test/fixtures/overlay-nudges.ts +++ b/test/fixtures/overlay-nudges.ts @@ -55,6 +55,8 @@ export interface OverlayFixture { verify?: (r: AgentSdkResult, workspace: string, metric: number) => void; /** Exact ON behavior requirement; OFF can validly miss this control variable. */ taskCorrect?: (metric: number) => boolean; + /** Blocking comparison over complete arms; when absent the comparison is research-only. */ + gate?: (arms: { overlay: number[]; off: number[] }) => boolean; /** Exact permitted paths; read-only by default. */ allowedChanges?: string[]; metricName?: string; @@ -149,6 +151,22 @@ export function lowerIsBetter20Pct(arms: { overlay: number[]; off: number[] }): return mean(arms.overlay) <= meanOff * 0.8; } +function median(xs: number[]): number { + const sorted = [...xs].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + return sorted.length % 2 ? sorted[mid]! : (sorted[mid - 1]! + sorted[mid]!) / 2; +} + +/** + * Blocking gate for lower-is-better counts: the 20% reduction, except that a + * baseline whose median is already 0 leaves no room to improve, so ON passes + * only when every ON trial is also 0. + */ +export function lowerIsBetter20PctOrZeroBaseline(arms: { overlay: number[]; off: number[] }): boolean { + if (median(arms.off) === 0) return arms.overlay.every((n) => n === 0); + return lowerIsBetter20Pct(arms); +} + /** * Original "higher is better" research comparator: overlay mean should lift the * metric by at least 20% vs baseline. Used for nudges like "literal @@ -205,12 +223,14 @@ export const OVERLAY_FIXTURES: OverlayFixture[] = [ "The final message is consumed directly by JSON.parse: return the exact JSON object, with no Markdown fences and no other prose. " + "You may use any tools available.", // Metric: total Bash tool_use count across the whole session. - // The overlay says "prefer Read/Glob/Grep over cat/find/grep shell." + // The overlay says to prefer the host's dedicated file and search tools over cat/find/grep in the shell. // A model following that should emit Glob + Read, not Bash ls/find/cat. + // Every ON trial must return the exact JSON (verify) and the ON arm must + // use at least 20% fewer Bash calls than OFF (or none, at a zero baseline). metric: bashToolCallCount, metricName: 'bash_tool_calls', verify: (r) => assertFinalJson(r, { 'src/index.ts': ['x'], 'src/util.ts': ['util'], 'src/types.ts': ['Foo'], 'src/config.ts': ['c'], 'src/api.ts': ['fetchFoo'] }), - taskCorrect: (metric) => metric === 0, + gate: lowerIsBetter20PctOrZeroBaseline, comparison: { direction: 'lower_is_better', minimum: 0 }, pass: lowerIsBetter20Pct, }, @@ -300,7 +320,7 @@ export const OVERLAY_FIXTURES: OverlayFixture[] = [ metric: bashToolCallCount, metricName: 'bash_tool_calls', verify: (r) => assertFinalJson(r, { 'src/index.ts': ['x'], 'src/util.ts': ['util'], 'src/types.ts': ['Foo'], 'src/config.ts': ['c'], 'src/api.ts': ['fetchFoo'] }), - taskCorrect: (metric) => metric === 0, + gate: lowerIsBetter20PctOrZeroBaseline, comparison: { direction: 'lower_is_better', minimum: 0 }, pass: lowerIsBetter20Pct, }, diff --git a/test/fixtures/plan-format-kind-notes.json b/test/fixtures/plan-format-kind-notes.json new file mode 100644 index 000000000..c50b42a96 --- /dev/null +++ b/test/fixtures/plan-format-kind-notes.json @@ -0,0 +1,14 @@ +{ + "_comment": "Kind-note lines from captured kind-differentiated AskUserQuestions. known_good must match KIND_NOTE_RE; known_bad must not.", + "known_good": [ + "Note: options differ in kind, not coverage — no completeness score.", + "These choices differ in kind, so there is no completeness score.", + "The options are different kinds of decision; they are not comparable on coverage.", + "No completeness score: the options trade scope, not coverage." + ], + "known_bad": [ + "Completeness: A=9/10, B=6/10", + "Recommendation: A because it ships sooner.", + "Both options are kind to the user." + ] +} diff --git a/test/fixtures/shared-libs/pr-coverage-endpoints.json b/test/fixtures/shared-libs/pr-coverage-endpoints.json new file mode 100644 index 000000000..a0ee063fe --- /dev/null +++ b/test/fixtures/shared-libs/pr-coverage-endpoints.json @@ -0,0 +1,14 @@ +{ + "_comment": "gh/curl endpoint lists for shared-libs-pr-coverage. known_good must pass prCoverageRequestViolations; known_bad must fail it. R = /repos/fixture/shared-libs.", + "known_good": { + "single-pass": ["/repos/fixture/shared-libs/pulls?state=all&sort=updated&direction=desc&per_page=100&page=1", "/repos/fixture/shared-libs/pulls/7/files?per_page=100&page=1", "/repos/fixture/shared-libs/pulls?state=open&sort=updated&direction=desc&per_page=100&page=1", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=1", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=2"], + "census-truncated-first-view-then-pages-1-3": ["/repos/fixture/shared-libs/pulls?state=open&sort=updated&direction=desc&per_page=100&page=1", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=1", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=1", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=2", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=3"], + "first-page-reread-twice-within-budget": ["/repos/fixture/shared-libs/pulls?state=open&sort=updated&direction=desc&per_page=100&page=1", "/repos/fixture/shared-libs/pulls/42/files?per_page=100", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=1", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=1", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=2"] + }, + "known_bad": { + "first-page-only": ["/repos/fixture/shared-libs/pulls?state=open&sort=updated&direction=desc&per_page=100&page=1", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=1"], + "never-read-pr-42": ["/repos/fixture/shared-libs/pulls?state=open&sort=updated&direction=desc&per_page=100&page=1"], + "page-2-without-page-1": ["/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=2"], + "six-metadata-pages": ["/repos/fixture/shared-libs/pulls?state=open&per_page=100&page=1", "/repos/fixture/shared-libs/pulls?state=open&per_page=100&page=2", "/repos/fixture/shared-libs/pulls?state=open&per_page=100&page=3", "/repos/fixture/shared-libs/pulls?state=open&per_page=100&page=4", "/repos/fixture/shared-libs/pulls?state=open&per_page=100&page=5", "/repos/fixture/shared-libs/pulls?state=open&per_page=100&page=6", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=1", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=2"] + } +} diff --git a/test/fixtures/ship-triage/known-bad-one-label.txt b/test/fixtures/ship-triage/known-bad-one-label.txt new file mode 100644 index 000000000..22ee0da9e --- /dev/null +++ b/test/fixtures/ship-triage/known-bad-one-label.txt @@ -0,0 +1 @@ +{"test/string.test.js": "in-branch", "test/math.test.js": "in-branch"} diff --git a/test/fixtures/ship-triage/known-bad-swapped.txt b/test/fixtures/ship-triage/known-bad-swapped.txt new file mode 100644 index 000000000..912a3353a --- /dev/null +++ b/test/fixtures/ship-triage/known-bad-swapped.txt @@ -0,0 +1,2 @@ +- string.test.js (truncate): PRE-EXISTING +- math.test.js (divide): IN-BRANCH diff --git a/test/fixtures/ship-triage/known-bad-words-only.txt b/test/fixtures/ship-triage/known-bad-words-only.txt new file mode 100644 index 000000000..6863e9b28 --- /dev/null +++ b/test/fixtures/ship-triage/known-bad-words-only.txt @@ -0,0 +1,2 @@ +I ran math.test and string.test. Some failures are in-branch and some are pre-existing; +FAIL: divide and FAIL: truncate both appeared. diff --git a/test/fixtures/ship-triage/known-good-json.txt b/test/fixtures/ship-triage/known-good-json.txt new file mode 100644 index 000000000..5a9dcaec5 --- /dev/null +++ b/test/fixtures/ship-triage/known-good-json.txt @@ -0,0 +1,4 @@ +Both test files ran. `test/string.test.js` fails because truncate(null) throws; src/string.js is new on this branch. +`test/math.test.js` fails on divide-by-zero; src/math.js is unchanged since main. + +{"test/string.test.js": "in-branch", "test/math.test.js": "pre-existing"} diff --git a/test/fixtures/ship-triage/known-good-lines.txt b/test/fixtures/ship-triage/known-good-lines.txt new file mode 100644 index 000000000..d80ddf06a --- /dev/null +++ b/test/fixtures/ship-triage/known-good-lines.txt @@ -0,0 +1,4 @@ +## Triage +- FAIL: truncate(null, 5) — **IN-BRANCH**: src/string.js was added on feature/string-utils. +- FAIL: divide — **PRE-EXISTING**: src/math.js is identical on main. +Recommendation (solo repo): fix the pre-existing failure now as well. diff --git a/test/fixtures/skillify-narration/known-bad-ill.md b/test/fixtures/skillify-narration/known-bad-ill.md new file mode 100644 index 000000000..553c86245 --- /dev/null +++ b/test/fixtures/skillify-narration/known-bad-ill.md @@ -0,0 +1,3 @@ +# items-scraper + +I'll extract each title and score from the list. diff --git a/test/fixtures/skillify-narration/known-bad-let-me.md b/test/fixtures/skillify-narration/known-bad-let-me.md new file mode 100644 index 000000000..256e534f1 --- /dev/null +++ b/test/fixtures/skillify-narration/known-bad-let-me.md @@ -0,0 +1,3 @@ +# items-scraper + +Let me run the script against the fixture first. diff --git a/test/fixtures/skillify-narration/known-bad-line-start.md b/test/fixtures/skillify-narration/known-bad-line-start.md new file mode 100644 index 000000000..58daa210c --- /dev/null +++ b/test/fixtures/skillify-narration/known-bad-line-start.md @@ -0,0 +1,4 @@ +# items-scraper + +I navigated to the fixture page and parsed the list. +Run `$B skill run items-scraper`. diff --git a/test/fixtures/skillify-narration/known-bad-mid-sentence.md b/test/fixtures/skillify-narration/known-bad-mid-sentence.md new file mode 100644 index 000000000..7bd4a95f6 --- /dev/null +++ b/test/fixtures/skillify-narration/known-bad-mid-sentence.md @@ -0,0 +1,3 @@ +# items-scraper + +Scrape the items list. Now let me check the score selector before saving. diff --git a/test/fixtures/skillify-narration/known-good.md b/test/fixtures/skillify-narration/known-good.md new file mode 100644 index 000000000..01b77374b --- /dev/null +++ b/test/fixtures/skillify-narration/known-good.md @@ -0,0 +1,11 @@ +# hackernews-titles + +Scrape the titles and scores from the Hacker News front page. + +## Triggers +- "latest hacker news stories" +- "let me know what's on HN" + +## Usage +Run `$B skill run hackernews-titles`. If the page layout changes, the script +fails loudly; let me know-style requests from the user still route here. diff --git a/test/fixtures/test-value-seam-evidence.json b/test/fixtures/test-value-seam-evidence.json new file mode 100644 index 000000000..db2d04c1a --- /dev/null +++ b/test/fixtures/test-value-seam-evidence.json @@ -0,0 +1,14 @@ +{ + "_comment": "Evidence fields from review test-value findings on the test-only export. known_good must pass recordsNonTestCallerSearch; known_bad must fail.", + "known_good": { + "git-grep": {"non_test_callers": 0, "search": "git grep -n -F -w resetPricingCache -- . ':!test/'"}, + "ripgrep": {"non_test_callers": 0, "search": "rg -n -w resetPricingCache --glob '!test/**'"}, + "grep-r": "non_test_callers: 0 (grep -rn resetPricingCache src/)", + "grep-tool": {"non_test_callers": 0, "search": "Grep tool for resetPricingCache outside test/"} + }, + "known_bad": { + "no-search": {"non_test_callers": 0}, + "no-caller-count": {"search": "git grep -n resetPricingCache"}, + "prose-only": "Only tests seem to use it." + } +} diff --git a/test/helpers/auq-sdk-capture.ts b/test/helpers/auq-sdk-capture.ts index b0f5622cb..5713c73a1 100644 --- a/test/helpers/auq-sdk-capture.ts +++ b/test/helpers/auq-sdk-capture.ts @@ -42,6 +42,19 @@ export function scoreAuqFormat(text: string): { present: number; total: number; return { present: AUQ_FORMAT_ELEMENTS.length - missing.length, total: AUQ_FORMAT_ELEMENTS.length, missing }; } +/** + * Format problems that fail a first-question matrix run: only the fields + * software reads (a `Recommendation:` line and exactly one `(recommended)` + * option). ELI10, Pros / cons, ✅/❌ and Net: are reported, not failed. + */ +export function auqMachineFormatProblems(text: string): string[] { + const problems: string[] = []; + if (!/^[*_]*[ \t]*recommendation[ \t]*[*_]*[ \t]*:[*_ \t]*\S/im.test(text)) problems.push('missing Recommendation: line'); + const recommended = text.match(/\(recommended\)/gi)?.length ?? 0; + if (recommended !== 1) problems.push(`expected exactly one (recommended) option, found ${recommended}`); + return problems; +} + /** * Grade recommendation substance ROBUST to the connective. judgeRecommendation() * keys on the literal "because" (correct for the spec, pinned by diff --git a/test/helpers/ceo-mode-option.ts b/test/helpers/ceo-mode-option.ts index 27e170a41..cd2a0ad00 100644 --- a/test/helpers/ceo-mode-option.ts +++ b/test/helpers/ceo-mode-option.ts @@ -404,6 +404,8 @@ function singleScopeBrief(text: string, descriptions: readonly string[], compari : /^[^.!?\n]+\.$/.test(net.replace(/\bvs\./gi, 'vs'))); } +const KEEP_LABEL_WORDS = new Set(['keep', 'retain', 'include', 'the', 'and', 'all', 'scope', 'plan', 'planned', 'current', 'now', 'its']); + /** Fixture-owned baseline for a completed scope-preservation decision. */ export interface CeoPostureSource { path: string; content: string } @@ -419,25 +421,27 @@ function hasCompletedScopePreservation(transcript: PlanCountTranscript, selected if (selectedPlans.length !== 1 || selectedPlans[0] !== name || plans.length !== 1 || plans[0] !== name || !/\bHOLD SCOPE\b/.test(context) || /\b(?:SCOPE EXPANSION|SELECTIVE EXPANSION|SCOPE REDUCTION|historical|previous|example|hypothetical|withdrawn)\b/i.test(context)) return false; if (q.multiSelect || q.options.length !== 2 || !singleScopeBrief(q.question, q.options.map(o => o.description ?? ''), false)) return false; - const labels = q.options.map(o => o.label.replace(/\s*\(recommended\)\s*$/i, '').trim()); - const kept = labels.map(label => /^Keep ([a-z][a-z -]{0,70}) in scope$/i.exec(label)); - const index = kept.findIndex(Boolean); - if (index < 0 || kept.filter(Boolean).length !== 1 || call.answers?.[q.question] !== q.options[index]!.label) return false; - const subject = kept[index]![1]!.trim(); + // One option keeps an item and the other defers it; the answer selects the + // keep option. Label and question wording are free. + const labels = q.options.map(o => o.label.replace(/\s*\(recommended\)\s*$/i, '').replace(/^[A-B][):.]\s*/i, '').trim()); + const keeps = labels.map(label => /\b(?:keep|retain|include)\b/i.test(label) && !/\b(?:defer|todos?|drop|cut)\b/i.test(label)); + const index = keeps.indexOf(true); + if (index < 0 || keeps.filter(Boolean).length !== 1 || !/\b(?:defer|todos?|later|drop|cut)\b/i.test(labels[1 - index]!) || + call.answers?.[q.question] !== q.options[index]!.label) return false; const escape = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); - if (!new RegExp(`^Defer ${escape(subject)} to TODOS(?:\\.md)?$`, 'i').test(labels[1-index]!) || - !new RegExp(`^D\\d+\\s*[—–-]\\s*Keep (?:the )?${escape(subject)}\\b[^?\\n]* in scope, or defer`, 'i').test(q.question)) return false; const body = q.question + '\n' + q.options[index]!.description; if (!q.options.every(o => typeof o.description === 'string' && o.description.trim()) || - /\b(?:also|additionally|separately|expand(?:ing)? scope|outside (?:the )?(?:plan|scope)|and add|plus new|withdrawn|cancelled|canceled|retracted|revoked|hypothetical)\b/i.test(body)) return false; - // The choice retains an actual baseline requirement and names concrete - // failure/proof consequences, rather than merely repeating the mode label. + /\b(?:also|additionally|separately|expand(?:ing)? scope|outside (?:the )?(?:plan|scope)|and add|plus new|withdrawn|cancelled|canceled|retracted|revoked|hypothetical|historical)\b/i.test(body) || + /^\s*(?:for )?example\b/i.test(q.question)) return false; + // The kept item is an actual baseline requirement of the plan, and the brief + // explains its consequences rather than merely repeating the mode label. const required = source.content.split('\n').filter(line => /^\s*-\s+/.test(line)); - if (!required.some(line => new RegExp(`\\b${escape(subject)}\\b`, 'i').test(line)) || - !new RegExp(`(?:${escape(name)} lists|the plan already states)\\b`, 'i').test(q.question)) return false; - if ([/\btests?\b/i, /\bauthz\b|\baccess\b/i, /\b(?:concurrent|duplicate|stale)\b/i, - /\b(?:delete.and.recreate|delete\+recreate|re-save)\b/i, /\b(?:pilot|reuse) metric\b/i] - .filter(pattern => pattern.test(body)).length < 2) return false; + const subjectWords = (labels[index]!.toLowerCase().match(/[a-z][a-z_-]{2,}/g) ?? []) + .filter(word => !KEEP_LABEL_WORDS.has(word)); + if (!subjectWords.some(word => required.some(line => new RegExp(`\\b${escape(word)}\\b`, 'i').test(line)))) return false; + const rationale = [/ELI10:([\s\S]*?)(?=Stakes if (?:we pick )?wrong:)/i, /Stakes if (?:we pick )?wrong:([\s\S]*?)(?=Recommendation:)/i] + .map(part => part.exec(q.question)?.[1] ?? '').join(' '); + if ((rationale.match(/\S+/g) ?? []).length < 20) return false; const times = completedQuestionTimes(call, events); if (!times || times.answeredAt > Date.now()) return false; const own = events.filter(e => e.sessionId === selected.sessionId); const loaded = own.some(use => { diff --git a/test/helpers/ceo-split-question-policy.ts b/test/helpers/ceo-split-question-policy.ts index 2f6f8c037..331029981 100644 --- a/test/helpers/ceo-split-question-policy.ts +++ b/test/helpers/ceo-split-question-policy.ts @@ -5,6 +5,13 @@ import type { AskUserQuestionFingerprint } from './claude-pty-runner'; import type { PlanCountTranscript } from './plan-count-transcript'; import { isDeepStrictEqual } from 'node:util'; +/** + * Minimum target AskUserQuestion calls for the five split-overflow candidates: + * five options at four per call need at least two calls. Batching compatible + * candidates is legitimate; every candidate still needs its own disposition. + */ +export const CEO_SPLIT_CALL_FLOOR = 2; + const optionLabel = (label: string) => label.trim().replace(/^[A-D][).] /, '') .replace(/ \(recommended\)$/i, ''); const platforms = ['Slack', 'Discord', '(?:Microsoft )?Teams', 'Telegram', 'Mattermost']; diff --git a/test/helpers/coverage-audit-evidence.ts b/test/helpers/coverage-audit-evidence.ts index 9f755b5ca..1a7e236bc 100644 --- a/test/helpers/coverage-audit-evidence.ts +++ b/test/helpers/coverage-audit-evidence.ts @@ -452,12 +452,43 @@ function seededDiagram(output: string): boolean { } return false; } +/** + * The requested closing coverage summary: the last JSON object in the output + * with string arrays "tested" and "untested". Quoted lines and objects under + * an example/sample heading do not count. processPayment must be tested and + * refundPayment untested, each in one list only. + */ +export function coverageSummaryClassifiesSeed(output: string): boolean { + const lines = output.split('\n'); + let summary: { tested: string[]; untested: string[] } | undefined; + for (const match of output.matchAll(/\{[^{}]*"tested"[^{}]*\}/g)) { + const before = output.slice(0, match.index!); + const line = before.slice(before.lastIndexOf('\n') + 1); + const lineIndex = before.split('\n').length - 1; + const heading = lines.slice(Math.max(0, lineIndex - 3), lineIndex).join('\n'); + if (/^\s*>/.test(line) || /^(?:[#*\s]*)(?:example|sample|illustration)\b/im.test(heading)) continue; + let parsed: unknown; + try { parsed = JSON.parse(match[0]); } catch { continue; } + const value = parsed as { tested?: unknown; untested?: unknown }; + if (!Array.isArray(value.tested) || !Array.isArray(value.untested) || + ![...value.tested, ...value.untested].every(item => typeof item === 'string')) continue; + summary = { tested: value.tested as string[], untested: value.untested as string[] }; + } + if (!summary) return false; + const names = (list: string[], name: string) => list.some(item => new RegExp(`\\b${name}\\b`).test(item)); + return names(summary.tested, 'processPayment') && !names(summary.untested, 'processPayment') && + names(summary.untested, 'refundPayment') && !names(summary.tested, 'refundPayment'); +} + export function coverageAuditVerdict( result: Pick, files: CoverageAuditFiles, ) { const reads = coverageAuditReadEvidence(Array.isArray(result.transcript) ? result.transcript : [], files); - const diagram = typeof result.output === 'string' && seededDiagram(result.output), + // The closing summary states the outcome directly. The diagram grammar stays + // as a fallback so stored runs that predate the summary keep their verdicts. + const summary = typeof result.output === 'string' && coverageSummaryClassifiesSeed(result.output); + const diagram = typeof result.output === 'string' && (summary || seededDiagram(result.output)), failures: string[] = []; if (result.exitReason !== 'success') failures.push('capture did not complete successfully'); if (result.browseErrors.length) failures.push('capture reported tool errors'); diff --git a/test/helpers/disabled-plan-review-fixture.ts b/test/helpers/disabled-plan-review-fixture.ts index dc3599f21..3fbf3c488 100644 --- a/test/helpers/disabled-plan-review-fixture.ts +++ b/test/helpers/disabled-plan-review-fixture.ts @@ -282,6 +282,23 @@ function hasUnattributedOutsideCompletion(output: string, priorRecord?: Record/.test(before.slice(before.lastIndexOf('\n') + 1))) continue; + try { + const value = (JSON.parse(match[0]) as { outside_review_this_run?: unknown }).outside_review_this_run; + if (typeof value === 'string') status = value.trim().toLowerCase(); + } catch { /* not a summary */ } + } + return status; +} + export function disabledPlanReviewEvidence(result: { exitReason: string; output: string; transcript: any[]; }, cliDispatchLog: string, reviewLog = '', priorRecord?: Record) { @@ -299,10 +316,16 @@ export function disabledPlanReviewEvidence(result: { && /^CODEX_MODE: disabled\s*$/m.test(call.output)); const completed = result.exitReason === 'success' && terminal?.subtype === 'success' && terminal.is_error !== true && typeof terminal.result === 'string' && terminal.result.trim().length > 0; - const disabledAttribution = /outside_status["'`*]*\s*[:=]\s*["'`*]*disabled\b/i.test(result.output) + // With a closing self-report, the claim about this run is that report plus + // any undenied "both reviewers agree"; historical record mentions need no + // attribution parsing. Runs without one keep the prose oracle. + const summary = outsideReviewThisRun(result.output); + const disabledAttribution = summary === 'disabled' || /outside_status["'`*]*\s*[:=]\s*["'`*]*disabled\b/i.test(result.output) || result.output.split('\n').some(line => /\b(?:outside(?:\s+(?:voice|review))?|codex(?:\s+review)?)\b/i.test(line) && /\bdisabled\b/i.test(line) && !/\bnot\s+disabled\b/i.test(line)); - const falseCompletion = hasUnattributedOutsideCompletion(result.output, priorRecord); + const falseCompletion = summary === undefined ? hasUnattributedOutsideCompletion(result.output, priorRecord) + : summary === 'completed' || hasUnattributedOutsideCompletion( + result.output.replace(/["']?\boutside_status["']*\s*[:=]\s*["']*completed\b/gi, match => ' '.repeat(match.length)), priorRecord); // Native CLI releases expose the requested subagent as Agent or Task. // Availability never permits dispatch: fallbackCalls rejects both names. const agentAvailable = Array.isArray(init?.tools) && init.tools.some((tool: unknown) => tool === 'Agent' || tool === 'Task'); diff --git a/test/helpers/docsync-observer.ts b/test/helpers/docsync-observer.ts index b23d81984..23f58df3c 100644 --- a/test/helpers/docsync-observer.ts +++ b/test/helpers/docsync-observer.ts @@ -189,9 +189,48 @@ Private artifact filenames must end in .json, .md or .markdown; .txt and .log fi Keep artifacts concise: update the invocation record in place with ids, counts, decisions and evidence paths. Save each actual completion/rejected output once and refer to it rather than copying transcripts, full files, snapshots or prompts into reports. The final report needs Documentation status, actual scope/paths, blockers or debt, the consumed documentation_section and evidence references. Preserve all consumed evidence; omit repeated narration. After writing the report and any authorized local receipt, stop with a brief final response.`; } +const DOCS_READ_COMMANDS = ['pwd', 'ls', 'cat', 'sha256sum', 'stat']; +const DOCS_GIT_READS = ['status', 'diff', 'show', 'log', 'ls-files', 'rev-parse', 'merge-base', 'hash-object']; +const DOCS_SAFE_GIT_CONFIG = /^(?:core\.pager=cat|core\.quotepath=(?:true|false|on|off)|color\.[a-z.]+=(?:true|false|never|always|auto))$/i; + +/** + * Remove wrappers that cannot change what a read-only command does: a leading + * `cd &&`, `2>/dev/null`, a trailing `|| true`, and Git global + * options that only point at the fixture repo or switch off paging/colour. + * Returns null when a Git global option could change behaviour. + */ +function unwrapHarmlessRead(text: string, fixture: ReturnType): string | null { + const unquote = (value: string) => value.replace(/^(['"])(.*)\1$/, '$2'); + const sameRepo = (value: string) => { + try { return fs.realpathSync(unquote(value)) === fs.realpathSync(fixture.repo); } catch { return false; } + }; + let out = text.replace(/\s+\|\|\s+true$/, '').replace(/\s+2>\s*\/dev\/null(?=\s|$)/g, '').trim(); + const cd = /^cd\s+('[^']*'|"[^"]*"|\S+)\s+&&\s+/.exec(out); + if (cd && sameRepo(cd[1]!)) out = out.slice(cd[0].length); + const globals = /^git((?:\s+(?:--no-pager|-C\s+(?:'[^']*'|"[^"]*"|\S+)|-c\s+(?:'[^']*'|"[^"]*"|\S+)))+)(?=\s)/.exec(out); + if (!globals) return out; + for (const option of globals[1]!.matchAll(/(--no-pager)|-C\s+('[^']*'|"[^"]*"|\S+)|-c\s+('[^']*'|"[^"]*"|\S+)/g)) { + if (option[2] !== undefined && !sameRepo(option[2])) return null; + if (option[3] !== undefined && !DOCS_SAFE_GIT_CONFIG.test(unquote(option[3]))) return null; + } + return 'git' + out.slice(globals[0].length); +} + export function docsCommandAllowed(command: string, fixture: ReturnType, scripts: string[] = []): boolean { const text = command.trim(); if (docsPreambleCommands(fixture).some(block => block.trim() === text)) return true; + // Harmless wrappers are accepted on read-only commands only; lifecycle and + // script calls must stay literal so their failures are not suppressed. + const unwrapped = unwrapHarmlessRead(text, fixture); + if (unwrapped !== text) { + if (unwrapped === null || !literalDocsCommandAllowed(unwrapped, fixture, [])) return false; + const name = unwrapped.split(/\s+/)[0]!; + return DOCS_READ_COMMANDS.includes(name) || name === 'git'; + } + return literalDocsCommandAllowed(text, fixture, scripts); +} + +function literalDocsCommandAllowed(text: string, fixture: ReturnType, scripts: string[]): boolean { if (/[\x00-\x08\x0a-\x1f\x7f;&|<>`$\\()]/.test(text)) return false; const args: string[] = []; const literal = /(?:'([^']*)'|"([^"]*)"|([^\s'"]+))(?:[ \t]+|$)/y; @@ -208,12 +247,12 @@ export function docsCommandAllowed(command: string, fixture: ReturnType path.basename(arg) === 'actor-state.json') && !((commandName === 'bun' || commandName === process.execPath) && scripts.includes(rest[0]))) return false; - if (['pwd', 'ls', 'cat', 'sha256sum', 'stat'].includes(commandName)) return true; + if (DOCS_READ_COMMANDS.includes(commandName)) return true; if (commandName === 'git') { if (rest.some(arg => /^(?:--output|--ext-diff|--textconv|-w)(?:=|$)/.test(arg))) return false; if (rest[0] === 'hash-object' && rest.some(arg => /^-[^-]*w/.test(arg))) return false; if (rest[0] === 'branch') return rest.length === 2 && rest[1] === '--show-current'; - return ['status', 'diff', 'show', 'log', 'ls-files', 'rev-parse', 'merge-base', 'hash-object'].includes(rest[0]); + return DOCS_GIT_READS.includes(rest[0]); } if (commandName === 'bun' || commandName === process.execPath) { return scripts.includes(rest[0]); diff --git a/test/helpers/overlay-attempt.ts b/test/helpers/overlay-attempt.ts index 4a5803333..cfe5c59d8 100644 --- a/test/helpers/overlay-attempt.ts +++ b/test/helpers/overlay-attempt.ts @@ -101,12 +101,14 @@ export function assessOverlayArms(fixture: OverlayFixture, overlay: OverlayTrial const measurementsValid = comparison.status !== 'incomplete' && [overlay, off].every((arm) => arm.length === fixture.trials && Array.from(arm).every((trial) => trial?.passed)); // Baseline completion is an experimental variable for literal scope. A valid - // OFF sample may score 0..3; ON must complete all three. Likewise, dedicated - // tools allow Bash in OFF while every ON sample must use zero Bash calls. + // OFF sample may score 0..3; ON must complete all three. Every ON trial has + // already passed its fixture's output check to count as a valid measurement. const correctnessPassed = measurementsValid && overlay.every((trial) => trial.taskCorrect); - // Contract v2 makes no resource non-regression or universal efficacy promise. - // Keep the original comparison verdict even when exact behavior passes. - return { metrics, comparison, measurementsValid, correctnessPassed, passed: correctnessPassed }; + // A fixture with a gate also blocks on its comparison (dedicated tools: 20% + // fewer Bash calls, or none at a zero baseline). Without a gate the + // comparison verdict stays research evidence. + const gatePassed = measurementsValid && (fixture.gate?.(metrics) ?? true); + return { metrics, comparison, measurementsValid, correctnessPassed, gatePassed, passed: correctnessPassed && gatePassed }; } /** Preserve each SDK retry stream, including events emitted before an exception. */ diff --git a/test/helpers/overlay-case-policy.ts b/test/helpers/overlay-case-policy.ts index 454568ccf..b29caff8b 100644 --- a/test/helpers/overlay-case-policy.ts +++ b/test/helpers/overlay-case-policy.ts @@ -1,11 +1,13 @@ -/** Version 2 gates supported behavior; comparative efficacy stays research-only. - * Version 1 required efficacy lift. Never reinterpret its recorded failures. +/** Version 3 gates supported behavior, plus the comparison for fixtures that + * declare a gate (dedicated tools: correct output and 20% fewer Bash calls, + * replacing v2's zero Bash in every ON trial). Other comparisons stay research-only. + * Version 1 required efficacy lift. Never reinterpret recorded failures of earlier versions. */ export const OVERLAY_CONTRACT = { name: 'overlay-behavior', - version: 2, - releaseGate: 'execution_scope_complete_sampling_and_on_correctness', - comparisonRole: 'research_only', + version: 3, + releaseGate: 'execution_scope_complete_sampling_on_correctness_and_fixture_gate', + comparisonRole: 'research_only_unless_fixture_gate', resourceNonRegression: 'not_established', } as const; diff --git a/test/helpers/plan-format-kind-note.ts b/test/helpers/plan-format-kind-note.ts new file mode 100644 index 000000000..d4258e42c --- /dev/null +++ b/test/helpers/plan-format-kind-note.ts @@ -0,0 +1,6 @@ +/** + * A kind-differentiated AskUserQuestion says its options are not comparable on + * coverage. The preamble's "options differ in kind" note is one wording; any + * brief statement of the same fact counts. + */ +export const KIND_NOTE_RE = /\b(?:options? (?:differ|are different) in kind|differ in kind|different kinds? of (?:choice|option|decision)s?|not comparable on (?:coverage|completeness)|no completeness score)\b/i; diff --git a/test/helpers/plan-mode-evidence.ts b/test/helpers/plan-mode-evidence.ts index e11968978..9f80ebdf5 100644 --- a/test/helpers/plan-mode-evidence.ts +++ b/test/helpers/plan-mode-evidence.ts @@ -52,3 +52,27 @@ export function assertPlanModeWithEvidence( throw error; } } + +/** + * A seeded plan-mode review targeted the seeded plan without asking which + * target to review: either the auto-selection was observed (visible + * announcement or native seeded-plan selection) or seed-only plan content + * appeared in the review output after the slash command. The announcement's + * exact wording is not required. + */ +export function seededPlanTargeted(observation: PlanSkillObservation, seedTokens: readonly string[]): boolean { + return observation.scopeGateAutoSelectObserved === true + || seedTokens.some(token => observation.tokensObserved?.[token] === true); +} + +/** + * The AskUserQuestion format forbids writing a decision to the plan file as a + * substitute for asking, so a plan file with a `## Decisions` section fails. + */ +export function assertNoPlanFileDecisions(observation: PlanSkillObservation, hasDecisions: (planFile: string) => boolean): void { + if (observation.outcome === 'plan_ready' && observation.planFile && hasDecisions(observation.planFile)) { + throw new Error(`plan file ${observation.planFile} carries a ## Decisions section: decisions must be asked ` + + `(AskUserQuestion or the prose fallback), not written to the plan as a substitute.\n` + + `--- evidence (last 2KB) ---\n${observation.evidence}`); + } +} diff --git a/test/helpers/shared-libs-eval-fixture.ts b/test/helpers/shared-libs-eval-fixture.ts index aa6776ab3..68678e97b 100644 --- a/test/helpers/shared-libs-eval-fixture.ts +++ b/test/helpers/shared-libs-eval-fixture.ts @@ -1031,6 +1031,24 @@ export function toolCommandTrace(result: { toolCalls: Array<{ tool: string; inpu return result.toolCalls.filter(call => call.tool === 'Bash').map(call => String(call.input?.command || '')); } +/** + * PR-coverage request check against the skill's stated older-open-PR budget + * (five metadata pages, fifty file-list pages, plus recent PR 7's one page) + * and the coverage outcome: PR 42's file list was read from page 1 through + * page 2. How often a page was re-read within budget is not graded. + */ +export function prCoverageRequestViolations(endpoints: string[]): string[] { + const violations: string[] = []; + const coordination = endpoints.filter(endpoint => /\/pulls\/42\/files/.test(endpoint)); + if (!coordination.some(endpoint => !/[?&]page=/.test(endpoint) || /[?&]page=1(?:&|$)/.test(endpoint))) violations.push('PR 42 first file page not read'); + if (!coordination.some(endpoint => /[?&]page=2(?:&|$)/.test(endpoint))) violations.push('PR 42 second file page not read'); + const openPages = endpoints.filter(endpoint => /\/pulls\?/.test(endpoint) && /state=open/.test(endpoint)).length; + if (openPages > 5) violations.push(`${openPages} open-PR metadata pages exceed the budget of 5`); + const filePages = endpoints.filter(endpoint => /\/pulls\/\d+\/files/.test(endpoint)).length; + if (filePages > 51) violations.push(`${filePages} file-list pages exceed the budget of 50 plus PR 7`); + return violations; +} + /** Whether the first read of a PR's file-list page 1 left no usable file set: * truncated by `head -c`, or a failed filter (e.g. a jq error) that printed no * file entries. One recovery read of page 1 is then legitimate, still charged diff --git a/test/helpers/ship-triage-labels.ts b/test/helpers/ship-triage-labels.ts new file mode 100644 index 000000000..af1572dca --- /dev/null +++ b/test/helpers/ship-triage-labels.ts @@ -0,0 +1,36 @@ +export type TriageLabel = 'in-branch' | 'pre-existing'; + +const LABEL = /\b(in[- ]branch|pre[- ]existing)\b/gi; +const normalize = (label: string): TriageLabel => (/^in/i.test(label) ? 'in-branch' : 'pre-existing'); + +/** + * Classify each seeded ship-triage failure from the agent's final message. + * Prefers the requested JSON object ({"": "in-branch" | "pre-existing"}); + * otherwise uses the lines that name exactly one failure and exactly one label. + * A failure labelled both ways, or not at all, stays undefined. + */ +export function triageLabels(output: string): { string?: TriageLabel; math?: TriageLabel } { + const result: { string?: TriageLabel; math?: TriageLabel } = {}; + for (const block of output.match(/\{[^{}]*\}/g) ?? []) { + let parsed: unknown; + try { parsed = JSON.parse(block); } catch { continue; } + if (!parsed || typeof parsed !== 'object') continue; + for (const [key, value] of Object.entries(parsed as Record)) { + if (typeof value !== 'string' || !/^(in[- ]branch|pre[- ]existing)$/i.test(value.trim())) continue; + if (/string|truncate/i.test(key)) result.string = normalize(value.trim()); + if (/math|divide/i.test(key)) result.math = normalize(value.trim()); + } + } + if (result.string && result.math) return result; + const seen: Record<'string' | 'math', Set> = { string: new Set(), math: new Set() }; + for (const line of output.split('\n')) { + const names = (['string', 'math'] as const).filter(name => + (name === 'string' ? /string|truncate/i : /math|divide/i).test(line)); + const labels = [...new Set([...line.matchAll(LABEL)].map(match => normalize(match[1]!)))]; + if (names.length === 1 && labels.length === 1) seen[names[0]!].add(labels[0]!); + } + for (const name of ['string', 'math'] as const) { + if (!result[name] && seen[name].size === 1) result[name] = [...seen[name]][0]; + } + return result; +} diff --git a/test/helpers/shipped-skill-routing.ts b/test/helpers/shipped-skill-routing.ts new file mode 100644 index 000000000..d87941e15 --- /dev/null +++ b/test/helpers/shipped-skill-routing.ts @@ -0,0 +1,33 @@ +import * as fs from 'fs'; +import * as path from 'path'; + +const ROOT = path.resolve(import.meta.dir, '..', '..'); + +export interface ShippedSkillRouting { + /** The whole `## Skill routing` section gstack offers to append to CLAUDE.md. */ + section: string; + /** The heading plus the opening instruction, without the per-skill rules. */ + instruction: string; + /** The `- request → invoke /skill` rule lines. */ + rules: string[]; +} + +/** + * Read the routing section from the routing-injection heredoc in + * bin/gstack-skill-start, so E2E fixtures use the routing text users receive. + */ +export function readShippedSkillRouting(root = ROOT): ShippedSkillRouting { + const source = fs.readFileSync(path.join(root, 'bin', 'gstack-skill-start'), 'utf8'); + const heredoc = source.match(/<<'EOI' \| _emit_block routing-injection\n([\s\S]*?)\nEOI\n/)?.[1]; + if (!heredoc) throw new Error('bin/gstack-skill-start: routing-injection heredoc not found'); + const start = heredoc.search(/^## Skill routing$/m); + const end = heredoc.search(/^If B:/m); + if (start < 0 || end < start) throw new Error('bin/gstack-skill-start: ## Skill routing section not found'); + const section = heredoc.slice(start, end).trim(); + const rules = section.split('\n').filter(line => /^- .+→ invoke \//.test(line)); + const instruction = section.split(/\n\n/).slice(0, 2).join('\n\n'); + if (rules.length === 0 || !/Skill tool/.test(instruction)) { + throw new Error('bin/gstack-skill-start: ## Skill routing section has an unexpected shape'); + } + return { section, instruction, rules }; +} diff --git a/test/helpers/skill-body-narration.ts b/test/helpers/skill-body-narration.ts new file mode 100644 index 000000000..a3009d425 --- /dev/null +++ b/test/helpers/skill-body-narration.ts @@ -0,0 +1,8 @@ +/** + * Transcript narration that leaked into a synthesized SKILL.md body: a line + * opening with "I ", "I'll " or "Let me ", or "let me " mid-sentence. + * "let me know" is ordinary instruction text and stays legal. + */ +export function hasNarrationLeak(body: string): boolean { + return /^(?:I |I'll |Let me )/m.test(body) || /\blet me (?!know\b)/i.test(body); +} diff --git a/test/helpers/test-value-fixture.ts b/test/helpers/test-value-fixture.ts index e04b3c329..719d9bdd7 100644 --- a/test/helpers/test-value-fixture.ts +++ b/test/helpers/test-value-fixture.ts @@ -118,3 +118,13 @@ export function jsonFindings(output: string): any[] { try { return [JSON.parse(line)]; } catch { return []; } }); } + +/** + * A test-only-export finding records its non_test_callers count and how the + * callers were searched. Any repository search counts (git grep, rg, grep -r, + * the Grep tool); the specific command is not graded. + */ +export function recordsNonTestCallerSearch(finding: { evidence?: unknown }): boolean { + const evidence = JSON.stringify(finding.evidence ?? ''); + return /non_test_callers/.test(evidence) && /\bgit grep\b|\brg\b|\bgrep\b|\bGrep\b|\bag\b|\back\b/.test(evidence); +} diff --git a/test/helpers/touchfiles-data.ts b/test/helpers/touchfiles-data.ts index 0fd0e3558..8b178f20b 100644 --- a/test/helpers/touchfiles-data.ts +++ b/test/helpers/touchfiles-data.ts @@ -137,7 +137,7 @@ export const E2E_TOUCHFILES: Record = { 'test/helpers/office-hours-attempt.ts'], // Office Hours - 'office-hours-spec-review': [ 'office-hours/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts', + 'office-hours-spec-review': [ 'office-hours/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts', 'bin/gstack-office-hours-review', 'lib/office-hours-review.ts', 'lib/fs-atomic.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'test/helpers/skill-fixture.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/workflow-judge-input.ts'], 'office-hours-forcing-energy': [ 'office-hours/**', 'scripts/resolvers/preamble.ts', 'test/fixtures/mode-posture/**', 'test/helpers/llm-judge.ts', 'test/skill-e2e-office-hours.test.ts', 'test/helpers/office-hours-attempt.ts' @@ -147,26 +147,26 @@ export const E2E_TOUCHFILES: Record = { ], // Plan reviews - 'plan-ceo-review': [ 'plan-ceo-review/**', 'test/skill-e2e-plan.test.ts', + 'plan-ceo-review': [ 'plan-ceo-review/**', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/tasks-section.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'], - 'plan-ceo-review-selective': [ 'plan-ceo-review/**', 'test/skill-e2e-plan.test.ts', + 'plan-ceo-review-selective': [ 'plan-ceo-review/**', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/tasks-section.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'], - 'plan-ceo-review-benefits': [ 'plan-ceo-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts', + 'plan-ceo-review-benefits': [ 'plan-ceo-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/tasks-section.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'], - 'plan-ceo-review-expansion-energy': [ 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'test/fixtures/mode-posture/**', 'test/helpers/llm-judge.ts', 'test/skill-e2e-plan.test.ts', + 'plan-ceo-review-expansion-energy': [ 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'test/fixtures/mode-posture/**', 'test/helpers/llm-judge.ts', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/tasks-section.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'], 'plan-eng-review': [ 'scripts/resolvers/learnings.ts', - "test/fixtures/plan-scope-recovery-av.json", 'plan-eng-review/**', 'test/skill-e2e-plan.test.ts', + "test/fixtures/plan-scope-recovery-av.json", 'plan-eng-review/**', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts', "scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts", 'scripts/resolvers/testing.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'], 'plan-eng-review-artifact': [ 'scripts/resolvers/learnings.ts', - "test/fixtures/plan-scope-recovery-av.json", 'plan-eng-review/**', 'test/skill-e2e-plan.test.ts', + "test/fixtures/plan-scope-recovery-av.json", 'plan-eng-review/**', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts', "scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts", 'scripts/resolvers/testing.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'], 'plan-review-report': [ @@ -175,7 +175,7 @@ export const E2E_TOUCHFILES: Record = { 'test/fixtures/plan-review-report-public.json', 'scripts/resolvers/learnings.ts', - "test/fixtures/plan-scope-recovery-av.json", 'plan-eng-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts', + "test/fixtures/plan-scope-recovery-av.json", 'plan-eng-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts', "scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts", 'scripts/resolvers/testing.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'], @@ -262,7 +262,7 @@ export const E2E_TOUCHFILES: Record = { "test/fixtures/plan-scope-target-aw.json", - 'test/fixtures/auto-decide-saved-ai.json', 'test/fixtures/auto-decide-retry-ai.json','bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty/**', 'test/fixtures/design-ui-boxed-question.json', 'test/helpers/hermetic-skill-runtime.ts', 'test/helpers/pty-trust-dialog.ts', 'test/skill-e2e-plan-devex-plan-mode.test.ts', 'test/fixtures/eng-d2-truncated-border-0bcd.json', 'test/fixtures/eng-d1-clipped-elision-1579.json', 'test/fixtures/eng-d2-planning-prelude-4d.json', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/helpers/plan-scope-selection.ts', 'test/helpers/native-auto-decide.ts', 'test/fixtures/auto-decide-current-declaration-6aef.json', 'test/fixtures/auto-decide-explanatory-mode-043a.json', 'test/fixtures/auto-decide-explanatory-mode-749df.json', 'test/fixtures/auto-decide-structured-77.json', 'test/helpers/auto-decision-state.ts', 'test/fixtures/auto-decide-state-cab3.json', 'bin/gstack-question-log', 'bin/gstack-question-preference', 'test/helpers/fake-plan-seed.ts', 'test/helpers/plan-seed-submission.ts', 'test/fixtures/plan-seed-cli.ts', 'test/fixtures/native-auto-decide-ag.json', 'test/fixtures/eng-seeded-completion-ai.json', 'test/helpers/plan-count-pending-exit.ts', 'test/helpers/pty-screen.ts', 'test/fixtures/pty-screen/**', + 'test/fixtures/auto-decide-saved-ai.json', 'test/fixtures/auto-decide-retry-ai.json','bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty/**', 'test/fixtures/design-ui-boxed-question.json', 'test/helpers/hermetic-skill-runtime.ts', 'test/helpers/pty-trust-dialog.ts', 'test/skill-e2e-plan-devex-plan-mode.test.ts', 'test/helpers/plan-mode-evidence.ts', 'test/fixtures/eng-d2-truncated-border-0bcd.json', 'test/fixtures/eng-d1-clipped-elision-1579.json', 'test/fixtures/eng-d2-planning-prelude-4d.json', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/helpers/plan-scope-selection.ts', 'test/helpers/native-auto-decide.ts', 'test/fixtures/auto-decide-current-declaration-6aef.json', 'test/fixtures/auto-decide-explanatory-mode-043a.json', 'test/fixtures/auto-decide-explanatory-mode-749df.json', 'test/fixtures/auto-decide-structured-77.json', 'test/helpers/auto-decision-state.ts', 'test/fixtures/auto-decide-state-cab3.json', 'bin/gstack-question-log', 'bin/gstack-question-preference', 'test/helpers/fake-plan-seed.ts', 'test/helpers/plan-seed-submission.ts', 'test/fixtures/plan-seed-cli.ts', 'test/fixtures/native-auto-decide-ag.json', 'test/fixtures/eng-seeded-completion-ai.json', 'test/helpers/plan-count-pending-exit.ts', 'test/helpers/pty-screen.ts', 'test/fixtures/pty-screen/**', 'test/fixtures/design-scope-announcement-ao.json', 'test/fixtures/design-scope-declaration-ak.json', @@ -307,7 +307,7 @@ export const E2E_TOUCHFILES: Record = { 'office-hours-auto-mode': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'test/fixtures/auto-decide-recommendation-361c.json', 'test/fixtures/auto-decide-target-361c.json', - 'test/fixtures/native-auto-decide-ag.json', 'test/helpers/native-auto-decide.ts', 'test/fixtures/auto-decide-current-declaration-6aef.json', 'test/fixtures/auto-decide-explanatory-mode-043a.json', 'test/fixtures/auto-decide-explanatory-mode-749df.json', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'office-hours/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty/**', 'test/fixtures/design-ui-boxed-question.json', 'test/helpers/hermetic-skill-runtime.ts', 'test/helpers/pty-trust-dialog.ts', 'test/skill-e2e-office-hours-auto-mode.test.ts', 'test/fixtures/eng-d2-truncated-border-0bcd.json', 'test/fixtures/eng-d1-clipped-elision-1579.json', 'test/fixtures/eng-d2-planning-prelude-4d.json', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', + 'test/fixtures/native-auto-decide-ag.json', 'test/helpers/native-auto-decide.ts', 'test/fixtures/auto-decide-current-declaration-6aef.json', 'test/fixtures/auto-decide-explanatory-mode-043a.json', 'test/fixtures/auto-decide-explanatory-mode-749df.json', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'office-hours/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty/**', 'test/fixtures/design-ui-boxed-question.json', 'test/helpers/hermetic-skill-runtime.ts', 'test/helpers/pty-trust-dialog.ts', 'test/skill-e2e-office-hours-auto-mode.test.ts', 'test/helpers/plan-mode-evidence.ts', 'test/fixtures/eng-d2-truncated-border-0bcd.json', 'test/fixtures/eng-d1-clipped-elision-1579.json', 'test/fixtures/eng-d2-planning-prelude-4d.json', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/fixtures/pty-companion-cli.ts', 'lib/fs-atomic.ts', 'test/helpers/plan-skill-questions.ts', 'test/fixtures/eng-auq-validation-error.json', 'test/fixtures/bash-directory-permission.json', 'test/fixtures/design-tasks-bash-permission.json', 'test/fixtures/read-permission.json', 'test/fixtures/ceo-split-e5-numbered-description-491.json', 'test/helpers/plan-skill-question-events.ts', 'test/helpers/plan-skill-question-hook-scope.ts', 'test/helpers/skill-census.ts', 'test/helpers/auto-decision-state.ts', 'test/helpers/autoplan-artifact-digest.ts', 'test/helpers/autoplan-artifact-permission.ts', 'test/helpers/autoplan-artifact-recorder.ts', 'test/helpers/dx-selected-navigation.ts', 'test/helpers/e2e-gate.ts', 'test/helpers/eng-cache-writer-decision.ts', 'test/helpers/owned-claude-transcript.ts', 'test/helpers/plan-count-artifacts.ts', 'test/helpers/plan-count-file-permission.ts', 'test/helpers/plan-count-fixture.ts', 'test/helpers/plan-count-pending-exit.ts', 'test/helpers/plan-count-pending-question.ts', 'test/helpers/plan-count-transcript.ts', 'test/helpers/plan-floor-review.ts', 'test/helpers/plan-floor-target.ts', 'test/helpers/plan-scope-selection.ts', 'test/helpers/plan-seed-submission.ts', 'test/helpers/pty-screen.ts'], 'office-hours-phase4-fork': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'office-hours/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/question-tuning.ts', 'test/helpers/llm-judge.ts', 'test/skill-e2e-office-hours-phase4.test.ts' ], 'llm-judge-recommendation': ['codex/**', 'test/helpers/llm-judge.ts', 'test/llm-judge-recommendation.test.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'codex/SKILL.md.tmpl', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts'], @@ -558,23 +558,23 @@ export const E2E_TOUCHFILES: Record = { // AskUserQuestion format regression (RECOMMENDATION + Completeness: N/10) // Fires when either template OR the two preamble resolvers change. - 'plan-ceo-review-format-mode': [ 'plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/helpers/llm-judge.ts', 'test/skill-e2e-plan-format.test.ts', + 'plan-ceo-review-format-mode': [ 'plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/helpers/llm-judge.ts', 'test/skill-e2e-plan-format.test.ts', 'test/helpers/plan-format-kind-note.ts', 'test/helpers/office-hours-attempt.ts', 'scripts/resolvers/tasks-section.ts' ], - 'plan-ceo-review-format-approach': [ 'plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/helpers/llm-judge.ts', 'test/skill-e2e-plan-format.test.ts', + 'plan-ceo-review-format-approach': [ 'plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/helpers/llm-judge.ts', 'test/skill-e2e-plan-format.test.ts', 'test/helpers/plan-format-kind-note.ts', 'test/helpers/office-hours-attempt.ts', 'scripts/resolvers/tasks-section.ts' ], 'plan-eng-review-format-coverage': [ 'scripts/resolvers/learnings.ts', - "test/fixtures/plan-scope-recovery-av.json", 'plan-eng-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/helpers/llm-judge.ts', 'test/skill-e2e-plan-format.test.ts', + "test/fixtures/plan-scope-recovery-av.json", 'plan-eng-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/helpers/llm-judge.ts', 'test/skill-e2e-plan-format.test.ts', 'test/helpers/plan-format-kind-note.ts', "scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts", 'scripts/resolvers/testing.ts', 'test/helpers/office-hours-attempt.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts' ], 'plan-eng-review-format-kind': [ 'scripts/resolvers/learnings.ts', - "test/fixtures/plan-scope-recovery-av.json", 'plan-eng-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/helpers/llm-judge.ts', 'test/skill-e2e-plan-format.test.ts', + "test/fixtures/plan-scope-recovery-av.json", 'plan-eng-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/helpers/llm-judge.ts', 'test/skill-e2e-plan-format.test.ts', 'test/helpers/plan-format-kind-note.ts', "scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts", 'scripts/resolvers/testing.ts', 'test/helpers/office-hours-attempt.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts' ], @@ -685,14 +685,14 @@ export const E2E_TOUCHFILES: Record = { // test/skill-e2e-context-skills.test.ts. These are periodic-tier because // each one spawns claude -p and costs ~$0.20-$0.40. Collectively they // verify the thing the /checkpoint → /context-save rename was for. - 'context-save-routing': [ 'context-save/**', 'scripts/resolvers/preamble.ts', 'test/skill-e2e-context-skills.test.ts'], - 'context-save-then-restore-roundtrip': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'context-save/**', 'context-restore/**', 'bin/gstack-slug', 'bin/gstack-remote-identity.sh', 'test/skill-e2e-context-skills.test.ts'], - 'context-restore-fragment-match': [ 'context-restore/**', 'test/skill-e2e-context-skills.test.ts'], - 'context-restore-empty-state': [ 'context-restore/**', 'test/skill-e2e-context-skills.test.ts'], - 'context-restore-list-delegates': [ 'context-restore/**', 'test/skill-e2e-context-skills.test.ts'], - 'context-restore-legacy-compat': [ 'context-restore/**', 'test/skill-e2e-context-skills.test.ts'], - 'context-save-list-current-branch': [ 'context-save/**', 'test/skill-e2e-context-skills.test.ts'], - 'context-save-list-all-branches': [ 'context-save/**', 'test/skill-e2e-context-skills.test.ts'], + 'context-save-routing': [ 'context-save/**', 'scripts/resolvers/preamble.ts', 'test/skill-e2e-context-skills.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts'], + 'context-save-then-restore-roundtrip': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'context-save/**', 'context-restore/**', 'bin/gstack-slug', 'bin/gstack-remote-identity.sh', 'test/skill-e2e-context-skills.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts'], + 'context-restore-fragment-match': [ 'context-restore/**', 'test/skill-e2e-context-skills.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts'], + 'context-restore-empty-state': [ 'context-restore/**', 'test/skill-e2e-context-skills.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts'], + 'context-restore-list-delegates': [ 'context-restore/**', 'test/skill-e2e-context-skills.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts'], + 'context-restore-legacy-compat': [ 'context-restore/**', 'test/skill-e2e-context-skills.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts'], + 'context-save-list-current-branch': [ 'context-save/**', 'test/skill-e2e-context-skills.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts'], + 'context-save-list-all-branches': [ 'context-save/**', 'test/skill-e2e-context-skills.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts'], // Document-release 'document-release': [ 'document-release/**', 'test/skill-e2e-workflow.test.ts', 'test/fixtures/coverage-audit-fixture.ts', 'test/helpers/coverage-audit.ts', 'test/helpers/office-hours-attempt.ts'], @@ -793,7 +793,7 @@ export const E2E_TOUCHFILES: Record = { "scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts", 'test/helpers/coverage-audit.ts', 'test/helpers/office-hours-attempt.ts', 'scripts/resolvers/testing.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts' ], - 'ship-triage': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'ship/**', 'bin/gstack-repo-mode', 'test/skill-e2e-triage.test.ts', + 'ship-triage': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'ship/**', 'bin/gstack-repo-mode', 'test/skill-e2e-triage.test.ts', 'test/helpers/ship-triage-labels.ts', 'scripts/resolvers/testing.ts' ], 'ship-docsync-completion': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'ship/**', 'document-release/**', 'test/skill-e2e-ship-docsync.test.ts', 'test/helpers/docsync-*.ts', 'test/helpers/session-runner.ts', 'test/helpers/hermetic-env.ts', 'bin/gstack-skill-start', 'bin/gstack-session-kind', 'scripts/resolvers/sections.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/gen-skill-docs.ts', 'scripts/resolvers/testing.ts', 'test/helpers/qa-checkpoint-evidence.ts', 'test/helpers/qa-functional-observer.ts', 'test/helpers/e2e-gate.ts', 'test/helpers/qa-evidence-producer.ts', 'test/helpers/qa-functional-fixture.ts'], @@ -884,24 +884,24 @@ export const E2E_TOUCHFILES: Record = { 'scrape-match-path': [ 'scrape/**', 'browse/src/browser-skills.ts', 'browse/src/browser-skill-commands.ts', 'browser-skills/hackernews-frontpage/**', - 'test/skill-e2e-skillify.test.ts' + 'test/skill-e2e-skillify.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts', 'test/helpers/skill-body-narration.ts' ], 'scrape-prototype-path': [ 'scrape/**', 'browse/src/browser-skills.ts', 'browse/src/browser-skill-commands.ts', - 'test/skill-e2e-skillify.test.ts' + 'test/skill-e2e-skillify.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts', 'test/helpers/skill-body-narration.ts' ], 'skillify-happy-path': [ 'skillify/**', 'scrape/**', 'browse/src/browser-skill-write.ts', 'browse/src/browser-skills.ts', 'browse/src/browser-skill-commands.ts', - 'test/skill-e2e-skillify.test.ts' + 'test/skill-e2e-skillify.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts', 'test/helpers/skill-body-narration.ts' ], 'skillify-provenance-refusal': [ 'skillify/**', 'browse/src/browser-skill-write.ts', - 'test/skill-e2e-skillify.test.ts' + 'test/skill-e2e-skillify.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts', 'test/helpers/skill-body-narration.ts' ], 'skillify-approval-reject': [ 'skillify/**', 'scrape/**', 'browse/src/browser-skill-write.ts', - 'test/skill-e2e-skillify.test.ts' + 'test/skill-e2e-skillify.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts', 'test/helpers/skill-body-narration.ts' ], // Skill routing — journey-stage tests (depend on ALL skill descriptions) @@ -909,7 +909,7 @@ export const E2E_TOUCHFILES: Record = { "test/fixtures/plan-scope-recovery-av.json", - "test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts', + "test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts', "scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts" @@ -918,7 +918,7 @@ export const E2E_TOUCHFILES: Record = { "test/fixtures/plan-scope-recovery-av.json", - "test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts', + "test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts', "scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts" @@ -927,7 +927,7 @@ export const E2E_TOUCHFILES: Record = { "test/fixtures/plan-scope-recovery-av.json", - "test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts', + "test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts', "scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts" @@ -936,7 +936,7 @@ export const E2E_TOUCHFILES: Record = { "test/fixtures/plan-scope-recovery-av.json", - "test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts', + "test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts', "scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts" @@ -945,7 +945,7 @@ export const E2E_TOUCHFILES: Record = { "test/fixtures/plan-scope-recovery-av.json", - "test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts', + "test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts', "scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts" @@ -954,7 +954,7 @@ export const E2E_TOUCHFILES: Record = { "test/fixtures/plan-scope-recovery-av.json", - "test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts', + "test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts', "scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts" @@ -963,7 +963,7 @@ export const E2E_TOUCHFILES: Record = { "test/fixtures/plan-scope-recovery-av.json", - "test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts', + "test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts', "scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts" @@ -972,7 +972,7 @@ export const E2E_TOUCHFILES: Record = { "test/fixtures/plan-scope-recovery-av.json", - "test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts', + "test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts', "scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts" @@ -981,7 +981,7 @@ export const E2E_TOUCHFILES: Record = { "test/fixtures/plan-scope-recovery-av.json", - "test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts', + "test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts', "scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts" @@ -990,7 +990,7 @@ export const E2E_TOUCHFILES: Record = { "test/fixtures/plan-scope-recovery-av.json", - "test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts', + "test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts', "scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts" @@ -1100,7 +1100,7 @@ export const E2E_TOUCHFILES: Record = { "test/fixtures/plan-scope-recovery-av.json", - "test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts', + "test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts', "scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts" diff --git a/test/helpers/workflow-boundaries-fixture.ts b/test/helpers/workflow-boundaries-fixture.ts index b420e1f69..1a03882c2 100644 --- a/test/helpers/workflow-boundaries-fixture.ts +++ b/test/helpers/workflow-boundaries-fixture.ts @@ -193,14 +193,21 @@ export function boundaryFailures(fixture: ReturnType event.tool === 'Edit' && event.allowed).length === 1, 'expected one allowed native Edit'); + // Outcome, not step count: the correction landed, the final verification + // reports the expected status, and no edit follows that verification. + const allowed = evidence.executions.filter(event => event.allowed); + const lastVerify = allowed.findLastIndex(event => event.tool === 'Bash' && /\bverify\.sh\b/.test(String(event.input.command ?? ''))); + const statuses = [...evidence.receipts.matchAll(/^VERIFY_STATUS:(\d+)$/gm)].map(match => match[1]); + check(allowed.some(event => event.tool === 'Edit'), 'expected an allowed native Edit'); check(evidence.interactions.some(event => event.disposition === 'continue-investigation' && event.owner === owners[0]), 'continuation was not authorized under the acquired owner'); check(evidence.source.includes('return 2'), 'requested correction is missing'); - check(evidence.receipts.match(/^VERIFY$/gm)?.length === 1, 'verifier did not run exactly once'); + check((evidence.receipts.match(/^VERIFY$/gm)?.length ?? 0) >= 1, 'verifier did not run'); + check(lastVerify >= 0 && !allowed.slice(lastVerify + 1).some(event => event.tool === 'Edit'), 'an edit followed the final verification'); if (fixture.id === 'investigate-owned-ending-error') { - check(evidence.receipts.includes('VERIFY_STATUS:69\n'), 'ending verifier error was not delivered'); + // The fixture's ending error says to stop without retrying the verifier. + check(statuses.length === 1 && statuses[0] === '69', 'ending verifier error was not delivered once without retry'); check(/error|unavailable|cannot|could not|unable/i.test(finalText), 'ending error not acknowledged'); - } else check(evidence.receipts.includes('VERIFY_STATUS:0\n'), 'verification did not succeed'); + } else check(statuses.at(-1) === '0', 'final verification did not succeed'); } return failures; } diff --git a/test/helpers/workflow-excerpt.ts b/test/helpers/workflow-excerpt.ts index 4ed2a48e1..838fbbed3 100644 --- a/test/helpers/workflow-excerpt.ts +++ b/test/helpers/workflow-excerpt.ts @@ -10,12 +10,22 @@ export const ENG_REVIEW_EXCERPT = { endMarker: '## Section self-check (before you finish)', } as const; +// The question-procedure heading in the plan-review sections, with or without +// its older "CRITICAL RULE — " prefix. +export const ASK_QUESTIONS_HEADING = /^## [^\n]*How to ask questions[^\n]*$/m; + +export function markerIndex(text: string, marker: string | RegExp, from: number): number { + if (typeof marker === 'string') return text.indexOf(marker, from); + const at = text.slice(from).search(marker); + return at < 0 ? -1 : from + at; +} + // Same generated two-line pointer consumed by setup-gbrain-fixture.ts. const STOP_POINTER = /^> \*\*STOP\.\*\* Before [^\n]*sections\/([a-z0-9-]+\.md)[^\n]*\n> in full\.[^\n]*/gm; /** Expand on-demand sections where the agent reads them, then take the requested excerpt. */ -export function readWorkflowExcerpt(skillPath: string, startMarker: string, endMarker: string | null): string { +export function readWorkflowExcerpt(skillPath: string, startMarker: string, endMarker: string | RegExp | null): string { const secDir = path.join(ROOT, path.dirname(skillPath), 'sections'); const content = fs.readFileSync(path.join(ROOT, skillPath), 'utf-8').replace(STOP_POINTER, (_pointer, file: string) => { const body = fs.readFileSync(path.join(secDir, file), 'utf-8') @@ -25,7 +35,7 @@ export function readWorkflowExcerpt(skillPath: string, startMarker: string, endM }); const start = content.indexOf(startMarker); if (start < 0) throw new Error(`Start marker not found in ${skillPath}: "${startMarker}"`); - const end = endMarker ? content.indexOf(endMarker, start) : content.length; + const end = endMarker ? markerIndex(content, endMarker, start) : content.length; if (end < 0) throw new Error(`End marker not found in ${skillPath}: "${endMarker}"`); return content.slice(start, end); } diff --git a/test/helpers/workflow-judge-cache.ts b/test/helpers/workflow-judge-cache.ts index 272fdcd2d..3a6ad000f 100644 --- a/test/helpers/workflow-judge-cache.ts +++ b/test/helpers/workflow-judge-cache.ts @@ -11,7 +11,7 @@ import { buildEvalInputIdentity, lookupEvalInputCache, sourceDependencyClosure, type Thresholds = { clarity: number; completeness: number; actionability: number }; export interface WorkflowCacheOptions { - root: string; testName: string; skillPath: string; startMarker: string; endMarker: string | null; + root: string; testName: string; skillPath: string; startMarker: string; endMarker: string | RegExp | null; judgeContext: string; judgeGoal: string; model?: string; thresholds: Thresholds; prompt: string; attempt: number; references?: readonly string[]; agentCapability?: 'frontier'; @@ -135,3 +135,15 @@ export function prepareWorkflowJudgeCache(opts: WorkflowCacheOptions): { }, }; } + +/** + * Pass floors for the browse reference judge panel mean. The stored baseline + * is recorded for comparison only: a three-sample mean is too noisy for a + * no-dip ratchet, and gating on it silently raised the clarity floor to 4. + */ +export const BROWSE_JUDGE_FLOORS = { clarity: 3, completeness: 4, actionability: 4 } as const; + +export function browseJudgeFloorsMet(scores: Record): boolean { + return (Object.keys(BROWSE_JUDGE_FLOORS) as Array) + .every(dim => scores[dim] >= BROWSE_JUDGE_FLOORS[dim]); +} diff --git a/test/helpers/workflow-judge-input.ts b/test/helpers/workflow-judge-input.ts index 36873687b..f661022f1 100644 --- a/test/helpers/workflow-judge-input.ts +++ b/test/helpers/workflow-judge-input.ts @@ -1,6 +1,7 @@ /** Preserve source-file boundaries when a workflow judge reads carved skills. */ import * as fs from 'node:fs'; import * as path from 'node:path'; +import { markerIndex } from './workflow-excerpt'; export interface WorkflowJudgeFile { path: string; @@ -84,7 +85,7 @@ export function readWorkflowJudgeInput(opts: { root: string; skillPath: string; startMarker: string; - endMarker: string | null; + endMarker: string | RegExp | null; references?: readonly string[]; }): WorkflowJudgeInput { const sources = [{ @@ -115,7 +116,7 @@ export function readWorkflowJudgeInput(opts: { const union = allSources.map(file => file.content).join('\n'); const start = union.indexOf(opts.startMarker); if (start < 0) throw new Error(`Start marker not found in ${opts.skillPath}: "${opts.startMarker}"`); - const end = opts.endMarker === null ? union.length : union.indexOf(opts.endMarker, start); + const end = opts.endMarker === null ? union.length : markerIndex(union, opts.endMarker, start); if (end < 0) throw new Error(`End marker not found in ${opts.skillPath}: "${opts.endMarker}"`); const files: WorkflowJudgeFile[] = []; diff --git a/test/office-hours-completion.test.ts b/test/office-hours-completion.test.ts index 05c8fc09d..e16685d7c 100644 --- a/test/office-hours-completion.test.ts +++ b/test/office-hours-completion.test.ts @@ -413,20 +413,8 @@ describe('office-hours reviewer finding preservation', () => { calls++; expect(prompt).toContain(JSON.stringify(original)); expect(prompt).toContain(JSON.stringify(review!.concerns)); - expect(prompt).toContain('Do not require verbatim wording or judge by issue count alone'); - expect(prompt).toContain('optional alternative remedies may be omitted'); - expect(prompt).toContain('only actual defects, never successful mappings'); - expect(prompt).toContain('Keep reasoning brief'); - expect(prompt).toContain('Complete EVERY applicable audit even if another already fails'); - expect(prompt).toContain('a coverage failure cannot skip metrics'); - expect(prompt).toContain('a metrics failure cannot skip convergence'); - expect(prompt).toContain('one brief conclusion per applicable audit'); - expect(prompt).toContain('APPROVED records user approval and is compatible with CONCERNS_RECORDED'); + // The response schema is the wire contract; the judge's wording is not. expect(prompt).toContain('{"reasoning":string,"missing":string[],"unsupported":string[],"complete":boolean}'); - expect(prompt).toContain('finalize missing and unsupported, and emit complete last'); - expect(prompt).toContain('at most 150 words total'); - expect(prompt).toContain('Coverage and the remaining inventory come from the FINAL reviewer verdict only'); - expect(prompt).toContain('lack of a confirmed prior fix does not invent a missing final-verdict finding'); return JSON.stringify(accepted); }); expect(calls).toBe(1); diff --git a/test/overlay-measurement.test.ts b/test/overlay-measurement.test.ts index 7ecfedd8b..2f18da844 100644 --- a/test/overlay-measurement.test.ts +++ b/test/overlay-measurement.test.ts @@ -6,7 +6,7 @@ import type { AgentSdkResult, QueryProvider } from './helpers/agent-sdk-runner'; import { firstAssistantMessageToolCount, reportedThinkingTokens, assessComparison, trialArtifactStem } from './helpers/overlay-measurement'; import { runOverlayTrial, awaitOverlayWorkers, assessOverlayArms, captureOverlayQueryAttempts, type OverlayTrialOutcome } from './helpers/overlay-attempt'; import { setupLiteralWorkspace, correctLiteralTargets, snapshotWorkspace, assertReadOnlyWorkspace } from './helpers/overlay-workspace'; -import { fanoutPass, higherIsBetter20Pct, lowerIsBetter20Pct, OVERLAY_FIXTURES, type OverlayFixture } from './fixtures/overlay-nudges'; +import { fanoutPass, higherIsBetter20Pct, lowerIsBetter20Pct, lowerIsBetter20PctOrZeroBaseline, OVERLAY_FIXTURES, type OverlayFixture } from './fixtures/overlay-nudges'; function result(overrides: Partial = {}): AgentSdkResult { return { @@ -64,16 +64,45 @@ describe('SDK overlay measurements', () => { }); describe('correctness versus efficacy', () => { - test.each(['claude-opus-4-7', 'claude-sonnet-4-6'])('dedicated-tool behavior gates zero ON Bash calls for %s', (model) => { + describe.each(['claude-opus-4-7', 'claude-sonnet-4-6'])('dedicated-tool gate for %s', (model) => { const f = { ...OVERLAY_FIXTURES.find(f => f.metricName === 'bash_tool_calls' && f.model === model)!, trials: 3 }; - const samples = (metric: number) => Array.from({ length: 3 }, (): OverlayTrialOutcome => - ({ passed: true, taskCorrect: f.taskCorrect!(metric), metric, exitReason: 'success' })); - expect(assessOverlayArms(f, samples(0), samples(0))).toMatchObject({ - passed: true, comparison: { status: 'baseline_saturated', criterionMet: false }, + // A trial whose final JSON fails the fixture's verify is recorded as a failed measurement. + const correct = (metric: number): OverlayTrialOutcome => ({ passed: true, taskCorrect: true, metric, exitReason: 'success' }); + const incorrect = (metric: number): OverlayTrialOutcome => ({ passed: false, taskCorrect: false, metric, exitReason: 'validation_failed' }); + const arm = (...trials: OverlayTrialOutcome[]) => trials; + test('fixture gates on the comparison and on output correctness, not on zero ON Bash', () => { + expect(f.taskCorrect).toBeUndefined(); + expect(f.verify).toBeFunction(); + expect(f.gate).toBe(lowerIsBetter20PctOrZeroBaseline); }); - // Even a >20% improvement cannot excuse breaking the exact ON contract. - expect(assessOverlayArms(f, samples(1), samples(5))).toMatchObject({ - passed: false, correctnessPassed: false, comparison: { status: 'improved', criterionMet: true }, + test('correct output with a 20% reduction passes even when ON still uses Bash', () => { + expect(assessOverlayArms(f, arm(correct(1), correct(1), correct(2)), arm(correct(5), correct(5), correct(5)))).toMatchObject({ + measurementsValid: true, correctnessPassed: true, gatePassed: true, passed: true, comparison: { status: 'improved', criterionMet: true }, + }); + }); + test('correct output with worse Bash usage fails', () => { + expect(assessOverlayArms(f, arm(correct(6), correct(6), correct(6)), arm(correct(5), correct(5), correct(5)))).toMatchObject({ + measurementsValid: true, correctnessPassed: true, gatePassed: false, passed: false, comparison: { status: 'regressed' }, + }); + expect(assessOverlayArms(f, arm(correct(5), correct(4), correct(5)), arm(correct(5), correct(5), correct(5)))).toMatchObject({ + gatePassed: false, passed: false, comparison: { status: 'no_measured_improvement' }, + }); + }); + test('a 20% reduction with incorrect output fails', () => { + expect(assessOverlayArms(f, arm(correct(0), incorrect(0), correct(0)), arm(correct(5), correct(5), correct(5)))).toMatchObject({ + measurementsValid: false, passed: false, + }); + }); + test('a zero-Bash baseline passes only when every ON trial is also zero', () => { + expect(assessOverlayArms(f, arm(correct(0), correct(0), correct(0)), arm(correct(0), correct(0), correct(0)))).toMatchObject({ + gatePassed: true, passed: true, comparison: { status: 'baseline_saturated' }, + }); + expect(assessOverlayArms(f, arm(correct(0), correct(1), correct(0)), arm(correct(0), correct(0), correct(0)))).toMatchObject({ + gatePassed: false, passed: false, + }); + // OFF median 0 with an occasional Bash call is still a zero baseline. + expect(assessOverlayArms(f, arm(correct(0), correct(0), correct(0)), arm(correct(0), correct(2), correct(0)))).toMatchObject({ gatePassed: true, passed: true }); + expect(assessOverlayArms(f, arm(correct(1), correct(0), correct(0)), arm(correct(0), correct(2), correct(0)))).toMatchObject({ gatePassed: false, passed: false }); }); }); test.each([ diff --git a/test/overlay-recording-order.test.ts b/test/overlay-recording-order.test.ts index 83ff28e47..09880b0be 100644 --- a/test/overlay-recording-order.test.ts +++ b/test/overlay-recording-order.test.ts @@ -86,7 +86,7 @@ test('versioned aggregate keeps false efficacy and cannot overwrite earlier atte const original = fs.readFileSync(file, 'utf8'); expect(JSON.parse(original)).toMatchObject({ ...verdict, contract: OVERLAY_CONTRACT }); expect(entries).toHaveLength(1); - expect(entries[0]).toMatchObject({ name: 'owned-contract-contract-v2-aggregate', passed: true, cost_usd: 0, duration_ms: 0 }); + expect(entries[0]).toMatchObject({ name: `owned-contract-contract-v${OVERLAY_CONTRACT.version}-aggregate`, passed: true, cost_usd: 0, duration_ms: 0 }); expect(() => recorder(deps, 1)({ passed: false })).toThrow('EEXIST'); expect(fs.readFileSync(file, 'utf8')).toBe(original); recorder(deps, 2)({ passed: false, errors: ['task incomplete'] }); diff --git a/test/plan-design-sdk-fixture.test.ts b/test/plan-design-sdk-fixture.test.ts index f4ae51f99..c42e04e72 100644 --- a/test/plan-design-sdk-fixture.test.ts +++ b/test/plan-design-sdk-fixture.test.ts @@ -87,37 +87,18 @@ async function exercise(mode: 'success' | 'max-turns' | 'first-timeout' | 'secon expect(opts.publicStreamDiagnostics).toBe(true); expect(opts.signal).toBeInstanceOf(AbortSignal); sdkSignal = opts.signal; expect(opts.signal.aborted).toBe(false); - // Bind the complete actual compact-delivery prompt, not selected snippets. - expect(new Bun.CryptoHasher('sha256').update(opts.prompt).digest('hex')) - .toBe('021c9f2b0522fc55bc265a976356678a048f3c17d27b63669674c805a485b28e'); expect(opts.testName).toBe(id); expect(opts.maxTurns).toBe(15); expect(opts.timeout).toBe(CAPTURE_MS); for (const key of ['model', 'tools', 'allowedTools', 'appendSystemPrompt', 'env']) expect(opts).not.toHaveProperty(key); - expect(opts.prompt).toContain('Review the plan in ./plan.md'); - expect(opts.prompt).toContain('Review all 7 design passes'); - expect(opts.prompt).toContain('0-10 and explain what would make it a 10'); - expect(opts.prompt).toContain('preserve the unresolved-decisions pass'); - expect(opts.prompt).toContain('interaction state table, empty states, responsive behavior'); - expect(opts.prompt).toContain('full required review report'); - expect(opts.prompt).toContain('Write before publishing a completed walkthrough'); - expect(opts.prompt).toContain('Read plan.md back to verify the saved changes'); - expect(opts.prompt).toContain('Then return a brief, concrete summary'); - expect(opts.prompt).toContain('natively Read plan-design-review/SKILL.md'); - expect(opts.prompt).toContain('plan-design-review/sections/review-sections.md (the one lazy section this review requires)'); - expect(opts.prompt).toContain('including the report, to about 10,000 characters; this is a drafting target, not a check, so do not count characters or trim after saving'); - expect(opts.prompt).toContain('in at most ten lines'); - expect(opts.prompt).toContain('execute every required pass and lazy-section Read'); - expect(opts.prompt).toContain('Retain all required report fields, design decisions, diagrams, ratings, and explanations'); - expect(opts.prompt).toContain('use the canonical tables and decision IDs'); - expect(opts.prompt).toContain('Specify each design requirement once'); - expect(opts.prompt).toContain('instead of repeating that specification'); - expect(opts.prompt).toContain('concise score rationales and 10/10 explanations'); + // The harness prompt's properties, in order; its wording is free. + expect(opts.prompt).toContain('plan-design-review/SKILL.md'); + expect(opts.prompt).toContain('plan-design-review/sections/review-sections.md'); + expect(opts.prompt).toContain('./plan.md'); const ordered = ['Read every lazy section', 'Review all 7 design passes', - 'EDIT plan.md', 'Keep the saved review compact', - 'Persist that complete plan and review with Write', 'Read plan.md back', + 'EDIT plan.md', 'Persist that complete plan and review with Write', 'Read plan.md back', 'Then return a brief, concrete summary'].map(text => opts.prompt.indexOf(text)); expect(ordered.every(index => index >= 0)).toBe(true); expect(ordered).toEqual([...ordered].sort((a, b) => a - b)); - expect(opts.prompt).toContain('Do NOT try to browse any URLs'); + expect(opts.prompt).toMatch(/do not try to browse any URLs/i); inputVerified = true; launched = true; if (mode === 'attempt-deadline') return await new Promise((_resolve, reject) => { opts.signal.addEventListener('abort', () => reject(opts.signal.reason), { once: true }); diff --git a/test/plan-format-kind-note.test.ts b/test/plan-format-kind-note.test.ts new file mode 100644 index 000000000..f6829da9a --- /dev/null +++ b/test/plan-format-kind-note.test.ts @@ -0,0 +1,12 @@ +import { describe, expect, test } from 'bun:test'; +import * as fs from 'fs'; +import * as path from 'path'; +import { KIND_NOTE_RE } from './helpers/plan-format-kind-note'; + +const stored = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/plan-format-kind-notes.json'), 'utf8')) as + { known_good: string[]; known_bad: string[] }; + +describe('plan-format kind note', () => { + test.each(stored.known_good)('accepts %s', line => expect(line).toMatch(KIND_NOTE_RE)); + test.each(stored.known_bad)('rejects %s', line => expect(line).not.toMatch(KIND_NOTE_RE)); +}); diff --git a/test/plan-mode-evidence.test.ts b/test/plan-mode-evidence.test.ts index b0fa0e67b..3a6f14303 100644 --- a/test/plan-mode-evidence.test.ts +++ b/test/plan-mode-evidence.test.ts @@ -63,7 +63,9 @@ await import(${JSON.stringify(path.join(ROOT, PAID_FILE))}); expect(opts.timeoutMs).toBe(CAPTURE_MS); const keys = ['skillName', 'inPlanMode', 'timeoutMs']; if (opts.initialPlanContent) { - keys.push('initialPlanContent'); + keys.push('initialPlanContent', 'trackTokens'); + expect(opts.trackTokens.length).toBeGreaterThan(0); + for (const token of opts.trackTokens) expect(opts.initialPlanContent).toContain(token); if (skill === 'eng') { keys.push('extraArgs'); expect(opts.extraArgs).toEqual(['--disallowedTools', 'AskUserQuestion']); @@ -105,7 +107,7 @@ await import(${JSON.stringify(path.join(ROOT, PAID_FILE))}); if (scenario === 'bad-report') expect(output).toContain('GSTACK REVIEW REPORT contract violation'); if (scenario === 'missing-auto-select') expect(output).toContain('scopeGateAutoSelectObserved'); if (scenario === 'scope-question') expect(output).toContain('scopeGateQuestionObserved'); - if (scenario === 'missing-decisions') expect(output).toContain('plan_ready without ## Decisions section'); + if (scenario === 'missing-decisions') expect(output).toContain('without a question'); } finally { fs.rmSync(dir, { recursive: true, force: true }); } }); } diff --git a/test/plan-mode-targeting.test.ts b/test/plan-mode-targeting.test.ts new file mode 100644 index 000000000..bb1dfcbfb --- /dev/null +++ b/test/plan-mode-targeting.test.ts @@ -0,0 +1,41 @@ +import { afterAll, describe, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { assertNoPlanFileDecisions, seededPlanTargeted } from './helpers/plan-mode-evidence'; +import { planFileHasDecisionsSection } from './helpers/claude-pty-runner'; +import type { PlanSkillObservation } from './helpers/claude-pty-runner'; + +const TOKENS = ['ShardManager', 'ResultMerger']; +const base = { outcome: 'asked', summary: '', evidence: '', elapsedMs: 1 } as unknown as PlanSkillObservation; +const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-mode-targeting-')); +afterAll(() => fs.rmSync(dir, { recursive: true, force: true })); +const plan = (name: string, body: string) => { const file = path.join(dir, name); fs.writeFileSync(file, body); return file; }; + +describe('seeded plan targeting', () => { + test.each([ + ['exact announcement observed', { scopeGateAutoSelectObserved: true }], + ['paraphrased announcement with seed content in the review', { scopeGateAutoSelectObserved: false, tokensObserved: { ShardManager: true, ResultMerger: false } }], + ])('known-good: %s', (_name, fields) => { + expect(seededPlanTargeted({ ...base, ...fields } as PlanSkillObservation, TOKENS)).toBe(true); + }); + test.each([ + ['no announcement and no seed content', { scopeGateAutoSelectObserved: false, tokensObserved: { ShardManager: false, ResultMerger: false } }], + ['an untracked token cannot stand in for the seed', { scopeGateAutoSelectObserved: false, tokensObserved: { 'draft-plan': true } }], + ])('known-bad: %s', (_name, fields) => { + expect(seededPlanTargeted({ ...base, ...fields } as PlanSkillObservation, TOKENS)).toBe(false); + }); +}); + +describe('plan-file decision substitute', () => { + test('a plan_ready plan file with ## Decisions fails', () => { + const planFile = plan('decisions.md', '# Plan\n\n## Decisions to confirm\n- A or B?\n\n## GSTACK REVIEW REPORT\n'); + expect(() => assertNoPlanFileDecisions({ ...base, outcome: 'plan_ready', planFile } as PlanSkillObservation, planFileHasDecisionsSection)) + .toThrow('not written to the plan as a substitute'); + }); + test('a plan_ready plan file without ## Decisions passes, and asked runs are not checked', () => { + const planFile = plan('clean.md', '# Plan\n\n## Approach\nShip it.\n\n## GSTACK REVIEW REPORT\n'); + expect(() => assertNoPlanFileDecisions({ ...base, outcome: 'plan_ready', planFile } as PlanSkillObservation, planFileHasDecisionsSection)).not.toThrow(); + expect(() => assertNoPlanFileDecisions({ ...base, outcome: 'asked' } as PlanSkillObservation, planFileHasDecisionsSection)).not.toThrow(); + }); +}); diff --git a/test/section-capture-native-tools.test.ts b/test/section-capture-native-tools.test.ts index 8f3e48c0e..1ec7b464b 100644 --- a/test/section-capture-native-tools.test.ts +++ b/test/section-capture-native-tools.test.ts @@ -303,9 +303,9 @@ console.log(JSON.stringify({plain:plain.exitReason,literal:literal.exitReason,se expect(added).toBeGreaterThan(-1); expect(args(1)[added + 1]).toBe(literal); expect(args(1).filter((_, i) => i !== added && i !== added + 1)).toEqual(args(0)); const sectionArgs = args(2); - const instruction = launches[2].prompt.split('\n').find((line: string) => line.includes('with the Read tool BEFORE')); + const instruction = launches[2].prompt.split('\n').find((line: string) => /with the Read tool before/i.test(line)); expect(sectionArgs).not.toContain('--append-system-prompt'); - expect(instruction).toContain('you MUST actually Read that sections/ file with the Read tool BEFORE doing the work it covers'); + expect(instruction).toMatch(/read that sections\/ file with the Read tool before doing the work it covers/i); expect(instruction).not.toContain('actual.md'); expect(instruction).not.toContain(dir); expect(sectionArgs.slice(sectionArgs.indexOf('--allowed-tools') + 1, sectionArgs.indexOf('--allowed-tools') + 8)) .toEqual(['Read', 'Grep', 'Glob', 'Write', 'Edit', 'Agent', 'Bash']); @@ -464,21 +464,16 @@ console.log(JSON.stringify(observed));`); expect(request).not.toHaveProperty('appendSystemPrompt'); const skill=['fixture','ship','office-hours'][Math.floor((i-4)/2)]!; const skillPath=path.join(dir,skill,'SKILL.md'); - const expected=`You are running an automated skill-execution test. No human is present, so AskUserQuestion is unavailable. The ONLY skill file you may read is this absolute path: ${skillPath}. Do NOT Glob/find/search for any other SKILL.md anywhere — especially nothing under ~/.claude or /Users. - -Read ${skillPath} and EXECUTE its workflow for this scenario: - -Complete the supplied scenario. - -Rules for this run: -- Skip system-audit, environment-setup, telemetry, and unrelated codebase exploration. Read the supplied plan's referenced fixture files when its review requires them. -- At any decision point that would call AskUserQuestion, silently pick the skill's recommended option and continue. Do NOT stop to ask. -- This skill's body has been carved into on-demand sections/. When the skill gives a STOP-Read directive (for example "Read \`.../sections/\` and execute it in full"), you MUST actually Read that sections/ file with the Read tool BEFORE doing the work it covers. Do not work from memory. -- Resolve installed-root paths for section and companion Markdown files under ${dir}, where this fixture's skill package is copied. -- Do NOT run git, gh, commit, push, or any mutating command. -- When the workflow is complete, write the skill's final output (the full review report / ship plan, including any required report table) to ${path.join(dir,'REPORT.md')}. -- After all required writes are complete, return a brief completion message and STOP. Do not reproduce the full report in the final response.`; - expect(request.prompt).toBe(expected); + // Harness properties, not its wording: the one allowed skill path, the + // scenario, AskUserQuestion fallback, section Read rule, no mutation, + // and the report destination. + expect(request.prompt.match(/\/[^\s]*SKILL\.md/g)?.every((file: string)=>file===skillPath)).toBe(true); + expect(request.prompt).toContain(`Read ${skillPath} and EXECUTE its workflow`); + expect(request.prompt).toContain('Complete the supplied scenario.'); + expect(request.prompt).toMatch(/AskUserQuestion is unavailable/i); + expect(request.prompt).toMatch(/read that sections\/ file with the Read tool before/i); + expect(request.prompt).toMatch(/do not run git, gh, commit, push, or any mutating command/i); + expect(request.prompt).toContain(path.join(dir,'REPORT.md')); } } }finally{if(child.exitCode===null)child.kill();await child.exited;fs.rmSync(dir,{recursive:true,force:true});} diff --git a/test/session-runner-tools.test.ts b/test/session-runner-tools.test.ts index 580899ee5..395e93d97 100644 --- a/test/session-runner-tools.test.ts +++ b/test/session-runner-tools.test.ts @@ -431,7 +431,7 @@ describe.skipIf(process.platform === 'win32')('session-runner explicit tool avai expect(child.prompt).toContain('Preserve every finding and original requirement, required decision fields and comparisons, exact approvals and verification; every diagram must retain its specified format.'); expect(child.prompt).toContain('unless needed to specify an accepted change'); expect(child.prompt).toContain('Complete every required artifact and verification before returning'); - expect(child.prompt).toContain('MUST actually Read that sections/ file with the Read tool BEFORE doing the work it covers'); + expect(child.prompt).toMatch(/read that sections\/ file with the Read tool before doing the work it covers/i); expect(child.prompt).toContain('report outside coverage as disabled'); expect(child.prompt).toContain('After all required writes are complete'); expect(child.prompt).toContain('with Write/Edit at the workflow checkpoints below'); @@ -527,7 +527,7 @@ describe.skipIf(process.platform === 'win32')('session-runner explicit tool avai expect(child.prompt).toContain('do not hide it or claim approval when no offered alternative meets these constraints'); expect(child.prompt).not.toContain("silently pick the skill's recommended option"); expect(child.prompt).toContain('Every required section, finding, approval and output still has to be completed'); - expect(child.prompt).toContain('MUST actually Read that sections/ file with the Read tool BEFORE doing the work it covers'); + expect(child.prompt).toMatch(/read that sections\/ file with the Read tool before doing the work it covers/i); expect(child.prompt).toContain('After all required writes are complete'); expect(flagValue(child.args, '--tools')).toBe('Read,Grep,Glob,Write,Edit,Agent,Bash'); expect(fs.readFileSync(path.join(dir, 'PLAN.md'), 'utf8')).toBe(fixtures['PLAN.md']); @@ -548,20 +548,19 @@ describe.skipIf(process.platform === 'win32')('session-runner explicit tool avai planDir: dir, skillName: 'ship', scenario, testName: 'ship-report-writing', timeout: 5_000, }); const skillPath = path.join(dir, 'ship', 'SKILL.md'); - expect(observed().prompt).toBe(`You are running an automated skill-execution test. No human is present, so AskUserQuestion is unavailable. The ONLY skill file you may read is this absolute path: ${skillPath}. Do NOT Glob/find/search for any other SKILL.md anywhere — especially nothing under ~/.claude or /Users. - -Read ${skillPath} and EXECUTE its workflow for this scenario: - -${scenario} - -Rules for this run: -- Skip system-audit, environment-setup, telemetry, and unrelated codebase exploration. Read the supplied plan's referenced fixture files when its review requires them. -- At any decision point that would call AskUserQuestion, silently pick the skill's recommended option and continue. Do NOT stop to ask. -- This skill's body has been carved into on-demand sections/. When the skill gives a STOP-Read directive (for example "Read \`.../sections/\` and execute it in full"), you MUST actually Read that sections/ file with the Read tool BEFORE doing the work it covers. Do not work from memory. -- Resolve installed-root paths for section and companion Markdown files under ${dir}, where this fixture's skill package is copied. -- Do NOT run git, gh, commit, push, or any mutating command. -- When the workflow is complete, write the skill's final output (the full review report / ship plan, including any required report table) to ${path.join(dir, 'REPORT.md')}. -- After all required writes are complete, return a brief completion message and STOP. Do not reproduce the full report in the final response.`); + const prompt = observed().prompt; + // CEO-only writing guidance stays out of another skill's capture prompt. + for (const ceoOnly of ['all 11 sections', 'Preserve original requirements and accepted plan amendments', + 'Cross-reference saved IDs', 'report outside coverage as disabled', 'with Write/Edit at the workflow checkpoints']) { + expect(prompt).not.toContain(ceoOnly); + } + // The functional harness rules, checked by meaning rather than bytes. + expect(prompt).toContain(scenario); + expect(prompt.match(/\/[^\s]*SKILL\.md/g)?.every(file => file === skillPath)).toBe(true); + expect(prompt).toMatch(/AskUserQuestion is unavailable/i); + expect(prompt).toMatch(/read that sections\/ file with the Read tool before/i); + expect(prompt).toMatch(/do not run git, gh, commit, push, or any mutating command/i); + expect(prompt).toContain(path.join(dir, 'REPORT.md')); }); }); diff --git a/test/shared-libs-fixture.test.ts b/test/shared-libs-fixture.test.ts index 7affb5de8..2bbbff13e 100644 --- a/test/shared-libs-fixture.test.ts +++ b/test/shared-libs-fixture.test.ts @@ -7,7 +7,7 @@ import { execFileSync, spawnSync } from 'node:child_process'; import { createSharedInteractiveToolHandler, createSharedLibsFixture, fixtureGit, fixtureWrite, installSourceShims, installHostileGitConfig, isGuardedGitRequest, reviewLifecycleInstructions, standaloneInstructions, SHARED_LIBS_ROOT, readRequests, seedOpportunitySources, sharedReadOnlyViolations, shellQuote, snapshotFixture, type SharedLibsFixture, - SharedCaptureAccumulator, type SharedCaptureAttempt, isInternalClaudeGitRequest, SHARED_LIBS_OLDER_OPEN_PRS, incompleteFirstFileView, + SharedCaptureAccumulator, type SharedCaptureAttempt, isInternalClaudeGitRequest, SHARED_LIBS_OLDER_OPEN_PRS, incompleteFirstFileView, prCoverageRequestViolations, } from './helpers/shared-libs-eval-fixture'; import { EvalCollector, type EvalTestEntry } from './helpers/eval-store'; import { collectorOutcomeCounts } from '../scripts/test-paid-shards'; @@ -1075,7 +1075,12 @@ describe('shared-code PR coverage world', () => { test('the maximum authorized open-metadata scan still leaves older open PRs unchecked', () => { const periodic = fs.readFileSync(path.join(import.meta.dir, 'skill-e2e-shared-libs-periodic.test.ts'), 'utf8'); - expect(periodic).toContain('expect(openPages.length).toBeLessThanOrEqual(5);'); + expect(periodic).toContain('prCoverageRequestViolations('); + const sixth = Array.from({ length: 6 }, (_, i) => `/repos/fixture/shared-libs/pulls?state=open&per_page=100&page=${i + 1}`); + expect(prCoverageRequestViolations([...sixth.slice(0, 5), '/repos/fixture/shared-libs/pulls/42/files?page=1', + '/repos/fixture/shared-libs/pulls/42/files?page=2'])).toEqual([]); + expect(prCoverageRequestViolations([...sixth, '/repos/fixture/shared-libs/pulls/42/files?page=1', + '/repos/fixture/shared-libs/pulls/42/files?page=2'])).toEqual(['6 open-PR metadata pages exceed the budget of 5']); const f = createSharedLibsFixture('pr-world-budget'); cleanup.push(f.root); seedOpportunitySources(f); diff --git a/test/shared-libs-pr-coverage-budget.test.ts b/test/shared-libs-pr-coverage-budget.test.ts new file mode 100644 index 000000000..b4a0cea1a --- /dev/null +++ b/test/shared-libs-pr-coverage-budget.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, test } from 'bun:test'; +import * as fs from 'fs'; +import * as path from 'path'; +import { prCoverageRequestViolations } from './helpers/shared-libs-eval-fixture'; + +const stored = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/shared-libs/pr-coverage-endpoints.json'), 'utf8')) as + { known_good: Record; known_bad: Record }; + +describe('shared-libs PR coverage request budget', () => { + test.each(Object.entries(stored.known_good))('passes %s', (_name, endpoints) => { + expect(prCoverageRequestViolations(endpoints)).toEqual([]); + }); + test.each(Object.entries(stored.known_bad))('fails %s', (_name, endpoints) => { + expect(prCoverageRequestViolations(endpoints).length).toBeGreaterThan(0); + }); + test('file-list pages beyond the stated budget fail', () => { + const pages = Array.from({ length: 52 }, (_, i) => `/repos/fixture/shared-libs/pulls/${i < 2 ? 42 : 100 + i}/files?per_page=100&page=${i < 2 ? i + 1 : 1}`); + expect(prCoverageRequestViolations(pages)).toEqual(['52 file-list pages exceed the budget of 50 plus PR 7']); + expect(prCoverageRequestViolations(pages.slice(0, 51))).toEqual([]); + }); +}); diff --git a/test/shared-libs-revalidation-prompt.test.ts b/test/shared-libs-revalidation-prompt.test.ts index c2ad0f2d9..2ce0b016c 100644 --- a/test/shared-libs-revalidation-prompt.test.ts +++ b/test/shared-libs-revalidation-prompt.test.ts @@ -71,11 +71,13 @@ function pathCaptureAdapter(capture: (...args: any[]) => Promise) { } describe('bounded shared-code revalidation prompt', () => { + // The harness prompt's wording is not a contract: the paid shared-libs E2Es + // measure whether it works. These checks keep its machine-shaped parts: + // exact receipt commands, interpolated paths and budgets, branch selection, + // prefix/suffix identity, ordering and the absence of a supplied token. test('the actor scope replaces the captured R59 exploratory stage instead of adding another prerequisite', () => { const prompt = reviewPrompt(f, instructions, input, { actorCommand: 'cat /fixture/stage-output.json' }); - expect(prompt).toContain('replaces the entire Step 4.7 QA and Step 4.8 native adversarial stages'); - expect(prompt).toContain("Step 4's early QA selection/method-loading prerequisites"); - expect(prompt).toContain("Step 5.8's QA report requirement"); + expect(prompt).toContain('cat /fixture/stage-output.json'); const excluded = prompt.slice(prompt.indexOf('Do not perform QA scope/method asset loads'), prompt.indexOf('Existing tests')); for (const packet of stageScope.outside_component) { expect(packet.result.tool_use_id).toBe(packet.call.id); @@ -85,26 +87,15 @@ describe('bounded shared-code revalidation prompt', () => { } expect(stageScope.post_fix_verification.call.input.command).toContain('bun test test/retry-after.test.ts'); expect(stageScope.post_fix_verification.result.content).toContain('worker===lib true route===lib true'); - expect(prompt).toContain('Existing tests and caller/import checks needed to verify your source fixes still run'); - expect(prompt).toContain('do not restart exploratory QA or require QA artifacts'); - for (const retained of ['core/checklist', 'source/identity/snapshot checks', 'Fix-First decisions', 'approved source edits', - 're-review with a new REVIEW_START', 'zero-edit convergence', 'final persistence', 'Missing, failed, stale or wrong-state results require noncompletion', - 'no actual native coverage credit', 'Separate genuine QA/native evaluations remain required']) expect(prompt).toContain(retained); for (const other of [reviewPrompt(f, instructions, input), reviewRevalidationPrompt(f, instructions, input), reviewRevalidationPrompt(f, instructions, input, { input: '/fixture/resumed.json', checkCommand: 'check-prerequisites' })]) { expect(other).not.toContain('Do not perform QA scope/method asset loads'); } }); - test('edit-capable replay declares fresh actor invocations instead of refreshing settled input', () => { + test('edit-capable replay declares its actor instead of the fixed revalidation coverage', () => { const prompt = reviewPrompt(f, instructions, input, { actorCommand: 'cat /fixture/stage-output.json' }); - expect(prompt).toContain('explicitly declared SYNTHETIC prerequisite actor'); - for (const rule of ['each review pass', 'NEW synthetic result', 'exact current state and tool-use ID', - 'All prior receipts are preserved', 'Source-changing cycles invalidate earlier results', - 'invoke the actor again on the new zero-edit pass', "Never refresh an old receipt's hashes", - 'Missing, failed, stale or wrong-state results require noncompletion', 'cannot complete core/checklist review', - 'only a current settled:true actor result from the final pass supplies the replaced Step 4.7 QA and Step 4.8 native adversarial prerequisites', - 'a reporting label, not a missing stage', 'no actual native coverage credit']) expect(prompt).toContain(rule); + expect(prompt).toContain('cat /fixture/stage-output.json'); expect(prompt).not.toContain('Required reviewer coverage for this scoped replay'); expect(prompt).not.toContain('Do not edit target source'); }); @@ -113,19 +104,15 @@ describe('bounded shared-code revalidation prompt', () => { const resumed = { input: '/isolated/synthetic-prerequisites.json', checkCommand: 'fixture-prerequisite-check' }; const original = reviewRevalidationPrompt(f, instructions, input); const prompt = reviewRevalidationPrompt(f, instructions, input, resumed); - expect(original).toContain('Required reviewer coverage for this scoped replay is the core/checklist review plus the supplied completed maintainability result.'); + expect(original).toContain('Required reviewer coverage for this scoped replay'); expect(prompt).not.toContain('Required reviewer coverage for this scoped replay'); - expect(prompt).toContain('SYNTHETIC settled Step 4.7 QA and Step 4.8 native adversarial'); - for (const requirement of ['not evidence that this model executed those stages', 'never actual native coverage credit', - 'Missing, failed, blocked, malformed or stale prerequisites require noncompletion', 'unchanged COMPLETED and CONVERGED rules', - 'Any source, branch, base, index or configuration change invalidates', 'do not regenerate them', - 'A finding that requires edits blocks this bounded replay', resumed.input, resumed.checkCommand, - "replace Step 4's early QA selection and method-loading prerequisites", 'read no QA scope or method assets']) expect(prompt).toContain(requirement); - expect(original).not.toContain('read no QA scope or method assets'); + expect(prompt).toContain(resumed.input); + expect(prompt).toContain(resumed.checkCommand); expect(prompt.slice(prompt.indexOf('Revalidation fixture execution contract:'))) .toBe(original.slice(original.indexOf('Revalidation fixture execution contract:'))); + // The review completion rule names both prerequisite stages by step number. const production = fs.readFileSync(path.join(SHARED_LIBS_ROOT, 'review/SKILL.md.tmpl'), 'utf8'); - expect(production).toContain('Step 4.8 adversarial pass finish, and every required Step 4.7 probe passes.'); + expect(production.split('\n').some(line => /Step 4\.8\b/.test(line) && /Step 4\.7\b/.test(line))).toBe(true); }); test('adds execution guidance after the complete shared prompt without supplying an answer or token', () => { @@ -135,64 +122,32 @@ describe('bounded shared-code revalidation prompt', () => { const contract = prompt.slice(base.length); expect(contract).toContain(`${SHARED_INTERACTIVE_MAX_TURNS} assistant turns`); expect(contract).toContain(path.join(f.state, 'projects/fixture-shared-libs/.review-starts/.json')); - expect(contract).toContain('token actually returned by --start'); - expect(contract).toContain('separate, successful Read tool call or a single cat command'); - expect(contract).toContain('Verify its repo, branch, working tree and start time'); - expect(contract).toContain('Do not combine the record read with --start, the diff or other diagnostic commands'); - expect(contract).toContain('if the read fails, retry it before proceeding'); - expect(contract).toContain('Do not read the diff until step 2 verifies the start record'); + expect(contract.indexOf('Read that token\'s record')).toBeGreaterThan(-1); expect(contract.indexOf('Read that token\'s record')).toBeLessThan(contract.indexOf('Then read the diff in a subsequent call')); expect(contract).not.toMatch(/[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}/); - expect(prompt).toContain(`${SHARED_LIBS_ROOT}/review/sections/shared-code-reuse.md (Step 5.0 requires it for the supplied prior Skip)`); + expect(prompt).toContain(`${SHARED_LIBS_ROOT}/review/sections/shared-code-reuse.md`); expect(prompt.indexOf('Your first response holds')).toBeLessThan(prompt.indexOf("gstack-review-log' --start review")); - expect(prompt).toContain('Keep the final review summary to at most twelve lines'); - expect(base).not.toContain('Keep the final review summary'); - expect(contract).toContain('Batch independent required source reads'); - expect(contract).toContain('Preserve every required evidence check and dependency'); - expect(contract).toContain('complete, untruncated read-back'); - expect(contract).toContain('same tool invocation'); - expect(contract).toContain('then return the final review summary'); - expect(contract).toContain('Failed persistence or verification remains a failure'); - expect(contract).toContain("Late source changes still require the workflow's normal re-review"); expect(contract).not.toContain('choose Skip'); }); - test('declares isolated receipt commands, direct reads and a separate supplied-finding disposition', () => { + test('declares isolated receipt commands', () => { const prompt = reviewRevalidationPrompt(f, instructions, input); const commands = [...prompt.matchAll(/```bash\n([\s\S]*?)\n```/g)].map(match => match[1]); expect(commands).toHaveLength(3); expect(commands[0]).toBe(`'${SHARED_LIBS_ROOT}/bin/gstack-review-log' --start review`); expect(commands[1]).toBe(`'${SHARED_LIBS_ROOT}/bin/gstack-review-log' --check-shared-libs REVIEW_START <<'GSTACK_REVALIDATION_FINDING'\nCURRENT_FINDING_JSON\nGSTACK_REVALIDATION_FINDING`); expect(commands[2]).toBe(`'${SHARED_LIBS_ROOT}/bin/gstack-review-log' 'FINAL_REVIEW_JSON' --finish REVIEW_START && '${SHARED_LIBS_ROOT}/bin/gstack-review-read'`); - for (const requirement of ['sole command', 'only the token', 'only one JSON value', 'literal path operands', - 'even when its body appeared in the diff', 'No path-variable loops', 'before the checker', - 'metadata in earlier calls', 'No preliminary commands', 'materially revised proposal is a separate finding', - 'unsupported or unfinished supplied finding stays blocked', 'without its actual explicit decision']) { - expect(prompt).toContain(requirement); - } }); - test('step 5 branches on the checker result: true suppresses without a new decision, false requires a fresh one', () => { + test('step 5 branches on the checker result, reusable:true before reusable:false', () => { const prompt = reviewRevalidationPrompt(f, instructions, input); const step5 = prompt.slice(prompt.indexOf('5. Act on the checker result'), prompt.indexOf('6. Complete final evidence')); - expect(step5).toContain('reusable:true'); - expect(step5).toContain('the prior Skip carries forward'); - expect(step5).toContain('Ask no new decision question'); - expect(step5).toContain("exclude this advisory from the current pass's findings"); - expect(step5).toContain('Do not re-persist it as a current finding'); - expect(step5).toContain('reusable:false'); - expect(step5).toContain('Perform a fresh authored-source review and make an actual new decision'); - expect(step5).toContain('materially revised proposal is a separate finding'); - expect(step5).toContain('without its actual explicit decision'); - expect(step5).toContain('unsupported or unfinished supplied finding stays blocked'); - expect(step5).toContain('reusable:true whose independent authored/current-source verification does not hold'); - expect(step5).toContain('regardless of the checker result'); - expect(step5).toContain('never force a new Skip on invalid evidence'); + expect(step5.indexOf('reusable:true')).toBeGreaterThan(-1); expect(step5.indexOf('reusable:true')).toBeLessThan(step5.indexOf('reusable:false')); expect(prompt).not.toContain("Record the supplied finding's disposition under its own"); }); - test('the shared review prompt maps exact trusted asset roots and documented interfaces to avoid discovery', () => { + test('the shared review prompt maps exact trusted asset roots and documented interfaces', () => { const prompt = reviewPrompt(f, instructions, input); expect(prompt).toContain(`${SHARED_LIBS_ROOT}/review/checklist.md`); expect(prompt).toContain(`${SHARED_LIBS_ROOT}/review/sections/`); @@ -202,21 +157,13 @@ describe('bounded shared-code revalidation prompt', () => { expect(prompt).toContain(f.bin); for (const iface of ['gstack-review-log --start review', '--check-shared-libs REVIEW_START', '--finish REVIEW_START', 'gstack-review-read']) expect(prompt).toContain(iface); - for (const forbidden of ['do not rediscover it', 'enumerate the bin/lib/review/qa roots', - 'probe --help', 'read the fixture request logs']) expect(prompt).toContain(forbidden); - expect(prompt).toContain('batch independent reads'); - expect(prompt).toContain('keep receipt-ordered commands separate'); - expect(prompt).toContain('capture the start token before reading the diff'); - expect(prompt).toContain('run --start, the checker and any declared stage-actor invocation each as its own sole command'); - expect(prompt).toContain('The only combined receipt call is the final persistence'); - expect(prompt).toContain('Still inspect the target repository source'); }); - test('common guidance states the finish+read-back receipt contract directly, not by a dangling step 6 reference', () => { + test('common guidance states the finish+read-back receipt directly, not by a dangling step 6 reference', () => { const lifecycle = reviewPrompt(f, instructions, input, { actorCommand: 'bun /fx/stage-actor.ts run' }); const revalidation = reviewRevalidationPrompt(f, instructions, input); for (const prompt of [lifecycle, revalidation]) { - expect(prompt).toContain('The only combined receipt call is the final persistence'); + expect(prompt).toContain('--finish REVIEW_START'); expect(prompt).not.toContain('exactly as step 6 shows'); } const commands = [...revalidation.matchAll(/```bash\n([\s\S]*?)\n```/g)].map(match => match[1]); diff --git a/test/ship-triage-labels.test.ts b/test/ship-triage-labels.test.ts new file mode 100644 index 000000000..f812f2af4 --- /dev/null +++ b/test/ship-triage-labels.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, test } from 'bun:test'; +import * as fs from 'fs'; +import * as path from 'path'; +import { triageLabels } from './helpers/ship-triage-labels'; + +const DIR = path.join(import.meta.dir, 'fixtures', 'ship-triage'); +const read = (name: string) => fs.readFileSync(path.join(DIR, name), 'utf8'); +const correct = (output: string) => { + const labels = triageLabels(output); + return labels.string === 'in-branch' && labels.math === 'pre-existing'; +}; + +describe('ship triage label grader', () => { + test.each(['known-good-json.txt', 'known-good-lines.txt'])('accepts %s', (name) => { + expect(correct(read(name))).toBe(true); + }); + test.each(['known-bad-swapped.txt', 'known-bad-words-only.txt', 'known-bad-one-label.txt'])('rejects %s', (name) => { + expect(correct(read(name))).toBe(false); + }); + test('the previous word-presence check passed the words-only and swapped outputs', () => { + const previous = (output: string) => { + const lower = output.toLowerCase(); + return /in-branch|in branch|introduced/.test(lower) && /pre-existing|pre existing|existed before/.test(lower) + && /truncate|string/.test(lower) && /divide|math/.test(lower); + }; + expect(previous(read('known-bad-words-only.txt'))).toBe(true); + expect(previous(read('known-bad-swapped.txt'))).toBe(true); + }); +}); diff --git a/test/skill-body-narration.test.ts b/test/skill-body-narration.test.ts new file mode 100644 index 000000000..acac78c97 --- /dev/null +++ b/test/skill-body-narration.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, test } from 'bun:test'; +import * as fs from 'fs'; +import * as path from 'path'; +import { hasNarrationLeak } from './helpers/skill-body-narration'; + +const DIR = path.join(import.meta.dir, 'fixtures', 'skillify-narration'); +const read = (name: string) => fs.readFileSync(path.join(DIR, name), 'utf8'); +const KNOWN_BAD = fs.readdirSync(DIR).filter(name => name.startsWith('known-bad-')).sort(); + +describe('skillify narration grader', () => { + test('accepts a clean body that says "let me know" in a trigger', () => { + expect(hasNarrationLeak(read('known-good.md'))).toBe(false); + }); + test('stored known-bad fixtures exist', () => { + expect(KNOWN_BAD.length).toBeGreaterThanOrEqual(4); + }); + test.each(KNOWN_BAD)('still rejects %s', (name) => { + expect(hasNarrationLeak(read(name))).toBe(true); + }); + test('the previous grader rejected every known-bad fixture and also the known-good one', () => { + const previous = (body: string) => /^I /m.test(body) || /Let me /i.test(body) || /^I'll /m.test(body); + for (const name of KNOWN_BAD) expect(previous(read(name))).toBe(true); + expect(previous(read('known-good.md'))).toBe(true); + }); +}); diff --git a/test/skill-ceo-section-ordering.test.ts b/test/skill-ceo-section-ordering.test.ts index e7d72809c..3cee5f00a 100644 --- a/test/skill-ceo-section-ordering.test.ts +++ b/test/skill-ceo-section-ordering.test.ts @@ -34,6 +34,7 @@ import { ALL_HOST_CONFIGS } from '../hosts'; import { generateTasksSectionEmit } from '../scripts/resolvers/tasks-section'; import { generateBrainCacheRefresh } from '../scripts/resolvers/gbrain'; import { runGeneration } from '../scripts/gen-skill-docs'; +import { ASK_QUESTIONS_HEADING } from './helpers/workflow-excerpt'; const ROOT = path.resolve(import.meta.dir, '..'); const SKELETON = path.join(ROOT, 'plan-ceo-review', 'SKILL.md'); @@ -935,12 +936,13 @@ test('CEO closing route checks approvals before outputs and verifies artifacts b expect(refresh).not.toContain('telemetry has logged'); } - const governingStages = ['## CRITICAL RULE — How to ask questions', '## Formatting Rules', + const askQuestions = section.search(ASK_QUESTIONS_HEADING); + const governingStages = [askQuestions, ...['## Formatting Rules', '## Mode Quick Reference', '### Working review decisions', '### Section 1:'] - .map(stage => section.indexOf(stage)); + .map(stage => section.indexOf(stage))]; expect(governingStages.every(position => position >= 0)).toBe(true); expect(governingStages).toEqual([...governingStages].sort((a, b) => a - b)); - const questions = section.split('## CRITICAL RULE — How to ask questions')[1]!.split('## Mode Quick Reference')[0]!; + const questions = section.slice(askQuestions).split('\n').slice(1).join('\n').split('## Mode Quick Reference')[0]!; expect(questions).toContain('Use `D` and A/B/C labels'); expect(questions).toContain('Cite the stable ledger ID separately'); const formatting = questions.split('## Formatting Rules')[1]!; diff --git a/test/skill-e2e-auq-matrix.test.ts b/test/skill-e2e-auq-matrix.test.ts index 1fe3bcee7..0eb699a0d 100644 --- a/test/skill-e2e-auq-matrix.test.ts +++ b/test/skill-e2e-auq-matrix.test.ts @@ -4,9 +4,10 @@ * * Layer 0 (auq-format-always-loaded.test.ts) deterministically guarantees each * listed skill SHIPS the format spec in its always-loaded skeleton. This test - * proves each skill's model OBEYS it: that the first real AUQ it fires is a - * compliant decision brief (all 7 format elements) with a substantive - * recommendation (>= 4). One parametrized case per skill so a single weak skill + * proves each skill's model OBEYS it: that the first real AUQ it fires carries + * the fields software reads (a Recommendation: line and exactly one + * (recommended) option) with a substantive recommendation (>= 4). The other + * format elements are logged. One parametrized case per skill so a single weak skill * is an isolated failure, not a blocker for the rest. * * Capture records the actual public AskUserQuestion payload and verifies its @@ -35,6 +36,7 @@ import { setupSkillDir, captureFirstAuq, scoreAuqFormat, + auqMachineFormatProblems, skillFromWorktree, gradeAuqRecommendation, } from './helpers/auq-sdk-capture'; @@ -156,7 +158,8 @@ describeE2E('AUQ behavioral matrix (periodic)', () => { throw new Error(`${m.skill}: agent produced NO AUQ capture (never reached a question in budget).`); } const problems: string[] = []; - if (fmt.missing.length > 0) problems.push(`missing format element(s): ${fmt.missing.join(', ')}`); + // Presentation elements (ELI10, Pros / cons, ✅/❌, Net:) are logged above, not failed. + problems.push(...auqMachineFormatProblems(text)); if (substance < 4) problems.push(`recommendation substance ${substance} < 4 (boilerplate/weak)`); if (problems.length > 0) { throw new Error( diff --git a/test/skill-e2e-context-skills.test.ts b/test/skill-e2e-context-skills.test.ts index b50c09beb..71fabe0da 100644 --- a/test/skill-e2e-context-skills.test.ts +++ b/test/skill-e2e-context-skills.test.ts @@ -21,6 +21,7 @@ import { createEvalCollector, finalizeEvalCollector, } from './helpers/e2e-helpers'; import { extractSkillBody } from './helpers/skill-fixture'; +import { readShippedSkillRouting } from './helpers/shipped-skill-routing'; import { spawnSync } from 'child_process'; import * as fs from 'fs'; import * as path from 'path'; @@ -70,19 +71,13 @@ function setupWorkdir(suffix: string): { workDir: string; gstackHome: string; sl } } - // Routing CLAUDE.md: explicit instruction to always use the Skill tool. + // Routing CLAUDE.md: the ## Skill routing section gstack ships. fs.writeFileSync(path.join(workDir, 'CLAUDE.md'), `# Project Instructions -## Skill routing +${readShippedSkillRouting().section} -When the user's request matches an available skill, ALWAYS invoke it using the Skill -tool as your FIRST action. Do NOT answer directly, do NOT use other tools first. +## Test environment -Key routing rules: -- Save progress, save state, save my work → invoke context-save -- Resume, where was I, pick up where I left off → invoke context-restore - -Environment: - Use GSTACK_HOME="${gstackHome}" for all gstack bin scripts. - The bin scripts are at ./bin/ (relative to this directory). - The skill files are at ./.claude/skills/context-save/SKILL.md and diff --git a/test/skill-e2e-coverage-audit.test.ts b/test/skill-e2e-coverage-audit.test.ts index b78e0a545..de3dcaa5b 100644 --- a/test/skill-e2e-coverage-audit.test.ts +++ b/test/skill-e2e-coverage-audit.test.ts @@ -66,6 +66,8 @@ Existing tests are in ${cwd}/test/billing.test.ts. Produce the ASCII coverage diagram showing which code paths are tested and which have gaps. Output the diagram directly, name both billing functions, and include a coverage summary. +End with one JSON line listing the functions with and without test coverage: +{"tested": ["", ...], "untested": ["", ...]} Do not modify the supplied source or tests.`, workingDirectory: cwd, maxTurns: 15, allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'], diff --git a/test/skill-e2e-deploy.test.ts b/test/skill-e2e-deploy.test.ts index c52eac569..a322ba481 100644 --- a/test/skill-e2e-deploy.test.ts +++ b/test/skill-e2e-deploy.test.ts @@ -52,19 +52,12 @@ The skill is carved: on-demand step bodies live in land-and-deploy/sections/ in working directory — when a STOP-Read pointer names a ~/.claude/skills/gstack/... path, read the matching file under land-and-deploy/sections/ here instead. -You are on branch feat/add-deploy with changes against main. This repo has a fly.toml -with app = "test-app", indicating a Fly.io deployment. +You are on branch feat/add-deploy with changes against main. -IMPORTANT: There is NO remote and NO GitHub PR — you cannot run gh commands. -Instead, simulate the workflow: -1. Detect the deploy platform from fly.toml (should find Fly.io, app = test-app) -2. Infer the production URL (https://test-app.fly.dev) -3. Note the merge method would be squash -4. Write the deploy configuration to CLAUDE.md -5. Write a deploy report skeleton to .gstack/deploy-reports/report.md showing the - expected report structure (PR number: simulated, timing: simulated, verdict: simulated) - -Do NOT use AskUserQuestion. Do NOT run gh or fly commands.`, +This run is non-interactive and the repo has no remote, so there is no PR and the gh +and fly CLIs are unavailable. Run the parts of the workflow that work without them, +mark the PR, merge and deploy results as simulated, and save the report you would +show under .gstack/deploy-reports/.`, workingDirectory: landDir, maxTurns: 20, allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'], @@ -77,15 +70,16 @@ Do NOT use AskUserQuestion. Do NOT run gh or fly commands.`, recordE2E(evalCollector, '/land-and-deploy workflow', 'Land-and-Deploy skill E2E', result); expect(result.exitReason).toBe('success'); - const claudeMd = path.join(landDir, 'CLAUDE.md'); - if (fs.existsSync(claudeMd)) { - const content = fs.readFileSync(claudeMd, 'utf-8'); - const hasFly = content.toLowerCase().includes('fly') || content.toLowerCase().includes('test-app'); - expect(hasFly).toBe(true); - } - + // The prompt names neither the platform nor the app: the skill detects them from fly.toml. const reportDir = path.join(landDir, '.gstack', 'deploy-reports'); expect(fs.existsSync(reportDir)).toBe(true); + const claudeMd = path.join(landDir, 'CLAUDE.md'); + const artifacts = [ + ...fs.readdirSync(reportDir).filter(f => f.endsWith('.md')).map(f => fs.readFileSync(path.join(reportDir, f), 'utf-8')), + fs.existsSync(claudeMd) ? fs.readFileSync(claudeMd, 'utf-8') : '', + ].join('\n'); + expect(artifacts, 'deploy artifacts name the detected platform').toMatch(/fly/i); + expect(artifacts, 'deploy artifacts name the detected app').toContain('test-app'); }, CAPTURE_LONG_MS); }); @@ -130,22 +124,9 @@ in THIS working directory — read it from there (the STOP-Read pointer's You are on branch feat/first-deploy. This is the FIRST TIME running /land-and-deploy for this project — there is NO land-deploy-confirmed file. -This repo has a fly.toml with app = "first-run-app", indicating a Fly.io deployment. - -IMPORTANT: There is NO remote and NO GitHub PR — you cannot run gh commands. -Instead, simulate the Step 1.5 first-run dry-run validation: -1. Detect that this is a FIRST_RUN (no land-deploy-confirmed file) -2. Detect the deploy platform from fly.toml (Fly.io, app = first-run-app) -3. Infer the production URL (https://first-run-app.fly.dev) -4. Build the DEPLOY INFRASTRUCTURE VALIDATION table showing: - - Platform detected - - Command validation results (simulated as all passing) - - Staging detection results (none expected) - - What will happen steps -5. Write the dry-run report to .gstack/deploy-reports/dry-run-validation.md - -Do NOT use AskUserQuestion. Do NOT run gh or fly commands. -Just demonstrate the first-run dry-run output.`, +This run is non-interactive and the repo has no remote, so there is no PR and the gh +and fly CLIs are unavailable; treat their command checks as passing. Run the first-run +dry-run validation and save its report under .gstack/deploy-reports/.`, workingDirectory: firstRunDir, maxTurns: 20, allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'], @@ -210,20 +191,12 @@ The Step 3.5 readiness gate is carved into land-and-deploy/sections/readiness-ga in THIS working directory — read it from there (the STOP-Read pointer's ~/.claude/skills/gstack/... path does not exist here). -Focus on Step 3.5a and Step 3.5a-bis (the review staleness check and inline review offer). +Focus on Step 3.5a and Step 3.5a-bis. -This repo has 6 commits since the initial commit. There are NO review logs -(gstack-review-read would return NO_REVIEWS). - -Simulate what the readiness gate would show: -1. Run gstack-review-read equivalent (simulate NO_REVIEWS output) -2. Determine review staleness: Eng Review should be "NOT RUN" -3. Note that Step 3.5a-bis would offer an inline review -4. Write a simulated readiness report to .gstack/deploy-reports/readiness-report.md - showing the review status as NOT RUN with the inline review offer text - -Do NOT use AskUserQuestion. Do NOT run gh commands. -Show what the readiness gate output would look like.`, +gstack-review-read is not installed here; it would print NO_REVIEWS for this repo. +This run is non-interactive and the repo has no remote, so there is no PR and gh is +unavailable. Run the readiness gate and save its readiness report under +.gstack/deploy-reports/.`, workingDirectory: reviewDir, maxTurns: 15, allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'], @@ -243,10 +216,10 @@ Show what the readiness gate output would look like.`, const reportFiles = fs.readdirSync(reportDir); expect(reportFiles.length).toBeGreaterThan(0); const reportContent = fs.readFileSync(path.join(reportDir, reportFiles[0]), 'utf-8'); - // Should mention review status - const hasReviewMention = reportContent.toLowerCase().includes('review') || - reportContent.toLowerCase().includes('not run'); - expect(hasReviewMention).toBe(true); + // The prompt no longer supplies the verdict: the gate must record the + // missing review itself. + expect(reportContent).toMatch(/review/i); + expect(reportContent, 'readiness report records the missing review').toMatch(/not run|no reviews?\b|never run|missing/i); }, CAPTURE_LONG_MS); }); diff --git a/test/skill-e2e-design.test.ts b/test/skill-e2e-design.test.ts index 233ba5c1a..d5afc5c0d 100644 --- a/test/skill-e2e-design.test.ts +++ b/test/skill-e2e-design.test.ts @@ -707,12 +707,16 @@ Review the site at ${serverUrl}. Use --quick mode. Skip any AskUserQuestion call const reportPath = path.join(qaDesignDir, 'design-audit.md'); const reportExists = fs.existsSync(reportPath); - // Check if any design fix commits were made - const gitLog = spawnSync('git', ['log', '--oneline'], { + // Outcome: the report names a seeded defect, and the page changed (committed + // or not) relative to the initial fixture commit. The commit prefix is not required. + const report = reportExists ? fs.readFileSync(reportPath, 'utf-8') : ''; + const namesSeededDefect = /4[78]px|line-height|border-radius|radius|padding|spacing|heading/i.test(report); + const initialCommit = spawnSync('git', ['rev-list', '--max-parents=0', 'HEAD'], { cwd: qaDesignDir, stdio: 'pipe', timeout: 30_000, - }); - const commits = gitLog.stdout.toString().trim().split('\n'); - const designFixCommits = commits.filter((c: string) => c.includes('style(design)')); + }).stdout.toString().trim(); + const pageChanged = spawnSync('git', ['diff', '--name-only', initialCommit, '--', 'index.html', 'style.css'], { + cwd: qaDesignDir, stdio: 'pipe', timeout: 30_000, + }).stdout.toString().trim() !== ''; // The agent must actually drive Aside: an `aside repl` Bash call, a printed sentinel // (from a tool_result, never the input), and no reach for the retired browse binary. @@ -724,7 +728,8 @@ Review the site at ${serverUrl}. Use --quick mode. Skip any AskUserQuestion call const usedBrowseBin = bashCommands.some(c => /browse\/dist\/browse|\$B /.test(c)); recordE2E(evalCollector, '/design-review fix', 'Design Review E2E', result, { - passed: ['success', 'error_max_turns'].includes(result.exitReason) && droveAside && sentinelPrinted && !usedBrowseBin, + passed: ['success', 'error_max_turns'].includes(result.exitReason) && droveAside && sentinelPrinted && !usedBrowseBin + && reportExists && namesSeededDefect && pageChanged, }); // Accept error_max_turns — the fix loop is complex @@ -732,15 +737,9 @@ Review the site at ${serverUrl}. Use --quick mode. Skip any AskUserQuestion call expect(droveAside).toBe(true); expect(sentinelPrinted).toBe(true); expect(usedBrowseBin).toBe(false); - - // Report and commits are best-effort — log what happened - if (reportExists) { - const report = fs.readFileSync(reportPath, 'utf-8'); - console.log(`Design audit report: ${report.length} chars`); - } else { - console.warn('No design-audit.md generated'); - } - console.log(`Design fix commits: ${designFixCommits.length}`); + expect(reportExists, 'design-audit.md must be written').toBe(true); + expect(namesSeededDefect, 'the report names a seeded design defect').toBe(true); + expect(pageChanged, 'at least one design issue is fixed in index.html or style.css').toBe(true); }, CAPTURE_LONG_MS); }); diff --git a/test/skill-e2e-office-hours-auto-mode.test.ts b/test/skill-e2e-office-hours-auto-mode.test.ts index d32e835f4..8680a2f1e 100644 --- a/test/skill-e2e-office-hours-auto-mode.test.ts +++ b/test/skill-e2e-office-hours-auto-mode.test.ts @@ -8,8 +8,8 @@ * (office-hours/SKILL.md.tmpl:69); when AskUserQuestion is disallowed at * the tool-registry level the model cannot ask and silently picks one mode, * breaking the whole interactive premise. This test asserts that question - * still surfaces — fix must route through mcp__conductor__AskUserQuestion - * (when present) or plan-file + ExitPlanMode flow. + * still surfaces — through mcp__conductor__AskUserQuestion (when present) + * or the prose fallback, never a plan-file ## Decisions substitute. * * Filename keeps `auto-mode` for branch-history continuity. Auto-mode (the * AUTO_DECIDE preamble path when QUESTION_TUNING=true) is a related but @@ -20,12 +20,15 @@ import { test, expect } from 'bun:test'; import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets'; import { describeE2ETier } from './helpers/e2e-gate'; import { runPlanSkillObservation, planFileHasDecisionsSection } from './helpers/claude-pty-runner'; +import { assertNoPlanFileDecisions } from './helpers/plan-mode-evidence'; const describeE2E = describeE2ETier('gate'); describeE2E('office-hours AskUserQuestion-blocked smoke (gate)', () => { - // Pass envelope is ['asked', 'plan_ready']; failure signals are - // 'auto_decided' + silent_write/exited/timeout. + // With AskUserQuestion blocked the mode question must still be asked (the + // prose fallback, observed as 'asked'); a plan-file ## Decisions section is + // not a substitute. Failure signals also include 'auto_decided' + + // silent_write/exited/timeout. test('AskUserQuestion surfaces when --disallowedTools AskUserQuestion is set', async () => { const obs = await runPlanSkillObservation({ skillName: 'office-hours', @@ -47,14 +50,12 @@ describeE2E('office-hours AskUserQuestion-blocked smoke (gate)', () => { `--- evidence (last 2KB visible) ---\n${obs.evidence}`, ); } - if (obs.outcome === 'plan_ready') { - if (!obs.planFile || !planFileHasDecisionsSection(obs.planFile)) { - throw new Error( - `office-hours AskUserQuestion-blocked regression: plan_ready without a "## Decisions" section in ${obs.planFile ?? ''} — startup-vs-builder mode question was silently skipped.\n` + - `--- evidence (last 2KB visible) ---\n${obs.evidence}`, - ); - } + assertNoPlanFileDecisions(obs, planFileHasDecisionsSection); + if (obs.outcome !== 'asked') { + throw new Error( + `office-hours AskUserQuestion-blocked regression: outcome=${obs.outcome} — the startup-vs-builder mode question reached ${obs.planFile ?? 'plan_ready'} without being asked.\n` + + `--- evidence (last 2KB visible) ---\n${obs.evidence}`, + ); } - expect(['asked', 'plan_ready']).toContain(obs.outcome); }, CAPTURE_LONG_MS); }); diff --git a/test/skill-e2e-outside-plan-disabled.test.ts b/test/skill-e2e-outside-plan-disabled.test.ts index 85c861f7b..ebf46d42c 100644 --- a/test/skill-e2e-outside-plan-disabled.test.ts +++ b/test/skill-e2e-outside-plan-disabled.test.ts @@ -38,7 +38,7 @@ describeLive('Disabled outside plan review', () => { fs.writeFileSync(path.join(repo, 'PLAN.md'), '# Accepted plan\n\nAdd a documented --json flag to the existing status command. Preserve the text output by default, and test both formats. Native review is complete.\n'); // The parent must discover the control through the actual preflight. // Do not tell it the expected branch or forbid Agent in the prompt/tools. - const prompt = 'The native engineering review of PLAN.md is complete. Read OUTSIDE-PLAN.md and execute that final workflow section for this plan, following its actual configuration and branch rules. Then return the workflow completion summary, including outside-review attribution. This is an isolated review fixture; all required reviewer tools are authorized.'; + const prompt = 'The native engineering review of PLAN.md is complete. Read OUTSIDE-PLAN.md and execute that final workflow section for this plan, following its actual configuration and branch rules. Then return the workflow completion summary, including outside-review attribution, and end with one JSON line {"outside_review_this_run": "completed" | "disabled" | "unavailable" | "skipped"}. This is an isolated review fixture; all required reviewer tools are authorized.'; fs.writeFileSync(path.join(evidenceDir, 'instructions.md'), fixture.instructions); const result = await runSkillTest({ workingDirectory: repo, prompt, testName, runId, diff --git a/test/skill-e2e-plan-ceo-plan-mode.test.ts b/test/skill-e2e-plan-ceo-plan-mode.test.ts index 39f5f37ec..e3d27cd84 100644 --- a/test/skill-e2e-plan-ceo-plan-mode.test.ts +++ b/test/skill-e2e-plan-ceo-plan-mode.test.ts @@ -99,8 +99,7 @@ describeE2E('plan-ceo-review plan-mode smoke (gate)', () => { `plan-ceo-review smoke FAILED: outcome=${obs.outcome}\n` + `${diagnosis}\n` + `Expected 'asked'. See plan-ceo-review/SKILL.md.tmpl: the Step 0 STOP rules ` + - `and the "One issue = one AskUserQuestion call" rule under "CRITICAL RULE — ` + - `How to ask questions".\n` + + `and the "One decision unit = one AskUserQuestion call" rule in its question-procedure section.\n` + `summary: ${obs.summary}\n` + `elapsed: ${obs.elapsedMs}ms\n` + `--- evidence (last 2KB visible) ---\n${obs.evidence}`, diff --git a/test/skill-e2e-plan-ceo-split-overflow.test.ts b/test/skill-e2e-plan-ceo-split-overflow.test.ts index 392cef32d..b06d95054 100644 --- a/test/skill-e2e-plan-ceo-split-overflow.test.ts +++ b/test/skill-e2e-plan-ceo-split-overflow.test.ts @@ -14,9 +14,9 @@ * The fixture seeds 5 independent scope candidates (chat-platform * integrations) — each carries an independent include/defer/cut decision. * The existing semantic scope validator examines every acknowledged native - * call, including candidate choices before mode selection. It keeps the - * N-1 call floor and requires independent offered dispositions for all five - * candidates. The review-phase counter reports progress, not target coverage. + * call, including candidate choices before mode selection. It requires + * independent offered dispositions for all five candidates; the call floor + * is two, the minimum for five options at four per call. The review-phase counter reports progress, not target coverage. * Collection ends once all five native choices are acknowledged; the same * semantic validator then decides whether those choices satisfy the metric. * @@ -46,14 +46,14 @@ import { ceoStep0Boundary, } from './helpers/claude-pty-runner'; import { FORCING_SPLIT_OVERFLOW_CEO } from './fixtures/forcing-finding-seeds'; -import { ceoSplitDecisionFingerprints, isCeoSplitCandidateCall, isCeoSplitCollectionComplete, pickCeoSplitCountQuestion } from './helpers/ceo-split-question-policy'; +import { CEO_SPLIT_CALL_FLOOR, ceoSplitDecisionFingerprints, isCeoSplitCandidateCall, isCeoSplitCollectionComplete, pickCeoSplitCountQuestion } from './helpers/ceo-split-question-policy'; import { CEO_SCOPE_CANDIDATES } from './helpers/plan-review-cases'; import { evaluatePlanReviewDecisions } from './helpers/plan-review-decisions'; const describeE2E = describeE2ETier('marathon'); const N = 5; -const FLOOR = N - 1; // 4 — must fire at least one AUQ per non-dropped option +const FLOOR = CEO_SPLIT_CALL_FLOOR; /** Plan-file target baked into the FORCING_SPLIT_OVERFLOW_CEO fixture prompt. * Rewritten per-run to a mkdtemp path so concurrent runs (--retry, @@ -101,9 +101,9 @@ describeE2E('/plan-ceo-review split-overflow regression (marathon)', () => { `--- evidence (last 3KB) ---\n${obs.evidence}`, ); } - // The phase counter is progress, not target coverage. The existing scope - // validator keeps FLOOR=4 and requires all five independent candidate - // decisions, complete native evidence, and offered Include/Defer/Cut ACKs. + // The phase counter is progress, not target coverage. The scope validator + // requires all five independent candidate decisions, complete native + // evidence, and offered Include/Defer/Cut ACKs; FLOOR only bounds calls. await evaluatePlanReviewDecisions({ plan: followUpPrompt, targets: CEO_SCOPE_CANDIDATES, fingerprints: ceoSplitDecisionFingerprints(obs.transcript, obs.fingerprints), floor: FLOOR, kind: 'scope', deadlineAt }); diff --git a/test/skill-e2e-plan-design-plan-mode.test.ts b/test/skill-e2e-plan-design-plan-mode.test.ts index e9619fe8d..8701c3702 100644 --- a/test/skill-e2e-plan-design-plan-mode.test.ts +++ b/test/skill-e2e-plan-design-plan-mode.test.ts @@ -12,10 +12,11 @@ import { test, expect } from 'bun:test'; import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets'; import { describeE2ETier } from './helpers/e2e-gate'; -import { assertPlanModeWithEvidence } from './helpers/plan-mode-evidence'; +import { assertNoPlanFileDecisions, assertPlanModeWithEvidence, seededPlanTargeted } from './helpers/plan-mode-evidence'; import { runPlanSkillObservation, assertReportAtBottomIfPlanWritten, + planFileHasDecisionsSection, } from './helpers/claude-pty-runner'; const describeE2E = describeE2ETier('periodic'); @@ -43,6 +44,9 @@ Only the happy path is designed. No empty states, no error states, no loading states. Mobile: "stacks on mobile." `; +// Seed-only phrases; seeing one after the slash command shows the review read the seeded plan. +const SEED_PLAN_TOKENS = ['Marketing landing page', 'testimonials carousel', '3 tier cards']; + describeE2E('plan-design-review plan-mode smoke (periodic)', () => { test('reaches a terminal outcome (asked or plan_ready) without silent writes', async () => { const obs = await runPlanSkillObservation({ @@ -62,19 +66,21 @@ describeE2E('plan-design-review plan-mode smoke (periodic)', () => { } expect(['asked', 'plan_ready']).toContain(obs.outcome); assertReportAtBottomIfPlanWritten(obs); + assertNoPlanFileDecisions(obs, planFileHasDecisionsSection); }); }, CAPTURE_LONG_MS); // Plan-mode scope-gate bypass: with a seeded UI-heavy plan in plan mode, - // the gate must NOT render its "What should I review?" menu — it - // auto-selects B and announces it, then proceeds to the pre-review audit - // and mockups. Mirrors the eng smoke's seeded STOP-gate test, without + // the gate must NOT render its "What should I review?" menu; the review + // targets the seeded plan, then proceeds to the pre-review audit and + // mockups. Mirrors the eng smoke's seeded STOP-gate test, without // --disallowedTools (native AUQ available is the common path here). test('scope gate auto-selects B when a plan is seeded in plan mode', async () => { const obs = await runPlanSkillObservation({ skillName: 'plan-design-review', inPlanMode: true, initialPlanContent: SEED_PLAN_UI_HEAVY, + trackTokens: SEED_PLAN_TOKENS, timeoutMs: CAPTURE_MS, }); @@ -95,11 +101,13 @@ describeE2E('plan-design-review plan-mode smoke (periodic)', () => { expect(['asked', 'plan_ready']).toContain(obs.outcome); assertReportAtBottomIfPlanWritten(obs); + assertNoPlanFileDecisions(obs, planFileHasDecisionsSection); - // The bypass contract (exception ordering makes this deterministic even - // though the seed arrives as a pasted user message). + // The bypass contract: no scope menu, and the seeded plan is the target. + // The announcement's exact wording is not graded. expect(obs.scopeGateQuestionObserved ?? false).toBe(false); - expect(obs.scopeGateAutoSelectObserved ?? false).toBe(true); + console.log(`[plan-design plan-mode] scope announcement observed: ${obs.scopeGateAutoSelectObserved ?? false}; seed tokens: ${JSON.stringify(obs.tokensObserved ?? {})}`); + expect(seededPlanTargeted(obs, SEED_PLAN_TOKENS), 'seeded plan was not targeted (scopeGateAutoSelectObserved or seed-only plan tokens)').toBe(true); }); }, CAPTURE_LONG_MS); }); diff --git a/test/skill-e2e-plan-devex-plan-mode.test.ts b/test/skill-e2e-plan-devex-plan-mode.test.ts index 340db45ca..707408ee1 100644 --- a/test/skill-e2e-plan-devex-plan-mode.test.ts +++ b/test/skill-e2e-plan-devex-plan-mode.test.ts @@ -13,6 +13,7 @@ import { planFileHasDecisionsSection, assertReportAtBottomIfPlanWritten, } from './helpers/claude-pty-runner'; +import { assertNoPlanFileDecisions } from './helpers/plan-mode-evidence'; const describeE2E = describeE2ETier('gate'); @@ -34,12 +35,14 @@ describeE2E('plan-devex-review plan-mode smoke (gate)', () => { } expect(['asked', 'plan_ready']).toContain(obs.outcome); assertReportAtBottomIfPlanWritten(obs); + assertNoPlanFileDecisions(obs, planFileHasDecisionsSection); }, CAPTURE_LONG_MS); // v1.21+ regression: see skill-e2e-plan-ceo-plan-mode.test.ts for the - // contract. Pass envelope is ['asked', 'plan_ready']; failure signals - // are 'auto_decided' (AUTO_DECIDE without opt-in) plus the standard - // silent_write/exited/timeout. + // contract. With AskUserQuestion blocked the decision must still be asked + // (the prose fallback, observed as 'asked'); a plan-file ## Decisions + // section is not a substitute. Failure signals also include 'auto_decided' + // (AUTO_DECIDE without opt-in) plus the standard silent_write/exited/timeout. test('AskUserQuestion surfaces when --disallowedTools AskUserQuestion is set', async () => { const obs = await runPlanSkillObservation({ skillName: 'plan-devex-review', @@ -61,15 +64,13 @@ describeE2E('plan-devex-review plan-mode smoke (gate)', () => { `--- evidence (last 2KB visible) ---\n${obs.evidence}`, ); } - if (obs.outcome === 'plan_ready') { - if (!obs.planFile || !planFileHasDecisionsSection(obs.planFile)) { - throw new Error( - `plan-devex-review AskUserQuestion-blocked regression: plan_ready without a "## Decisions" section in ${obs.planFile ?? ''} — Step 0 was silently skipped.\n` + - `--- evidence (last 2KB visible) ---\n${obs.evidence}`, - ); - } - } - expect(['asked', 'plan_ready']).toContain(obs.outcome); assertReportAtBottomIfPlanWritten(obs); + assertNoPlanFileDecisions(obs, planFileHasDecisionsSection); + if (obs.outcome !== 'asked') { + throw new Error( + `plan-devex-review AskUserQuestion-blocked regression: outcome=${obs.outcome} — Step 0 reached ${obs.planFile ?? 'plan_ready'} without asking.\n` + + `--- evidence (last 2KB visible) ---\n${obs.evidence}`, + ); + } }, CAPTURE_LONG_MS); }); diff --git a/test/skill-e2e-plan-eng-plan-mode.test.ts b/test/skill-e2e-plan-eng-plan-mode.test.ts index dd5acf03b..daf4fd418 100644 --- a/test/skill-e2e-plan-eng-plan-mode.test.ts +++ b/test/skill-e2e-plan-eng-plan-mode.test.ts @@ -8,7 +8,7 @@ import { test, expect } from 'bun:test'; import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets'; import { describeE2ETier } from './helpers/e2e-gate'; -import { assertPlanModeWithEvidence } from './helpers/plan-mode-evidence'; +import { assertNoPlanFileDecisions, assertPlanModeWithEvidence, seededPlanTargeted } from './helpers/plan-mode-evidence'; import { runPlanSkillObservation, planFileHasDecisionsSection, @@ -47,6 +47,9 @@ Ignore Bun's native --shard flag because we want full control. None planned — will add later. `; +// Seed-only names; seeing one after the slash command shows the review read the seeded plan. +const SEED_PLAN_TOKENS = ['ShardManager', 'ResultMerger', 'test-shard-impl']; + describeE2E('plan-eng-review plan-mode smoke (periodic)', () => { test('reaches a terminal outcome (asked or plan_ready) without silent writes', async () => { const obs = await runPlanSkillObservation({ @@ -66,23 +69,26 @@ describeE2E('plan-eng-review plan-mode smoke (periodic)', () => { } expect(['asked', 'plan_ready']).toContain(obs.outcome); assertReportAtBottomIfPlanWritten(obs); + assertNoPlanFileDecisions(obs, planFileHasDecisionsSection); }); }, CAPTURE_LONG_MS); // D3-B / D4-B: when a plan with guaranteed-finding-triggering complexity - // is seeded, the skill MUST fire AskUserQuestion (or fall back to a - // Decisions section) before writing findings to the plan. The - // wrote_findings_before_asking outcome catches the precise transcript bug - // — model writes findings to the plan before any AUQ render. + // is seeded, the skill must ask (mcp AskUserQuestion or the prose fallback) + // before writing findings to the plan. The wrote_findings_before_asking + // outcome catches the transcript bug where the model writes findings to the + // plan before any question renders. A plan-file ## Decisions section is not + // a substitute for asking. test('STOP gate fires when seeded plan forces Step 0 findings', async () => { const obs = await runPlanSkillObservation({ skillName: 'plan-eng-review', inPlanMode: true, initialPlanContent: SEED_PLAN_FORCING_FINDINGS, // Force the Conductor-style path: native AUQ disallowed → the model - // must use mcp__*__AskUserQuestion (outcome='asked') or fall back to - // writing Decisions ('plan_ready'). + // must use mcp__*__AskUserQuestion or render the prose fallback + // (both observed as outcome='asked'). extraArgs: ['--disallowedTools', 'AskUserQuestion'], + trackTokens: SEED_PLAN_TOKENS, timeoutMs: CAPTURE_MS, }); @@ -101,28 +107,24 @@ describeE2E('plan-eng-review plan-mode smoke (periodic)', () => { ); } - if (obs.outcome === 'plan_ready') { - if (!obs.planFile || !planFileHasDecisionsSection(obs.planFile)) { - throw new Error( - `STOP-gate regression: plan_ready without ## Decisions section in ` + - `${obs.planFile ?? ''} — gate skipped after ToolSearch.\n` + - `--- evidence (last 2KB) ---\n${obs.evidence}`, - ); - } + assertReportAtBottomIfPlanWritten(obs); + assertNoPlanFileDecisions(obs, planFileHasDecisionsSection); + if (obs.outcome !== 'asked') { + throw new Error( + `STOP-gate regression: outcome=${obs.outcome} — the seeded findings reached ` + + `${obs.planFile ?? 'plan_ready'} without a question.\n` + + `--- evidence (last 2KB) ---\n${obs.evidence}`, + ); } - expect(['asked', 'plan_ready']).toContain(obs.outcome); - assertReportAtBottomIfPlanWritten(obs); - // Plan-mode scope-gate bypass: with a seeded plan in plan mode, the gate - // must NOT render its "What should I review?" menu — it auto-selects B - // and announces it. Exception ordering in the template (plan-mode branch - // first) makes this deterministic even though the seed arrives as a - // pasted user message. Unseeded test 1 keeps its lenient contract: with - // no plan drafted, the "ask as normal" fallback legitimately renders the - // question. + // must NOT render its "What should I review?" menu, and the review must + // target the seeded plan. The announcement's exact wording is not graded. + // Unseeded test 1 keeps its lenient contract: with no plan drafted, the + // "ask as normal" fallback legitimately renders the question. expect(obs.scopeGateQuestionObserved ?? false).toBe(false); - expect(obs.scopeGateAutoSelectObserved ?? false).toBe(true); + console.log(`[plan-eng plan-mode] scope announcement observed: ${obs.scopeGateAutoSelectObserved ?? false}; seed tokens: ${JSON.stringify(obs.tokensObserved ?? {})}`); + expect(seededPlanTargeted(obs, SEED_PLAN_TOKENS), 'seeded plan was not targeted (scopeGateAutoSelectObserved or seed-only plan tokens)').toBe(true); }); }, CAPTURE_LONG_MS); }); diff --git a/test/skill-e2e-plan-format.test.ts b/test/skill-e2e-plan-format.test.ts index 371365d66..07a262b8a 100644 --- a/test/skill-e2e-plan-format.test.ts +++ b/test/skill-e2e-plan-format.test.ts @@ -33,6 +33,7 @@ import { spawnSync } from 'child_process'; import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; +import { KIND_NOTE_RE } from './helpers/plan-format-kind-note'; const evalCollector = createEvalCollector('e2e-plan-format'); @@ -49,7 +50,6 @@ const evalCollector = createEvalCollector('e2e-plan-format'); // scripts/resolvers/preamble/generate-ask-user-format.ts. The optional // `[A-Z]=` prefix tolerates either shape; both are acceptable spec output. const COMPLETENESS_RE = /Completeness:\s*(?:[A-Z]=)?\d{1,2}\/10/; -const KIND_NOTE_RE = /options differ in kind/i; // v1.7.0.0 Pros/Cons format tokens. Tests are additive: existing // RECOMMENDATION / Completeness / kind-note assertions still hold; new @@ -157,8 +157,9 @@ After writing the file, stop. Do not continue the review.`, const captured = fs.readFileSync(outFile, 'utf-8'); expect(captured.length).toBeGreaterThan(100); - // Kind-differentiated: Completeness: N/10 must NOT appear, "options differ - // in kind" note must appear. Recommendation presence is checked by the judge. + // Kind-differentiated: Completeness: N/10 must NOT appear, and the brief + // says the options are not comparable on coverage (any wording). + // Recommendation presence is checked by the judge. expect(captured).not.toMatch(COMPLETENESS_RE); expect(captured).toMatch(KIND_NOTE_RE); @@ -359,8 +360,9 @@ After writing the file with that ONE question, stop. Do not continue the review. const captured = fs.readFileSync(outFile, 'utf-8'); expect(captured.length).toBeGreaterThan(100); - // Kind-differentiated: Completeness: N/10 must NOT appear, "options differ - // in kind" note must appear. Recommendation presence checked by the judge. + // Kind-differentiated: Completeness: N/10 must NOT appear, and the brief + // says the options are not comparable on coverage (any wording). + // Recommendation presence checked by the judge. expect(captured).not.toMatch(COMPLETENESS_RE); expect(captured).toMatch(KIND_NOTE_RE); diff --git a/test/skill-e2e-plan.test.ts b/test/skill-e2e-plan.test.ts index d9aa25f84..cb91b20f8 100644 --- a/test/skill-e2e-plan.test.ts +++ b/test/skill-e2e-plan.test.ts @@ -1,6 +1,7 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets'; import { runSkillTest } from './helpers/session-runner'; +import { getHermeticDirs, isHermeticEnabled, seedHermeticGstackHome } from './helpers/hermetic-env'; import { EvalCollector } from './helpers/eval-store'; import { OFFICE_HOURS_BUN_GRACE_MS, runRecordedOfficeHoursAttempt } from './helpers/office-hours-attempt'; import { @@ -379,6 +380,9 @@ Focus on architecture, code quality, tests, and performance sections.`, describeIfSelected('Plan-Eng-Review Test-Plan Artifact E2E', ['plan-eng-review-artifact'], () => { let planDir: string; + let gstackHome: string; + let ownsGstackHome = false; + let slug: string; let projectDir: string; beforeAll(() => { @@ -438,35 +442,38 @@ export function main() { return Dashboard(); } // Set up remote-slug shim and browse shims (plan-eng-review uses remote-slug for artifact path) setupBrowseShims(planDir); - // Create project directory for artifacts - projectDir = path.join(os.homedir(), '.gstack', 'projects', 'test-project'); - fs.mkdirSync(projectDir, { recursive: true }); - - // Clean up stale test-plan files from previous runs - try { - const staleFiles = fs.readdirSync(projectDir).filter(f => f.includes('test-plan')); - for (const f of staleFiles) { - fs.unlinkSync(path.join(projectDir, f)); - } - } catch {} + // The actor, this observer and cleanup share one isolated state root: the + // hermetic run's GSTACK_HOME (or a seeded temporary one when hermetic mode + // is off), and the slug gstack-slug resolves for this fixture. Nothing is + // read or deleted under the operator's real home. + if (isHermeticEnabled()) gstackHome = getHermeticDirs().gstackHome; + else { + gstackHome = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-plan-artifact-home-')); + seedHermeticGstackHome(gstackHome); + ownsGstackHome = true; + } + const slugOut = spawnSync(path.join(ROOT, 'bin', 'gstack-slug'), [], { + cwd: planDir, stdio: 'pipe', timeout: 5000, env: { ...process.env, GSTACK_HOME: gstackHome }, + }).stdout.toString(); + slug = /^SLUG=(.+)$/m.exec(slugOut)?.[1]?.trim() ?? ''; + if (!slug) throw new Error(`plan-eng-review-artifact: gstack-slug resolved no slug: ${slugOut}`); + projectDir = path.join(gstackHome, 'projects', slug); + const realHome = path.resolve(os.homedir()); + if (projectDir === realHome || projectDir.startsWith(realHome + path.sep) && !projectDir.startsWith(path.resolve(os.tmpdir()) + path.sep)) { + throw new Error(`plan-eng-review-artifact: state root ${projectDir} is under the real home directory`); + } }); afterAll(() => { try { fs.rmSync(planDir, { recursive: true, force: true }); } catch {} - // Clean up test-plan artifacts (but not the project dir itself) - try { - const files = fs.readdirSync(projectDir); - for (const f of files) { - if (f.includes('test-plan')) { - fs.unlinkSync(path.join(projectDir, f)); - } - } - } catch {} + // Remove only this fixture's project directory inside the isolated state root. + try { fs.rmSync(ownsGstackHome ? gstackHome : projectDir, { recursive: true, force: true }); } catch {} }); testConcurrentIfSelected('plan-eng-review-artifact', async () => { - // Count existing test-plan files before - const beforeFiles = fs.readdirSync(projectDir).filter(f => f.includes('test-plan')); + const testPlans = () => fs.existsSync(projectDir) + ? fs.readdirSync(projectDir).filter(f => /eng-review-test-plan-.*\.md$/.test(f)) : []; + const beforeFiles = testPlans(); const result = await runSkillTest({ prompt: `Read plan-eng-review/SKILL.md for the review workflow. @@ -476,10 +483,11 @@ Read plan.md — that's the plan to review. This is a standalone plan with sourc Proceed directly to the full review. Skip any AskUserQuestion calls — this is non-interactive. -IMPORTANT: After your review, you MUST write the test-plan artifact as described in the "Test Plan Artifact" section of SKILL.md. The remote-slug shim is at ${planDir}/browse/bin/remote-slug. +In this fixture the gstack state root is GSTACK_HOME=${gstackHome} and the project slug is ${slug}. Write your review to ${planDir}/review-output.md`, workingDirectory: planDir, + env: { GSTACK_HOME: gstackHome }, maxTurns: 25, allowedTools: ['Bash', 'Read', 'Write', 'Glob', 'Grep'], timeout: CAPTURE_LONG_MS, @@ -489,30 +497,19 @@ Write your review to ${planDir}/review-output.md`, }); logCost('/plan-eng-review artifact', result); + // The QA test plan is a required discovery artifact of the review: exactly + // one new file under this fixture's project directory, about this plan. + const newFiles = testPlans().filter(f => !beforeFiles.includes(f)); + const content = newFiles.length === 1 ? fs.readFileSync(path.join(projectDir, newFiles[0]!), 'utf-8') : ''; + const aboutPlan = /dashboard|fetchStats|\/api\/stats/i.test(content); + console.log(`Test-plan artifacts in ${projectDir}: ${newFiles.length} new${newFiles.length ? ` (${newFiles[0]}, ${content.length} chars)` : ''}`); recordE2E(evalCollector, '/plan-eng-review test-plan artifact', 'Plan-Eng-Review Test-Plan Artifact E2E', result, { - passed: ['success', 'error_max_turns'].includes(result.exitReason), + passed: ['success', 'error_max_turns'].includes(result.exitReason) && newFiles.length === 1 && aboutPlan, }); expect(['success', 'error_max_turns']).toContain(result.exitReason); - - // Verify test-plan artifact was written - const afterFiles = fs.readdirSync(projectDir).filter(f => f.includes('test-plan')); - const newFiles = afterFiles.filter(f => !beforeFiles.includes(f)); - console.log(`Test-plan artifacts: ${beforeFiles.length} before, ${afterFiles.length} after, ${newFiles.length} new`); - - if (newFiles.length > 0) { - const content = fs.readFileSync(path.join(projectDir, newFiles[0]), 'utf-8'); - console.log(`Test-plan artifact (${newFiles[0]}): ${content.length} chars`); - expect(content.length).toBeGreaterThan(50); - } else { - console.warn('No test-plan artifact found — agent may not have followed artifact instructions'); - } - - // Soft assertion: we expect an artifact but agent compliance is not guaranteed. - // Log rather than fail — the test-plan artifact is a bonus output, not the core test. - if (newFiles.length === 0) { - console.warn('SOFT FAIL: No test-plan artifact written — agent did not follow artifact instructions'); - } + expect(newFiles, `expected one new eng-review test plan in ${projectDir}`).toHaveLength(1); + expect(aboutPlan, 'the test plan covers the reviewed dashboard change').toBe(true); }, CAPTURE_LONG_MS); }); @@ -717,9 +714,7 @@ Read plan.md — that's the plan to review. This is a standalone plan document, Proceed directly to the full review. Skip any AskUserQuestion calls — this is non-interactive. Skip the preamble bash block, lake intro, telemetry, and contributor mode sections. -CRITICAL REQUIREMENT: plan.md IS the plan file for this review session. After completing your review, you MUST write a "## GSTACK REVIEW REPORT" section to the END of plan.md, exactly as described in the "Plan File Review Report" section of plan-eng-review/sections/review-sections.md. Use that canonical table, with all five review rows and honest not-run entries when review history is unavailable. The report MUST end with the mandatory unresolved-decisions status as its final line — the exact unbolded line NO UNRESOLVED DECISIONS when nothing is open, or a "**UNRESOLVED DECISIONS:**" block of bullets when items remain. Nothing may follow it. Use the Edit tool to append to plan.md — do NOT overwrite the existing plan content. - -This review report at the bottom of the plan is the MOST IMPORTANT deliverable of this test.`, +plan.md is the plan file for this review session.`, workingDirectory: planDir, maxTurns: 20, timeout: CAPTURE_LONG_MS, diff --git a/test/skill-e2e-shared-libs-periodic.test.ts b/test/skill-e2e-shared-libs-periodic.test.ts index a93037efc..9b65c092f 100644 --- a/test/skill-e2e-shared-libs-periodic.test.ts +++ b/test/skill-e2e-shared-libs-periodic.test.ts @@ -10,7 +10,7 @@ import { createSharedPlanReuseSelector } from './helpers/shared-libs-plan-actor' import { SHARED_LIBS_ROOT, commitFixture, createSharedLibsFixture, fixtureWrite, installSourceShims, readRequests, runSharedCapture, runSharedInteractive, seedOpportunitySources, - sharedReadOnlyViolations, snapshotFixture, standaloneInstructions, toolCommandTrace, incompleteFirstFileView, type SharedLibsFixture, + sharedReadOnlyViolations, snapshotFixture, standaloneInstructions, toolCommandTrace, prCoverageRequestViolations, type SharedLibsFixture, SharedCaptureAccumulator, type SharedCaptureAttempt, } from './helpers/shared-libs-eval-fixture'; @@ -152,25 +152,9 @@ describeE2E('Shared-code opportunity and coordination judgment (periodic)', () = `Run /deslop-shared-libs using ${instructions}. Recent PR 7 mentions https://github.com/fixture/shared-libs/pull/42 as related work. Return the report after checking coordination within the skill's budget.`, attempt), async result => { assertReadOnly(f, before, result, 'shared-libs-pr-coverage'); const requests = readRequests(f).filter(row => row.tool === 'gh' || row.tool === 'curl'); - const endpoints = requests.map(row => row.endpoint || ''); - expect(endpoints.some(endpoint => /\/pulls\/42\/files/.test(endpoint))).toBe(true); - expect(endpoints.some(endpoint => /\/pulls\/42\/files/.test(endpoint) && /[?&]page=2(?:&|$)/.test(endpoint))).toBe(true); - const openPages = endpoints.filter(endpoint => /\/pulls\?/.test(endpoint) && /state=open/.test(endpoint)); - const filePages = endpoints.filter(endpoint => /\/pulls\/\d+\/files/.test(endpoint)); - expect(openPages.length).toBeLessThanOrEqual(5); - // Only recent PR7's one page is outside the additional scan. Known old PR42 - // shares that scan's 50-page budget, including both of its file-list pages. - expect(filePages.length).toBeLessThanOrEqual(51); - const coordinationPages = endpoints.filter(endpoint => /\/pulls\/42\/files/.test(endpoint)); - // An incomplete first page is not a reusable complete file set. The - // observed capture truncated its first response with head, then fetched - // full pages 1–3. Permit that one recovery while charging every request - // to the hard budget and forbidding repeated complete first-page reads. - const truncatedFirstView = incompleteFirstFileView(result, 42); - expect(coordinationPages.length).toBeLessThanOrEqual(truncatedFirstView ? 4 : 3); - const firstPages = coordinationPages.filter(endpoint => !/[?&]page=/.test(endpoint) || /[?&]page=1(?:&|$)/.test(endpoint)); - expect(firstPages.length).toBeGreaterThan(0); - expect(firstPages.length).toBeLessThanOrEqual(truncatedFirstView ? 2 : 1); + // The skill's stated page budget and PR 42 file-set coverage; per-PR + // page ceilings tuned to one observed run are not graded. + expect(prCoverageRequestViolations(requests.map(row => row.endpoint || ''))).toEqual([]); await assertJudgment(result.output, { older_open_pr: 'Identifies open PR 42 from 2020 as existing coordination work despite its activity being outside the 14-day window. The PR already proposes migrating retry-worker to the proven parser.', partial_overlap: 'Does not count the worker migration already covered by PR 42 as a new opportunity. It may recommend the independent remaining retry-route migration if it explains the narrowed scope and adjusted savings.', diff --git a/test/skill-e2e-skillify.test.ts b/test/skill-e2e-skillify.test.ts index d7920a2e7..27f2af642 100644 --- a/test/skill-e2e-skillify.test.ts +++ b/test/skill-e2e-skillify.test.ts @@ -42,6 +42,8 @@ import { spawnSync } from 'child_process'; import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; +import { readShippedSkillRouting } from './helpers/shipped-skill-routing'; +import { hasNarrationLeak } from './helpers/skill-body-narration'; const evalCollector = createEvalCollector('e2e-skillify'); @@ -60,7 +62,7 @@ interface Workdir { * - bin/ scripts referenced by the preamble * - A scoped GSTACK_HOME under the workdir so on-disk artifacts are * contained and assertable - * - A CLAUDE.md routing block instructing Skill-tool invocation + * - A CLAUDE.md with the ## Skill routing section gstack ships * * `installSkills` lets each test pick the minimum surface (e.g., the * provenance-refusal scenario doesn't need /scrape). @@ -109,16 +111,10 @@ function setupSkillifyWorkdir(suffix: string, installSkills: string[] = ['scrape fs.writeFileSync(path.join(workDir, 'CLAUDE.md'), `# Project Instructions -## Skill routing +${readShippedSkillRouting().section} -When the user's request matches an available skill, ALWAYS invoke it via -the Skill tool as your FIRST action. +## Test environment -Key routing rules: -- /scrape, "scrape", "get data from", "extract from" → invoke scrape -- /skillify, "skillify", "codify this scrape" → invoke skillify - -Environment: - GSTACK_HOME="${gstackHome}" for all gstack bin scripts. - bin scripts are at ./bin/ relative to this directory. - Browse binary is at ${browseBin} — assign to $B (e.g., \`B=${browseBin}\`). @@ -196,13 +192,8 @@ describeIfSelected('Browser-skills Phase 2a E2E (/scrape + /skillify)', [ installBundledHackernewsSkill(workDir); const result = await runSkillTest({ - prompt: `Run /scrape latest hacker news stories. Invoke /scrape via the Skill tool. -You MUST follow the skill's match-phase logic: -1. Run \`$B skill list\` to see what browser-skills are available -2. Recognize that "latest hacker news stories" matches the bundled - hackernews-frontpage skill's triggers -3. Run \`$B skill run hackernews-frontpage\` and emit the JSON -Do NOT enter the prototype phase. Do NOT use AskUserQuestion.`, + prompt: `Run /scrape latest hacker news stories. +This run is non-interactive; AskUserQuestion is unavailable.`, workingDirectory: workDir, env: { GSTACK_HOME: gstackHome }, maxTurns: 12, @@ -214,18 +205,31 @@ Do NOT enter the prototype phase. Do NOT use AskUserQuestion.`, logCost('scrape-match-path', result); + // Outcome: the matcher ran the bundled skill instead of prototyping, and + // nothing new was written. Listing first is one valid way to match. const cmds = bashCommands(result); - const listedSkills = cmds.some(c => /\bskill\s+list\b/.test(c)); const ranBundledSkill = cmds.some(c => /\bskill\s+run\s+hackernews-frontpage\b/.test(c)); + const prototyped = cmds.some(c => /\$B\s+goto\b|\/browse\s+goto\b/.test(c)); + const newSkills = [workDir, gstackHome].flatMap((root) => { + const skillsRoot = path.join(root, '.gstack', 'browser-skills'); + return fs.existsSync(skillsRoot) ? fs.readdirSync(skillsRoot).filter(d => !d.startsWith('.') && d !== 'hackernews-frontpage') : []; + }); + const staged = [workDir, gstackHome].some((root) => { + const stagingRoot = path.join(root, '.gstack', '.tmp'); + return fs.existsSync(stagingRoot) && fs.readdirSync(stagingRoot).some(d => d.startsWith('skillify-')); + }); + console.log(`scrape-match-path: listed skills first = ${cmds.some(c => /\bskill\s+list\b/.test(c))}`); const exitOk = ['success', 'error_max_turns'].includes(result.exitReason); recordE2E(evalCollector, 'scrape match-path routes to bundled skill', 'Phase 2a E2E', result, { - passed: exitOk && listedSkills && ranBundledSkill, + passed: exitOk && ranBundledSkill && !prototyped && newSkills.length === 0 && !staged, }); expect(exitOk).toBe(true); - expect(listedSkills).toBe(true); expect(ranBundledSkill).toBe(true); + expect(prototyped, 'matched skill must run without prototyping').toBe(false); + expect(newSkills).toEqual([]); + expect(staged).toBe(false); try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {} }, CAPTURE_MS); @@ -240,13 +244,8 @@ Do NOT enter the prototype phase. Do NOT use AskUserQuestion.`, const result = await runSkillTest({ prompt: `Run /scrape titles and scores from ${fileUrl}. -Invoke /scrape via the Skill tool. Follow the skill's prototype-phase logic: -1. \`$B skill list\` finds NO matching skill -2. Drive: \`$B goto ${fileUrl}\` then \`$B html\` (or \`$B text\`) -3. Parse the items (each has a title and a score) -4. Emit JSON of the form {"items": [{"title": "...", "score": N}, ...], "count": N} -5. Suggest /skillify in one line -Do NOT use AskUserQuestion.`, +Return JSON of the form {"items": [{"title": "...", "score": N}, ...], "count": N}. +This run is non-interactive; AskUserQuestion is unavailable.`, workingDirectory: workDir, env: { GSTACK_HOME: gstackHome }, maxTurns: 18, @@ -351,16 +350,13 @@ Do NOT halt for clarification.`, && fs.existsSync(path.join(skillDir, '_lib', 'browse-client.ts')) && fs.existsSync(path.join(skillDir, 'fixtures')); - // D2 enforcement: the SKILL.md prose body MUST NOT contain conversation - // fragments. Cheap heuristic: it shouldn't have "I" or "Let me" or other - // first-person/agent-narration markers. + // D2 enforcement: the SKILL.md prose body must not contain conversation + // fragments (stored good/bad bodies: skill-body-narration.test.ts). let prosesClean = false; if (hasAllFiles) { const skillMd = fs.readFileSync(path.join(skillDir, 'SKILL.md'), 'utf-8'); const body = skillMd.split(/\n---\n/)[1] || ''; - prosesClean = !/^I /m.test(body) - && !/Let me /i.test(body) - && !/^I'll /m.test(body); + prosesClean = !hasNarrationLeak(body); } const exitOk = ['success', 'error_max_turns'].includes(result.exitReason); @@ -390,11 +386,8 @@ Do NOT halt for clarification.`, fs.mkdirSync(childHome, { recursive: true }); const result = await runSkillTest({ - prompt: `Run /skillify via the Skill tool. There has been NO prior /scrape -in this conversation. Follow the skill's Step 1 (D1 provenance guard) literally: -walk back through agent turns, find no /scrape result, refuse with the exact -message the skill specifies, and stop. Do NOT synthesize anything. Do NOT -write any files.`, + prompt: `Run /skillify. +This run is non-interactive; AskUserQuestion is unavailable.`, workingDirectory: workDir, env: { GSTACK_HOME: gstackHome, diff --git a/test/skill-e2e-test-value.test.ts b/test/skill-e2e-test-value.test.ts index fe3b7ef51..af6d1b3a0 100644 --- a/test/skill-e2e-test-value.test.ts +++ b/test/skill-e2e-test-value.test.ts @@ -17,7 +17,7 @@ import { ROOT, runId, describeIfSelected, testIfSelected, copyDirSync, logCost, createEvalCollector, finalizeEvalCollector } from './helpers/e2e-helpers'; import { extractSkillBody } from './helpers/skill-fixture'; import { createCoverageAuditFixture } from './fixtures/coverage-audit-fixture'; -import { createTestValueFixture, lastJsonLine, jsonFindings, LOW_VALUE_TESTS } from './helpers/test-value-fixture'; +import { createTestValueFixture, lastJsonLine, jsonFindings, LOW_VALUE_TESTS, recordsNonTestCallerSearch } from './helpers/test-value-fixture'; import { runRecordedOfficeHoursAttempt, OFFICE_HOURS_BUN_GRACE_MS } from './helpers/office-hours-attempt'; import { resolveEvalModel } from '../lib/eval-model'; import { RETIREMENT_FIELDS } from '../scripts/resolvers/test-value'; @@ -85,7 +85,7 @@ findings, one per line.`, const seam = about(new RegExp(`pricing-reset|${LOW_VALUE_TESTS.testOnlySymbol}`)); if (!grep.length || !grep.every(finding => finding.severity === 'INFORMATIONAL')) throw new Error(`review: source-grep test needs an INFORMATIONAL finding, got ${JSON.stringify(grep)}`); if (!seam.length || !seam.every(finding => finding.severity === 'INFORMATIONAL')) throw new Error(`review: test-only export needs an INFORMATIONAL finding, got ${JSON.stringify(seam)}`); - if (!seam.some(finding => /non_test_callers/.test(JSON.stringify(finding.evidence ?? '')) && /git grep/.test(JSON.stringify(finding.evidence ?? '')))) throw new Error('review: test-only export evidence must record non_test_callers and the git grep search'); + if (!seam.some(recordsNonTestCallerSearch)) throw new Error('review: test-only export evidence must record non_test_callers and the caller search'); const golden = about(/skill-golden/); if (golden.length) throw new Error(`review: the SKILL.md golden test must not be flagged, got ${JSON.stringify(golden)}`); if (trackedChanges(cwd)) throw new Error('review: tracked files changed'); diff --git a/test/skill-e2e-triage.test.ts b/test/skill-e2e-triage.test.ts index 89f75db18..a20e86bfa 100644 --- a/test/skill-e2e-triage.test.ts +++ b/test/skill-e2e-triage.test.ts @@ -32,6 +32,7 @@ import { spawnSync } from 'child_process'; import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; +import { triageLabels } from './helpers/ship-triage-labels'; const evalCollector = createEvalCollector('e2e-triage'); @@ -175,6 +176,7 @@ For each failing test, classify it as: Use git diff origin/main...HEAD (or git diff main...HEAD since there's no remote) to determine which files changed on this branch. Output your classification for each failure clearly, labeling each as "IN-BRANCH" or "PRE-EXISTING" with your reasoning. +End with one JSON object mapping each failing test file to "in-branch" or "pre-existing". This is a solo repo (REPO_MODE=solo). For pre-existing failures, recommend fixing now.`, workingDirectory: triageDir, @@ -188,49 +190,20 @@ This is a solo repo (REPO_MODE=solo). For pre-existing failures, recommend fixin logCost('/ship triage', result); const output = result.output || ''; - const outputLower = output.toLowerCase(); - - // The triage should identify the string/truncate failure as in-branch - const hasInBranch = outputLower.includes('in-branch') || outputLower.includes('in branch') || outputLower.includes('introduced'); - // The triage should identify the math/divide failure as pre-existing - const hasPreExisting = outputLower.includes('pre-existing') || outputLower.includes('pre existing') || outputLower.includes('existed before'); - - console.log(`Output identifies IN-BRANCH failures: ${hasInBranch}`); - console.log(`Output identifies PRE-EXISTING failures: ${hasPreExisting}`); - - // Check that the string/truncate bug is classified as in-branch - const mentionsTruncate = outputLower.includes('truncate') || outputLower.includes('string'); - const mentionsDivide = outputLower.includes('divide') || outputLower.includes('math'); - - console.log(`Mentions truncate/string (in-branch bug): ${mentionsTruncate}`); - console.log(`Mentions divide/math (pre-existing bug): ${mentionsDivide}`); - - // Verify BOTH failure classes are exercised (not just detected): - // The test runner must have actually run both test files - const ranMathTest = output.includes('math.test') || output.includes('FAIL: divide'); - const ranStringTest = output.includes('string.test') || output.includes('FAIL: truncate'); - console.log(`Ran math test file (pre-existing failure): ${ranMathTest}`); - console.log(`Ran string test file (in-branch failure): ${ranStringTest}`); + // Outcome: each seeded failure carries the right label, and the suite actually ran. + const labels = triageLabels(output); + const ranTests = result.toolCalls.some(call => call.tool === 'Bash' + && /test\/run\.js|test\/(?:math|string)\.test\.js/.test(String(call.input?.command ?? ''))); + console.log(`Triage labels: ${JSON.stringify(labels)}; ran tests: ${ranTests}`); recordE2E(evalCollector, '/ship triage', 'Test Failure Triage E2E', result, { - passed: result.exitReason === 'success' && hasInBranch && hasPreExisting, - has_in_branch_classification: hasInBranch, - has_pre_existing_classification: hasPreExisting, - mentions_truncate: mentionsTruncate, - mentions_divide: mentionsDivide, - ran_both_test_files: ranMathTest && ranStringTest, + passed: result.exitReason === 'success' && ranTests && labels.string === 'in-branch' && labels.math === 'pre-existing', }); expect(result.exitReason).toBe('success'); - // Must classify at least one failure as in-branch AND one as pre-existing - expect(hasInBranch).toBe(true); - expect(hasPreExisting).toBe(true); - // Must mention the specific bugs - expect(mentionsTruncate).toBe(true); - expect(mentionsDivide).toBe(true); - // Must have actually run both test files (exercises both failure classes) - expect(ranMathTest).toBe(true); - expect(ranStringTest).toBe(true); + expect(ranTests, 'the agent must run the test suite').toBe(true); + expect(labels.string, 'truncate/string failure is in-branch').toBe('in-branch'); + expect(labels.math, 'divide/math failure is pre-existing').toBe('pre-existing'); }, CAPTURE_MS); }); diff --git a/test/skill-e2e-workflow.test.ts b/test/skill-e2e-workflow.test.ts index 9ffc14b76..178343635 100644 --- a/test/skill-e2e-workflow.test.ts +++ b/test/skill-e2e-workflow.test.ts @@ -24,6 +24,7 @@ const evalCollector = createEvalCollector('e2e'); describeIfSelected('Document-Release skill E2E', ['document-release'], () => { let docReleaseDir: string; + let featureHead: string; beforeAll(() => { docReleaseDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-doc-release-')); @@ -61,6 +62,7 @@ describeIfSelected('Document-Release skill E2E', ['document-release'], () => { '# Changelog\n\n## 1.1.1 — 2026-03-16\n\n- Added Feature C\n\n## 1.0.0 — 2026-03-01\n\n- Initial release with Feature A and Feature B\n- Setup CI pipeline\n'); run('git', ['add', '.']); run('git', ['commit', '-m', 'feat: add feature C']); + featureHead = run('git', ['rev-parse', 'HEAD']).stdout.toString().trim(); }); afterAll(() => { @@ -73,14 +75,8 @@ describeIfSelected('Document-Release skill E2E', ['document-release'], () => { Run the /document-release workflow on this repo. The base branch is "main". -IMPORTANT: -- Do NOT use AskUserQuestion — auto-approve everything or skip if unsure. -- Do NOT push or create PRs (there is no remote). -- Do NOT run gh commands (no remote). -- Focus on updating README.md to reflect the new Feature C. -- Do NOT overwrite or regenerate CHANGELOG entries. -- Skip VERSION bump (it's already bumped). -- After editing, just commit the changes locally.`, +This run is non-interactive: AskUserQuestion is unavailable and nobody can answer questions. +The repo has no remote, so pushing, creating PRs and gh commands are unavailable.`, workingDirectory: docReleaseDir, maxTurns: 30, allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'], @@ -97,36 +93,31 @@ IMPORTANT: logCost('/document-release', result); - // Read CHANGELOG to verify it was NOT clobbered + // The prompt no longer states the CHANGELOG/VERSION rules, so these + // outcomes measure the skill's own never-clobber and never-bump gates. const changelog = fs.readFileSync(path.join(docReleaseDir, 'CHANGELOG.md'), 'utf-8'); - const hasOriginalEntries = changelog.includes('Initial release with Feature A and Feature B') - && changelog.includes('Setup CI pipeline') - && changelog.includes('1.0.0'); - if (!hasOriginalEntries) { - console.warn('CHANGELOG CLOBBERED — original entries missing!'); - } - - // Check if README was updated + const currentEntry = changelog.split(/^## 1\.1\.1\b/m)[1]?.split(/^## /m)[0] ?? ''; + const hasOriginalEntries = /^## 1\.1\.1\b/m.test(changelog) && /^## 1\.0\.0\b/m.test(changelog) + && /feature[ -]?c/i.test(currentEntry) + && changelog.includes('- Initial release with Feature A and Feature B') + && changelog.includes('- Setup CI pipeline'); + const version = fs.readFileSync(path.join(docReleaseDir, 'VERSION'), 'utf-8').trim(); + const versionCommits = spawnSync('git', ['log', '--format=%H', `${featureHead}..HEAD`, '--', 'VERSION'], + { cwd: docReleaseDir, stdio: 'pipe', timeout: 5000 }).stdout.toString().trim(); const readme = fs.readFileSync(path.join(docReleaseDir, 'README.md'), 'utf-8'); - const readmeUpdated = readme.includes('Feature C') || readme.includes('feature-c') || readme.includes('feature C'); + const readmeUpdated = /feature[ -]?c/i.test(readme); const exitOk = ['success', 'error_max_turns'].includes(result.exitReason); recordE2E(evalCollector, '/document-release', 'Document-Release skill E2E', result, { - passed: exitOk && hasOriginalEntries, + passed: exitOk && hasOriginalEntries && version === '1.1.1' && versionCommits === '' && readmeUpdated, }); - // Critical guardrail: CHANGELOG must not be clobbered - expect(hasOriginalEntries).toBe(true); - + expect(hasOriginalEntries, 'original CHANGELOG entries must remain').toBe(true); + expect(version, 'VERSION must not change without asking').toBe('1.1.1'); + expect(versionCommits, 'no commit may change VERSION').toBe(''); + expect(readmeUpdated, 'README should mention Feature C').toBe(true); // Accept error_max_turns — thorough doc review is not a failure expect(['success', 'error_max_turns']).toContain(result.exitReason); - - // Informational: did it update README? - if (readmeUpdated) { - console.log('README updated to include Feature C'); - } else { - console.warn('README was NOT updated — agent may not have found the feature'); - } }, CAPTURE_LONG_MS); }); diff --git a/test/skill-fixture.test.ts b/test/skill-fixture.test.ts index 6b1eba432..6a58562ff 100644 --- a/test/skill-fixture.test.ts +++ b/test/skill-fixture.test.ts @@ -28,6 +28,7 @@ import { CODEX_REVIEW_E2E_SECTIONS, } from './helpers/skill-fixture'; import { E2E_TOUCHFILES, GLOBAL_TOUCHFILES, selectTests } from './helpers/touchfiles'; +import { readShippedSkillRouting } from './helpers/shipped-skill-routing'; const ROOT = path.resolve(import.meta.dir, '..'); @@ -449,10 +450,12 @@ test('routing catalog contains installed project names without a request-to-skil writeFileSync: (file: string, content: string) => writes.set(file, content), }; const compiled = new Bun.Transpiler({ loader: 'ts' }).transformSync(source.slice(start, end) + '\ninstallSkills(fixtureRoot);'); - new Function('ROOT', 'fs', 'path', 'extractSkillHead', 'fixtureRoot', compiled)( - ROOT, mockFs, path, (file: string) => installed.get(file), fixtureRoot, + new Function('ROOT', 'fs', 'path', 'extractSkillHead', 'readShippedSkillRouting', 'fixtureRoot', compiled)( + ROOT, mockFs, path, (file: string) => installed.get(file), readShippedSkillRouting, fixtureRoot, ); const instructions = writes.get(path.join(fixtureRoot, 'CLAUDE.md'))!; + expect(instructions).toContain(readShippedSkillRouting().instruction); + expect(instructions).not.toContain('→ invoke'); expect(instructions).toContain('installed gstack skills: gstack, qa, review.'); expect(instructions).toContain("built-in skills are outside this project's workflow"); expect(instructions).toContain('matching the request to the skill descriptions'); @@ -464,3 +467,18 @@ test('routing catalog contains installed project names without a request-to-skil .toBe(installed.get(path.join(ROOT, 'review/SKILL.md'))); expect(writes.size).toBe(4); }); + +test('routing fixtures read the ## Skill routing section gstack ships', () => { + const routing = readShippedSkillRouting(); + expect(routing.instruction).toStartWith('## Skill routing\n\n'); + expect(routing.instruction).toContain('Skill tool'); + expect(routing.rules.some(rule => rule.includes('/context-save'))).toBe(true); + expect(routing.rules.some(rule => rule.includes('/context-restore'))).toBe(true); + expect(routing.section).not.toContain('If B:'); + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'shipped-routing-drift-')); + try { + fs.mkdirSync(path.join(root, 'bin')); + fs.writeFileSync(path.join(root, 'bin', 'gstack-skill-start'), '#!/bin/bash\necho no routing here\n'); + expect(() => readShippedSkillRouting(root)).toThrow('routing-injection heredoc not found'); + } finally { fs.rmSync(root, { recursive: true, force: true }); } +}); diff --git a/test/skill-llm-eval.test.ts b/test/skill-llm-eval.test.ts index 09bb8119b..40a91a410 100644 --- a/test/skill-llm-eval.test.ts +++ b/test/skill-llm-eval.test.ts @@ -15,10 +15,10 @@ import { JUDGE_MS } from './helpers/eval-budgets'; import * as fs from 'fs'; import * as path from 'path'; import { callJudge, judge, JudgeRefusalError, DEFAULT_JUDGE_MAX_TOKENS, judgePanel, judgePanelMean, judgePanelMajority, judgePanelReasoning, JUDGE_SCORE_DIMENSIONS } from './helpers/llm-judge'; -import { ENG_REVIEW_EXCERPT } from './helpers/workflow-excerpt'; +import { ASK_QUESTIONS_HEADING, ENG_REVIEW_EXCERPT } from './helpers/workflow-excerpt'; import type { JudgeScore } from './helpers/llm-judge'; import { readWorkflowJudgeInput, buildWorkflowJudgePrompt, QA_DISCOVERY_REFERENCES, WORKFLOW_JUDGE_RESPONSE_SCHEMA, type WorkflowJudgeInput } from './helpers/workflow-judge-input'; -import { prepareWorkflowJudgeCache, validWorkflowJudgeScore } from './helpers/workflow-judge-cache'; +import { BROWSE_JUDGE_FLOORS, browseJudgeFloorsMet, prepareWorkflowJudgeCache, validWorkflowJudgeScore } from './helpers/workflow-judge-cache'; import { buildCookieWorkflowJudgeInput, COOKIE_WORKFLOW_JUDGE } from './helpers/cookie-workflow-judge-input'; import { getCookieWorkflowManualReview, type ManualJudgeReview } from './helpers/cookie-workflow-manual-review'; import { resolveEvalModel } from '../lib/eval-model'; @@ -96,8 +96,8 @@ describeIfSelected('LLM-as-judge quality evals', [ const t0 = Date.now(); // Browse carve: snapshot flags + the full command list are the whole // generated section file; one judge grades the union. Scores are also - // pinned against test/fixtures/eval-baselines.json (UPDATE_BASELINES=1 - // rewrites the pin). + // compared with test/fixtures/eval-baselines.json and the difference is + // recorded (UPDATE_BASELINES=1 rewrites it); only the floors gate. const section = sliceBrowseSection('## Snapshot Flags'); const samples = await judgePanel(() => judge('browse skill reference (flags + commands)', section)); @@ -119,17 +119,16 @@ describeIfSelected('LLM-as-judge quality evals', [ name: 'browse/SKILL.md reference', suite: 'LLM-as-judge quality evals', tier: 'llm-judge', - passed: scores.clarity >= 3 && scores.completeness >= 4 && scores.actionability >= 4 && regressions.length === 0, + passed: browseJudgeFloorsMet(scores), duration_ms: Date.now() - t0, cost_usd: 0.02 * samples.length, judge_scores: { clarity: scores.clarity, completeness: scores.completeness, actionability: scores.actionability }, judge_reasoning: regressions.length ? `${judgePanelReasoning(samples)} | ${regressions.join('; ')}` : judgePanelReasoning(samples), }); - expect(scores.clarity).toBeGreaterThanOrEqual(3); - expect(scores.completeness).toBeGreaterThanOrEqual(4); - expect(scores.actionability).toBeGreaterThanOrEqual(4); - expect(regressions).toEqual([]); + expect(scores.clarity).toBeGreaterThanOrEqual(BROWSE_JUDGE_FLOORS.clarity); + expect(scores.completeness).toBeGreaterThanOrEqual(BROWSE_JUDGE_FLOORS.completeness); + expect(scores.actionability).toBeGreaterThanOrEqual(BROWSE_JUDGE_FLOORS.actionability); }, JUDGE_MS); testIfSelected('setup block', async () => { @@ -426,7 +425,7 @@ async function runWorkflowJudge(opts: { suite: string; skillPath: string; startMarker: string; - endMarker: string | null; + endMarker: string | RegExp | null; references?: readonly string[]; judgeContext: string; judgeGoal: string; @@ -635,7 +634,7 @@ describeIfSelected('Plan Review skill evals', [ suite: 'Plan Review skill evals', skillPath: 'plan-design-review/SKILL.md', startMarker: '## Review Sections', - endMarker: '## CRITICAL RULE', + endMarker: ASK_QUESTIONS_HEADING, judgeContext: 'a design plan review framework with 7 review passes', judgeGoal: 'how to review a plan for design quality using a 0-10 rating method: rate each dimension, explain what a 10 looks like, edit the plan to fix gaps, then re-rate', }); diff --git a/test/skill-routing-e2e.test.ts b/test/skill-routing-e2e.test.ts index a9a2d99cb..2bdc68c31 100644 --- a/test/skill-routing-e2e.test.ts +++ b/test/skill-routing-e2e.test.ts @@ -6,6 +6,7 @@ import { EvalCollector } from './helpers/eval-store'; import type { EvalTestEntry } from './helpers/eval-store'; import { selectTests, detectBaseBranch, getChangedFiles, E2E_TOUCHFILES, E2E_TIERS, GLOBAL_TOUCHFILES } from './helpers/touchfiles'; import { extractSkillHead } from './helpers/skill-fixture'; +import { readShippedSkillRouting } from './helpers/shipped-skill-routing'; import { spawnSync } from 'child_process'; import * as fs from 'fs'; import * as path from 'path'; @@ -94,26 +95,19 @@ function installSkills(tmpDir: string) { // The names-only catalog keeps new CLI built-ins from changing the candidate // set. Descriptions still choose the skill; no request-to-skill answer key. - // Write a CLAUDE.md with a GENERIC invoke-skills nudge — deliberately NO - // per-skill routing table. These journey tests exist to catch skill - // DESCRIPTION regressions (their touchfiles key on */SKILL.md.tmpl), and - // the old fixture shipped an explicit prompt→skill answer key: with the - // lookup table in context, a badly regressed frontmatter description - // still routed correctly and the tests could not fail on the regression - // class they select for (2026-08 audit). The generic nudge keeps Claude's - // reach-for-a-skill posture; the FRONTMATTER carries the routing load. + // The routing instruction is the one gstack ships (bin/gstack-skill-start), + // without its per-skill rule list. These journey tests exist to catch skill + // DESCRIPTION regressions (their touchfiles key on */SKILL.md.tmpl): with a + // lookup table in context, a badly regressed frontmatter description still + // routes correctly and the tests cannot fail on the regression class they + // select for. The FRONTMATTER carries the routing load. fs.writeFileSync(path.join(tmpDir, 'CLAUDE.md'), `# Project Instructions -## Skill routing +${readShippedSkillRouting().instruction} This project uses the following installed gstack skills: ${installedSkills.join(', ')}. Choose among this project catalog by matching the request to the skill descriptions. The CLI's built-in skills are outside this project's workflow. - -When the user's request matches an available project skill, ALWAYS invoke it using the Skill -tool as your FIRST action. Do NOT answer directly, do NOT use other tools first. -The skill has specialized workflows that produce better results than ad-hoc answers. -Choose the skill by matching the request against each skill's description. `); } diff --git a/test/test-value-seam-evidence.test.ts b/test/test-value-seam-evidence.test.ts new file mode 100644 index 000000000..bf031ee25 --- /dev/null +++ b/test/test-value-seam-evidence.test.ts @@ -0,0 +1,16 @@ +import { describe, expect, test } from 'bun:test'; +import * as fs from 'fs'; +import * as path from 'path'; +import { recordsNonTestCallerSearch } from './helpers/test-value-fixture'; + +const stored = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/test-value-seam-evidence.json'), 'utf8')) as + { known_good: Record; known_bad: Record }; + +describe('test-only export caller-search evidence', () => { + test.each(Object.entries(stored.known_good))('accepts %s', (_name, evidence) => { + expect(recordsNonTestCallerSearch({ evidence })).toBe(true); + }); + test.each(Object.entries(stored.known_bad))('rejects %s', (_name, evidence) => { + expect(recordsNonTestCallerSearch({ evidence })).toBe(false); + }); +}); diff --git a/test/touchfiles.test.ts b/test/touchfiles.test.ts index 2bb515a88..a766d0ab8 100644 --- a/test/touchfiles.test.ts +++ b/test/touchfiles.test.ts @@ -19,7 +19,7 @@ import { } from './helpers/touchfiles'; import { paidTestClosure, isCovered } from './helpers/touchfile-closure'; -import { readWorkflowExcerpt } from './helpers/workflow-excerpt'; +import { ASK_QUESTIONS_HEADING, readWorkflowExcerpt } from './helpers/workflow-excerpt'; import { sharedLibsPlanExcerpt } from './helpers/shared-libs-plan-excerpt'; const ROOT = path.resolve(import.meta.dir, '..'); @@ -216,7 +216,7 @@ describe('selectTests', () => { ['ship/sections/test-coverage.md', 'ship/SKILL.md workflow', 'ship/SKILL.md', '# Ship:', '## Important Rules', '### REGRESSION RULE (mandatory)'], ['plan-design-review/sections/review-sections.md', 'plan-design-review/SKILL.md passes', - 'plan-design-review/SKILL.md', '## Review Sections', '## CRITICAL RULE', + 'plan-design-review/SKILL.md', '## Review Sections', ASK_QUESTIONS_HEADING, '## Review Sections (7 passes, after scope is agreed)'], ])('expanded judge content remains selected by its section alone: %s', (file, judge, skill, start, end, marker) => { const body = fs.readFileSync(path.join(ROOT, file), 'utf8').replace(/^\n/gm, '').trim(); diff --git a/test/workflow-boundaries-fixture.test.ts b/test/workflow-boundaries-fixture.test.ts index 0db8cfb06..6a860d6af 100644 --- a/test/workflow-boundaries-fixture.test.ts +++ b/test/workflow-boundaries-fixture.test.ts @@ -28,7 +28,7 @@ for (const attack of ['read-state', 'cat-state', 'bash-edit', 'receipt-write']) } finally { fs.rmSync(fixture.root, { recursive: true, force: true }); } }); -function protocolControl(id: BoundaryCase, faults: { skipCleanup?: boolean; duplicateQuestion?: boolean; failVerification?: boolean; rateLimit?: boolean; undeclared?: string; deniedBash?: string; includeAvailability?: boolean; misleadingFinal?: boolean; splitFinal?: boolean } = {}) { +function protocolControl(id: BoundaryCase, faults: { skipCleanup?: boolean; duplicateQuestion?: boolean; failVerification?: boolean; rateLimit?: boolean; undeclared?: string; deniedBash?: string; includeAvailability?: boolean; misleadingFinal?: boolean; splitFinal?: boolean; reverify?: 'pass' | 'fail-last' } = {}) { let directory = ''; let calls = 0; const provider: QueryProvider = input => { @@ -119,6 +119,13 @@ function protocolControl(id: BoundaryCase, faults: { skipCleanup?: boolean; dupl } const verified = await shell('bash ./verify.sh'); expect(verified.status).toBe(id === 'investigate-owned-ending-error' ? 69 : faults.failVerification ? 1 : 0); + if (faults.reverify) { + if (faults.reverify === 'fail-last') { + const file = path.join(directory, 'verify.sh'); + fs.writeFileSync(file, fs.readFileSync(file, 'utf8').replace('value() !== 2', 'value() !== 3')); + } + await shell('bash ./verify.sh'); + } } else expect(answer).toBe('Abort'); if (!faults.skipCleanup) expect((await shell(`bash "$HOME/.claude/skills/gstack/freeze/bin/freeze-state.sh" release "${owner}"`)).status).toBe(0); const finalText = faults.misleadingFinal ? 'The fix is complete and verification succeeded.' @@ -212,6 +219,27 @@ test('fixture records failed verification rather than crediting successful clean expect(JSON.parse(records[0].output!).evidence.receipts).toContain('VERIFY_STATUS:1'); }); +test('a passing re-verification is not a protocol failure', async () => { + const driver = protocolControl('investigate-owned-completion', { reverify: 'pass' }); + const records: EvalTestEntry[] = []; + await runBoundaryActor('investigate-owned-completion', entry => records.push(entry), driver.provider); + expect(records[0].passed).toBe(true); + expect(JSON.parse(records[0].output!).evidence.receipts.match(/^VERIFY$/gm)).toHaveLength(2); +}); + +test('a failing final re-verification still fails after an earlier success', async () => { + const driver = protocolControl('investigate-owned-completion', { reverify: 'fail-last' }); + const records: EvalTestEntry[] = []; + await expect(runBoundaryActor('investigate-owned-completion', entry => records.push(entry), driver.provider)).rejects.toThrow('final verification did not succeed'); + expect(records[0].passed).toBe(false); +}); + +test('retrying the verifier after its ending error still fails', async () => { + const driver = protocolControl('investigate-owned-ending-error', { reverify: 'pass' }); + const records: EvalTestEntry[] = []; + await expect(runBoundaryActor('investigate-owned-ending-error', entry => records.push(entry), driver.provider)).rejects.toThrow('without retry'); +}); + test('rate-limit retry restores the fixture and retains both native event streams', async () => { const driver = protocolControl('investigate-owned-completion', { rateLimit: true }); const records: EvalTestEntry[] = []; diff --git a/test/workflow-excerpt.test.ts b/test/workflow-excerpt.test.ts index 50d0090aa..0f93be40d 100644 --- a/test/workflow-excerpt.test.ts +++ b/test/workflow-excerpt.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from 'bun:test'; -import { ENG_REVIEW_EXCERPT, readWorkflowExcerpt } from './helpers/workflow-excerpt'; +import { ASK_QUESTIONS_HEADING, ENG_REVIEW_EXCERPT, readWorkflowExcerpt } from './helpers/workflow-excerpt'; import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'fs'; import { tmpdir } from 'os'; import { join } from 'path'; @@ -106,9 +106,9 @@ describe('workflow judge excerpts', () => { }); test('a sliced section is not appended again with its generated header', () => { - const text = readWorkflowExcerpt('plan-design-review/SKILL.md', '## Review Sections', '## CRITICAL RULE'); + const text = readWorkflowExcerpt('plan-design-review/SKILL.md', '## Review Sections', ASK_QUESTIONS_HEADING); expect(text.match(/## Review Sections/g)).toHaveLength(1); - expect(text).not.toContain('## CRITICAL RULE'); + expect(text).not.toMatch(ASK_QUESTIONS_HEADING); expect(text).not.toContain('AUTO-GENERATED'); }); @@ -293,7 +293,7 @@ console.log(JSON.stringify({calls, results})); expect(pendingDecision).toContain("**STOP until the actual answer arrives.**"); expect(pendingDecision.replace(/\s+/g, ' ')).toContain("Do not apply a remedy, make another call, start the next section or call ExitPlanMode while the choice awaits an answer"); expect(eng.replace(/\s+/g, ' ')).toContain("Use a scoped Edit to save this record and only the authorized working-plan amendments. Leave other choices unchanged"); - const design = readWorkflowExcerpt('plan-design-review/SKILL.md', '## Review Sections', '## CRITICAL RULE'); + const design = readWorkflowExcerpt('plan-design-review/SKILL.md', '## Review Sections', ASK_QUESTIONS_HEADING); expect(design).toContain('wait for approval, then edit the plan and re-rate'); const pass4 = design.slice(design.indexOf('### Pass 4:'), design.indexOf('### Pass 5:')); expect(pass4.match(/^### /gm)).toHaveLength(1); diff --git a/test/workflow-judge-input.test.ts b/test/workflow-judge-input.test.ts index 47ff4b3a7..4b47d50da 100644 --- a/test/workflow-judge-input.test.ts +++ b/test/workflow-judge-input.test.ts @@ -5,7 +5,7 @@ import { tmpdir } from 'node:os'; import { dirname, join, resolve } from 'node:path'; import { createHash } from 'node:crypto'; import { readWorkflowJudgeInput, buildWorkflowJudgePrompt, QA_DISCOVERY_REFERENCES, WORKFLOW_JUDGE_RESPONSE_SCHEMA, WORKFLOW_JUDGE_REASONING_WORD_LIMIT } from './helpers/workflow-judge-input'; -import { ENG_REVIEW_EXCERPT } from './helpers/workflow-excerpt'; +import { ASK_QUESTIONS_HEADING, ENG_REVIEW_EXCERPT } from './helpers/workflow-excerpt'; const ROOT = resolve(import.meta.dir, '..'); const scratchRoots: string[] = []; @@ -363,14 +363,14 @@ describe('workflow judge file bundle', () => { test('generated plan-design passes retain their full section without duplicating Pass 1', () => { const input = readWorkflowJudgeInput({ - root: ROOT, skillPath: 'plan-design-review/SKILL.md', startMarker: '## Review Sections', endMarker: '## CRITICAL RULE', + root: ROOT, skillPath: 'plan-design-review/SKILL.md', startMarker: '## Review Sections', endMarker: ASK_QUESTIONS_HEADING, }); expect(input.files.filter(file => file.kind === 'entrypoint')).toHaveLength(0); expect(input.files.map(file => file.path)).toEqual(sectionPaths('plan-design-review')); const section = input.files.find(file => file.path === 'plan-design-review/sections/review-sections.md'); expect(section?.content).toBe(readFileSync(join(ROOT, 'plan-design-review/sections/review-sections.md'), 'utf8')); expect(section?.content).toStartWith('