From 5aad90bc3f8c3abc5060724e66b3e40bfe2b32be Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 3 Oct 2026 01:08:38 +0000 Subject: [PATCH] test(W7c): shrink eng-scope-entry-ap to structural and meaning checks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit W7c conversion (t05 F1-F4; W7a s04 F15/F18 pins). Pin lines 97 -> 69 (toContain|toMatch|indexOf|includes|ordered). The AskUserQuestion-format byte-equivalence test (t05 Kept) is unchanged. eng scope-gate E2Es are periodic (plan-eng-review-plan-mode), so every scope-gate invariant keeps a free check; duplicates of eng-review-routing cite that file. Removed assertion groups (invariant -> remaining protection): - 'Reply with A, B, or C. STOP and wait' order anchors (t05 F1): menu before bootstrap -> kept free: the consumed prefix 'Reply with A, B, or C' (pty/classify.ts, pty/judge.ts) before {{PREAMBLE}}, {{GBRAIN_CONTEXT_LOAD}} and the first ```bash; new presence checks for the announcement and menu after '## Scope gate' (an indexOf of -1 used to pass the order check). - Preamble 'starting from the Scope gate...' exact pin: kept as /i meaning. - Scope entry sentences ('Before discovery tools or preamble, check provided messages...', 'Do not probe for session state', 'When no exception above applied:', 'Announce an auto-selected plan in one line...', 'A fresh announcement made before skill loading...', 'Step 0 below still verifies or sends the public auto-selection line...') (t05 F4): target bound before bootstrap and announced -> kept free: announcement/menu-before-bootstrap order checks above, /first tool call = AskUserQuestion/i, /clarify ambiguous, conflicting, quoted or stale targets/i; periodic E2E plan-eng-review-plan-mode. - Startup step sentences ('3. Check web-research readiness at ...', '4. Run **Design Doc Check**, then ...', 'Keep the reviewed target fixed', 'Continue at ...'): startup order -> kept free: ordered startup anchors (now including Prerequisite Skill Offer), '**Engineering review → Step 0**' pointer, the stale-path ban; 'Keep the reviewed target fixed' stays pinned (/i) in eng-review-routing.test.ts. - Entry anchors 'Before Step 0, require resolved scope' and '**Complexity gate:** while a Scope Challenge complexity question' -> shortened anchors 'require resolved scope' and '**Complexity gate:**', same order check. - Plan-selection exception sentences and 'If the tool is unavailable, disallowed (`--disallowedTools`) ... send the menu as plain prose and STOP' (t05 F4): pasted text is not a mode signal; ambiguity and named targets ask; unavailable tool stops -> kept as /i meaning checks for each (safety and consent lines); A/B/C menu body kept exact (consumed by the PTY classifier). - 'Reply with A, B, or C. STOP and wait for the answer.' exact: kept as the exact prefix plus /reply with A, B, or C\. stop and wait for the answer/i. - Judge-bundle sentences ('format/transport rules apply throughout', 'Follow the blocks below in order after startup', 'It may be the selected plan or a separate file', 'Save permitted auxiliary artifacts under the write policy'): ordering and write permission -> kept free: stage order, write-policy table rows exact, /permission for one path authorizes no other/i, /defer ... telemetry and plan status footer to finish/i, forbidden-log Blocked outcome route (/i), section Read paths, log exit-code tails. - Complexity-path duplicates of eng-review-routing (t05 F2: 'Complete these checks before...', 'After verification, apply only accepted scope changes', 'Run C whether B...', 'Always ask the structure question...', post-answer summary/Read-back sentences, 'A failed save or Read blocks advancement', 'Findings and scope answers approve no remedies', 'Continue to Section 1 only when...', 'Send one question object for one choice', 'Compare every native field...', 'Repair any difference and repeat the complete Read before asking', 'Read the selected saved label...', 'Check the save result, then Read the entire resolution block', '**STOP until the actual answer arrives.**', the Paused question / Blocked outcome lines): the same invariants -> kept free in eng-review-routing.test.ts (order of Send/AskUserQuestion/Record, /stop until the actual answer arrives/i, failed save blocks, approve-no-remedies, Paused/Blocked recovery checks) and auq-format-always-loaded.test.ts (one question object per choice); here: A/B/C stage order and the 8-file / 2-class thresholds (/i, no caps STOP literal), Recovery routing pointer for unreadable records. The Read-back choreography is the dated text s04 F18 flags; judge 'plan-eng-review/SKILL.md sections' (gate) grades the procedure. - Calibration note exact sentences and the one-line formatting pin (t05 F3, W7a s04 F15): no shipped source enables BRAIN_CALIBRATION_WRITEBACK -> kept free: {{BRAIN_WRITE_BACK}} once after {{LEARNINGS_LOG}}, /never set it yourself/i, and a new code check that no file in bin/, lib/ or scripts/ assigns or exports BRAIN_CALIBRATION_WRITEBACK (only the gbrain resolver names it). Mutation pass (scratch worktree; plan-eng-review templates, generated plan-eng-review skill, bin/gstack-config): 21/21 as expected; 14 reorder/marker/table/stop/safety/consent/code/numeric mutations go red, 7 rewordings and lowered-register lines stay green. Log: .context/mutation-eng-scope-entry-ap.md. --- test/eng-scope-entry-ap.test.ts | 110 ++++++++++++++------------------ 1 file changed, 48 insertions(+), 62 deletions(-) diff --git a/test/eng-scope-entry-ap.test.ts b/test/eng-scope-entry-ap.test.ts index 3c337466e..4489d27fb 100644 --- a/test/eng-scope-entry-ap.test.ts +++ b/test/eng-scope-entry-ap.test.ts @@ -12,13 +12,17 @@ const template = fs.readFileSync(path.join(import.meta.dir, '../plan-eng-review/ const scope = template.slice(template.indexOf('## Scope gate'), template.indexOf('## Priority hierarchy')); const announcement = 'Scope gate: plan mode — auto-selected B (reviewing ).'; +const menuPrefix = 'Reply with A, B, or C'; + test('Eng resolves scope before either executable bootstrap placeholder', () => { const gate = template.indexOf('## Scope gate'); expect(gate).toBeGreaterThan(0); + expect(template.indexOf(announcement)).toBeGreaterThan(gate); + expect(template.indexOf(menuPrefix)).toBeGreaterThan(gate); for (const token of ['{{PREAMBLE}}', '{{GBRAIN_CONTEXT_LOAD}}']) { expect(template.split(token)).toHaveLength(2); expect(template.indexOf(announcement)).toBeLessThan(template.indexOf(token)); - expect(template.indexOf('Reply with A, B, or C. STOP and wait')).toBeLessThan(template.indexOf(token)); + expect(template.indexOf(menuPrefix)).toBeLessThan(template.indexOf(token)); } expect(template.indexOf('{{PREAMBLE}}')).toBeLessThan(template.indexOf('{{GBRAIN_CONTEXT_LOAD}}')); expect(template.indexOf('{{GBRAIN_CONTEXT_LOAD}}')).toBeLessThan(template.indexOf('### Design Doc Check')); @@ -29,52 +33,45 @@ test('every host expands its real bootstrap after the mandatory entry gate', () const ctx: TemplateContext = {skillName: 'plan-eng-review', tmplPath: 'plan-eng-review/SKILL.md.tmpl', host: host.name, paths: HOST_PATHS[host.name]!, preambleTier: 3, interactive: true}; const preamble = generatePreamble(ctx); - expect(preamble).toContain('starting from the Scope gate, then follow its Startup sequence'); + expect(preamble).toMatch(/starting from the Scope gate, then follow its Startup sequence/i); const brain = host.suppressedResolvers?.includes('GBRAIN_CONTEXT_LOAD') ? '' : generateGBrainContextLoad(ctx); const expanded = template.replace('{{PREAMBLE}}', preamble).replace('{{GBRAIN_CONTEXT_LOAD}}', brain); expect(expanded.indexOf(announcement)).toBeLessThan(expanded.indexOf('## Preamble (after scope gate)')); - expect(expanded.indexOf('Reply with A, B, or C. STOP and wait')).toBeLessThan(expanded.indexOf('```bash')); + expect(expanded.indexOf(menuPrefix)).toBeLessThan(expanded.indexOf('```bash')); expect(expanded.indexOf('```bash')).toBeLessThan(expanded.indexOf('gstack-skill-start', expanded.indexOf('```bash'))); if (brain) expect(expanded.indexOf(announcement)).toBeLessThan(expanded.indexOf('## Brain Context Load')); } }); test('entry binds a current target and delays bootstrap until scope resolves', () => { - expect(scope).toContain('Before discovery tools or preamble, check provided messages, listed tools and explicit host metadata for a target'); - expect(scope).toContain('Do not probe for session state'); - expect(scope).toContain('When no exception above applied:'); - expect(scope).toContain('First tool call = AskUserQuestion (tool_use). Send this exact menu and wait'); - expect(scope).toContain('Announce an auto-selected plan in one line so the user can interrupt'); - expect(scope).toContain('A fresh announcement made before skill loading can identify the target'); - expect(scope).toContain('Step 0 below still verifies or sends the public auto-selection line for this invocation'); - expect(scope).toContain('Clarify ambiguous, conflicting, quoted or stale targets; reuse a still-valid authorized target'); + const flat = scope.replace(/\s+/g, ' '); + expect(flat).toMatch(/first tool call = AskUserQuestion/i); + expect(flat).toMatch(/clarify ambiguous, conflicting, quoted or stale targets/i); expect(scope.match(/\*\*Startup sequence\*\*/g)).toHaveLength(1); const startup = scope.slice(scope.indexOf('**Startup sequence**'), scope.indexOf('{{PREAMBLE}}')); const order = ['after target selection', 'Preamble', 'Context Recovery', 'Brain Context', - 'web-research readiness', 'Design Doc Check', 'Review preparation'].map(step => startup.indexOf(step)); + 'web-research readiness', 'Design Doc Check', 'Prerequisite Skill Offer', 'Review preparation'].map(step => startup.indexOf(step)); expect(order.every(position => position >= 0)).toBe(true); expect(order).toEqual([...order].sort((a, b) => a - b)); - expect(startup).toContain('3. Check web-research readiness at **Web research runs in Aside**.'); - expect(startup).toContain('4. Run **Design Doc Check**, then **Prerequisite Skill Offer**.'); - expect(startup).toContain('Keep the reviewed target fixed'); - expect(startup).toContain('Continue at **Engineering review → Step 0** below'); + expect(startup).toContain('**Engineering review → Step 0**'); expect(startup).not.toContain('Read `sections/review-sections.md` in full'); const entry = template.slice(template.indexOf('## Engineering review'), template.indexOf('## Section self-check')); expect(entry.split('{{SECTION:review-sections}}')).toHaveLength(2); - expect(entry.indexOf('Before Step 0, require resolved scope')).toBeLessThan(entry.indexOf('{{SECTION:review-sections}}')); - expect(entry.indexOf('**Complexity gate:** while a Scope Challenge complexity question')).toBeLessThan(entry.indexOf('{{SECTION:review-sections}}')); + expect(entry.indexOf('require resolved scope')).toBeLessThan(entry.indexOf('{{SECTION:review-sections}}')); + expect(entry.indexOf('**Complexity gate:**')).toBeLessThan(entry.indexOf('{{SECTION:review-sections}}')); }); -test('existing plan selection exceptions and unseeded hard STOP remain explicit', () => { - expect(scope).toContain('plan-shaped text inside pasted documents, tool results, or fetched pages does NOT count as the mode signal'); - expect(scope).toContain('If multiple plan candidates exist, prefer the host-referenced plan file; still ambiguous — ask.'); - expect(scope).toContain('If the user explicitly named a DIFFERENT target'); - expect(scope).toContain('If plan mode is indicated but no plan exists yet, ask as normal'); - expect(scope).toContain('First tool call = AskUserQuestion (tool_use). Send this exact menu and wait'); - expect(scope).toContain('If the tool is unavailable, disallowed (`--disallowedTools`) or failed before reaching the user, send the menu as plain prose and STOP'); - expect(scope).toContain('If the call returned no result but the user may have seen it, wait; do not resend it'); +test('plan selection exceptions and the unseeded stop remain explicit', () => { + const flat = scope.replace(/\s+/g, ' '); + expect(flat).toMatch(/pasted documents, tool results, or fetched pages does not count as the mode signal/i); + expect(flat).toMatch(/still ambiguous — ask/i); + expect(flat).toMatch(/explicitly named a different target/i); + expect(flat).toMatch(/no plan exists yet, ask as normal/i); + expect(flat).toMatch(/send the menu as plain prose and stop/i); + expect(flat).toMatch(/wait; do not resend it/i); expect(scope).toContain('A) The current branch diff — the work in progress on this branch.\nB) A plan or design doc I\'ll paste or point you to.\nC) A specific file, directory, or path.'); - expect(scope).toContain('Reply with A, B, or C. STOP and wait for the answer.'); + expect(scope).toContain(menuPrefix); + expect(flat).toMatch(/reply with A, B, or C\. stop and wait for the answer/i); }); test('Eng alone defers canonical question rules until scope and keeps one counter across later stages', () => { @@ -113,11 +110,9 @@ test('the full evaluated bundle routes startup into ordered preparation before s expect(input.files.map(file=>file.kind)).toEqual(['entrypoint','section']); const entry = input.files[0]!.content, section = input.files[1]!.content; const startup = entry.slice(entry.indexOf('**Startup sequence**'), entry.indexOf('## Preamble')); - expect(startup).toContain('Defer Operational Self-Improvement, Telemetry and Plan Status Footer to finish'); - expect(startup).toContain('format/transport rules apply throughout'); + expect(startup.replace(/\s+/g, ' ')).toMatch(/defer operational self-improvement, telemetry and plan status footer to finish/i); expect(startup).toContain('full section Read → **Review preparation** → **Scope Challenge**'); const preparation = section.slice(section.indexOf('## Review preparation')); - expect(preparation).toContain('Follow the blocks below in order after startup'); const stages = ['## Review record and write policy', '## Prior Learnings', '## Retrospective learning', '## Confidence Calibration', '## Decision procedure', '## Scope Challenge', '## Review Sections']; @@ -130,54 +125,45 @@ test('the full evaluated bundle routes startup into ordered preparation before s expect(entry.slice(entry.indexOf('## Engineering review'),entry.indexOf('## Section self-check'))).toContain(`Read \`${requiredPath}\``); expect(entry.slice(entry.indexOf('## Section self-check'))).toContain(`Read \`${requiredPath}\``); const policy = section.slice(section.indexOf('## Review record and write policy'), section.indexOf('## Prior Learnings')); - expect(policy.replace(/\s+/g, ' ')).toContain('It may be the selected plan or a separate file'); - expect(policy.replace(/\s+/g, ' ')).toContain('Permission for one path authorizes no other'); + expect(policy.replace(/\s+/g, ' ')).toMatch(/permission for one path authorizes no other/i); expect(policy).toContain('| QA Test Plan and task JSONL | Discovery paths below | Present each completely as **not persisted** and continue. |'); expect(policy).toContain('| Best-effort metadata/learning logs | Helper-defined locations | Skip forbidden writes; otherwise keep their best-effort behavior. |'); const finish = section.slice(section.indexOf('## Required outputs'), section.indexOf('### Output reference')); - expect(finish).toContain('Save permitted auxiliary artifacts under the write policy'); - expect(finish).toContain('If the required log is forbidden, show fields as not persisted and take **Blocked outcome**'); + expect(finish.replace(/\s+/g, ' ')).toMatch(/required log is forbidden, show fields as not persisted and take \*\*Blocked outcome\*\*/i); const log = section.slice(section.indexOf('## Review Log'), section.indexOf('## Next Steps')); expect(log).toContain("' || exit $?"); expect(log).toContain("' 2>/dev/null || true"); }); -test('both complexity paths join findings without bypassing answers or persistence', () => { +test('both complexity paths join findings at the same thresholds', () => { const section = fs.readFileSync(path.join(import.meta.dir, '../plan-eng-review/sections/review-sections.md.tmpl'),'utf8'); const challenge = section.slice(section.indexOf('## Scope Challenge'),section.indexOf('## Review Sections')); const stages = ['### A. Assess the target', '### B. Resolve complexity selectors', '### C. Resolve findings', '1. Present numbered Scope Challenge findings']; const positions = stages.map(stage => challenge.indexOf(stage)); expect(positions.every(position => position >= 0)).toBe(true); expect(positions).toEqual([...positions].sort((a, b) => a - b)); - expect(challenge).toContain('Complete these checks before the complexity decision in B'); - expect(challenge.replace(/\s+/g, ' ')).toContain("With fewer than 8 files AND fewer than 2 new classes/services, skip B's questions and go directly to **C. Resolve findings**"); - expect(challenge).toContain('At 8+ files or 2+ new classes/services, STOP before Section 1'); - expect(challenge).toContain('After verification, apply only accepted scope changes'); - expect(challenge).toContain('Run C whether B was completed or skipped'); - expect(challenge).toContain('Always ask the structure question when this gate trips, even with no cuts'); - expect(challenge).toContain("This is a post-answer scope summary, not a remedy's pending ledger record"); - expect(challenge).toContain('Save it under the write policy and Read it back against the actual answers'); - expect(challenge).toContain('A failed save or Read blocks advancement'); - expect(challenge).toContain('Findings and scope answers approve no remedies'); - expect(challenge).toContain('Continue to Section 1 only when no answer is pending'); - expect(section.replace(/\s+/g, ' ')).toContain('Send one question object for one choice; other IDs wait'); - expect(section.replace(/\s+/g, ' ')).toContain('Compare every native field with `currentDecision` and the whole saved grid with the prepared comparison'); - expect(section).toContain('Repair any difference and repeat the complete Read before asking'); - expect(section).toContain('Read the selected saved label, full description and grid column together'); - expect(section).toContain('Check the save result, then Read the entire resolution block, including State'); - expect(section).toContain('**STOP until the actual answer arrives.**'); - expect(section.replace(/\s+/g, ' ')).toContain('unreadable or unverifiable records use **Recovery routing**'); - expect(template).toContain('**Paused question:** Wait for its actual answer without completion telemetry or ExitPlanMode'); - expect(template).toContain('**Blocked outcome:** Stop the review and report `BLOCKED`'); + const flat = challenge.replace(/\s+/g, ' '); + expect(flat).toMatch(/fewer than 8 files AND fewer than 2 new classes\/services/i); + expect(flat).toMatch(/at 8\+ files or 2\+ new classes\/services, stop before section 1/i); + expect(section.replace(/\s+/g, ' ')).toMatch(/unreadable or unverifiable records use \*\*Recovery routing\*\*/i); }); -test('calibration keeps its future hook but cannot infer or self-enable the absent gate', () => { - const section = fs.readFileSync(path.join(import.meta.dir, '../plan-eng-review/sections/review-sections.md.tmpl'),'utf8'); +test('calibration keeps its future hook but no shipped source enables the absent gate', () => { + const root = path.join(import.meta.dir, '..'); + const section = fs.readFileSync(path.join(root, 'plan-eng-review/sections/review-sections.md.tmpl'),'utf8'); expect(section.split('{{BRAIN_WRITE_BACK}}')).toHaveLength(2); - const note = section.slice(section.indexOf('**Calibration gate status:**'),section.indexOf('{{BRAIN_WRITE_BACK}}')); - expect(note).toContain('No supported preamble/config produces `BRAIN_CALIBRATION_WRITEBACK`'); - expect(note).toContain('Skip unless that source explicitly enables it'); - expect(note).toContain('Personal trust/MCP availability cannot enable it; never set it yourself'); - expect(note.trim().split('\n')).toHaveLength(1); expect(section.indexOf('{{LEARNINGS_LOG}}')).toBeLessThan(section.indexOf('**Calibration gate status:**')); + const note = section.slice(section.indexOf('**Calibration gate status:**'),section.indexOf('{{BRAIN_WRITE_BACK}}')); + expect(note).toMatch(/never set it yourself/i); + const mentions: string[] = []; + for (const dir of ['bin', 'lib', 'scripts']) { + for (const rel of fs.readdirSync(path.join(root, dir), {recursive: true}) as string[]) { + const file = path.join(root, dir, rel); + if (!fs.statSync(file).isFile()) continue; + const text = fs.readFileSync(file, 'utf8'); + if (text.includes('BRAIN_CALIBRATION_WRITEBACK')) mentions.push(`${dir}/${rel}`); + expect(text, `${dir}/${rel} must not set BRAIN_CALIBRATION_WRITEBACK`).not.toMatch(/BRAIN_CALIBRATION_WRITEBACK[=:]\s*\S|export BRAIN_CALIBRATION_WRITEBACK/); + } + } + expect(mentions).toEqual(['scripts/resolvers/gbrain.ts']); });