diff --git a/test/helpers/touchfiles-data.ts b/test/helpers/touchfiles-data.ts index 3f3e54830..2672eb8c3 100644 --- a/test/helpers/touchfiles-data.ts +++ b/test/helpers/touchfiles-data.ts @@ -130,17 +130,18 @@ export const E2E_TOUCHFILES: Record = { ], // Review Army (specialist dispatch) - 'review-army-migration-safety': [ 'review/**', 'test/fixtures/review-army-migration.sql', 'scripts/resolvers/review-army.ts', 'bin/gstack-diff-scope', 'test/skill-e2e-review-army.test.ts', 'test/helpers/office-hours-attempt.ts'], - 'review-army-perf-n-plus-one': [ 'review/**', 'scripts/resolvers/review-army.ts', 'bin/gstack-diff-scope', 'test/skill-e2e-review-army.test.ts', 'test/fixtures/review-n-plus-one-dispatch.json', 'test/fixtures/review-army-n-plus-one.rb', 'test/helpers/office-hours-attempt.ts'], - 'review-army-delivery-audit': [ 'review/**', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'scripts/resolvers/review-army.ts', 'test/skill-e2e-review-army.test.ts', 'test/helpers/office-hours-attempt.ts'], - 'review-army-quality-score': [ 'review/**', 'scripts/resolvers/review-army.ts', 'test/skill-e2e-review-army.test.ts', 'test/helpers/office-hours-attempt.ts'], - 'review-army-json-findings': [ 'review/**', 'scripts/resolvers/review-army.ts', 'test/skill-e2e-review-army.test.ts', 'test/helpers/office-hours-attempt.ts'], - 'review-army-red-team': [ 'review/**', 'scripts/resolvers/review-army.ts', 'test/skill-e2e-review-army.test.ts', + 'review-army-migration-safety': [ 'review/**', 'test/fixtures/review-army-migration.sql', 'scripts/resolvers/review-army.ts', 'bin/gstack-diff-scope', 'test/skill-e2e-review-army.test.ts', 'test/helpers/state-matrix-fixture.ts', 'test/fixtures/state-matrix-6792/**', 'test/helpers/office-hours-attempt.ts'], + 'review-army-perf-n-plus-one': [ 'review/**', 'scripts/resolvers/review-army.ts', 'bin/gstack-diff-scope', 'test/skill-e2e-review-army.test.ts', 'test/helpers/state-matrix-fixture.ts', 'test/fixtures/state-matrix-6792/**', 'test/fixtures/review-n-plus-one-dispatch.json', 'test/fixtures/review-army-n-plus-one.rb', 'test/helpers/office-hours-attempt.ts'], + 'review-army-delivery-audit': [ 'review/**', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'scripts/resolvers/review-army.ts', 'test/skill-e2e-review-army.test.ts', 'test/helpers/state-matrix-fixture.ts', 'test/fixtures/state-matrix-6792/**', 'test/helpers/office-hours-attempt.ts'], + 'review-army-quality-score': [ 'review/**', 'scripts/resolvers/review-army.ts', 'test/skill-e2e-review-army.test.ts', 'test/helpers/state-matrix-fixture.ts', 'test/fixtures/state-matrix-6792/**', 'test/helpers/office-hours-attempt.ts'], + 'review-army-json-findings': [ 'review/**', 'scripts/resolvers/review-army.ts', 'test/skill-e2e-review-army.test.ts', 'test/helpers/state-matrix-fixture.ts', 'test/fixtures/state-matrix-6792/**', 'test/helpers/office-hours-attempt.ts'], + 'review-army-red-team': [ 'review/**', 'scripts/resolvers/review-army.ts', 'test/skill-e2e-review-army.test.ts', 'test/helpers/state-matrix-fixture.ts', 'test/fixtures/state-matrix-6792/**', 'test/helpers/office-hours-attempt.ts' ], - 'review-army-simplification': [ 'review/**', 'scripts/resolvers/review-army.ts', 'test/fixtures/review-army-overbuild.js', 'test/fixtures/review-army-lean-complete.js', 'test/skill-e2e-review-army.test.ts', 'test/helpers/office-hours-attempt.ts'], - 'review-army-simplification-precision': [ 'review/**', 'scripts/resolvers/review-army.ts', 'test/fixtures/review-army-overbuild.js', 'test/fixtures/review-army-lean-complete.js', 'test/skill-e2e-review-army.test.ts', 'test/helpers/office-hours-attempt.ts'], - 'review-army-consensus': [ 'review/**', 'scripts/resolvers/review-army.ts', 'test/skill-e2e-review-army.test.ts', + 'review-army-simplification': [ 'review/**', 'scripts/resolvers/review-army.ts', 'test/fixtures/review-army-overbuild.js', 'test/fixtures/review-army-lean-complete.js', 'test/skill-e2e-review-army.test.ts', 'test/helpers/state-matrix-fixture.ts', 'test/fixtures/state-matrix-6792/**', 'test/helpers/office-hours-attempt.ts'], + 'review-army-simplification-precision': [ 'review/**', 'scripts/resolvers/review-army.ts', 'test/fixtures/review-army-overbuild.js', 'test/fixtures/review-army-lean-complete.js', 'test/skill-e2e-review-army.test.ts', 'test/helpers/state-matrix-fixture.ts', 'test/fixtures/state-matrix-6792/**', 'test/helpers/office-hours-attempt.ts'], + 'review-army-state-matrix': [ 'review/**', 'scripts/resolvers/review-army.ts', 'lib/state-matrix.ts', 'bin/gstack-state-matrix', 'test/fixtures/state-matrix-6792/**', 'test/helpers/state-matrix-fixture.ts', 'test/skill-e2e-review-army.test.ts', 'test/helpers/office-hours-attempt.ts'], + 'review-army-consensus': [ 'review/**', 'scripts/resolvers/review-army.ts', 'test/skill-e2e-review-army.test.ts', 'test/helpers/state-matrix-fixture.ts', 'test/fixtures/state-matrix-6792/**', 'test/helpers/office-hours-attempt.ts'], // Office Hours @@ -1212,6 +1213,7 @@ export const E2E_TIERS: Record = { 'review-army-consensus': 'periodic', // Multi-specialist agreement 'review-army-simplification': 'periodic', // Advisory lens quality benchmark 'review-army-simplification-precision': 'periodic', // False-flag noise benchmark + 'review-army-state-matrix': 'periodic', // State-machine round-trip matrix (field list 2, W3) // Office Hours // Brain-writeback E2E — periodic per cost (claude -p) + non-deterministic @@ -1702,6 +1704,7 @@ export const E2E_KINDS: Record = { 'review-army-consensus': 'behavior', 'review-army-simplification': 'behavior', 'review-army-simplification-precision': 'behavior', + 'review-army-state-matrix': 'behavior', 'office-hours-brain-writeback': 'behavior', 'gbrain-roundtrip-local': 'rule', 'sync-gbrain-read-ready': 'rule', @@ -1932,6 +1935,8 @@ export const BEHAVIOR_WHY: Record = { "Flagging the planted unnecessary structure is an advisory-lens quality judgment.", 'review-army-simplification-precision': "Staying silent on a lean diff is a false-flag noise benchmark; an occasional advisory is acceptable noise.", + 'review-army-state-matrix': + "How the live specialist words each finding and which line it cites varies run to run; a complete kind x owner action matrix with every untested cell reported, checked by gstack-state-matrix against the fixture's ground truth, stays the contract.", 'office-hours-forcing-energy': "The Q3 posture is scored by a live judge on generated prose; a single flat phrasing is tolerable.", 'office-hours-builder-wildness': diff --git a/test/skill-e2e-review-army.test.ts b/test/skill-e2e-review-army.test.ts index e31410dde..d8d5d0b88 100644 --- a/test/skill-e2e-review-army.test.ts +++ b/test/skill-e2e-review-army.test.ts @@ -8,6 +8,7 @@ import { logCost, recordE2E, createEvalCollector, finalizeEvalCollector, } from './helpers/e2e-helpers'; import { extractSkillSections, REVIEW_ARMY_E2E_SECTIONS, sliceBetween } from './helpers/skill-fixture'; +import { makeStateMatrixRepo, STATE_MACHINES_CHECKLIST, STATE_MATRIX_FIXTURE, type StateMatrixRepo } from './helpers/state-matrix-fixture'; import { spawnSync } from 'child_process'; import * as fs from 'fs'; import * as path from 'path'; @@ -584,6 +585,56 @@ Write ONLY JSON findings (no preamble) to ${dir}/findings.json`, }, JUDGE_MS); }); +// --- Review Army: State-machine round-trip matrix (field list 2, W3) --- +// Contract: on the #6792-shaped diff (two new proposal kinds that fall through +// to the remove path, tests only for add, remove and a new-claim replace), the +// live State Machines specialist, given Step 4.5's dispatch instructions +// extracted from review-army.md, returns a matrix that gstack-state-matrix +// check passes against the fixture's ground truth: every kind x owner action +// cell, each untested cell reported as a finding. The free regression on a +// captured run is test/state-matrix.test.ts. + +describeIfSelected('Review Army: State-machine matrix', ['review-army-state-matrix'], () => { + let repo: StateMatrixRepo; + + beforeAll(() => { + repo = makeStateMatrixRepo('skill-e2e-army-state-matrix-'); + const specDir = path.join(repo.work, 'review-specialists'); + fs.mkdirSync(specDir, { recursive: true }); + fs.copyFileSync(STATE_MACHINES_CHECKLIST, path.join(specDir, 'state-machines.md')); + }); + + afterAll(() => { try { fs.rmSync(repo.root, { recursive: true, force: true }); } catch {} }); + + testConcurrentIfSelected('review-army-state-matrix', async () => { + const dispatch = sliceBetween(readReviewSection('review-army.md'), '"You are a specialist code reviewer.', '**Subagent configuration:**') + .trim().replace(/^"|"$/g, '').replace(/\\"/g, '"') + .replace('{checklist path}', 'review-specialists/state-machines.md').replace('origin/', 'origin/main') + .replace('{TEST_FW}', 'unknown').replace('{STACK}', 'node').replace("{learnings or 'none'}", 'none'); + const out = path.join(repo.work, 'review-output.jsonl'); + const result = await runSkillTest({ + prompt: `You are the State Machines specialist that /review's Step 4.5 dispatched for branch trust-guard (gstack-state-matrix detect printed SCOPE_STATE_MACHINE=true). These are your dispatch instructions: + +${dispatch} + +Write your raw output, exactly as those instructions define it, to ${out} and nothing else. After saving it, finish with a brief acknowledgement.`, + workingDirectory: repo.work, + maxTurns: 20, + timeout: 180_000, + testName: 'review-army-state-matrix', + runId, + }); + + logCost('/review army state matrix', result); + recordE2E(evalCollector, '/review army state-machine matrix', 'Review Army', result); + expect(result.exitReason).toBe('success'); + expect(fs.existsSync(out)).toBe(true); + const check = spawnSync(path.join(ROOT, 'bin', 'gstack-state-matrix'), ['check', out, '--repo', repo.work, '--expect', path.join(STATE_MATRIX_FIXTURE, 'expect.json')], { cwd: repo.work, encoding: 'utf-8', timeout: 60_000 }); + expect(check.stdout, check.stdout + check.stderr).toMatch(/^STATE_MATRIX_CHECK: ok/m); + expect(check.status).toBe(0); + }, 240_000); +}); + // --- Review Army: Red Team (periodic) --- describeIfSelected('Review Army: Red Team', ['review-army-red-team'], () => { diff --git a/test/state-matrix.test.ts b/test/state-matrix.test.ts index 59cab815a..8b07a7c5f 100644 --- a/test/state-matrix.test.ts +++ b/test/state-matrix.test.ts @@ -8,8 +8,8 @@ * tests only for add, remove and a new-claim replace. `gstack-state-matrix * check` must pass it against the fixture's ground truth, and fail each * negative control derived from it. The paid behavior case that reruns the - * specialist live is `review-army-state-matrix`; it belongs in - * test/skill-e2e-review-army.test.ts and is not added here. + * specialist live is `review-army-state-matrix` in + * test/skill-e2e-review-army.test.ts. */ import { afterAll, describe, expect, test } from 'bun:test'; import { spawnSync } from 'node:child_process';