mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-05 00:03:15 +01:00
test(e2e): review-army-state-matrix behavior case on the #6792 fixture (field list 2, W3)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed
1 parent
9485c257e3
commit
463273b73c
3 files changed
+67
-11
No files matched your search
@@ -130,17 +130,18 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
],
|
||||
|
||||
// Review Army (specialist dispatch)
|
||||
'review-army-migration-safety': [ 'review/**', 'test/fixtures/review-army-migration.sql', 'scripts/resolvers/review-army.ts', 'bin/gstack-diff-scope', 'test/skill-e2e-review-army.test.ts', 'test/helpers/office-hours-attempt.ts'],
|
||||
'review-army-perf-n-plus-one': [ 'review/**', 'scripts/resolvers/review-army.ts', 'bin/gstack-diff-scope', 'test/skill-e2e-review-army.test.ts', 'test/fixtures/review-n-plus-one-dispatch.json', 'test/fixtures/review-army-n-plus-one.rb', 'test/helpers/office-hours-attempt.ts'],
|
||||
'review-army-delivery-audit': [ 'review/**', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'scripts/resolvers/review-army.ts', 'test/skill-e2e-review-army.test.ts', 'test/helpers/office-hours-attempt.ts'],
|
||||
'review-army-quality-score': [ 'review/**', 'scripts/resolvers/review-army.ts', 'test/skill-e2e-review-army.test.ts', 'test/helpers/office-hours-attempt.ts'],
|
||||
'review-army-json-findings': [ 'review/**', 'scripts/resolvers/review-army.ts', 'test/skill-e2e-review-army.test.ts', 'test/helpers/office-hours-attempt.ts'],
|
||||
'review-army-red-team': [ 'review/**', 'scripts/resolvers/review-army.ts', 'test/skill-e2e-review-army.test.ts',
|
||||
'review-army-migration-safety': [ 'review/**', 'test/fixtures/review-army-migration.sql', 'scripts/resolvers/review-army.ts', 'bin/gstack-diff-scope', 'test/skill-e2e-review-army.test.ts', 'test/helpers/state-matrix-fixture.ts', 'test/fixtures/state-matrix-6792/**', 'test/helpers/office-hours-attempt.ts'],
|
||||
'review-army-perf-n-plus-one': [ 'review/**', 'scripts/resolvers/review-army.ts', 'bin/gstack-diff-scope', 'test/skill-e2e-review-army.test.ts', 'test/helpers/state-matrix-fixture.ts', 'test/fixtures/state-matrix-6792/**', 'test/fixtures/review-n-plus-one-dispatch.json', 'test/fixtures/review-army-n-plus-one.rb', 'test/helpers/office-hours-attempt.ts'],
|
||||
'review-army-delivery-audit': [ 'review/**', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'scripts/resolvers/review-army.ts', 'test/skill-e2e-review-army.test.ts', 'test/helpers/state-matrix-fixture.ts', 'test/fixtures/state-matrix-6792/**', 'test/helpers/office-hours-attempt.ts'],
|
||||
'review-army-quality-score': [ 'review/**', 'scripts/resolvers/review-army.ts', 'test/skill-e2e-review-army.test.ts', 'test/helpers/state-matrix-fixture.ts', 'test/fixtures/state-matrix-6792/**', 'test/helpers/office-hours-attempt.ts'],
|
||||
'review-army-json-findings': [ 'review/**', 'scripts/resolvers/review-army.ts', 'test/skill-e2e-review-army.test.ts', 'test/helpers/state-matrix-fixture.ts', 'test/fixtures/state-matrix-6792/**', 'test/helpers/office-hours-attempt.ts'],
|
||||
'review-army-red-team': [ 'review/**', 'scripts/resolvers/review-army.ts', 'test/skill-e2e-review-army.test.ts', 'test/helpers/state-matrix-fixture.ts', 'test/fixtures/state-matrix-6792/**',
|
||||
'test/helpers/office-hours-attempt.ts'
|
||||
],
|
||||
'review-army-simplification': [ 'review/**', 'scripts/resolvers/review-army.ts', 'test/fixtures/review-army-overbuild.js', 'test/fixtures/review-army-lean-complete.js', 'test/skill-e2e-review-army.test.ts', 'test/helpers/office-hours-attempt.ts'],
|
||||
'review-army-simplification-precision': [ 'review/**', 'scripts/resolvers/review-army.ts', 'test/fixtures/review-army-overbuild.js', 'test/fixtures/review-army-lean-complete.js', 'test/skill-e2e-review-army.test.ts', 'test/helpers/office-hours-attempt.ts'],
|
||||
'review-army-consensus': [ 'review/**', 'scripts/resolvers/review-army.ts', 'test/skill-e2e-review-army.test.ts',
|
||||
'review-army-simplification': [ 'review/**', 'scripts/resolvers/review-army.ts', 'test/fixtures/review-army-overbuild.js', 'test/fixtures/review-army-lean-complete.js', 'test/skill-e2e-review-army.test.ts', 'test/helpers/state-matrix-fixture.ts', 'test/fixtures/state-matrix-6792/**', 'test/helpers/office-hours-attempt.ts'],
|
||||
'review-army-simplification-precision': [ 'review/**', 'scripts/resolvers/review-army.ts', 'test/fixtures/review-army-overbuild.js', 'test/fixtures/review-army-lean-complete.js', 'test/skill-e2e-review-army.test.ts', 'test/helpers/state-matrix-fixture.ts', 'test/fixtures/state-matrix-6792/**', 'test/helpers/office-hours-attempt.ts'],
|
||||
'review-army-state-matrix': [ 'review/**', 'scripts/resolvers/review-army.ts', 'lib/state-matrix.ts', 'bin/gstack-state-matrix', 'test/fixtures/state-matrix-6792/**', 'test/helpers/state-matrix-fixture.ts', 'test/skill-e2e-review-army.test.ts', 'test/helpers/office-hours-attempt.ts'],
|
||||
'review-army-consensus': [ 'review/**', 'scripts/resolvers/review-army.ts', 'test/skill-e2e-review-army.test.ts', 'test/helpers/state-matrix-fixture.ts', 'test/fixtures/state-matrix-6792/**',
|
||||
'test/helpers/office-hours-attempt.ts'],
|
||||
|
||||
// Office Hours
|
||||
@@ -1212,6 +1213,7 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic' | 'marathon'> = {
|
||||
'review-army-consensus': 'periodic', // Multi-specialist agreement
|
||||
'review-army-simplification': 'periodic', // Advisory lens quality benchmark
|
||||
'review-army-simplification-precision': 'periodic', // False-flag noise benchmark
|
||||
'review-army-state-matrix': 'periodic', // State-machine round-trip matrix (field list 2, W3)
|
||||
|
||||
// Office Hours
|
||||
// Brain-writeback E2E — periodic per cost (claude -p) + non-deterministic
|
||||
@@ -1702,6 +1704,7 @@ export const E2E_KINDS: Record<string, 'rule' | 'behavior' | 'judge'> = {
|
||||
'review-army-consensus': 'behavior',
|
||||
'review-army-simplification': 'behavior',
|
||||
'review-army-simplification-precision': 'behavior',
|
||||
'review-army-state-matrix': 'behavior',
|
||||
'office-hours-brain-writeback': 'behavior',
|
||||
'gbrain-roundtrip-local': 'rule',
|
||||
'sync-gbrain-read-ready': 'rule',
|
||||
@@ -1932,6 +1935,8 @@ export const BEHAVIOR_WHY: Record<string, string> = {
|
||||
"Flagging the planted unnecessary structure is an advisory-lens quality judgment.",
|
||||
'review-army-simplification-precision':
|
||||
"Staying silent on a lean diff is a false-flag noise benchmark; an occasional advisory is acceptable noise.",
|
||||
'review-army-state-matrix':
|
||||
"How the live specialist words each finding and which line it cites varies run to run; a complete kind x owner action matrix with every untested cell reported, checked by gstack-state-matrix against the fixture's ground truth, stays the contract.",
|
||||
'office-hours-forcing-energy':
|
||||
"The Q3 posture is scored by a live judge on generated prose; a single flat phrasing is tolerable.",
|
||||
'office-hours-builder-wildness':
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
logCost, recordE2E, createEvalCollector, finalizeEvalCollector,
|
||||
} from './helpers/e2e-helpers';
|
||||
import { extractSkillSections, REVIEW_ARMY_E2E_SECTIONS, sliceBetween } from './helpers/skill-fixture';
|
||||
import { makeStateMatrixRepo, STATE_MACHINES_CHECKLIST, STATE_MATRIX_FIXTURE, type StateMatrixRepo } from './helpers/state-matrix-fixture';
|
||||
import { spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
@@ -584,6 +585,56 @@ Write ONLY JSON findings (no preamble) to ${dir}/findings.json`,
|
||||
}, JUDGE_MS);
|
||||
});
|
||||
|
||||
// --- Review Army: State-machine round-trip matrix (field list 2, W3) ---
|
||||
// Contract: on the #6792-shaped diff (two new proposal kinds that fall through
|
||||
// to the remove path, tests only for add, remove and a new-claim replace), the
|
||||
// live State Machines specialist, given Step 4.5's dispatch instructions
|
||||
// extracted from review-army.md, returns a matrix that gstack-state-matrix
|
||||
// check passes against the fixture's ground truth: every kind x owner action
|
||||
// cell, each untested cell reported as a finding. The free regression on a
|
||||
// captured run is test/state-matrix.test.ts.
|
||||
|
||||
describeIfSelected('Review Army: State-machine matrix', ['review-army-state-matrix'], () => {
|
||||
let repo: StateMatrixRepo;
|
||||
|
||||
beforeAll(() => {
|
||||
repo = makeStateMatrixRepo('skill-e2e-army-state-matrix-');
|
||||
const specDir = path.join(repo.work, 'review-specialists');
|
||||
fs.mkdirSync(specDir, { recursive: true });
|
||||
fs.copyFileSync(STATE_MACHINES_CHECKLIST, path.join(specDir, 'state-machines.md'));
|
||||
});
|
||||
|
||||
afterAll(() => { try { fs.rmSync(repo.root, { recursive: true, force: true }); } catch {} });
|
||||
|
||||
testConcurrentIfSelected('review-army-state-matrix', async () => {
|
||||
const dispatch = sliceBetween(readReviewSection('review-army.md'), '"You are a specialist code reviewer.', '**Subagent configuration:**')
|
||||
.trim().replace(/^"|"$/g, '').replace(/\\"/g, '"')
|
||||
.replace('{checklist path}', 'review-specialists/state-machines.md').replace('origin/<base>', 'origin/main')
|
||||
.replace('{TEST_FW}', 'unknown').replace('{STACK}', 'node').replace("{learnings or 'none'}", 'none');
|
||||
const out = path.join(repo.work, 'review-output.jsonl');
|
||||
const result = await runSkillTest({
|
||||
prompt: `You are the State Machines specialist that /review's Step 4.5 dispatched for branch trust-guard (gstack-state-matrix detect printed SCOPE_STATE_MACHINE=true). These are your dispatch instructions:
|
||||
|
||||
${dispatch}
|
||||
|
||||
Write your raw output, exactly as those instructions define it, to ${out} and nothing else. After saving it, finish with a brief acknowledgement.`,
|
||||
workingDirectory: repo.work,
|
||||
maxTurns: 20,
|
||||
timeout: 180_000,
|
||||
testName: 'review-army-state-matrix',
|
||||
runId,
|
||||
});
|
||||
|
||||
logCost('/review army state matrix', result);
|
||||
recordE2E(evalCollector, '/review army state-machine matrix', 'Review Army', result);
|
||||
expect(result.exitReason).toBe('success');
|
||||
expect(fs.existsSync(out)).toBe(true);
|
||||
const check = spawnSync(path.join(ROOT, 'bin', 'gstack-state-matrix'), ['check', out, '--repo', repo.work, '--expect', path.join(STATE_MATRIX_FIXTURE, 'expect.json')], { cwd: repo.work, encoding: 'utf-8', timeout: 60_000 });
|
||||
expect(check.stdout, check.stdout + check.stderr).toMatch(/^STATE_MATRIX_CHECK: ok/m);
|
||||
expect(check.status).toBe(0);
|
||||
}, 240_000);
|
||||
});
|
||||
|
||||
// --- Review Army: Red Team (periodic) ---
|
||||
|
||||
describeIfSelected('Review Army: Red Team', ['review-army-red-team'], () => {
|
||||
|
||||
@@ -8,8 +8,8 @@
|
||||
* tests only for add, remove and a new-claim replace. `gstack-state-matrix
|
||||
* check` must pass it against the fixture's ground truth, and fail each
|
||||
* negative control derived from it. The paid behavior case that reruns the
|
||||
* specialist live is `review-army-state-matrix`; it belongs in
|
||||
* test/skill-e2e-review-army.test.ts and is not added here.
|
||||
* specialist live is `review-army-state-matrix` in
|
||||
* test/skill-e2e-review-army.test.ts.
|
||||
*/
|
||||
import { afterAll, describe, expect, test } from 'bun:test';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
|
||||
Reference in new issue
Block a user