From 7824a20316666b2e69ebade50157087ea6946630 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 31 Oct 2026 15:54:54 +0000 Subject: [PATCH] fix(review): drop the standalone Red Team on a change's first full pass (GSTA-167 PR A, no-loss replay V7) Round 7 arm V7 (handoffs/GSTA-167/noloss7-results.md): 32 of 41 blockers vs 33, p_NI 0.00074. Repins lib/review-depth.ts under replay V7; DEPTH_RULES moves. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- changelog.d/gsta-167-first-pass-red-team.md | 16 +++ docs/review-policy.md | 35 ++++++- lib/review-depth.ts | 16 ++- test/review-depth.test.ts | 105 +++++++++++++++++--- test/review-dispatch.test.ts | 24 +++++ test/review-packets-removed.test.ts | 3 +- test/review-pass.test.ts | 38 +++++-- test/review-targeted.test.ts | 6 +- 8 files changed, 215 insertions(+), 28 deletions(-) create mode 100644 changelog.d/gsta-167-first-pass-red-team.md diff --git a/changelog.d/gsta-167-first-pass-red-team.md b/changelog.d/gsta-167-first-pass-red-team.md new file mode 100644 index 000000000..f47b80914 --- /dev/null +++ b/changelog.d/gsta-167-first-pass-red-team.md @@ -0,0 +1,16 @@ +**`/ship` no longer runs the standalone Red Team on a change's first full review pass at gate class or past 200 runtime lines.** The preregistered no-loss replay (GSTA-167 Round 7, handoffs/GSTA-167/noloss7-results.md) removed those Red Team lines from the recorded reviewer outputs. The rest of the stack still found 32 of the 41 panel blockers against 33 with them, a one-sided non-inferiority p of 0.00074 at a margin of 4. The Red Team still runs on a first pass with a CRITICAL, on a small diff raised to standard, on every post-fix full pass and on the scoped pass. + +### Changed +- **`lib/review-depth.ts`**: on a first full pass (`fullPasses` 0), the gate-class term (a depth error included, since it runs as gate class) and the more-than-`RED_TEAM_LINES` term no longer add the Red Team. The plan prints `SKIP: red-team (first full pass; Round 7 no-loss)`. The CRITICAL and raised small-diff terms are unchanged, and post-fix full passes keep every term. The scoped pass, the fix-check and light depth's batched red-team checklist are unchanged. +- **`RED_TEAM_FIRST_PASS`** (false) joins the `DEPTH_RULES` hash, so the hash moves. + +### Docs +- `docs/review-policy.md`: the first-pass rule and its evidence, the depth table, the light-versus-standard inversion this leaves (light keeps the red-team checklist in its batched reader; a first standard or gate pass has no Red Team) and the follow-up arm light needs. + +### Tests +- `test/review-depth.test.ts`: separate histories for a first gate pass, a first pass reached through a depth error and a first pass past 200 lines (no Red Team), a first pass with a CRITICAL and a raised small diff (Red Team), post-fix full passes, the scoped pass and the fix-check, light's checklist, and the `DEPTH_RULES` move. +- `test/review-pass.test.ts`: the pass program creates no Red Team on a first full pass, and a failed Red Team on a post-fix pass stops it. `test/review-dispatch.test.ts`: a trimmed Red Team brief is `BRIEF_MISMATCH` and the exact re-send voids it. + +### Gate integrity +- Behavior change: the `DEPTH_RULES` hash moves. `test/review-packets-removed.test.ts` repins `lib/review-depth.ts` under the new `V7` replay. +- Changed assertions that expected a first-pass Red Team now assert none there and assert it after a fix: `test/review-depth.test.ts` (gate class and hit-rate gating, efforts, `--all-specialists`, GSTA-131 coverage, the six fail-closed matrix cases, the captured gate fixture, the standard plan); `test/review-pass.test.ts` (four pass-program tests move to a post-fix pass); `test/review-targeted.test.ts` (the dispatch-superset check allows exactly the first-pass Red Team to drop on its four first-pass full-depth fixtures). diff --git a/docs/review-policy.md b/docs/review-policy.md index 301d187e5..c038cc331 100644 --- a/docs/review-policy.md +++ b/docs/review-policy.md @@ -188,8 +188,8 @@ in a data file still schedules the outside voice. |---|---|---| | **light**, runtime 0 | docs and tests only, or only registered outputs | the core checklist only; the outside voice when the diff publishes numbers | | **light**, runtime 1–50 | `SMALL_DIFF_LINES` or fewer runtime lines | core, one **batched** specialist (the batched checklists below plus the Red Team prompt in one context), the native adversarial pass (kept: backtest inconclusive, GSTA-147 D5) and the outside voice on numbers | -| **standard** | more than 50 runtime lines | one batched specialist at `medium` and the Red Team at 200 runtime lines (or on a CRITICAL); the batched reader leaves docs and registered outputs out of its diff; adversarial and outside voice read everything | -| **full** | gate class, or any classifier error | one batched specialist on the whole diff, the Red Team always, no hit-rate gating | +| **standard** | more than 50 runtime lines | one batched specialist at `medium`; the Red Team on a CRITICAL or a raised small diff, and past 200 runtime lines only after a fix ([first full pass](#first-pass-red-team)); the batched reader leaves docs and registered outputs out of its diff; adversarial and outside voice read everything | +| **full** | gate class, or any classifier error | one batched specialist on the whole diff, no hit-rate gating; the Red Team on every post-fix full pass and on a CRITICAL, never on the first full pass otherwise ([first full pass](#first-pass-red-team)) | **Batched checklists (the lean ensemble, GSTA-167 part 2).** One specialist subagent reads Testing and Security on every code or gate diff, plus @@ -207,6 +207,32 @@ and the Red Team runs, so a raise never dispatches less than light did. When the next step is a scoped pass, the raise applies to the scoped set only. A fix-check is the same at every depth. + +**The first full pass has no standalone Red Team (GSTA-167 Round 7).** When a +change's first full pass (`fullPasses` 0) is gate class or has more than +`RED_TEAM_LINES` (200) runtime lines, the plan skips the Red Team and says why: +`SKIP: red-team (first full pass; Round 7 no-loss)`. A full plan reached through a +classifier error (`DEPTH_ERROR`) runs as gate class, so the same rule covers it; +Round 7 unit 3540 (an unmatched `gstack-diff-scope` path) is that case. The Red Team +still runs on a first pass with a CRITICAL (`--critical-found`) and on a small diff +raised to standard, on every post-fix full pass, and on the scoped pass. A +fix-check never had one. `RED_TEAM_FIRST_PASS` (false) records the rule in +`DEPTH_RULES`. + +The evidence is the preregistered no-loss replay +(handoffs/GSTA-167/noloss7-results.md, main cae783e). With the standalone Red Team +removed on the first full pass, the recorded reviewer stack found 32 of the 41 +panel blockers against 33 with it. The one loss was 3527 B1. The one-sided +non-inferiority p was 0.00074 at a margin of 4. That replay covered first full +passes only, so it certifies only this trigger; the post-fix, scoped, CRITICAL +and raised triggers stay until a fix-regression panel covers them. + +This leaves an inversion between depths. Light depth still gives its one batched +specialist the red-team checklist, while a first standard or gate pass has no Red +Team at all. Round 7 did not test light: its one light unit (3545) had no +red-team lines in the baseline. A follow-up arm, light without the red-team +checklist in the batched reader, is needed before light changes. + **Fail closed.** Each of these makes the depth full and is listed in `errors[]` with the input it could not read, a sanitized cause and the repair: a `gstack-diff-scope` error, a gate scan that is unavailable, a base registry that cannot be read or is @@ -270,6 +296,11 @@ the specialists, audits and watchers at the host default too ### Behavior changes +- A change's first full pass at gate class (a depth error included) or past 200 + runtime lines no longer dispatches the standalone Red Team + ([first full pass](#first-pass-red-team), GSTA-167 Round 7). This is a gate + edit: the `DEPTH_RULES` hash moves, so receipts written before it carry the old + `rules` value. - Pass 2 after a blocking fix on a diff below full depth, with a fix range of at most 150 runtime lines, is a [scoped pass](#scoped-pass): a fix-check, the Red Team and the native adversarial pass on the range instead of the specialist army. diff --git a/lib/review-depth.ts b/lib/review-depth.ts index 2cdbd8fd9..8a11ed5d0 100644 --- a/lib/review-depth.ts +++ b/lib/review-depth.ts @@ -27,6 +27,12 @@ export const FULL_PASS_CAP = 2; export const SMALL_DIFF_LINES = 50; export const RED_TEAM_LINES = 200; export const LARGE_DIFF_LINES = 100; +/** + * Whether a change's first full pass dispatches the standalone Red Team for gate class (a depth error included) or more than + * RED_TEAM_LINES runtime lines. Off since the Round 7 no-loss replay (handoffs/GSTA-167/noloss7-results.md); a CRITICAL, a + * raised small diff, post-fix full passes and the scoped pass keep it. + */ +export const RED_TEAM_FIRST_PASS = false; /** The largest fix range, in runtime lines, a pass 2 after a blocking fix reviews as a scoped pass (GSTA-152 D6); a starting value. */ export const SCOPED_RANGE_LINES = 150; /** The checklists the batched specialist always reads on code and gate class, whatever the depth or scope (GSTA-167 part 2). */ @@ -66,7 +72,7 @@ const DATA_CONFIG = [ /** A hash of the constants and patterns that decide a depth, not of this file's bytes: a comment edit keeps it, a threshold edit changes it. */ export const DEPTH_RULES = createHash('sha256').update(JSON.stringify({ - SMALL_DIFF_LINES, RED_TEAM_LINES, LARGE_DIFF_LINES, FULL_PASS_CAP, LIGHT_FIXED, BATCH_SCOPED, SCOPED_RANGE_LINES, ROLE_EFFORT, + SMALL_DIFF_LINES, RED_TEAM_LINES, RED_TEAM_FIRST_PASS, LARGE_DIFF_LINES, FULL_PASS_CAP, LIGHT_FIXED, BATCH_SCOPED, SCOPED_RANGE_LINES, ROLE_EFFORT, patterns: Object.entries({ GATE_PATH, AGENT_PROSE, TEST_PATH, RELEASE_METADATA, DOC_PATH: [DOC_PATH], DATA_PATH: [DATA_PATH], DATA_CONFIG }).map(([k, v]) => [k, v.map(r => `/${r.source}/${r.flags}`)]), })).digest('hex').slice(0, 16); @@ -405,10 +411,12 @@ export function planReview(i: PlanInput): ReviewPlan { const lines = full ? i.diffLines : Math.max(runtimeLines, raised ? SMALL_DIFF_LINES : 0); const checklists = batchChecklists(dcls, i, skipped); if (checklists.length) add('specialist', 'batched', specialist, full ? [] : docsAndOutputs, { checklists }); - const redTeam = dcls === 'gate' || (dcls === 'code' && (lines > RED_TEAM_LINES || !!i.criticalFound || raised)); - if (redTeam) add('red-team', 'red-team', 'full'); - else skipped.push(`red-team (${dcls} class${dcls === 'code' ? `, ${lines} runtime lines, no critical` : ''})`); const codeLike = dcls === 'code' || dcls === 'gate'; + const sized = dcls === 'gate' || (dcls === 'code' && lines > RED_TEAM_LINES); + const firstPassCut = sized && h.fullPasses === 0 && !RED_TEAM_FIRST_PASS; + if (codeLike && (!!i.criticalFound || raised || (sized && !firstPassCut))) add('red-team', 'red-team', 'full'); + else if (firstPassCut) skipped.push('red-team (first full pass; Round 7 no-loss)'); + else skipped.push(`red-team (${dcls} class${dcls === 'code' ? `, ${lines} runtime lines, no critical` : ''})`); if (h.fullPasses === 0 || (!i.adversarialRan && codeLike)) { if (codeLike) add('adversarial-native', 'adversarial', 'full'); else skipped.push(`adversarial-native (${dcls} class)`); diff --git a/test/review-depth.test.ts b/test/review-depth.test.ts index 801353112..34b989bf3 100644 --- a/test/review-depth.test.ts +++ b/test/review-depth.test.ts @@ -89,9 +89,10 @@ describe('planReview', () => { expect(batch(p)).toEqual(['testing', 'security']); expect(p.skipped).toContain('maintainability, performance, simplification (lean ensemble: read only when forced)'); }); - test('gate class forces red team and ignores hit-rate gating', () => { - const p = planReview(base({ gateTagged: 1, gated: ['security', 'state-machines'], scope: { SCOPE_BACKEND: 'true' } })); + test('gate class forces red team after a fix (not on the first full pass) and ignores hit-rate gating', () => { + const p = planReview(base({ gateTagged: 1, gated: ['security', 'state-machines'], scope: { SCOPE_BACKEND: 'true' }, history: { fullPasses: 1, last: 'full', fixedBlocking: true } })); expect(roles(p)).toContain('red-team:red-team:full'); + expect(roles(planReview(base({ gateTagged: 1, gated: ['security', 'state-machines'], scope: { SCOPE_BACKEND: 'true' } })))).not.toContain('red-team:red-team:full'); expect(batch(p)).toEqual(['testing', 'security', 'state-machines']); }); test('code class honours hit-rate gating and small-diff skip', () => { @@ -130,9 +131,10 @@ describe('planReview', () => { expect(d.filter(x => ['red-team', 'adversarial-native', 'fix-check'].includes(x.role)).every(x => x.effort === 'full')).toBe(true); expect(d.filter(x => x.role === 'outside-voice').every(x => x.effort === 'host')).toBe(true); } - const gate = planReview(base({ gateTagged: 1 })).dispatches; + const gate = planReview(base({ gateTagged: 1, history: { fullPasses: 1, last: 'full', fixedBlocking: true } })).dispatches; expect(gate.map(x => x.role)).toEqual(expect.arrayContaining(['specialist', 'red-team', 'adversarial-native'])); - const host = planReview(base({ gateTagged: 1, tiers: false })).dispatches; + const host = planReview(base({ gateTagged: 1, tiers: false, history: { fullPasses: 1, last: 'full', fixedBlocking: true } })).dispatches; + expect(host.map(x => x.role)).toContain('red-team'); expect(host.filter(x => x.role === 'specialist').every(x => x.effort === 'host')).toBe(true); expect(host.filter(x => x.role === 'red-team' || x.role === 'adversarial-native').every(x => x.effort === 'full')).toBe(true); }); @@ -161,24 +163,28 @@ describe('GSTA-131 pass-1 review repairs', () => { expect(scanNumbers({ 'test/a.test.ts': ['const n = 3;', 'for (let i = 0; i < 10; i++)', 'expect(x).toBe(0.5)'] }).publishes).toBe(false); expect(scanNumbers({ 'test/fixtures/bench.json': ['{ "ratio": 0.74 }'] }).publishes).toBe(true); }); - test('--all-specialists forces every specialist; a code diff past 200 lines or with a CRITICAL adds the Red Team', () => { + test('--all-specialists forces every specialist; a CRITICAL, or a code diff past 200 lines after a fix, adds the Red Team', () => { const all = planReview(base({ force: 'all' })); expect(all.dispatches.filter(d => d.role === 'specialist').length).toBe(1); expect([...batch(all)!].sort()).toEqual(['api-contract', 'data-migration', 'design', 'maintainability', 'performance', 'security', 'simplification', 'state-machines', 'testing']); const has = (o: Partial) => roles(planReview(base(o))).some(r => r.startsWith('red-team')); expect(has({ diffLines: 120 })).toBe(false); - expect(has({ diffLines: 201 })).toBe(true); + expect(has({ diffLines: 201 })).toBe(false); + expect(has({ diffLines: 201, history: { fullPasses: 1, last: 'full', fixedBlocking: true, rangeRuntimeLines: 201 } })).toBe(true); expect(has({ diffLines: 120, criticalFound: true })).toBe(true); }); }); describe('GSTA-131 coverage: Red Team and specialist arms', () => { const roles = (p: ReturnType) => p.dispatches.map(d => `${d.role}:${d.name}`); - test('Red Team runs on code past 200 lines or after a CRITICAL, and on every gate diff', () => { + test('Red Team runs after a CRITICAL, and after a fix on code past 200 lines and on every gate diff', () => { + const fixed = { fullPasses: 1, last: 'full', fixedBlocking: true, rangeRuntimeLines: 201 } as const; expect(roles(planReview(base({ diffLines: 120 })))).not.toContain('red-team:red-team'); - expect(roles(planReview(base({ diffLines: 201 })))).toContain('red-team:red-team'); + expect(roles(planReview(base({ diffLines: 201 })))).not.toContain('red-team:red-team'); + expect(roles(planReview(base({ diffLines: 201, history: fixed })))).toContain('red-team:red-team'); expect(roles(planReview(base({ diffLines: 120, criticalFound: true })))).toContain('red-team:red-team'); - expect(roles(planReview(base({ diffLines: 20, gateTagged: 1 })))).toContain('red-team:red-team'); + expect(roles(planReview(base({ diffLines: 20, gateTagged: 1 })))).not.toContain('red-team:red-team'); + expect(roles(planReview(base({ diffLines: 20, gateTagged: 1, history: fixed })))).toContain('red-team:red-team'); }); test('scope arms select their specialist, state machines default on when scope is unknown, and --force all reaches docs class', () => { const picked = (scope: Record) => batch(planReview(base({ scope: { SCOPE_STATE_MACHINE: 'false', ...scope } }))); @@ -236,7 +242,8 @@ describe('review depth: the matrix', () => { expect(p.depth).toBe('full'); expect(p.errors.map(e => e.code)).toContain(code); expect(p.codes).toEqual([...codes]); - expect(roles(p)).toContain('red-team:red-team:full'); + expect(roles(p)).not.toContain('red-team:red-team:full'); + expect(roles(planReview(mixed({ perPath: per({ 'src/app.ts': 5 }), ...(o as Partial), history: { fullPasses: 1, last: 'full', fixedBlocking: true } })))).toContain('red-team:red-team:full'); }); test.each([ ['a gate path', { paths: ['src/app.ts', '.github/workflows/ci.yml'] }], @@ -478,10 +485,11 @@ describe('lean ensemble: one batched specialist at standard and full depth (GSTA // Value: protects=the lean ensemble's reader set (testing and security always, the four scoped checklists when scope asks, // maintainability, performance and simplification only when forced) on captured diffs; fails_when=a plan dispatches more than // one specialist reader at standard or full, drops testing or security, or the Red Team or voices change; seam=planReview - test('the captured gate fixture plans one batched reader, the Red Team and both voices', () => { + test('the captured gate fixture plans one batched reader and both voices, and the Red Team after a fix', () => { const p = planReview(fixture('code')); expect([p.class, p.depth]).toEqual(['gate', 'full']); - expect(roles(p)).toEqual(['specialist:batched:medium', 'red-team:red-team:full', 'adversarial-native:adversarial:full', 'outside-voice:outside-voice:host']); + expect(roles(p)).toEqual(['specialist:batched:medium', 'adversarial-native:adversarial:full', 'outside-voice:outside-voice:host']); + expect(roles(planReview({ ...fixture('code'), history: { fullPasses: 1, last: 'full', fixedBlocking: true } }))).toEqual(['specialist:batched:medium', 'red-team:red-team:full', 'adversarial-native:adversarial:full', 'outside-voice:outside-voice:host']); expect(batch(p)).toEqual(['testing', 'security', 'state-machines']); expect(batch(planReview({ ...fixture('code'), scope: { ...fixture('code').scope, SCOPE_MIGRATIONS: 'true', SCOPE_API: 'true', SCOPE_FRONTEND: 'true', SCOPE_STATE_MACHINE: 'false' } }))).toEqual(['testing', 'security', 'data-migration', 'api-contract', 'design']); }); @@ -493,9 +501,80 @@ describe('lean ensemble: one batched specialist at standard and full depth (GSTA test('a standard code plan reads one batched reader on its own hunks, and a forced checklist joins it', () => { const p = planReview(base({ diffLines: 400, scope: { SCOPE_BACKEND: 'true', SCOPE_STATE_MACHINE: 'false' } })); expect(specialists(p).map(d => [d.name, d.checklists])).toEqual([['batched', ['testing', 'security']]]); - expect(roles(p)).toContain('red-team:red-team:full'); + expect(roles(p)).not.toContain('red-team:red-team:full'); + expect(roles(planReview(base({ diffLines: 400, scope: { SCOPE_BACKEND: 'true', SCOPE_STATE_MACHINE: 'false' }, history: { fullPasses: 1, last: 'full', fixedBlocking: true, rangeRuntimeLines: 400 } })))).toContain('red-team:red-team:full'); expect(batch(planReview(base({ force: ['performance', 'simplification'] })))).toEqual(['testing', 'security', 'performance', 'simplification']); expect(specialists(planReview(base({ tiers: false })))[0]!.effort).toBe('host'); }); }); + +describe('the first full pass drops the standalone Red Team (GSTA-167 Round 7 no-loss)', () => { + const rt = (p: ReturnType) => p.dispatches.some(d => d.role === 'red-team'); + const NOTE = 'red-team (first full pass; Round 7 no-loss)'; + const postFix = { fullPasses: 1, last: 'full', fixedBlocking: true, rangeRuntimeLines: 300 } as const; + // Value: protects=a change's first full pass at gate class (or full from a depth error) or past RED_TEAM_LINES dispatches no + // standalone Red Team and says why in SKIP; fails_when=the Round 7 cut is missing, too broad or unexplained; + // why_new=V7 passed (handoffs/GSTA-167/noloss7-results.md: 32 of 41 against 33, p_NI 0.00074); seam=planReview + test('a first full pass on gate class has no Red Team', () => { + const p = planReview(base({ diffLines: 20, gateTagged: 1 })); + expect([p.class, p.depth, p.next.step]).toEqual(['gate', 'full', 'full']); + expect(rt(p)).toBe(false); + expect(p.skipped).toContain(NOTE); + expect(roles(p)).toEqual(['specialist:batched:medium', 'adversarial-native:adversarial:full', 'outside-voice:outside-voice:host']); + }); + test('a first full pass reached through a depth error (gate class by fail-closed) has no Red Team', () => { + const p = planReview(base({ diffLines: 20, scope: { SCOPE_ERROR: 'unmatched path' } })); + expect([p.depth, p.errors.map(e => e.code)]).toEqual(['full', ['scope_error']]); + expect(rt(p)).toBe(false); + expect(p.skipped).toContain(NOTE); + }); + test('a first full pass on code past 200 runtime lines has no Red Team', () => { + const p = planReview(base({ diffLines: 201 })); + expect([p.class, p.depth, p.runtimeLines]).toEqual(['code', 'standard', 201]); + expect(rt(p)).toBe(false); + expect(p.skipped).toContain(NOTE); + }); + test('a first full pass with a CRITICAL keeps the Red Team, on code and gate class', () => { + for (const o of [{ diffLines: 120 }, { diffLines: 201 }, { diffLines: 20, gateTagged: 1 }]) expect(rt(planReview(base({ ...o, criticalFound: true })))).toBe(true); + }); + test('a pass-1 small diff raised to standard keeps the Red Team', () => { + const floored = planReview(base({ diffLines: 20, floor: 'standard' })); + expect([floored.depth, floored.runtimeLines, rt(floored)]).toEqual(['standard', 20, true]); + const critical = planReview(base({ diffLines: 20, criticalFound: true })); + expect([critical.depth, critical.stepDepth, rt(critical)]).toEqual(['light', 'standard', true]); + }); + test('a post-fix full pass keeps the Red Team on gate class, a depth error and code past 200 lines', () => { + for (const o of [{ diffLines: 20, gateTagged: 1 }, { diffLines: 20, scope: { SCOPE_ERROR: 'unmatched path' } }, { diffLines: 201 }]) { + const p = planReview(base({ ...o, history: postFix, adversarialRan: true })); + expect([p.next.step, rt(p)]).toEqual(['full', true]); + } + const touched = planReview(base({ diffLines: 20, gateTagged: 1, history: { fullPasses: 1, last: 'full', fixedBelow: true, touchedGate: true } })); + expect([touched.next.step, rt(touched)]).toEqual(['full', true]); + }); + test('a scoped pass keeps the Red Team; a fix-check never had one', () => { + const scoped = planReview(base({ diffLines: 201, history: { ...postFix, rangeRuntimeLines: 24, pass1Depth: 'standard' }, adversarialRan: true })); + expect([scoped.next.step, rt(scoped)]).toEqual(['scoped', true]); + const check = planReview(base({ diffLines: 201, history: { fullPasses: 1, last: 'full', fixedBelow: true } })); + expect([check.next.step, rt(check)]).toEqual(['fix-check', false]); + }); + test('light depth keeps the red-team checklist in the batched reader on a first pass', () => { + const p = planReview(base({ diffLines: 20 })); + expect(batch(p)).toContain('red-team'); + expect(p.skipped).not.toContain(NOTE); + }); + test('DEPTH_RULES changes when the first-pass Red Team rule does', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'depth-rules-red-team-')); + try { + const src = fs.readFileSync(path.join(ROOT, 'lib/review-depth.ts'), 'utf8'); + expect(src.includes('RED_TEAM_FIRST_PASS = false')).toBe(true); + fs.symlinkSync(path.join(ROOT, 'lib/ship-policy.ts'), path.join(dir, 'ship-policy.ts')); + fs.writeFileSync(path.join(dir, 'review-depth.ts'), src.replace('RED_TEAM_FIRST_PASS = false', 'RED_TEAM_FIRST_PASS = true')); + fs.writeFileSync(path.join(dir, 'print.ts'), "import { DEPTH_RULES } from './review-depth';\nconsole.log(DEPTH_RULES);\n"); + const res = spawnSync(process.execPath, ['--no-env-file', '--config=/dev/null', path.join(dir, 'print.ts')], { encoding: 'utf8', timeout: 30_000 }); + expect(res.status, res.stderr).toBe(0); + expect(res.stdout.trim()).not.toBe(DEPTH_RULES); + } finally { fs.rmSync(dir, { recursive: true, force: true }); } + }); +}); + diff --git a/test/review-dispatch.test.ts b/test/review-dispatch.test.ts index b76de4a38..75d52074b 100644 --- a/test/review-dispatch.test.ts +++ b/test/review-dispatch.test.ts @@ -112,6 +112,30 @@ describe('gstack-review-dispatch sent, collect, judge and verify: nothing change expect([v.status, v.stdout.includes('BRIEF specialist/testing MISMATCH'), v.stdout.includes('VERIFY: dirty')]).toEqual([1, true, true]); }); + // Value: protects=the Red Team, still dispatched on post-fix, scoped and CRITICAL passes after GSTA-167 Round 7, is held to the + // verbatim-brief rule: a trimmed send is BRIEF_MISMATCH and the exact re-send before any collect voids it; fails_when=a + // paraphrased Red Team brief counts as reviewed; why_new=PR A narrows when the Red Team runs; seam=render, sent and verify + test('a trimmed Red Team brief is BRIEF_MISMATCH; the exact re-send before collect voids it and verifies clean', () => { + const out = tmp(); + const ledger = path.join(out, 'review-ledger.jsonl'); + const findings = path.join(out, 'findings.txt'); + fs.writeFileSync(findings, '[CRITICAL] a.ts:3 off by one\n'); + const file = field(run(['render', '--role', 'red-team', '--out', out, '--diff-base', SHA, '--findings-file', findings, '--specialists', '1']).stdout, 'file'); + const trimmed = path.join(out, 'trimmed.txt'); + fs.writeFileSync(trimmed, fs.readFileSync(file, 'utf8').split('\n').slice(1).join('\n')); + const bad = run(['sent', '--ledger', ledger, '--role', 'red-team', '--sent', trimmed]); + expect([bad.status, bad.stdout.includes('BRIEF_MISMATCH: reader=red-team')]).toEqual([1, true]); + expect(run(['sent', '--ledger', ledger, '--role', 'red-team', '--sent', file]).status).toBe(0); + const merged = path.join(out, 'merged.jsonl'); + const o = path.join(out, 'red.out'); + fs.writeFileSync(o, 'NO FINDINGS\n'); + expect(run(['collect', '--ledger', ledger, '--role', 'red-team', '--output', o, '--merged', merged]).status).toBe(0); + fs.writeFileSync(path.join(out, 'judged.jsonl'), ''); + expect(run(['judge', '--ledger', ledger, '--merged', merged, '--judged', path.join(out, 'judged.jsonl')]).status).toBe(0); + const v = run(['verify', '--ledger', ledger]); + expect([v.status, v.stdout.includes('BRIEF red-team MATCH'), v.stdout.includes('VERIFY: clean briefs=1 outputs=1 voided=1')]).toEqual([0, true, true]); + }); + // Value: protects=verify checks each render as its own dispatch, so a two-pass ledger compares pass-1 sends with pass-1 briefs, and a mismatched send re-sent byte-exact before any collect is reported as voided, never hidden; a mismatched send that produced an output stays dirty; fails_when=a run-wide verify prints dirty for a clean two-pass run, a superseded mismatch blocks, voided is not printed, or an output collected under a mismatched send reads clean (GSTA-189 #36 replay: VERIFY dirty briefs=9 outputs=16, every output MATCH); // why_new=verify compared every send with the reader's last render; both passes render into one briefs directory and judge runs with relative paths, as a dispatch program does (pass-3 fix-check); seam=sent, collect and verify; no_claim=which pass a render belongs to beyond ledger order test('two passes verify per render; a superseded mismatch is voided=1; a mismatched send with an output stays dirty', () => { diff --git a/test/review-packets-removed.test.ts b/test/review-packets-removed.test.ts index ab7ff8240..24b78749a 100644 --- a/test/review-packets-removed.test.ts +++ b/test/review-packets-removed.test.ts @@ -56,6 +56,7 @@ const NO_LOSS_REPLAYS = { 'G1': "GSTA-167 variant G Arm G1: live no-loss replay of both prose voices with part 1's prompts, 2 passes, normalizer v2, paired rule against pooled C' (parts 1 and 3; specialists at medium as in C and C2)", 'L': "GSTA-167 variant G Arm L: live no-loss replay of one batched reader (testing and security plus scoped checklists) and the Red Team on its findings, 2 passes, scored L∘G1 against G1 on the paired rule (part 2)", 'GST-257': "GST-257 PR 1 dispatch-superset no-loss check (test/review-targeted.test.ts): on every fixture diff the branch dispatches main's reviewers at 8be7f21 unchanged plus only the targeted reader, and over the last 100 non-release commits on main it lost none; adds the targeted role and brief to the G1 and L bytes, whose existing reviewer text is unchanged", + 'V7': "GSTA-167 no-loss Round 7 arm V7 (handoffs/GSTA-167/noloss7-results.md): L∘G1 without the standalone Red Team on a change's first full pass found 32 of 41 panel blockers against R's 33, p_NI 0.00074 at delta 4; certifies only the first-full-pass gate-class and >200-line trigger in lib/review-depth.ts", } as const; type Replay = keyof typeof NO_LOSS_REPLAYS; const REVIEWER_BRIEFS: Record = { @@ -77,7 +78,7 @@ const REVIEWER_BRIEFS: Record = { 'review/specialists/testing.md': { sha: 'd6cc2bef7be562387d0c0a22fefe646dff9c295f1d34d8453a6cb496584150e4', replay: '#3638' }, 'review/specialists/targeted.md': { sha: '7f92a0d0a8319c3c0e076fb3b1a4689c45120b2be2c14dfc7d4cb03aafa9d6a5', replay: 'GST-257' }, 'scripts/resolvers/fix-check.ts': { sha: '712bf5b0c439e96bac32f19cbe25394dabb4c94aa66600100bfde9b100e15a08', replay: '#3638' }, - 'lib/review-depth.ts': { sha: '4542785a1e217d8dd054985978b42c055b9fd79082579cb7664f187f7066e48e', replay: 'GST-257' }, + 'lib/review-depth.ts': { sha: '807309b28c604951f0f2584d5d23f0fe54652fcdc87ad981094888c9df0a5c9a', replay: 'V7' }, }; describe('reviewer briefs stay at their no-loss-tested bytes (GSTA-167)', () => { diff --git a/test/review-pass.test.ts b/test/review-pass.test.ts index 9bfc99c89..a48643ee0 100644 --- a/test/review-pass.test.ts +++ b/test/review-pass.test.ts @@ -180,6 +180,8 @@ const AsyncFunction = Object.getPrototypeOf(async () => {}).constructor as new ( async function runPass(h: ReturnType, input: Record): Promise { return new AsyncFunction('tools', 'sleep', 'now', 'input', PROGRAM)(h.tools, h.sleep, h.now, { startedAt: await h.now(), laneModel: 'anthropic/claude-opus-5-5', ...input }); } +/** A post-fix full pass: the plan keeps the Red Team there; a first full pass has none (GSTA-167 Round 7). */ +const POST_FIX = ['--depth-floor', 'full', '--pass', '1', '--last', 'full', '--fixed-blocking']; const bin = (dir: string, env: NodeJS.ProcessEnv, a: string[]) => spawnSync(path.join(BIN, 'gstack-review-dispatch'), a, { cwd: dir, env, encoding: 'utf8', timeout: 60_000 }); describe('the targeted reader in a pass (GST-257 on main)', () => { @@ -225,12 +227,12 @@ describe('Capy pass program (fake host, real bins)', () => { expect(JSON.parse(body!)).toBe(PROGRAM); }); - test('one pass: specialists, then the Red Team on their collected lines; every prompt is the rendered brief; the ledger verifies clean', async () => { + test('one post-fix pass: specialists, then the Red Team on their collected lines; every prompt is the rendered brief; the ledger verifies clean', async () => { const { dir, env } = repo(); const out = path.join(dir, '.state/run/p1'); const h = host(dir, env, { output: t => (t.endsWith('specialist-batched') ? 'High: src/a.ts:3 the export is untested' : 'NO FINDINGS') }); const medium = { model_id: 'anthropic/claude-opus-5-5', reasoning_mode: 'medium' }; - const res = await runPass(h, { base: 'main', out, pass: 'x1p1', planFlags: ['--depth-floor', 'full'], models: { specialist: medium, 'red-team': medium } }); + const res = await runPass(h, { base: 'main', out, pass: 'x1p1', planFlags: POST_FIX, models: { specialist: medium, 'red-team': medium } }); expect(h.creates.every(c => JSON.stringify(c.model) === JSON.stringify(medium))).toBe(true); expect(res).toContain('NEXT: judge (Phase 4)'); const m = JSON.parse(fs.readFileSync(path.join(out, 'pass-manifest.json'), 'utf8')) as Manifest; @@ -247,17 +249,41 @@ describe('Capy pass program (fake host, real bins)', () => { expect(v.stdout).toContain('VERIFY: clean briefs=' + active.length); }, 180_000); + // Value: protects=the Round 7 cut reaches the pass program: a first full pass creates no Red Team reader and still verifies clean, + // while a post-fix pass whose Red Team fails stops instead of judging; fails_when=pass 1 creates red-team, or a failed Red Team + // on a kept path reaches judge; why_new=GSTA-167 PR A drops the first-pass Red Team; seam=plan through the pass program + test('a first full pass creates no Red Team reader; a failed Red Team on a post-fix pass stops the pass', async () => { + const { dir, env } = repo(); + const out = path.join(dir, '.state/run/p1'); + const h = host(dir, env); + const res = await runPass(h, { base: 'main', out, pass: 'x9p1', planFlags: ['--depth-floor', 'full'] }); + expect(res).toContain('SKIP: red-team (first full pass; Round 7 no-loss)'); + expect(res).toContain('NEXT: judge (Phase 4)'); + expect(h.creates.map(c => c.title)).toEqual(['ship x9p1 specialist-batched']); + const m = JSON.parse(fs.readFileSync(path.join(out, 'pass-manifest.json'), 'utf8')) as Manifest; + expect(m.readers.some(r => r.role === 'red-team')).toBe(false); + fs.writeFileSync(path.join(out, 'judged.jsonl'), ''); + expect(bin(dir, env, ['judge', '--ledger', m.ledger, '--merged', m.merged, '--judged', path.join(out, 'judged.jsonl')]).status).toBe(0); + expect(bin(dir, env, ['verify', '--ledger', m.ledger]).stdout).toContain('VERIFY: clean briefs=1'); + + const b = repo(); + const h2 = host(b.dir, b.env, { fail: t => t.endsWith('red-team') }); + const res2 = await runPass(h2, { base: 'main', out: path.join(b.dir, '.state/run/p1'), pass: 'x10p1', planFlags: POST_FIX }); + expect(res2).toMatch(/STOP: red-team \(subagent \d+\) failed/); + expect(res2).not.toContain('NEXT: judge'); + }, 240_000); + test('a pass past its 200 s window returns NEXT: poll and the rerun resumes without creating any reader twice', async () => { const { dir, env } = repo(); const out = path.join(dir, '.state/run/p1'); const h = host(dir, env, { pollsToFinish: 30 }); - const first = await runPass(h, { base: 'main', out, pass: 'x2p1', planFlags: ['--depth-floor', 'full'], logStart: ['review'] }); + const first = await runPass(h, { base: 'main', out, pass: 'x2p1', planFlags: POST_FIX, logStart: ['review'] }); expect(first).toContain('NEXT: poll'); expect(first).toMatch(/REVIEW_LOG_START: review token=\S+/); expect(first).toMatch(/RUNNING: specialist-batched state=created path=1/); const created = h.creates.length; let res = ''; - for (let i = 0; i < 6 && !res.includes('NEXT: judge'); i++) res = await runPass(h, { base: 'main', out, pass: 'x2p1', planFlags: ['--depth-floor', 'full'], logStart: ['review'] }); + for (let i = 0; i < 6 && !res.includes('NEXT: judge'); i++) res = await runPass(h, { base: 'main', out, pass: 'x2p1', planFlags: POST_FIX, logStart: ['review'] }); expect(res).toContain('NEXT: judge'); expect(res).not.toContain('REVIEW_PLAN:'); expect(res).not.toContain('REVIEW_LOG_START:'); @@ -285,7 +311,7 @@ describe('Capy pass program (fake host, real bins)', () => { test('a failed reader stops the pass before the Red Team; a reader refused at the cap is queued and started later', async () => { const { dir, env } = repo(); const h = host(dir, env, { fail: t => t.endsWith('specialist-batched') }); - const res = await runPass(h, { base: 'main', out: path.join(dir, '.state/run/p1'), pass: 'x5p1', planFlags: ['--depth-floor', 'full'] }); + const res = await runPass(h, { base: 'main', out: path.join(dir, '.state/run/p1'), pass: 'x5p1', planFlags: POST_FIX }); expect(res).toMatch(/STOP: specialist-batched \(subagent \d+\) failed/); expect(h.creates.some(c => c.title.endsWith('red-team'))).toBe(false); @@ -344,7 +370,7 @@ describe('Capy pass program (fake host, real bins)', () => { test('a rerun after a failed reader stops again instead of polling forever', async () => { const { dir, env } = repo(); const h = host(dir, env, { fail: t => t.endsWith('specialist-batched') }); - const input = { base: 'main', out: path.join(dir, '.state/run/p1'), pass: 'x12p1', planFlags: ['--depth-floor', 'full'] }; + const input = { base: 'main', out: path.join(dir, '.state/run/p1'), pass: 'x12p1', planFlags: POST_FIX }; expect(await runPass(h, input)).toMatch(/STOP: specialist-batched \(subagent \d+\) failed/); const again = await runPass(h, input); expect(again).toMatch(/STOP: specialist-batched \(subagent \d+\) failed; .*new --pass and --out/); diff --git a/test/review-targeted.test.ts b/test/review-targeted.test.ts index 3f0f69859..9f90b3dec 100644 --- a/test/review-targeted.test.ts +++ b/test/review-targeted.test.ts @@ -169,6 +169,8 @@ const MAIN: Record = 'money-pass2': { depth: 'light', classes: 'money', main: ['specialist batched effort=medium checklists=testing,security,state-machines', 'red-team red-team effort=full', 'adversarial-native adversarial effort=full', 'outside-voice outside-voice effort=host'] }, 'gate-repeat': { depth: 'full', classes: 'gate', main: ['specialist batched effort=medium checklists=testing,security,state-machines', 'red-team red-team effort=full'] }, }; +/** First full passes at full depth: GSTA-167 Round 7 (handoffs/GSTA-167/noloss7-results.md) drops main's standalone Red Team there and nothing else. */ +const ROUND7_FIRST_PASS = ['none-gate-path', 'secret-path', 'gate-tagged', 'auth-path']; function planRepo(f: Fixture) { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'targeted-plan-')); @@ -189,7 +191,7 @@ describe('gstack-review-plan prints TARGETED and dispatches the targeted reader' // standard and full depth and on fix-check, scoped and repeat steps, is still dispatched unchanged, and the only addition is the // targeted reader on a class diff; fails_when=the change drops, renames or re-efforts any reviewer, or adds the reader on a diff // with no class; seam=bin/gstack-review-plan end to end; no_claim=that the reader finds the defect (the paid replay's question) - test('the dispatch-superset no-loss check: on every fixture the branch dispatches main\'s reviewers plus only the targeted reader', () => { + test('the dispatch-superset no-loss check: on every fixture the branch dispatches main\'s reviewers plus only the targeted reader, less only the Round 7 first-pass Red Team', () => { expect(Object.keys(MAIN).sort()).toEqual(Object.keys(FIXTURES).sort()); expect(new Set(Object.values(MAIN).map(m => m.depth))).toEqual(new Set(['light', 'standard', 'full'])); for (const [name, f] of Object.entries(FIXTURES)) { @@ -199,7 +201,7 @@ describe('gstack-review-plan prints TARGETED and dispatches the targeted reader' const want = MAIN[name]!; expect(r.stdout, name).toContain(`REVIEW_DEPTH: ${want.depth} `); expect(got.filter(d => !want.main.includes(d)), name).toEqual(want.classes === 'none' ? [] : ['targeted targeted effort=full']); - expect(want.main.filter(d => !got.includes(d)), name).toEqual([]); + expect(want.main.filter(d => !got.includes(d)), name).toEqual(ROUND7_FIRST_PASS.includes(name) ? ['red-team red-team effort=full'] : []); expect(field(r.stdout, 'TARGETED:', 'classes'), name).toBe(want.classes); } }, 180_000);