Files
gstack/test/outside-review-severity.test.ts
T

115 lines
6.9 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/**
* CEO-28 / DX-11 / ENG-10: a review that lists high-severity findings is never
* `VERDICT: clean`. The Oct 7 /autoplan run's Codex CEO and DX outputs labeled
* seven and eight findings "High"/"Medium" without [Pn] tags, and the installed
* classifier printed `VERDICT: clean` / `FINDINGS: none` for both. The fixtures
* in test/fixtures/outside-review-severity/ reproduce those shapes (the
* findings come from the run's review record; the raw outputs were not kept),
* plus the eng output's [P1] tags, a table, and prose negative controls.
*/
import { describe, expect, test } from 'bun:test';
import { spawnSync } from 'node:child_process';
import * as fs from 'node:fs';
import * as path from 'node:path';
import { classifyOutsideReview, type OutsideGate } from '../lib/outside-review-result';
import { HOST_PATHS } from '../scripts/resolvers/types';
import { outsideVoiceCommand } from '../scripts/resolvers/outside-voice';
const ROOT = path.resolve(import.meta.dir, '..');
const LIB = path.join(ROOT, 'lib', 'outside-review-result.ts');
const FIX = path.join(ROOT, 'test', 'fixtures', 'outside-review-severity');
const fixture = (name: string) => path.join(FIX, name);
const read = (name: string) => fs.readFileSync(fixture(name), 'utf8');
const REC = 'Recommendation: fix it because the change loses data.';
describe('severity words count as findings in label position', () => {
for (const [name, highest] of [
['ceo-codex-shape.txt', 'P1'],
['dx-codex-shape.txt', 'P1'],
['eng-codex-tagged.txt', 'P1'],
['table-shape.txt', 'P1'],
] as const) {
test(`${name}: the CLI prints VERDICT: findings, FINDINGS: ${highest}, exit 3`, () => {
const r = spawnSync(process.execPath, [LIB, '--verdict', 'review', fixture(name)], { encoding: 'utf8', timeout: 10000 });
expect([r.status, r.stdout]).toEqual([3, `VERDICT: findings\nFINDINGS: ${highest}\n`]);
});
}
test('every label form ENG-10 names is read, with critical/high blocking and medium/low advisory', () => {
const cases: Array<[string, string | null, string]> = [
['Severity: High\nThe cap is ignored.', 'P1', 'findings'],
['Priority: critical — data loss', 'P0', 'findings'],
['Severity: **medium**', 'P2', 'clean'],
['High: the retry loop never stops.', 'P1', 'findings'],
['1. **High** — the cap is ignored.', 'P1', 'findings'],
['- [High] the cap is ignored.', 'P1', 'findings'],
['### Critical - secrets in logs', 'P0', 'findings'],
['High — the cap is ignored.', 'P1', 'findings'],
['The cap is ignored (**High**).', 'P1', 'findings'],
['| 1 | high | cap ignored |', 'P1', 'findings'],
['Low: rename the helper.', 'P3', 'clean'],
['Medium: the docs lag.\nLow: a typo.', 'P2', 'clean'],
['1. The guard trusts the payload without a flush check — High.', 'P1', 'findings'],
['2. The docs lag the new flag – Medium.', 'P2', 'clean'],
['- The retry loop never stops (high)', 'P1', 'findings'],
];
for (const [text, highest, verdict] of cases) {
const r = classifyOutsideReview({ text: `${text}\n${REC}`, gate: 'review' });
expect({ text, highest: r.findings.highest as string | null, verdict: r.verdict as string }).toEqual({ text, highest, verdict });
}
});
test('negative controls: severity words inside prose never count', () => {
const r = classifyOutsideReview({ text: read('prose-negative.txt'), gate: 'review' });
expect([r.verdict, r.findings.highest]).toEqual(['clean', null]);
for (const text of ['A high-level plan.', 'Highly consistent.', 'This is low-risk.', 'A medium-term follow-up.',
'There are no critical findings.', 'High availability: kept.', 'The high number of retries is fine.', 'Lower the limit.',
'Latency stays low.', 'The cost is high-ish - acceptable.', 'Keep the trade-off low.']) {
expect({ text, highest: classifyOutsideReview({ text: `${text}\nNo issues found.\n${REC}`, gate: 'review' }).findings.highest })
.toEqual({ text, highest: null });
}
});
test('a review with neither tags, severity labels nor a no-findings statement is unverified, never clean', () => {
const r = classifyOutsideReview({ text: read('prose-unlabeled.txt'), gate: 'review' });
expect([r.verdict, r.reason]).toEqual(['unverified', 'untagged_review']);
const cli = spawnSync(process.execPath, [LIB, '--verdict', 'review', fixture('prose-unlabeled.txt')], { encoding: 'utf8', timeout: 10000 });
expect(cli.status).toBe(4);
expect(cli.stdout).toContain('VERDICT: unverified\n');
expect(classifyOutsideReview({ text: `No issues found.\n${REC}`, gate: 'review' }).verdict).toBe('clean');
expect(classifyOutsideReview({ text: `I did not find any issues.\n${REC}`, gate: 'review' }).verdict).toBe('clean');
});
test('the structured gate reads severity labels too', () => {
expect(classifyOutsideReview({ text: 'High: race in cleanup.', gate: 'structured' }).verdict).toBe('findings');
expect(classifyOutsideReview({ text: 'Medium: stale comment.', gate: 'structured' }).verdict).toBe('clean');
});
test('a stored panel of real Codex outputs keeps its verdicts', () => {
const panel = path.join(ROOT, 'test', 'fixtures', 'codex-review');
const verdict = (name: string, gate: OutsideGate) => {
const r = classifyOutsideReview({ text: fs.readFileSync(path.join(panel, name), 'utf8'), gate });
return [r.verdict, r.findings.highest, r.reason ?? null];
};
expect(verdict('clean-review.stdout.txt', 'structured')).toEqual(['unverified', null, 'untagged_review']);
expect(verdict('p2-review.stdout.txt', 'structured')).toEqual(['clean', 'P2', null]);
expect(verdict('clean-review.stdout.txt', 'review')).toEqual(['unavailable', null, 'missing_markers']);
expect(verdict('p2-review.stdout.txt', 'review')).toEqual(['unavailable', 'P2', 'missing_markers']);
});
});
describe('design proposals use the proposal gate', () => {
test('a completed proposal needs only its Recommendation line; severity rules do not apply', () => {
const proposal = 'Use a dense triage table with sticky filters.\nRecommendation: table because operators compare many rows.';
expect(classifyOutsideReview({ text: proposal, gate: 'proposal' }).verdict).toBe('clean');
expect(classifyOutsideReview({ text: 'A table.', gate: 'proposal' }).reason).toBe('missing_markers');
expect(classifyOutsideReview({ text: proposal, gate: 'review' }).verdict).toBe('unverified');
});
test('design-direction invocations render the proposal gate; reviews keep the review gate', () => {
const ctx = { skillName: 'design-consultation', tmplPath: 'design-consultation/SKILL.md.tmpl', host: 'claude' as const, paths: HOST_PATHS.claude };
expect(outsideVoiceCommand(ctx, { timeoutMs: 300000, purpose: 'design-direction' })).toMatch(/ proposal "\$_OUTSIDE_TMP\/text"/);
expect(outsideVoiceCommand({ ...ctx, skillName: 'plan-ceo-review' }, { timeoutMs: 300000 })).toMatch(/ review "\$_OUTSIDE_TMP\/text"/);
});
});