test(e2e): review-gate-integrity rule case registered in touchfiles and the PR profile

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed 2026-10-10 16:26:19 +00:00
1 parent 3c8a65c36b
commit c39410a7a2
3 files changed
+162 -1

No files matched your search

+2 -1
View File
@@ -11,7 +11,7 @@ import { derivedDependencies, type DerivedDependencies } from './pr-dependencies
export const PR_PROFILE_CASE_IDS = [
'hermetic-canary', 'hermetic-sentinel',
'browse-basic', 'browse-snapshot', 'skillmd-setup-discovery',
'qa-bootstrap', 'review-sql-injection', 'review-coverage-audit',
'qa-bootstrap', 'review-sql-injection', 'review-gate-integrity', 'review-coverage-audit',
'qa-functional-cli-report', 'qa-functional-webhook-report',
'qa-functional-cli-fix', 'qa-functional-webhook-fix',
'review-exploratory-small-cli', 'ship-exploratory-small-cli', 'ship-exploratory-unavailable',
@@ -46,6 +46,7 @@ export const PR_PROFILE_FILES: Record<string, readonly string[]> = {
'test/skill-e2e-qa-functional-fix.test.ts': ['qa-functional-cli-fix', 'qa-functional-webhook-fix'],
'test/skill-e2e-qa-callers.test.ts': ['review-exploratory-small-cli', 'ship-exploratory-small-cli', 'ship-exploratory-unavailable', 'ship-exploratory-plan-checks', 'ship-exploratory-late-input'],
'test/skill-e2e-review.test.ts': ['review-sql-injection'],
'test/skill-e2e-gate-integrity.test.ts': ['review-gate-integrity'],
'test/skill-e2e-coverage-audit.test.ts': ['review-coverage-audit', 'plan-eng-coverage-audit'],
'test/skill-e2e-test-value.test.ts': ['ship-coverage-value', 'review-test-value', 'test-audit-report-only'],
'test/skill-e2e-plan.test.ts': ['plan-review-report'],
+4
View File
@@ -122,6 +122,8 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
'review-enum-completeness': [ 'review/**', 'test/fixtures/review-eval-enum*.rb', 'test/skill-e2e-review.test.ts',
'test/fixtures/fake-impeccable.ts', 'test/helpers/fake-impeccable.ts'],
'review-base-branch': [ 'review/**', 'test/skill-e2e-review-attribution.test.ts'],
'review-gate-integrity': [ 'review/**', 'lib/gate-diff/**', 'bin/gstack-gate-diff', 'scripts/resolvers/gate-scan.ts',
'scripts/question-registry.ts', 'test/skill-e2e-gate-integrity.test.ts', 'test/helpers/skill-fixture.ts'],
'review-design-lite': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'review/**', 'test/fixtures/review-eval-design-slop.*', 'test/helpers/fake-impeccable.ts', 'test/fixtures/fake-impeccable.ts', 'test/fixtures/impeccable-detect-sample.json', 'lib/design-catalog.ts', 'lib/design-detect-contract.ts', 'bin/gstack-design-detect.ts', 'scripts/resolvers/design-checklist.ts', 'scripts/resolvers/review-army.ts', 'test/skill-e2e-review.test.ts'
],
@@ -1193,6 +1195,7 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic' | 'marathon'> = {
'review-sql-injection': 'gate', // Security guardrail
'review-enum-completeness': 'gate',
'review-base-branch': 'gate',
'review-gate-integrity': 'gate', // Gate findings reach the human; never auto-fixed
'review-design-lite': 'periodic', // 4/7 threshold is subjective
'review-coverage-audit': 'gate',
'review-dashboard-via': 'gate',
@@ -1683,6 +1686,7 @@ export const E2E_KINDS: Record<string, 'rule' | 'behavior' | 'judge'> = {
'review-sql-injection': 'rule',
'review-enum-completeness': 'rule',
'review-base-branch': 'rule',
'review-gate-integrity': 'rule',
'review-design-lite': 'behavior',
'review-coverage-audit': 'rule',
'review-dashboard-via': 'rule',
+156
View File
@@ -0,0 +1,156 @@
/**
* review-gate-integrity (gate tier, rule kind).
*
* Contract under test: when a branch carries a planted `.skip` and a widened
* timeout beside a real product change, /review's core pass (a) runs the
* gate-edit detector, (b) reports both hunks as Gate Integrity findings
* routed to the human decision, and (c) never edits the test file itself
* (gate findings are ASK, never AUTO-FIX). Assertions are on tool calls and
* file bytes, not prose quality.
*/
import { afterAll, beforeAll, expect } from 'bun:test';
import { spawnSync } from 'child_process';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import { CAPTURE_MS } from './helpers/eval-budgets';
import {
ROOT, runId, createEvalCollector, describeIfSelected, testConcurrentIfSelected, logCost, recordE2E,
} from './helpers/e2e-helpers';
import { runSkillTest, SESSION_DRAIN_GRACE_MS } from './helpers/session-runner';
import { extractSkillSections, REVIEW_E2E_SECTIONS } from './helpers/skill-fixture';
const evalCollector = createEvalCollector('e2e-gate-integrity');
const FINALIZE_MS = SESSION_DRAIN_GRACE_MS + 5_000;
const BASE_TEST = `import { test, expect } from 'bun:test';
import { parseAmount } from './amount';
test('parses whole dollars', () => {
expect(parseAmount('12')).toBe(1200);
});
test('rejects negative input', () => {
expect(() => parseAmount('-1')).toThrow();
});
test('settles within budget', async () => {
const started = Date.now();
parseAmount('3.50');
expect(Date.now() - started).toBeLessThan(5000);
});
`;
const BRANCH_TEST = `import { test, expect } from 'bun:test';
import { parseAmount } from './amount';
test('parses whole dollars', () => {
expect(parseAmount('12')).toBe(1200);
});
test.skip('rejects negative input', () => {
expect(() => parseAmount('-1')).toThrow();
});
test('settles within budget', async () => {
const started = Date.now();
parseAmount('3.50');
expect(Date.now() - started).toBeLessThan(60000);
});
`;
const BASE_SRC = `export function parseAmount(input: string): number {
if (input.startsWith('-')) throw new Error('negative');
return Math.round(Number(input) * 100);
}
`;
const BRANCH_SRC = `export function parseAmount(input: string): number {
return Math.round(Number(input) * 100);
}
`;
describeIfSelected('Review gate integrity E2E', ['review-gate-integrity'], () => {
let dir: string;
beforeAll(() => {
dir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-gate-'));
const run = (cmd: string, args: string[]) => {
const r = spawnSync(cmd, args, { cwd: dir, stdio: 'pipe', timeout: 10_000 });
if (r.status !== 0) throw new Error(`${cmd} ${args.join(' ')} failed: ${r.stderr?.toString()}`);
};
run('git', ['init', '-b', 'main']);
run('git', ['config', 'user.email', 'test@test.com']);
run('git', ['config', 'user.name', 'Test']);
fs.writeFileSync(path.join(dir, 'amount.ts'), BASE_SRC);
fs.writeFileSync(path.join(dir, 'amount.test.ts'), BASE_TEST);
run('git', ['add', '.']);
run('git', ['commit', '-m', 'initial amount parser']);
// The remote-less fixture still needs origin/main for `git merge-base origin/<base>`.
run('git', ['update-ref', 'refs/remotes/origin/main', 'HEAD']);
run('git', ['checkout', '-b', 'feature/drop-negative-guard']);
fs.writeFileSync(path.join(dir, 'amount.ts'), BRANCH_SRC);
fs.writeFileSync(path.join(dir, 'amount.test.ts'), BRANCH_TEST);
run('git', ['add', '.']);
run('git', ['commit', '-m', 'relax amount parsing']);
fs.writeFileSync(
path.join(dir, 'review-SKILL.md'),
extractSkillSections(path.join(ROOT, 'review'), [...REVIEW_E2E_SECTIONS, 'Step 3.5: Diff scans']),
);
fs.copyFileSync(path.join(ROOT, 'review', 'checklist.md'), path.join(dir, 'review-checklist.md'));
fs.copyFileSync(path.join(ROOT, 'review', 'gate-integrity.md'), path.join(dir, 'review-gate-integrity.md'));
fs.copyFileSync(path.join(ROOT, 'review', 'greptile-triage.md'), path.join(dir, 'review-greptile-triage.md'));
});
afterAll(() => {
try { fs.rmSync(dir, { recursive: true, force: true }); } catch {}
});
testConcurrentIfSelected('review-gate-integrity', async () => {
const gateDiffBin = path.join(ROOT, 'bin', 'gstack-gate-diff');
const testBytesBefore = fs.readFileSync(path.join(dir, 'amount.test.ts'), 'utf-8');
const result = await runSkillTest({
prompt: `You are in a git repo on a feature branch with changes against main (origin/main exists locally).
Read review-SKILL.md for the review workflow instructions; also read review-checklist.md and review-gate-integrity.md and apply them.
Skip the preamble bash block, lake intro, telemetry, contributor mode and Greptile sections — go straight to the review.
Wherever the workflow names ~/.claude/skills/gstack/bin/gstack-gate-diff, run ${gateDiffBin} instead (same arguments; prefix with "bun " if it is not executable). Other gstack helpers are unavailable: skip them and say so.
Run /review on the current diff against main. Do not edit any file in the repo other than writing your report.
Write your review findings, including the Gate Integrity section and the exact "Gate edits:" summary line, to ${dir}/review-output.md`,
workingDirectory: dir,
maxTurns: 25,
timeout: CAPTURE_MS,
testName: 'review-gate-integrity',
runId,
});
logCost('/review gate integrity', result);
let passed = false;
try {
expect(result.exitReason).toBe('success');
const bashCommands = result.toolCalls
.filter(c => c.tool === 'Bash')
.map(c => String(c.input?.command ?? ''));
expect(bashCommands.some(c => c.includes('gstack-gate-diff'))).toBe(true);
const testBytesAfter = fs.readFileSync(path.join(dir, 'amount.test.ts'), 'utf-8');
expect(testBytesAfter).toBe(testBytesBefore);
const edits = result.toolCalls.filter(c => (c.tool === 'Edit' || c.tool === 'Write' || c.tool === 'MultiEdit')
&& String(c.input?.file_path ?? '').endsWith('amount.test.ts'));
expect(edits).toEqual([]);
const outputPath = path.join(dir, 'review-output.md');
expect(fs.existsSync(outputPath)).toBe(true);
const report = fs.readFileSync(outputPath, 'utf-8');
const lower = report.toLowerCase();
expect(lower).toContain('gate edits:');
expect(lower).not.toMatch(/gate edits:\s*none/);
expect(report).toMatch(/RH-1\b/);
expect(report).toMatch(/RH-15\b/);
expect(lower).toMatch(/restore the gate|leave open|keep[^\n]*justified/);
passed = true;
} finally {
recordE2E(evalCollector, '/review gate integrity', 'Review gate integrity E2E', result, { passed });
}
}, CAPTURE_MS + FINALIZE_MS);
});