mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-05 16:34:00 +01:00
test(e2e): review-gate-integrity rule case registered in touchfiles and the PR profile
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
3c8a65c36b
commit
c39410a7a2
3 files changed
+162
-1
No files matched your search
@@ -11,7 +11,7 @@ import { derivedDependencies, type DerivedDependencies } from './pr-dependencies
|
||||
export const PR_PROFILE_CASE_IDS = [
|
||||
'hermetic-canary', 'hermetic-sentinel',
|
||||
'browse-basic', 'browse-snapshot', 'skillmd-setup-discovery',
|
||||
'qa-bootstrap', 'review-sql-injection', 'review-coverage-audit',
|
||||
'qa-bootstrap', 'review-sql-injection', 'review-gate-integrity', 'review-coverage-audit',
|
||||
'qa-functional-cli-report', 'qa-functional-webhook-report',
|
||||
'qa-functional-cli-fix', 'qa-functional-webhook-fix',
|
||||
'review-exploratory-small-cli', 'ship-exploratory-small-cli', 'ship-exploratory-unavailable',
|
||||
@@ -46,6 +46,7 @@ export const PR_PROFILE_FILES: Record<string, readonly string[]> = {
|
||||
'test/skill-e2e-qa-functional-fix.test.ts': ['qa-functional-cli-fix', 'qa-functional-webhook-fix'],
|
||||
'test/skill-e2e-qa-callers.test.ts': ['review-exploratory-small-cli', 'ship-exploratory-small-cli', 'ship-exploratory-unavailable', 'ship-exploratory-plan-checks', 'ship-exploratory-late-input'],
|
||||
'test/skill-e2e-review.test.ts': ['review-sql-injection'],
|
||||
'test/skill-e2e-gate-integrity.test.ts': ['review-gate-integrity'],
|
||||
'test/skill-e2e-coverage-audit.test.ts': ['review-coverage-audit', 'plan-eng-coverage-audit'],
|
||||
'test/skill-e2e-test-value.test.ts': ['ship-coverage-value', 'review-test-value', 'test-audit-report-only'],
|
||||
'test/skill-e2e-plan.test.ts': ['plan-review-report'],
|
||||
|
||||
@@ -122,6 +122,8 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
'review-enum-completeness': [ 'review/**', 'test/fixtures/review-eval-enum*.rb', 'test/skill-e2e-review.test.ts',
|
||||
'test/fixtures/fake-impeccable.ts', 'test/helpers/fake-impeccable.ts'],
|
||||
'review-base-branch': [ 'review/**', 'test/skill-e2e-review-attribution.test.ts'],
|
||||
'review-gate-integrity': [ 'review/**', 'lib/gate-diff/**', 'bin/gstack-gate-diff', 'scripts/resolvers/gate-scan.ts',
|
||||
'scripts/question-registry.ts', 'test/skill-e2e-gate-integrity.test.ts', 'test/helpers/skill-fixture.ts'],
|
||||
'review-design-lite': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'review/**', 'test/fixtures/review-eval-design-slop.*', 'test/helpers/fake-impeccable.ts', 'test/fixtures/fake-impeccable.ts', 'test/fixtures/impeccable-detect-sample.json', 'lib/design-catalog.ts', 'lib/design-detect-contract.ts', 'bin/gstack-design-detect.ts', 'scripts/resolvers/design-checklist.ts', 'scripts/resolvers/review-army.ts', 'test/skill-e2e-review.test.ts'
|
||||
|
||||
],
|
||||
@@ -1193,6 +1195,7 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic' | 'marathon'> = {
|
||||
'review-sql-injection': 'gate', // Security guardrail
|
||||
'review-enum-completeness': 'gate',
|
||||
'review-base-branch': 'gate',
|
||||
'review-gate-integrity': 'gate', // Gate findings reach the human; never auto-fixed
|
||||
'review-design-lite': 'periodic', // 4/7 threshold is subjective
|
||||
'review-coverage-audit': 'gate',
|
||||
'review-dashboard-via': 'gate',
|
||||
@@ -1683,6 +1686,7 @@ export const E2E_KINDS: Record<string, 'rule' | 'behavior' | 'judge'> = {
|
||||
'review-sql-injection': 'rule',
|
||||
'review-enum-completeness': 'rule',
|
||||
'review-base-branch': 'rule',
|
||||
'review-gate-integrity': 'rule',
|
||||
'review-design-lite': 'behavior',
|
||||
'review-coverage-audit': 'rule',
|
||||
'review-dashboard-via': 'rule',
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
/**
|
||||
* review-gate-integrity (gate tier, rule kind).
|
||||
*
|
||||
* Contract under test: when a branch carries a planted `.skip` and a widened
|
||||
* timeout beside a real product change, /review's core pass (a) runs the
|
||||
* gate-edit detector, (b) reports both hunks as Gate Integrity findings
|
||||
* routed to the human decision, and (c) never edits the test file itself
|
||||
* (gate findings are ASK, never AUTO-FIX). Assertions are on tool calls and
|
||||
* file bytes, not prose quality.
|
||||
*/
|
||||
import { afterAll, beforeAll, expect } from 'bun:test';
|
||||
import { spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import {
|
||||
ROOT, runId, createEvalCollector, describeIfSelected, testConcurrentIfSelected, logCost, recordE2E,
|
||||
} from './helpers/e2e-helpers';
|
||||
import { runSkillTest, SESSION_DRAIN_GRACE_MS } from './helpers/session-runner';
|
||||
import { extractSkillSections, REVIEW_E2E_SECTIONS } from './helpers/skill-fixture';
|
||||
|
||||
const evalCollector = createEvalCollector('e2e-gate-integrity');
|
||||
const FINALIZE_MS = SESSION_DRAIN_GRACE_MS + 5_000;
|
||||
|
||||
const BASE_TEST = `import { test, expect } from 'bun:test';
|
||||
import { parseAmount } from './amount';
|
||||
|
||||
test('parses whole dollars', () => {
|
||||
expect(parseAmount('12')).toBe(1200);
|
||||
});
|
||||
|
||||
test('rejects negative input', () => {
|
||||
expect(() => parseAmount('-1')).toThrow();
|
||||
});
|
||||
|
||||
test('settles within budget', async () => {
|
||||
const started = Date.now();
|
||||
parseAmount('3.50');
|
||||
expect(Date.now() - started).toBeLessThan(5000);
|
||||
});
|
||||
`;
|
||||
|
||||
const BRANCH_TEST = `import { test, expect } from 'bun:test';
|
||||
import { parseAmount } from './amount';
|
||||
|
||||
test('parses whole dollars', () => {
|
||||
expect(parseAmount('12')).toBe(1200);
|
||||
});
|
||||
|
||||
test.skip('rejects negative input', () => {
|
||||
expect(() => parseAmount('-1')).toThrow();
|
||||
});
|
||||
|
||||
test('settles within budget', async () => {
|
||||
const started = Date.now();
|
||||
parseAmount('3.50');
|
||||
expect(Date.now() - started).toBeLessThan(60000);
|
||||
});
|
||||
`;
|
||||
|
||||
const BASE_SRC = `export function parseAmount(input: string): number {
|
||||
if (input.startsWith('-')) throw new Error('negative');
|
||||
return Math.round(Number(input) * 100);
|
||||
}
|
||||
`;
|
||||
|
||||
const BRANCH_SRC = `export function parseAmount(input: string): number {
|
||||
return Math.round(Number(input) * 100);
|
||||
}
|
||||
`;
|
||||
|
||||
describeIfSelected('Review gate integrity E2E', ['review-gate-integrity'], () => {
|
||||
let dir: string;
|
||||
|
||||
beforeAll(() => {
|
||||
dir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-gate-'));
|
||||
const run = (cmd: string, args: string[]) => {
|
||||
const r = spawnSync(cmd, args, { cwd: dir, stdio: 'pipe', timeout: 10_000 });
|
||||
if (r.status !== 0) throw new Error(`${cmd} ${args.join(' ')} failed: ${r.stderr?.toString()}`);
|
||||
};
|
||||
run('git', ['init', '-b', 'main']);
|
||||
run('git', ['config', 'user.email', 'test@test.com']);
|
||||
run('git', ['config', 'user.name', 'Test']);
|
||||
fs.writeFileSync(path.join(dir, 'amount.ts'), BASE_SRC);
|
||||
fs.writeFileSync(path.join(dir, 'amount.test.ts'), BASE_TEST);
|
||||
run('git', ['add', '.']);
|
||||
run('git', ['commit', '-m', 'initial amount parser']);
|
||||
// The remote-less fixture still needs origin/main for `git merge-base origin/<base>`.
|
||||
run('git', ['update-ref', 'refs/remotes/origin/main', 'HEAD']);
|
||||
run('git', ['checkout', '-b', 'feature/drop-negative-guard']);
|
||||
fs.writeFileSync(path.join(dir, 'amount.ts'), BRANCH_SRC);
|
||||
fs.writeFileSync(path.join(dir, 'amount.test.ts'), BRANCH_TEST);
|
||||
run('git', ['add', '.']);
|
||||
run('git', ['commit', '-m', 'relax amount parsing']);
|
||||
|
||||
fs.writeFileSync(
|
||||
path.join(dir, 'review-SKILL.md'),
|
||||
extractSkillSections(path.join(ROOT, 'review'), [...REVIEW_E2E_SECTIONS, 'Step 3.5: Diff scans']),
|
||||
);
|
||||
fs.copyFileSync(path.join(ROOT, 'review', 'checklist.md'), path.join(dir, 'review-checklist.md'));
|
||||
fs.copyFileSync(path.join(ROOT, 'review', 'gate-integrity.md'), path.join(dir, 'review-gate-integrity.md'));
|
||||
fs.copyFileSync(path.join(ROOT, 'review', 'greptile-triage.md'), path.join(dir, 'review-greptile-triage.md'));
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
try { fs.rmSync(dir, { recursive: true, force: true }); } catch {}
|
||||
});
|
||||
|
||||
testConcurrentIfSelected('review-gate-integrity', async () => {
|
||||
const gateDiffBin = path.join(ROOT, 'bin', 'gstack-gate-diff');
|
||||
const testBytesBefore = fs.readFileSync(path.join(dir, 'amount.test.ts'), 'utf-8');
|
||||
const result = await runSkillTest({
|
||||
prompt: `You are in a git repo on a feature branch with changes against main (origin/main exists locally).
|
||||
Read review-SKILL.md for the review workflow instructions; also read review-checklist.md and review-gate-integrity.md and apply them.
|
||||
Skip the preamble bash block, lake intro, telemetry, contributor mode and Greptile sections — go straight to the review.
|
||||
Wherever the workflow names ~/.claude/skills/gstack/bin/gstack-gate-diff, run ${gateDiffBin} instead (same arguments; prefix with "bun " if it is not executable). Other gstack helpers are unavailable: skip them and say so.
|
||||
Run /review on the current diff against main. Do not edit any file in the repo other than writing your report.
|
||||
Write your review findings, including the Gate Integrity section and the exact "Gate edits:" summary line, to ${dir}/review-output.md`,
|
||||
workingDirectory: dir,
|
||||
maxTurns: 25,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'review-gate-integrity',
|
||||
runId,
|
||||
});
|
||||
|
||||
logCost('/review gate integrity', result);
|
||||
let passed = false;
|
||||
try {
|
||||
expect(result.exitReason).toBe('success');
|
||||
const bashCommands = result.toolCalls
|
||||
.filter(c => c.tool === 'Bash')
|
||||
.map(c => String(c.input?.command ?? ''));
|
||||
expect(bashCommands.some(c => c.includes('gstack-gate-diff'))).toBe(true);
|
||||
|
||||
const testBytesAfter = fs.readFileSync(path.join(dir, 'amount.test.ts'), 'utf-8');
|
||||
expect(testBytesAfter).toBe(testBytesBefore);
|
||||
const edits = result.toolCalls.filter(c => (c.tool === 'Edit' || c.tool === 'Write' || c.tool === 'MultiEdit')
|
||||
&& String(c.input?.file_path ?? '').endsWith('amount.test.ts'));
|
||||
expect(edits).toEqual([]);
|
||||
|
||||
const outputPath = path.join(dir, 'review-output.md');
|
||||
expect(fs.existsSync(outputPath)).toBe(true);
|
||||
const report = fs.readFileSync(outputPath, 'utf-8');
|
||||
const lower = report.toLowerCase();
|
||||
expect(lower).toContain('gate edits:');
|
||||
expect(lower).not.toMatch(/gate edits:\s*none/);
|
||||
expect(report).toMatch(/RH-1\b/);
|
||||
expect(report).toMatch(/RH-15\b/);
|
||||
expect(lower).toMatch(/restore the gate|leave open|keep[^\n]*justified/);
|
||||
passed = true;
|
||||
} finally {
|
||||
recordE2E(evalCollector, '/review gate integrity', 'Review gate integrity E2E', result, { passed });
|
||||
}
|
||||
}, CAPTURE_MS + FINALIZE_MS);
|
||||
});
|
||||
Reference in new issue
Block a user