mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-22 08:01:32 +01:00
Pre-commit /verify fails on a timeout, and the review collector stops reading reviewers' scope lists as findings (GST-3201) (#4527)
Pre-commit /verify now fails a check that times out and runs its checks on one time budget; a reviewer's scope-list bullets no longer need an owner ruling to clear; stack tests fail when git can't list a PR's tests. Why: each let a result read clean with nothing proving it, or block on nothing (#4518, #4519). Goal: gates never pass on missing evidence. Nothing to do; `gstack-checks quick --timeout` is now one budget the checks share (540 s default). Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> Review-Finding: p1/outside-voice#1 reader=outside-voice Review-Finding: p1/outside-voice#2 reader=outside-voice
This commit is contained in:
1 parent
3cccf074ad
commit
bd449234aa
14 files changed
+197
-26
No files matched your search
@@ -102,12 +102,14 @@ describe('testVerdict', () => {
|
||||
});
|
||||
|
||||
describe('quickLine', () => {
|
||||
// Value: protects=the pre-commit line is red on a zero-run or incomplete selection and never green when nothing reached a result;
|
||||
// fails_when=ZERO_RUN/INCOMPLETE leave the red set, or an all-ENV/TIMEOUT run reads VERIFY PASS; why_new=the fixture test only
|
||||
// Value: protects=the pre-commit line is red on a zero-run, incomplete or timed-out check and never green when nothing reached a
|
||||
// result; fails_when=ZERO_RUN/INCOMPLETE/TIMEOUT leave the red set, or an all-ENV run reads VERIFY PASS; why_new=the fixture test only
|
||||
// reaches PASS, FAIL and UNMAPPED; seam=none; no_claim=the wording after the VERIFY token
|
||||
test('classifies red, not-run and passed outcomes', () => {
|
||||
const o = (name: string, verdict: Outcome['verdict']): Outcome => ({ name, verdict, ms: 100, detail: 'exit 1', lines: [] });
|
||||
expect(quickLine([o('typecheck', 'ENV'), o('tests', 'TIMEOUT')])).toEqual({ red: [], line: 'VERIFY skipped: nothing ran to a result (typecheck ENV, tests TIMEOUT)' });
|
||||
expect(quickLine([o('typecheck', 'ENV'), o('tests', 'ENV')])).toEqual({ red: [], line: 'VERIFY skipped: nothing ran to a result (typecheck ENV, tests ENV)' });
|
||||
expect(quickLine([o('typecheck', 'PASS'), o('tests', 'TIMEOUT')]).line).toBe('VERIFY FAIL: tests TIMEOUT');
|
||||
expect(quickLine([o('typecheck', 'ENV'), o('tests', 'TIMEOUT')]).red.map(r => r.name)).toEqual(['tests']);
|
||||
expect(quickLine([]).line).toBe('VERIFY skipped: nothing ran to a result (no checks)');
|
||||
expect(quickLine([o('typecheck', 'PASS'), o('tests', 'ZERO_RUN')]).line).toMatch(/^VERIFY FAIL: tests ZERO_RUN/);
|
||||
expect(quickLine([o('tests', 'INCOMPLETE')]).red.map(r => r.name)).toEqual(['tests']);
|
||||
@@ -241,6 +243,22 @@ describe('gstack-checks on a fixture repo', () => {
|
||||
expect(off.out).toStartWith('VERIFY skipped: verify_before_commit is off');
|
||||
}, 60_000);
|
||||
|
||||
// Value: protects=a check that times out fails the pre-commit pass, and the pass ends with its line inside its --timeout, so a
|
||||
// 10-minute Bash call always sees it; fails_when=a timed-out check reads VERIFY PASS (not run), or each check gets the whole
|
||||
// --timeout again; why_new=GST-3201; seam=the real quick pass on the fixture repo with a typecheck that never exits; no_claim=
|
||||
// Claude Code's own Bash limit
|
||||
test('quick: a timeout is a red, and --timeout bounds the whole pass', () => {
|
||||
const s = setup();
|
||||
commit(s.repo, { 'lib/util.js': 'exports.add = (a, b) => b + a;\n' });
|
||||
expect(s.checks(['trust']).status).toBe(0);
|
||||
write(s.repo.work, 'scripts/typecheck.js', 'setTimeout(() => {}, 30000);\n');
|
||||
const started = Date.now();
|
||||
const slow = s.checks(['quick', '--timeout', '3']);
|
||||
expect(slow.status).toBe(1);
|
||||
expect(slow.out.trim().split('\n').at(-1)).toMatch(/^VERIFY FAIL: typecheck TIMEOUT, tests TIMEOUT/);
|
||||
expect(Date.now() - started).toBeLessThan(20_000);
|
||||
}, 60_000);
|
||||
|
||||
// Value: protects=a selection that runs zero tests is not green; fails_when=exit 0 with no test counted reads PASS; why_new=the
|
||||
// verdict is new; seam=pregate's runner summary (a bats stub printing an empty TAP plan); no_claim=node --test, which counts
|
||||
// each file as a test, so an empty file there is not a zero run
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
/**
|
||||
* The collector read a reader's own scope list or passed-checks bullets as findings (GST-3201). On #4518 (p1, native adversarial
|
||||
* reader) "- a HIGH finding still refused;" in its list of what the tests cover collected as HIGH, and on #4519 (round 2, Red Team)
|
||||
* "- `missing_verdict` can't occur alongside P0 or P1 findings." in its list of checks that held collected as P0. Both replies
|
||||
* declared REVIEW_VERDICT: blocking=0, and both forced an owner ruling, since a list item was never a summary candidate the lead could
|
||||
* lower. Such a bullet now keeps its label and becomes one. Excerpts of the captured outputs below, as returned.
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import { judge, parseReply, type MergedLine } from '../lib/review-ledger';
|
||||
|
||||
const CAPTURED_4518_P1_ADVERSARIAL = [
|
||||
'REVIEWED_TREE: 5733be6b184a1f8387580c089b4c0413beeb304b /workspace/gstack',
|
||||
'REVIEW_VERDICT: blocking=0',
|
||||
'',
|
||||
"I found no blocking findings. The weakest point is that the hand-off replaces the whole body from a fresh read, so it is only as safe as the agent's habit of publishing each file before the next gstack step.",
|
||||
'',
|
||||
'**Scope.** I read the full non-fixture diff from f13dbb0d to HEAD (6 commits, GST-3183): `bin/gstack-post`, `bin/gstack-ship-receipt`, `lib/host-publish.ts`, `lib/gstack-post.ts`, `lib/greptile-early.ts`, `scripts/lib/windows-curation.ts`, the ship section prose and templates, `docs/troubleshooting.md` and the changelog fragment. I reviewed the fixtures and tests in summary mode only (diff stat and test names, no payload bytes): `test/capy-host-publish.test.ts` (+320, new), `test/greptile-early.test.ts` (+11), and a size-ledger JSON. They cover:',
|
||||
'- that no gh write happens under `GSTACK_CAPY=1` for stub, stop, claim, carry and every gstack-post op;',
|
||||
'- a negative control without Capy;',
|
||||
'- two-repo, fork (`gh repo set-default` or `GH_REPO`) and GitHub Enterprise destinations;',
|
||||
"- refusal when the repository can't be resolved;",
|
||||
'- a HIGH finding still refused;',
|
||||
'- a detached HEAD on pr-create;',
|
||||
"- a hand-off that can't be written;",
|
||||
'- greptile-early exiting 3 on a hand-off.',
|
||||
'',
|
||||
'**Findings**',
|
||||
'',
|
||||
"2. **claim's re-read check is now always true on Capy, so two runs can claim one grant** (`bin/gstack-ship-receipt:1152-1153`). INVESTIGATE. Under host publishing, `again` is built locally from `read.body` plus this run's own marker. That makes the `first?.run !== run` re-read check a tautology. Two runs racing on one grant both print `GRANT_CLAIMED` and both dispatch a granted round; only a later `history` call catches it. This is documented, but it weakens a merge-check guard. A fix needs a design call, for example a second `claim --verify` step after publishing. Severity: Low. Blocks: no.",
|
||||
'',
|
||||
'Recommendation: Ship as-is because the strongest finding, the stale whole-body hand-off at `bin/gstack-ship-receipt:1069` overwriting an unpublished run entry, only happens when the agent skips the documented publish-before-next-step order, and no test or production path fails when that order is followed.',
|
||||
].join('\n');
|
||||
|
||||
const CAPTURED_4519_R2_RED_TEAM = [
|
||||
'REVIEWED_TREE: 42315eacf53a28493d84f02f9c3701e295a04733 /workspace/gstack',
|
||||
'REVIEW_VERDICT: blocking=0',
|
||||
'',
|
||||
'I found 2 informational findings and nothing that blocks the merge. I read the non-fixture source diff in full. I reviewed the three changed test files in summary mode only, by their stats and test titles, so that part of the coverage is reduced. They add 118 lines covering: the final-failure-decides ordering, 402 in a request id or a duration versus 402 as a status, Claude Code error subtypes and its budget stop, reviewer-file versus closing-message selection (GST-3188), and refusing a prompt that is also an output file.',
|
||||
'',
|
||||
'The integration paths I checked hold up:',
|
||||
"- Claude Code's budget stop reaches `execution()` as `ERROR: <CLAUDE_CODE_BUDGET_STOP …>`, the last stderr line, so `codexErrorLines` picks it up. It gets matched before `QUOTA_FAILURE`.",
|
||||
"- The phase names and hex sha in `autoplan-prompts` `RESULT_FORMAT` match the runner's receipt regex. `parseReceipt` doesn't lowercase the expected phase, but the phases are already lowercase.",
|
||||
"- `missing_verdict` can't occur alongside P0 or P1 findings.",
|
||||
'',
|
||||
JSON.stringify({ severity: 'INFORMATIONAL', confidence: 5, path: 'lib/outside-review-result.ts', line: 73, category: 'red-team', summary: "The tighter BUDGET_FAILURE regex no longer recognizes some real 402 spellings that the old one caught, such as 'Request failed with status code (402)' and 'got 402 from proxy' (both checked at runtime: false). Those now fall through to execution_failed. The gate still fails closed, but the user gets generic advice instead of the budget_exhausted fix.", fix: "Allow '(' in the separator class, e.g. [\\s:=\"'(]{0,4}, or accept that these spellings are rare.", fingerprint: 'lib/outside-review-result.ts:73:red-team', specialist: 'red-team' }),
|
||||
].join('\n');
|
||||
|
||||
const rows = (reader: string, out: string) => parseReply(reader, out, true, true).findings.map(f => [f.label, f.raw.slice(0, 40), !!f.summary]);
|
||||
const reply = (verdict: string, line: string) => ['REVIEWED_TREE: 42315eacf53a28493d84f02f9c3701e295a04733 /workspace/gstack', verdict, '', line].join('\n');
|
||||
const ZERO = 'REVIEW_VERDICT: blocking=0';
|
||||
/** The merged lines collect writes for a reply, then lead judgment lowering every line to INFORMATIONAL with a reason. */
|
||||
const lowered = (reader: string, out: string) => {
|
||||
const merged: MergedLine[] = parseReply(reader, out, true, true).findings.map((f, i) => ({ id: `${reader}#${i + 1}`, reader, label: f.label, raw: f.raw, output_sha256: 'x', ...(f.summary ? { summary_candidate: true as const } : {}) }));
|
||||
return judge(merged, merged.map(m => ({ id: m.id, severity: 'INFORMATIONAL', reason: 'the reader lists what passed; it reports no defect' })));
|
||||
};
|
||||
|
||||
describe('a reader\'s scope and passed-checks bullets under a blocking=0 verdict', () => {
|
||||
// Value: protects=a bullet that names severities only as a kind of finding, in a reply declaring blocking=0, is a summary candidate
|
||||
// the lead may lower with a reason, and still blocks until then; fails_when=#4518's or #4519's bullet cannot be lowered without an
|
||||
// owner ruling again, or a bullet asserting a blocker ("Two P0 findings remain") loses its label; why_new=both forced owner
|
||||
// rulings (GST-3201); seam=parseReply with the verdict ask and judge, as collect and the lead use them; no_claim=/ship's Step 11 prose
|
||||
test('#4518: the scope-list bullet keeps HIGH as a summary candidate; the real finding keeps Severity: Low', () => {
|
||||
expect(rows('adversarial-native', CAPTURED_4518_P1_ADVERSARIAL)).toEqual([
|
||||
['HIGH', '- a HIGH finding still refused;', true],
|
||||
['Low', "2. **claim's re-read check is now always", false],
|
||||
]);
|
||||
});
|
||||
|
||||
test('#4519: the passed-check bullet keeps P0 as a summary candidate, so the lead can lower it with a reason', () => {
|
||||
const r = parseReply('red-team', CAPTURED_4519_R2_RED_TEAM, true, true);
|
||||
expect(r.findings.map(f => [f.label, !!f.summary])).toEqual([['P0', true], ['INFORMATIONAL', false]]);
|
||||
expect(r.verdict).toEqual({ state: 'valid', parsed: 1, declared: 0, agrees: false });
|
||||
expect(lowered('red-team', CAPTURED_4519_R2_RED_TEAM)).toMatchObject({ ok: true, dismissed: ['red-team#1'], downgraded: [] });
|
||||
expect(lowered('adversarial-native', CAPTURED_4518_P1_ADVERSARIAL)).toMatchObject({ dismissed: ['adversarial-native#1'] });
|
||||
});
|
||||
|
||||
test('negative controls: a bullet asserting a blocker keeps its label and blocks until judged; other shapes collect as before', () => {
|
||||
for (const line of ['- Two P0 findings remain unfixed.', '- The P0 finding is that the gate drops rows.']) {
|
||||
expect(rows('red-team', reply(ZERO, line))).toEqual([['P0', line.slice(0, 40), true]]);
|
||||
expect(parseReply('red-team', reply(ZERO, line), true, true).verdict).toMatchObject({ parsed: 1, agrees: false });
|
||||
}
|
||||
expect(rows('red-team', reply('REVIEW_VERDICT: blocking=1', "- `missing_verdict` can't occur alongside P0 or P1 findings."))[0]).toEqual(['P0', "- `missing_verdict` can't occur alongsid", false]);
|
||||
expect(parseReply('red-team', "- `missing_verdict` can't occur alongside P0 or P1 findings.", true, false).findings.map(f => [f.label, !!f.summary])).toEqual([['P0', false]]);
|
||||
expect(rows('adversarial-native', CAPTURED_4518_P1_ADVERSARIAL.replace('- a HIGH finding still refused;', '- a HIGH finding still refused at lib/host-publish.ts:40;'))[0]![2]).toBe(false);
|
||||
expect(rows('red-team', reply(ZERO, '- P0: the gate drops a HIGH finding.'))[0]).toEqual(['P0', '- P0: the gate drops a HIGH finding.', false]);
|
||||
expect(rows('red-team', reply(ZERO, '- a HIGH finding: the cache key drops the tree.'))[0]![2]).toBe(false);
|
||||
expect(rows('red-team', reply(ZERO, '- a HIGH finding still refused. Severity: High'))[0]![2]).toBe(false);
|
||||
expect(rows('adversarial-native', reply(ZERO, '- a HIGH finding is dropped, FIXABLE.'))[0]).toEqual(['CRITICAL', '- a HIGH finding is dropped, FIXABLE.', false]);
|
||||
expect(rows('red-team', reply(ZERO, '- the HIGH path crashes when the ledger is empty.'))[0]![2]).toBe(false);
|
||||
expect(lowered('red-team', reply(ZERO, '- P0: the gate drops a HIGH finding.'))).toMatchObject({ ok: false, downgraded: ['red-team#1'] });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,29 @@
|
||||
/**
|
||||
* train build --dry-run --stack-tests read a git failure listing a PR's added files as "no new test files" and passed the slot
|
||||
* (GST-3201; the round-1 /ship note on #4502, crew/lib/ship-queue-stack-tests.ts). An unread list is not an empty one.
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as path from 'node:path';
|
||||
import { newTestFiles, stackTestLine, stackTests } from '../crew/lib/ship-queue-stack-tests';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const MISSING = '0000000000000000000000000000000000000001';
|
||||
|
||||
describe('stack tests fail closed when git cannot list the added files', () => {
|
||||
// Value: protects=a slot whose added test files cannot be listed is red, not "no new test files"; fails_when=a git failure reads
|
||||
// as an empty list again; why_new=the fail-open had no test; seam=the real git diff on this checkout with a base that does not
|
||||
// exist; no_claim=running the selected files on a prefix worktree
|
||||
test('a base git cannot resolve is an error, and the slot fails with it', () => {
|
||||
const listed = newTestFiles(ROOT, MISSING, 'HEAD');
|
||||
expect('error' in listed && listed.error).toBeTruthy();
|
||||
const [r] = stackTests(ROOT, MISSING, [{ pr: '1', head: 'HEAD', ready_at: '', screen: 'none' } as any], [{ index: 0, commit: 'HEAD', tree: '', pr: null }, { index: 1, commit: 'HEAD', tree: '', pr: '1' }]);
|
||||
expect(r!.ok).toBe(false);
|
||||
expect(stackTestLine(r!)).toStartWith('STACK_TESTS: #1 T1 FAIL: cannot list the test files it adds: ');
|
||||
});
|
||||
|
||||
test('negative control: a readable empty list still passes as no new test files', () => {
|
||||
expect(newTestFiles(ROOT, 'HEAD', 'HEAD')).toEqual({ files: [] });
|
||||
const [r] = stackTests(ROOT, 'HEAD', [{ pr: '1', head: 'HEAD', ready_at: '', screen: 'none' } as any], [{ index: 0, commit: 'HEAD', tree: '', pr: null }, { index: 1, commit: 'HEAD', tree: '', pr: '1' }]);
|
||||
expect(stackTestLine(r!)).toBe('STACK_TESTS: #1 T1 no new test files');
|
||||
});
|
||||
});
|
||||
@@ -181,11 +181,11 @@ const CHILD_PROCESS = String.raw`['"](?:node:)?child_process['"]`;
|
||||
|
||||
/** Whether this source takes spawnSync from node:child_process (named, aliased, namespace, default or require). */
|
||||
export function importsRawSpawnSync(text: string): boolean {
|
||||
for (const m of text.matchAll(new RegExp(String.raw`import\s*\{([^}]*)\}\s*from\s*${CHILD_PROCESS}`, 'g'))) {
|
||||
for (const m of text.matchAll(new RegExp(String.raw`import\s*(?:\w+\s*,\s*)?\{([^}]*)\}\s*from\s*${CHILD_PROCESS}`, 'g'))) {
|
||||
if (/(?:^|[,{\s])spawnSync\b/.test(m[1].replace(/\btype\s+spawnSync\b/g, ''))) return true;
|
||||
}
|
||||
for (const m of text.matchAll(new RegExp(String.raw`import\s+(?:\*\s+as\s+)?(\w+)\s+from\s*${CHILD_PROCESS}`, 'g'))) {
|
||||
if (new RegExp(String.raw`\b${m[1]}\.spawnSync\b`).test(text)) return true;
|
||||
for (const m of text.matchAll(new RegExp(String.raw`import\s+(?:\*\s+as\s+)?(\w+)(?:\s*,\s*(?:\{[^}]*\}|\*\s+as\s+(\w+)))?\s+from\s*${CHILD_PROCESS}`, 'g'))) {
|
||||
if (m.slice(1).some(name => name && new RegExp(String.raw`\b${name}\.spawnSync\b`).test(text))) return true;
|
||||
}
|
||||
return new RegExp(String.raw`require\(\s*${CHILD_PROCESS}\s*\)`).test(text) && /\bspawnSync\b/.test(text);
|
||||
}
|
||||
@@ -227,6 +227,10 @@ describe('raw spawnSync import ratchet', () => {
|
||||
expect(importsRawSpawnSync("import {\n spawn,\n spawnSync,\n} from 'node:child_process';")).toBe(true);
|
||||
expect(importsRawSpawnSync("import * as cp from 'node:child_process';\ncp.spawnSync('git', [], { timeout: 1000 });")).toBe(true);
|
||||
expect(importsRawSpawnSync("const { spawnSync } = require('child_process');")).toBe(true);
|
||||
expect(importsRawSpawnSync("import cp, { spawnSync } from 'node:child_process';")).toBe(true);
|
||||
expect(importsRawSpawnSync("import cp, { spawn } from 'node:child_process';\ncp.spawnSync('git', [], { timeout: 1000 });")).toBe(true);
|
||||
expect(importsRawSpawnSync("import cp, * as all from 'node:child_process';\nall.spawnSync('git', [], { timeout: 1000 });")).toBe(true);
|
||||
expect(importsRawSpawnSync("import cp, { spawn } from 'node:child_process';\ncp.spawn('git', []);")).toBe(false);
|
||||
expect(importsRawSpawnSync("import type { SpawnSyncReturns } from 'node:child_process';")).toBe(false);
|
||||
expect(importsRawSpawnSync("import { spawn, type spawnSync } from 'node:child_process';")).toBe(false);
|
||||
expect(importsRawSpawnSync("import * as cp from 'node:child_process';\ncp.spawn('git', []);")).toBe(false);
|
||||
|
||||
Reference in new issue
Block a user