mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-22 08:01:32 +01:00
Pre-commit /verify fails on a timeout, and the review collector stops reading reviewers' scope lists as findings (GST-3201) (#4527)
Pre-commit /verify now fails a check that times out and runs its checks on one time budget; a reviewer's scope-list bullets no longer need an owner ruling to clear; stack tests fail when git can't list a PR's tests. Why: each let a result read clean with nothing proving it, or block on nothing (#4518, #4519). Goal: gates never pass on missing evidence. Nothing to do; `gstack-checks quick --timeout` is now one budget the checks share (540 s default). Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> Review-Finding: p1/outside-voice#1 reader=outside-voice Review-Finding: p1/outside-voice#2 reader=outside-voice
This commit is contained in:
1 parent
3cccf074ad
commit
bd449234aa
14 files changed
+197
-26
No files matched your search
+6
-3
@@ -21,7 +21,9 @@ const USAGE = `Usage: gstack-checks <detect|trust|run|status|quick> [options]
|
||||
runs nothing
|
||||
quick [--timeout <s>] the pre-commit pass: run's checks and tests with no consent prompt, ending with one line,
|
||||
VERIFY PASS|FAIL|skipped: <what>; untrusted commands, no pregate context or
|
||||
verify_before_commit=off skip instead of stopping; exit 1 only on FAIL
|
||||
verify_before_commit=off skip instead of stopping; exit 1 only on FAIL, and a TIMEOUT
|
||||
is a FAIL; --timeout (default 540) is one budget the checks share, and the test
|
||||
stage starts with what is left
|
||||
|
||||
--base <branch> the branch pregate selects against (default: origin/HEAD's branch, else main)
|
||||
|
||||
@@ -48,6 +50,7 @@ function loadOrRefuse(root: string, args: string[], timeoutMs: number): Loaded |
|
||||
const QUICK_RED_LINES = 10;
|
||||
|
||||
async function quick(root: string | null, args: string[], timeoutMs: number): Promise<number> {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
const skip = (why: string) => { process.stdout.write(`VERIFY skipped: ${why}\n`); return 0; };
|
||||
if (readConfigKey('verify_before_commit') === 'off') return skip('verify_before_commit is off (gstack-config set verify_before_commit on)');
|
||||
if (!root) return skip('not inside a git repository');
|
||||
@@ -60,7 +63,7 @@ async function quick(root: string | null, args: string[], timeoutMs: number): Pr
|
||||
if (nothingToRun(l)) return skip('the repo declares no checks or tests');
|
||||
if (!isTrusted(root, l.set)) return skip("the repo's check commands are not trusted yet; type /verify once to review them");
|
||||
for (const signal of ['SIGINT', 'SIGTERM'] as const) process.once(signal, () => { cancelActive(); process.exit(signal === 'SIGINT' ? 130 : 143); });
|
||||
const { outcomes } = await runChecks(l, { root, timeoutMs });
|
||||
const { outcomes } = await runChecks(l, { root, timeoutMs, deadline });
|
||||
const { line, red } = quickLine(outcomes);
|
||||
for (const o of red) for (const text of o.lines.slice(-QUICK_RED_LINES)) process.stdout.write(` | ${text}\n`);
|
||||
process.stdout.write(line + '\n');
|
||||
@@ -81,7 +84,7 @@ async function main(argv: string[]): Promise<number> {
|
||||
}
|
||||
const root = repoRoot();
|
||||
if (!root && verb !== 'quick') { process.stderr.write('gstack-checks: not inside a git repository\n'); return 3; }
|
||||
const timeoutS = Number(flag(args, '--timeout') ?? 600);
|
||||
const timeoutS = Number(flag(args, '--timeout') ?? (verb === 'quick' ? 540 : 600));
|
||||
if (!Number.isFinite(timeoutS) || timeoutS <= 0) { process.stderr.write('gstack-checks: --timeout needs a positive number of seconds\n'); return 64; }
|
||||
if (verb === 'quick') return quick(root, args, timeoutS * 1000);
|
||||
const l = loadOrRefuse(root, args, timeoutS * 1000);
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
- **The pre-commit /verify check now fails a check that times out, and its checks share one time budget (GST-3201).** A check that ran out of time used to be listed as "not run", so the commit still got `VERIFY PASS`. Now it is `VERIFY FAIL: <check> TIMEOUT`. `gstack-checks quick --timeout` (540 seconds by default) is now one budget for the whole pass instead of a fresh allowance per check: each check gets the time that is left, and one that can't start in time is a TIMEOUT. The test stage starts with the time that is left, but its runner can still run past it, so a very slow test run can still outlast Claude Code's 10-minute command limit.
|
||||
- **/ship's review collector no longer reads a reviewer's own scope list or list of passed checks as findings (GST-3201).** A bullet such as "a HIGH finding still refused" in the list of what the tests cover, or "`missing_verdict` can't occur alongside P0 or P1 findings" in the list of checks that held, collected as a HIGH or P0 blocker even though the reviewer declared `REVIEW_VERDICT: blocking=0`. When the reviewer declares no blockers, a plain bullet with no file:line, no Severity or Blocks field and no severity other than one naming findings as a kind keeps its label but is marked for the lead to lower with a written reason, so a bullet that really reports an open blocker still holds the merge until someone judges it.
|
||||
- **Merge-queue stack tests fail when git cannot list a PR's added test files (GST-3201).** That failure used to read as "no new test files" and pass the slot.
|
||||
@@ -9,7 +9,8 @@
|
||||
* with the test command (`bun --no-env-file --config=/dev/null test` by
|
||||
* default; `--stack-tests-command "<cmd>"` for another runner, the files
|
||||
* appended), and the worktree is removed. A PR that adds no test file is
|
||||
* reported as such and never fails.
|
||||
* reported as such and never fails; one whose added files git cannot list
|
||||
* fails, since an unread list is not an empty one.
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
@@ -25,10 +26,11 @@ const TEST_FILE = /[._](?:test|spec)\.[cm]?[jt]sx?$/;
|
||||
|
||||
export interface StackTestResult { pr: string; prefix: number; files: string[]; ok: boolean; detail: string }
|
||||
|
||||
/** Test files the PR adds relative to the base (added, not modified). */
|
||||
export function newTestFiles(repoRoot: string, baseSha: string, head: string): string[] {
|
||||
/** Test files the PR adds relative to the base (added, not modified), or git's error when it cannot list them. */
|
||||
export function newTestFiles(repoRoot: string, baseSha: string, head: string): { files: string[] } | { error: string } {
|
||||
const r = git(repoRoot, ['diff', '--name-only', '--diff-filter=A', `${baseSha}...${head}`]);
|
||||
return r.ok ? r.out.split('\n').map(s => s.trim()).filter(f => f && TEST_FILE.test(f)) : [];
|
||||
if (!r.ok) return { error: r.err.slice(0, 160) || `git diff exited ${r.status}` };
|
||||
return { files: r.out.split('\n').map(s => s.trim()).filter(f => f && TEST_FILE.test(f)) };
|
||||
}
|
||||
|
||||
export function stackTests(repoRoot: string, baseSha: string, slots: TrainSlot[], prefixes: TrainPrefix[], command: string[] = DEFAULT_STACK_TEST_COMMAND, timeoutMs = 600_000): StackTestResult[] {
|
||||
@@ -36,7 +38,9 @@ export function stackTests(repoRoot: string, baseSha: string, slots: TrainSlot[]
|
||||
for (const [i, slot] of slots.entries()) {
|
||||
const prefix = prefixes[i + 1];
|
||||
if (!prefix) break;
|
||||
const files = newTestFiles(repoRoot, baseSha, slot.head);
|
||||
const listed = newTestFiles(repoRoot, baseSha, slot.head);
|
||||
if ('error' in listed) { results.push({ pr: slot.pr, prefix: prefix.index, files: [], ok: false, detail: `cannot list the test files it adds: ${listed.error}` }); continue; }
|
||||
const { files } = listed;
|
||||
if (!files.length) { results.push({ pr: slot.pr, prefix: prefix.index, files, ok: true, detail: 'no new test files' }); continue; }
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-stack-tests-'));
|
||||
try {
|
||||
@@ -58,6 +62,7 @@ export function stackTests(repoRoot: string, baseSha: string, slots: TrainSlot[]
|
||||
|
||||
/** One `STACK_TESTS:` line per slot. */
|
||||
export function stackTestLine(r: StackTestResult): string {
|
||||
if (!r.ok && !r.files.length) return `STACK_TESTS: #${r.pr} T${r.prefix} FAIL: ${r.detail}`;
|
||||
if (!r.files.length) return `STACK_TESTS: #${r.pr} T${r.prefix} no new test files`;
|
||||
return `STACK_TESTS: #${r.pr} T${r.prefix} ${r.files.length} new test file(s) ${r.ok ? 'pass' : 'FAIL'} (${r.files.join(', ')})${r.ok ? '' : `: ${r.detail}`}`;
|
||||
}
|
||||
@@ -575,7 +575,7 @@ export async function buildTrain(ctx: QueueContext, o: BuildOptions): Promise<Tr
|
||||
const red = results.filter(r => !r.ok);
|
||||
if (red.length) {
|
||||
const again = `${Q} train build ${prsArgs(t.slots, t.explicit_order)} --dry-run --stack-tests`;
|
||||
return outFailed({ code: 'QUEUE_TRAIN_STACK_RED', message: red.map(r => `#${r.pr}'s new tests fail on the integrated prefix T${r.prefix} (${r.files.join(', ')})`).join('; ') + '; they pass on the PR head alone only if nothing before it in the train changes what they read' },
|
||||
return outFailed({ code: 'QUEUE_TRAIN_STACK_RED', message: red.map(r => (r.files.length ? `#${r.pr}'s new tests fail on the integrated prefix T${r.prefix} (${r.files.join(', ')})` : `#${r.pr}: ${r.detail}`)).join('; ') + '; they pass on the PR head alone only if nothing before it in the train changes what they read' },
|
||||
[...dry, 'DRY-RUN: nothing written, pushed or dispatched'], `fix ${red.map(r => `#${r.pr}'s tests against T${r.prefix}`).join(' and ')}, or reorder with --order, then ${again}`, 'none');
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1114,7 +1114,10 @@ ones, and the arm lost blockers. `bin/gstack-review-dispatch` closes both ends:
|
||||
`blocking=0` still collects as CRITICAL, as a summary candidate the lead lowers with a reason. An automatic relabel
|
||||
was tried and removed (gstack#4353): beside a denial line such as "No other CRITICAL issues.", it let a real
|
||||
unanchored FIXABLE blocker drop to FIXABLE while the count still agreed. A reader that undercounts a real blocker
|
||||
therefore loses nothing without a written reason. `verify` re-parses each output the way its
|
||||
therefore loses nothing without a written reason. The same holds for a reader's scope list (GST-3201): when the valid line declares
|
||||
`blocking=0`, a single-line list item with no structure (no ` | ` table cell, Severity or Blocks field, or file:line) and no
|
||||
severity label other than one naming findings as a kind ("a HIGH finding still refused", "P0 or P1 findings") keeps its own
|
||||
label and is marked a summary candidate, so the lead lowers it with a reason; "Two P0 findings remain unfixed" has the same shape and still blocks until judged. `verify` re-parses each output the way its
|
||||
render asked, prints `VERIFY_VERDICT: agree=<a> asked=<n>` and one `VERDICT <reader> AGREES|DISAGREES|MISSING|MALFORMED`
|
||||
row per checked reply. These are informational: a disagreeing count has already added or marked its rows, so
|
||||
they never make a ledger dirty. When collect prints `agrees=no` or `declared=malformed`, read the reply, then judge the
|
||||
|
||||
+3
-1
@@ -1166,7 +1166,9 @@ to the base, with a name ending in `.test`, `_test`, `.spec` or `_spec` and a
|
||||
JS or TS extension, bun's own rule, so a helper or fixture under `test/` is not
|
||||
one) on the integrated prefix that includes it, T<k>, in a scratch worktree, and prints
|
||||
`STACK_TESTS: #<pr> T<k> <n> new test file(s) pass|FAIL (<files>)` or `no new
|
||||
test files`. A failure is `QUEUE_TRAIN_STACK_RED` (nothing written, pushed or
|
||||
test files`. When git cannot list the files a PR adds, the line is
|
||||
`STACK_TESTS: #<pr> T<k> FAIL: cannot list the test files it adds: <git error>`,
|
||||
since an unread list is not an empty one. A failure is `QUEUE_TRAIN_STACK_RED` (nothing written, pushed or
|
||||
dispatched): the tests pass on the PR head alone only because nothing before
|
||||
it in the train had changed what they read, so fix them against that prefix or
|
||||
reorder with `--order`. The default command is `bun --no-env-file
|
||||
|
||||
+1
-1
@@ -670,7 +670,7 @@ The skill no longer installs, and `./setup` removes an existing gstack-owned `in
|
||||
|
||||
It asks once before running any repo command (`gstack-checks trust`) and asks again when a command changes. A touched file no test covers is `UNMAPPED`, a selection that ran zero tests is `ZERO_RUN`, and neither counts as green. A missing tool or service is reported, not worked around. It stops after five runs or when the same failure comes back twice, and asks for a focused failing test and a root-cause trace before another fix to a stubborn red. When everything passes, `gstack-checks status` prints `READY_FOR_SHIP tree=<tree>` without rerunning anything, and you run `/ship`. It never reviews, bumps versions, pushes or opens a PR.
|
||||
|
||||
Since Claude Code 2.1.286, any installed skill named `verify` makes Claude Code tell the model to run it right before each `git commit` (except docs-only and tests-only commits). When the model calls `/verify` that way, without you typing it or asking for checks, it gets a quick check-only pass instead of the loop: one `gstack-checks quick` call that runs the same checks and selected tests, edits nothing, adds no tests, asks nothing and skips gstack's startup steps, then ends with one line, `VERIFY PASS`, `VERIFY FAIL` or `VERIFY skipped: <reason>`. A skip (no `origin` to select tests against, commands you haven't trusted yet, nothing declared) lets the commit go ahead; a `FAIL` makes the model stop and ask you before committing. The arguments `quick` or `pre-commit` ask for the same pass. `gstack-config set verify_before_commit off` turns the pass into an immediate `VERIFY skipped`; installing with `./setup --prefix` (`/gstack-verify`) removes Claude Code's instruction altogether.
|
||||
Since Claude Code 2.1.286, any installed skill named `verify` makes Claude Code tell the model to run it right before each `git commit` (except docs-only and tests-only commits). When the model calls `/verify` that way, without you typing it or asking for checks, it gets a quick check-only pass instead of the loop: one `gstack-checks quick` call that runs the same checks and selected tests, edits nothing, adds no tests, asks nothing and skips gstack's startup steps, then ends with one line, `VERIFY PASS`, `VERIFY FAIL` or `VERIFY skipped: <reason>`. A check that runs out of time is a `FAIL` (`<check> TIMEOUT`). The checks share one `--timeout` budget, and the test stage starts with what is left. A skip (no `origin` to select tests against, commands you haven't trusted yet, nothing declared) lets the commit go ahead; a `FAIL` makes the model stop and ask you before committing. The arguments `quick` or `pre-commit` ask for the same pass. `gstack-config set verify_before_commit off` turns the pass into an immediate `VERIFY skipped`; installing with `./setup --prefix` (`/gstack-verify`) removes Claude Code's instruction altogether.
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -7165,7 +7165,9 @@ rule; a helper or fixture under `test/` is not one) on the integrated prefix tha
|
||||
T<k> (the base plus every slot up to and including that PR), in a scratch
|
||||
worktree, and they failed. The `STACK_TESTS: #<pr> T<k> <n> new test file(s)
|
||||
FAIL (<files>): <failing assertion>` line names the files and the runner's
|
||||
failing lines. Those tests pass on the PR head alone only because nothing
|
||||
failing lines. A `STACK_TESTS: #<pr> T<k> FAIL: cannot list the test files it
|
||||
adds: <git error>` line means git could not list the files the PR adds; fix
|
||||
what the git error names and rerun the dry run. Otherwise, those tests pass on the PR head alone only because nothing
|
||||
before it in the train had changed what they read (a PR stacked on another
|
||||
PR's test files or fixtures hit this on 2026-10-26). Nothing was written,
|
||||
pushed or dispatched; the ordinary dry run (without `--stack-tests`) does not
|
||||
|
||||
+10
-5
@@ -16,7 +16,9 @@
|
||||
* (GST-3150). Same checks and selection as run, but it never asks for consent:
|
||||
* an untrusted command set, a missing policy source or the verify_before_commit
|
||||
* switch ends it with one `VERIFY skipped: <reason>` line, and a not-run result
|
||||
* (ENV, TIMEOUT, UNMAPPED, TOO_BROAD) is named, never a red.
|
||||
* (ENV, UNMAPPED, TOO_BROAD) is named, never a red. A TIMEOUT is a red: a check
|
||||
* that never finished proved nothing. The checks share one deadline, and the
|
||||
* test stage starts with what is left (its runner can still run past it).
|
||||
*/
|
||||
import { createHash } from 'node:crypto';
|
||||
import * as fs from 'node:fs';
|
||||
@@ -255,7 +257,8 @@ export function testVerdict(r: CheckResult): Pick<Outcome, 'verdict' | 'detail'
|
||||
return { verdict: 'INCOMPLETE', detail };
|
||||
}
|
||||
|
||||
export interface RunOptions { root: string; only?: string[]; timeoutMs: number; env?: NodeJS.ProcessEnv }
|
||||
/** deadline (epoch ms) bounds the whole run: each check gets the time left, and one that cannot start before it is a TIMEOUT. */
|
||||
export interface RunOptions { root: string; only?: string[]; timeoutMs: number; deadline?: number; env?: NodeJS.ProcessEnv }
|
||||
export interface Loaded { ctx: PregateContext; checks: Check[]; set: string }
|
||||
const basisOf = (ctx: PregateContext) => sha256(JSON.stringify([ctx.baseRef, ctx.baseSha, ctx.configLabel, ctx.configDigest ?? '']));
|
||||
|
||||
@@ -302,8 +305,10 @@ export async function runChecks(l: Loaded, o: RunOptions): Promise<{ outcomes: O
|
||||
const dir = path.join(path.dirname(file), 'logs');
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
const outcomes: Outcome[] = [];
|
||||
for (const check of l.checks) if (want(check.name)) outcomes.push(await runCommand(l.ctx, check, dir, o.timeoutMs));
|
||||
if (want(TESTS)) outcomes.push(await runTests(l.ctx));
|
||||
const left = () => Math.min(o.timeoutMs, (o.deadline ?? Infinity) - Date.now());
|
||||
const spent = (name: string): Outcome => ({ name, verdict: 'TIMEOUT', ms: 0, detail: 'not started: the run\'s time budget ran out', lines: [] });
|
||||
for (const check of l.checks) if (want(check.name)) outcomes.push(left() > 0 ? await runCommand(l.ctx, check, dir, left()) : spent(check.name));
|
||||
if (want(TESTS)) outcomes.push(left() > 0 ? await runTests({ ...l.ctx, timeoutMs: Math.min(l.ctx.timeoutMs, left()) }) : spent(TESTS));
|
||||
const after = wtreeFingerprint({ cwd: o.root, env: o.env });
|
||||
const moved = 'error' in after || after.tree !== l.ctx.wtree;
|
||||
const final = moved ? outcomes.map(x => (x.verdict === 'PASS' ? { ...x, verdict: 'INCOMPLETE' as Verdict, detail: 'the working tree changed during the run; rerun' } : x)) : outcomes;
|
||||
@@ -321,7 +326,7 @@ export async function runChecks(l: Loaded, o: RunOptions): Promise<{ outcomes: O
|
||||
|
||||
/** The quick pass's last line: FAIL on a red, PASS when something passed and nothing is red, else skipped. */
|
||||
export function quickLine(outcomes: readonly Outcome[]): { line: string; red: Outcome[] } {
|
||||
const red = outcomes.filter(o => o.verdict === 'FAIL' || o.verdict === 'ZERO_RUN' || o.verdict === 'INCOMPLETE');
|
||||
const red = outcomes.filter(o => o.verdict === 'FAIL' || o.verdict === 'ZERO_RUN' || o.verdict === 'INCOMPLETE' || o.verdict === 'TIMEOUT');
|
||||
const passed = outcomes.filter(o => o.verdict === 'PASS');
|
||||
const notRun = outcomes.filter(o => o.verdict !== 'PASS' && !red.includes(o)).map(o => `${o.name} ${o.verdict}`);
|
||||
const secs = (outcomes.reduce((sum, o) => sum + o.ms, 0) / 1000).toFixed(1);
|
||||
|
||||
@@ -329,6 +329,7 @@ export function parseFindings(reader: string, output: string, fixableRaises = tr
|
||||
|
||||
export interface VerdictCheck { state: 'valid' | 'missing' | 'malformed'; parsed: number; declared?: number; agrees?: boolean; why?: string }
|
||||
export interface ReplyFinding { label: string; raw: string; summary?: true; synthetic?: true }
|
||||
const FINDING_KIND = new RegExp(`(?<![A-Za-z0-9])${SEV_LIST}\\s+findings?(?![A-Za-z0-9])(?!\\s*:)`, 'g');
|
||||
const blockingRow = (f: { label: string; raw: string }) => (rank(f.label) ?? 0) >= 4 && !statesNoBlocks(f.raw);
|
||||
|
||||
/**
|
||||
@@ -337,7 +338,10 @@ const blockingRow = (f: { label: string; raw: string }) => (rank(f.label) ?? 0)
|
||||
* line lowers nothing by itself. Declared more than parsed adds CRITICAL rows, one per unmatched at= location and one for the
|
||||
* rest; declared fewer marks each blocking row with no anchor (no list item, heading, field, lead label or JSON; a file:line in prose
|
||||
* is not one, and in a NO FINDINGS reply a list item is not one either, GST-2972) and no Blocks: yes a summary candidate, whether a CRITICAL or P0 word or the native voice's FIXABLE raise made it block. It still
|
||||
* blocks until lead judgment lowers it with a reason.
|
||||
* blocks until lead judgment lowers it with a reason. A reply whose valid line declares blocking=0 marks the same way, at its own label,
|
||||
* each single-line bullet with no structure, no file:line and no label beyond severities naming findings as a kind (`- a HIGH finding still
|
||||
* refused;`, `- \`missing_verdict\` can't occur alongside P0 or P1 findings.`, #4518 and #4519): a scope list or passed check the lead
|
||||
* may lower with a reason, though `- Two P0 findings remain unfixed.` has the same shape, so it still blocks until judged.
|
||||
*/
|
||||
export function parseReply(reader: string, output: string, fixableRaises = true, asked = false): { findings: ReplyFinding[]; verdict?: VerdictCheck } {
|
||||
const verdict = asked ? parseVerdict(output) : undefined;
|
||||
@@ -350,7 +354,8 @@ export function parseReply(reader: string, output: string, fixableRaises = true,
|
||||
}
|
||||
const n = verdict.blocking;
|
||||
const parsed = rows.filter(blockingRow).length;
|
||||
const findings: ReplyFinding[] = rows.map(f => ({ label: f.label, raw: f.raw, ...(n < parsed && blockingRow(f) && !f.anchored && !BLOCKS_YES.test(f.raw) ? { summary: true as const } : {}) }));
|
||||
const kindOnly = (f: { raw: string; shaped: boolean }) => n === 0 && !f.shaped && /^[-*+]\s/.test(f.raw) && !f.raw.includes(' | ') && !ANY_LABEL.test(f.raw.replace(FINDING_KIND, ' '));
|
||||
const findings: ReplyFinding[] = rows.map(f => ({ label: f.label, raw: f.raw, ...((n < parsed && blockingRow(f) && !f.anchored && !BLOCKS_YES.test(f.raw)) || kindOnly(f) ? { summary: true as const } : {}) }));
|
||||
if (n > parsed) {
|
||||
const loci = new Set(rows.filter(blockingRow).map(f => lineLocus(f.raw)));
|
||||
const unmatched = verdict.at.map(a => a.replace(/^\.\//, '')).filter(a => !loci.has(a)).slice(0, n - parsed);
|
||||
|
||||
@@ -102,12 +102,14 @@ describe('testVerdict', () => {
|
||||
});
|
||||
|
||||
describe('quickLine', () => {
|
||||
// Value: protects=the pre-commit line is red on a zero-run or incomplete selection and never green when nothing reached a result;
|
||||
// fails_when=ZERO_RUN/INCOMPLETE leave the red set, or an all-ENV/TIMEOUT run reads VERIFY PASS; why_new=the fixture test only
|
||||
// Value: protects=the pre-commit line is red on a zero-run, incomplete or timed-out check and never green when nothing reached a
|
||||
// result; fails_when=ZERO_RUN/INCOMPLETE/TIMEOUT leave the red set, or an all-ENV run reads VERIFY PASS; why_new=the fixture test only
|
||||
// reaches PASS, FAIL and UNMAPPED; seam=none; no_claim=the wording after the VERIFY token
|
||||
test('classifies red, not-run and passed outcomes', () => {
|
||||
const o = (name: string, verdict: Outcome['verdict']): Outcome => ({ name, verdict, ms: 100, detail: 'exit 1', lines: [] });
|
||||
expect(quickLine([o('typecheck', 'ENV'), o('tests', 'TIMEOUT')])).toEqual({ red: [], line: 'VERIFY skipped: nothing ran to a result (typecheck ENV, tests TIMEOUT)' });
|
||||
expect(quickLine([o('typecheck', 'ENV'), o('tests', 'ENV')])).toEqual({ red: [], line: 'VERIFY skipped: nothing ran to a result (typecheck ENV, tests ENV)' });
|
||||
expect(quickLine([o('typecheck', 'PASS'), o('tests', 'TIMEOUT')]).line).toBe('VERIFY FAIL: tests TIMEOUT');
|
||||
expect(quickLine([o('typecheck', 'ENV'), o('tests', 'TIMEOUT')]).red.map(r => r.name)).toEqual(['tests']);
|
||||
expect(quickLine([]).line).toBe('VERIFY skipped: nothing ran to a result (no checks)');
|
||||
expect(quickLine([o('typecheck', 'PASS'), o('tests', 'ZERO_RUN')]).line).toMatch(/^VERIFY FAIL: tests ZERO_RUN/);
|
||||
expect(quickLine([o('tests', 'INCOMPLETE')]).red.map(r => r.name)).toEqual(['tests']);
|
||||
@@ -241,6 +243,22 @@ describe('gstack-checks on a fixture repo', () => {
|
||||
expect(off.out).toStartWith('VERIFY skipped: verify_before_commit is off');
|
||||
}, 60_000);
|
||||
|
||||
// Value: protects=a check that times out fails the pre-commit pass, and the pass ends with its line inside its --timeout, so a
|
||||
// 10-minute Bash call always sees it; fails_when=a timed-out check reads VERIFY PASS (not run), or each check gets the whole
|
||||
// --timeout again; why_new=GST-3201; seam=the real quick pass on the fixture repo with a typecheck that never exits; no_claim=
|
||||
// Claude Code's own Bash limit
|
||||
test('quick: a timeout is a red, and --timeout bounds the whole pass', () => {
|
||||
const s = setup();
|
||||
commit(s.repo, { 'lib/util.js': 'exports.add = (a, b) => b + a;\n' });
|
||||
expect(s.checks(['trust']).status).toBe(0);
|
||||
write(s.repo.work, 'scripts/typecheck.js', 'setTimeout(() => {}, 30000);\n');
|
||||
const started = Date.now();
|
||||
const slow = s.checks(['quick', '--timeout', '3']);
|
||||
expect(slow.status).toBe(1);
|
||||
expect(slow.out.trim().split('\n').at(-1)).toMatch(/^VERIFY FAIL: typecheck TIMEOUT, tests TIMEOUT/);
|
||||
expect(Date.now() - started).toBeLessThan(20_000);
|
||||
}, 60_000);
|
||||
|
||||
// Value: protects=a selection that runs zero tests is not green; fails_when=exit 0 with no test counted reads PASS; why_new=the
|
||||
// verdict is new; seam=pregate's runner summary (a bats stub printing an empty TAP plan); no_claim=node --test, which counts
|
||||
// each file as a test, so an empty file there is not a zero run
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
/**
|
||||
* The collector read a reader's own scope list or passed-checks bullets as findings (GST-3201). On #4518 (p1, native adversarial
|
||||
* reader) "- a HIGH finding still refused;" in its list of what the tests cover collected as HIGH, and on #4519 (round 2, Red Team)
|
||||
* "- `missing_verdict` can't occur alongside P0 or P1 findings." in its list of checks that held collected as P0. Both replies
|
||||
* declared REVIEW_VERDICT: blocking=0, and both forced an owner ruling, since a list item was never a summary candidate the lead could
|
||||
* lower. Such a bullet now keeps its label and becomes one. Excerpts of the captured outputs below, as returned.
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import { judge, parseReply, type MergedLine } from '../lib/review-ledger';
|
||||
|
||||
const CAPTURED_4518_P1_ADVERSARIAL = [
|
||||
'REVIEWED_TREE: 5733be6b184a1f8387580c089b4c0413beeb304b /workspace/gstack',
|
||||
'REVIEW_VERDICT: blocking=0',
|
||||
'',
|
||||
"I found no blocking findings. The weakest point is that the hand-off replaces the whole body from a fresh read, so it is only as safe as the agent's habit of publishing each file before the next gstack step.",
|
||||
'',
|
||||
'**Scope.** I read the full non-fixture diff from f13dbb0d to HEAD (6 commits, GST-3183): `bin/gstack-post`, `bin/gstack-ship-receipt`, `lib/host-publish.ts`, `lib/gstack-post.ts`, `lib/greptile-early.ts`, `scripts/lib/windows-curation.ts`, the ship section prose and templates, `docs/troubleshooting.md` and the changelog fragment. I reviewed the fixtures and tests in summary mode only (diff stat and test names, no payload bytes): `test/capy-host-publish.test.ts` (+320, new), `test/greptile-early.test.ts` (+11), and a size-ledger JSON. They cover:',
|
||||
'- that no gh write happens under `GSTACK_CAPY=1` for stub, stop, claim, carry and every gstack-post op;',
|
||||
'- a negative control without Capy;',
|
||||
'- two-repo, fork (`gh repo set-default` or `GH_REPO`) and GitHub Enterprise destinations;',
|
||||
"- refusal when the repository can't be resolved;",
|
||||
'- a HIGH finding still refused;',
|
||||
'- a detached HEAD on pr-create;',
|
||||
"- a hand-off that can't be written;",
|
||||
'- greptile-early exiting 3 on a hand-off.',
|
||||
'',
|
||||
'**Findings**',
|
||||
'',
|
||||
"2. **claim's re-read check is now always true on Capy, so two runs can claim one grant** (`bin/gstack-ship-receipt:1152-1153`). INVESTIGATE. Under host publishing, `again` is built locally from `read.body` plus this run's own marker. That makes the `first?.run !== run` re-read check a tautology. Two runs racing on one grant both print `GRANT_CLAIMED` and both dispatch a granted round; only a later `history` call catches it. This is documented, but it weakens a merge-check guard. A fix needs a design call, for example a second `claim --verify` step after publishing. Severity: Low. Blocks: no.",
|
||||
'',
|
||||
'Recommendation: Ship as-is because the strongest finding, the stale whole-body hand-off at `bin/gstack-ship-receipt:1069` overwriting an unpublished run entry, only happens when the agent skips the documented publish-before-next-step order, and no test or production path fails when that order is followed.',
|
||||
].join('\n');
|
||||
|
||||
const CAPTURED_4519_R2_RED_TEAM = [
|
||||
'REVIEWED_TREE: 42315eacf53a28493d84f02f9c3701e295a04733 /workspace/gstack',
|
||||
'REVIEW_VERDICT: blocking=0',
|
||||
'',
|
||||
'I found 2 informational findings and nothing that blocks the merge. I read the non-fixture source diff in full. I reviewed the three changed test files in summary mode only, by their stats and test titles, so that part of the coverage is reduced. They add 118 lines covering: the final-failure-decides ordering, 402 in a request id or a duration versus 402 as a status, Claude Code error subtypes and its budget stop, reviewer-file versus closing-message selection (GST-3188), and refusing a prompt that is also an output file.',
|
||||
'',
|
||||
'The integration paths I checked hold up:',
|
||||
"- Claude Code's budget stop reaches `execution()` as `ERROR: <CLAUDE_CODE_BUDGET_STOP …>`, the last stderr line, so `codexErrorLines` picks it up. It gets matched before `QUOTA_FAILURE`.",
|
||||
"- The phase names and hex sha in `autoplan-prompts` `RESULT_FORMAT` match the runner's receipt regex. `parseReceipt` doesn't lowercase the expected phase, but the phases are already lowercase.",
|
||||
"- `missing_verdict` can't occur alongside P0 or P1 findings.",
|
||||
'',
|
||||
JSON.stringify({ severity: 'INFORMATIONAL', confidence: 5, path: 'lib/outside-review-result.ts', line: 73, category: 'red-team', summary: "The tighter BUDGET_FAILURE regex no longer recognizes some real 402 spellings that the old one caught, such as 'Request failed with status code (402)' and 'got 402 from proxy' (both checked at runtime: false). Those now fall through to execution_failed. The gate still fails closed, but the user gets generic advice instead of the budget_exhausted fix.", fix: "Allow '(' in the separator class, e.g. [\\s:=\"'(]{0,4}, or accept that these spellings are rare.", fingerprint: 'lib/outside-review-result.ts:73:red-team', specialist: 'red-team' }),
|
||||
].join('\n');
|
||||
|
||||
const rows = (reader: string, out: string) => parseReply(reader, out, true, true).findings.map(f => [f.label, f.raw.slice(0, 40), !!f.summary]);
|
||||
const reply = (verdict: string, line: string) => ['REVIEWED_TREE: 42315eacf53a28493d84f02f9c3701e295a04733 /workspace/gstack', verdict, '', line].join('\n');
|
||||
const ZERO = 'REVIEW_VERDICT: blocking=0';
|
||||
/** The merged lines collect writes for a reply, then lead judgment lowering every line to INFORMATIONAL with a reason. */
|
||||
const lowered = (reader: string, out: string) => {
|
||||
const merged: MergedLine[] = parseReply(reader, out, true, true).findings.map((f, i) => ({ id: `${reader}#${i + 1}`, reader, label: f.label, raw: f.raw, output_sha256: 'x', ...(f.summary ? { summary_candidate: true as const } : {}) }));
|
||||
return judge(merged, merged.map(m => ({ id: m.id, severity: 'INFORMATIONAL', reason: 'the reader lists what passed; it reports no defect' })));
|
||||
};
|
||||
|
||||
describe('a reader\'s scope and passed-checks bullets under a blocking=0 verdict', () => {
|
||||
// Value: protects=a bullet that names severities only as a kind of finding, in a reply declaring blocking=0, is a summary candidate
|
||||
// the lead may lower with a reason, and still blocks until then; fails_when=#4518's or #4519's bullet cannot be lowered without an
|
||||
// owner ruling again, or a bullet asserting a blocker ("Two P0 findings remain") loses its label; why_new=both forced owner
|
||||
// rulings (GST-3201); seam=parseReply with the verdict ask and judge, as collect and the lead use them; no_claim=/ship's Step 11 prose
|
||||
test('#4518: the scope-list bullet keeps HIGH as a summary candidate; the real finding keeps Severity: Low', () => {
|
||||
expect(rows('adversarial-native', CAPTURED_4518_P1_ADVERSARIAL)).toEqual([
|
||||
['HIGH', '- a HIGH finding still refused;', true],
|
||||
['Low', "2. **claim's re-read check is now always", false],
|
||||
]);
|
||||
});
|
||||
|
||||
test('#4519: the passed-check bullet keeps P0 as a summary candidate, so the lead can lower it with a reason', () => {
|
||||
const r = parseReply('red-team', CAPTURED_4519_R2_RED_TEAM, true, true);
|
||||
expect(r.findings.map(f => [f.label, !!f.summary])).toEqual([['P0', true], ['INFORMATIONAL', false]]);
|
||||
expect(r.verdict).toEqual({ state: 'valid', parsed: 1, declared: 0, agrees: false });
|
||||
expect(lowered('red-team', CAPTURED_4519_R2_RED_TEAM)).toMatchObject({ ok: true, dismissed: ['red-team#1'], downgraded: [] });
|
||||
expect(lowered('adversarial-native', CAPTURED_4518_P1_ADVERSARIAL)).toMatchObject({ dismissed: ['adversarial-native#1'] });
|
||||
});
|
||||
|
||||
test('negative controls: a bullet asserting a blocker keeps its label and blocks until judged; other shapes collect as before', () => {
|
||||
for (const line of ['- Two P0 findings remain unfixed.', '- The P0 finding is that the gate drops rows.']) {
|
||||
expect(rows('red-team', reply(ZERO, line))).toEqual([['P0', line.slice(0, 40), true]]);
|
||||
expect(parseReply('red-team', reply(ZERO, line), true, true).verdict).toMatchObject({ parsed: 1, agrees: false });
|
||||
}
|
||||
expect(rows('red-team', reply('REVIEW_VERDICT: blocking=1', "- `missing_verdict` can't occur alongside P0 or P1 findings."))[0]).toEqual(['P0', "- `missing_verdict` can't occur alongsid", false]);
|
||||
expect(parseReply('red-team', "- `missing_verdict` can't occur alongside P0 or P1 findings.", true, false).findings.map(f => [f.label, !!f.summary])).toEqual([['P0', false]]);
|
||||
expect(rows('adversarial-native', CAPTURED_4518_P1_ADVERSARIAL.replace('- a HIGH finding still refused;', '- a HIGH finding still refused at lib/host-publish.ts:40;'))[0]![2]).toBe(false);
|
||||
expect(rows('red-team', reply(ZERO, '- P0: the gate drops a HIGH finding.'))[0]).toEqual(['P0', '- P0: the gate drops a HIGH finding.', false]);
|
||||
expect(rows('red-team', reply(ZERO, '- a HIGH finding: the cache key drops the tree.'))[0]![2]).toBe(false);
|
||||
expect(rows('red-team', reply(ZERO, '- a HIGH finding still refused. Severity: High'))[0]![2]).toBe(false);
|
||||
expect(rows('adversarial-native', reply(ZERO, '- a HIGH finding is dropped, FIXABLE.'))[0]).toEqual(['CRITICAL', '- a HIGH finding is dropped, FIXABLE.', false]);
|
||||
expect(rows('red-team', reply(ZERO, '- the HIGH path crashes when the ledger is empty.'))[0]![2]).toBe(false);
|
||||
expect(lowered('red-team', reply(ZERO, '- P0: the gate drops a HIGH finding.'))).toMatchObject({ ok: false, downgraded: ['red-team#1'] });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,29 @@
|
||||
/**
|
||||
* train build --dry-run --stack-tests read a git failure listing a PR's added files as "no new test files" and passed the slot
|
||||
* (GST-3201; the round-1 /ship note on #4502, crew/lib/ship-queue-stack-tests.ts). An unread list is not an empty one.
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as path from 'node:path';
|
||||
import { newTestFiles, stackTestLine, stackTests } from '../crew/lib/ship-queue-stack-tests';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const MISSING = '0000000000000000000000000000000000000001';
|
||||
|
||||
describe('stack tests fail closed when git cannot list the added files', () => {
|
||||
// Value: protects=a slot whose added test files cannot be listed is red, not "no new test files"; fails_when=a git failure reads
|
||||
// as an empty list again; why_new=the fail-open had no test; seam=the real git diff on this checkout with a base that does not
|
||||
// exist; no_claim=running the selected files on a prefix worktree
|
||||
test('a base git cannot resolve is an error, and the slot fails with it', () => {
|
||||
const listed = newTestFiles(ROOT, MISSING, 'HEAD');
|
||||
expect('error' in listed && listed.error).toBeTruthy();
|
||||
const [r] = stackTests(ROOT, MISSING, [{ pr: '1', head: 'HEAD', ready_at: '', screen: 'none' } as any], [{ index: 0, commit: 'HEAD', tree: '', pr: null }, { index: 1, commit: 'HEAD', tree: '', pr: '1' }]);
|
||||
expect(r!.ok).toBe(false);
|
||||
expect(stackTestLine(r!)).toStartWith('STACK_TESTS: #1 T1 FAIL: cannot list the test files it adds: ');
|
||||
});
|
||||
|
||||
test('negative control: a readable empty list still passes as no new test files', () => {
|
||||
expect(newTestFiles(ROOT, 'HEAD', 'HEAD')).toEqual({ files: [] });
|
||||
const [r] = stackTests(ROOT, 'HEAD', [{ pr: '1', head: 'HEAD', ready_at: '', screen: 'none' } as any], [{ index: 0, commit: 'HEAD', tree: '', pr: null }, { index: 1, commit: 'HEAD', tree: '', pr: '1' }]);
|
||||
expect(stackTestLine(r!)).toBe('STACK_TESTS: #1 T1 no new test files');
|
||||
});
|
||||
});
|
||||
@@ -181,11 +181,11 @@ const CHILD_PROCESS = String.raw`['"](?:node:)?child_process['"]`;
|
||||
|
||||
/** Whether this source takes spawnSync from node:child_process (named, aliased, namespace, default or require). */
|
||||
export function importsRawSpawnSync(text: string): boolean {
|
||||
for (const m of text.matchAll(new RegExp(String.raw`import\s*\{([^}]*)\}\s*from\s*${CHILD_PROCESS}`, 'g'))) {
|
||||
for (const m of text.matchAll(new RegExp(String.raw`import\s*(?:\w+\s*,\s*)?\{([^}]*)\}\s*from\s*${CHILD_PROCESS}`, 'g'))) {
|
||||
if (/(?:^|[,{\s])spawnSync\b/.test(m[1].replace(/\btype\s+spawnSync\b/g, ''))) return true;
|
||||
}
|
||||
for (const m of text.matchAll(new RegExp(String.raw`import\s+(?:\*\s+as\s+)?(\w+)\s+from\s*${CHILD_PROCESS}`, 'g'))) {
|
||||
if (new RegExp(String.raw`\b${m[1]}\.spawnSync\b`).test(text)) return true;
|
||||
for (const m of text.matchAll(new RegExp(String.raw`import\s+(?:\*\s+as\s+)?(\w+)(?:\s*,\s*(?:\{[^}]*\}|\*\s+as\s+(\w+)))?\s+from\s*${CHILD_PROCESS}`, 'g'))) {
|
||||
if (m.slice(1).some(name => name && new RegExp(String.raw`\b${name}\.spawnSync\b`).test(text))) return true;
|
||||
}
|
||||
return new RegExp(String.raw`require\(\s*${CHILD_PROCESS}\s*\)`).test(text) && /\bspawnSync\b/.test(text);
|
||||
}
|
||||
@@ -227,6 +227,10 @@ describe('raw spawnSync import ratchet', () => {
|
||||
expect(importsRawSpawnSync("import {\n spawn,\n spawnSync,\n} from 'node:child_process';")).toBe(true);
|
||||
expect(importsRawSpawnSync("import * as cp from 'node:child_process';\ncp.spawnSync('git', [], { timeout: 1000 });")).toBe(true);
|
||||
expect(importsRawSpawnSync("const { spawnSync } = require('child_process');")).toBe(true);
|
||||
expect(importsRawSpawnSync("import cp, { spawnSync } from 'node:child_process';")).toBe(true);
|
||||
expect(importsRawSpawnSync("import cp, { spawn } from 'node:child_process';\ncp.spawnSync('git', [], { timeout: 1000 });")).toBe(true);
|
||||
expect(importsRawSpawnSync("import cp, * as all from 'node:child_process';\nall.spawnSync('git', [], { timeout: 1000 });")).toBe(true);
|
||||
expect(importsRawSpawnSync("import cp, { spawn } from 'node:child_process';\ncp.spawn('git', []);")).toBe(false);
|
||||
expect(importsRawSpawnSync("import type { SpawnSyncReturns } from 'node:child_process';")).toBe(false);
|
||||
expect(importsRawSpawnSync("import { spawn, type spawnSync } from 'node:child_process';")).toBe(false);
|
||||
expect(importsRawSpawnSync("import * as cp from 'node:child_process';\ncp.spawn('git', []);")).toBe(false);
|
||||
|
||||
Reference in new issue
Block a user