/** * Lane D (GSTA-112, E6): a pregate failure the environment caused (a runner * binary missing from PATH, a network error before any test ran, a quota * refusal from the remote gate, a guard that cannot start) is classified and * reported apart from a failure the diff caused, under its own result code * `PREGATE_ENVIRONMENT` with a kind and the evidence, so a lane can waive by * cause instead of by test name (truther #617). A failing test whose log also * mentions a network error is still a diff failure: the runner's own summary * decides, never the log alone. */ import { afterEach, describe, expect, test } from 'bun:test'; import * as fs from 'node:fs'; import * as path from 'node:path'; import { EXIT } from '../lib/headless-artifacts'; import { classifyCommandEnvironment, classifyEnvironment, commandBinary } from '../lib/pregate/environment'; import { verifyPregate } from '../lib/pregate/index'; import { BASE_PREGATE, cleanup, commit, makeFixtureRepo, runBin, type FixtureRepo } from './helpers/pregate-fixture'; const repos: FixtureRepo[] = []; const fixture = (pregate: Record, baseFiles?: Record) => { const r = makeFixtureRepo({ pregate, baseFiles }); repos.push(r); return r; }; afterEach(() => { while (repos.length) cleanup(repos.pop()!); }); const pregate = (repo: FixtureRepo, args: string[]) => runBin(repo, 'gstack-pregate', ['--base', 'main', '--allow-repo-commands', '--only', 'lanes,guards', ...args]); const readOut = (repo: FixtureRepo) => JSON.parse(fs.readFileSync(path.join(repo.work, '.gstack/tmp/pregate.json'), 'utf8')); const check = (repo: FixtureRepo, id: string) => readOut(repo).checks.find((c: any) => c.id === id); const UTIL_EDIT = { 'lib/util.js': 'exports.add = (a, b) => b + a;\n' }; describe('classifyEnvironment', () => { // Value: protects=the classifier's vocabulary and its silence on ordinary failures; fails_when=a plain assertion failure or a // non-zero exit with a test summary reads as environment, or exit 127 / ENOTFOUND / quota text does not; why_new=new module; // seam=none; no_claim=completeness of the vocabulary test('names missing-binary, network, quota and disk, and stays null for an ordinary failure', () => { expect(classifyEnvironment('bash: line 1: vitest: command not found', 127)?.kind).toBe('missing-binary'); expect(classifyEnvironment('', 127)).toEqual({ kind: 'missing-binary', evidence: 'exit 127: the shell found no such command', proof: 'launch' }); expect(classifyEnvironment('write failed: Disk quota exceeded (EDQUOT)', 1)).toEqual({ kind: 'disk', evidence: 'write failed: Disk quota exceeded (EDQUOT)', proof: 'output' }); expect(classifyEnvironment("'nope' is not recognized as an internal or external command", 1)?.kind).toBe('missing-binary'); expect(classifyEnvironment('Error: getaddrinfo ENOTFOUND registry.example.invalid', 1)).toEqual({ kind: 'network', evidence: 'getaddrinfo ENOTFOUND registry.example.invalid', proof: 'output' }); expect(classifyEnvironment('curl: (7) Failed to connect to host port 443: Connection refused', 7)?.kind).toBe('network'); expect(classifyEnvironment('ubicloud: vcpu quota exceeded (528/512)', 1)?.kind).toBe('quota'); expect(classifyEnvironment('HTTP 429 Too Many Requests', 1)?.kind).toBe('quota'); expect(classifyEnvironment('ENOSPC: no space left on device, write', 1)?.kind).toBe('disk'); expect(classifyEnvironment('AssertionError: expected 3 to be 4', 1)).toBeNull(); expect(classifyEnvironment('✖ fetches the thing\n ECONNREFUSED 127.0.0.1:3000\nℹ tests 1\nℹ fail 1', 1)?.kind).toBe('network'); }); // Value: protects=exit 127 (or "command not found" text) is the machine's only when the command's own binary is off PATH: a script // the diff controls that exits 127 is the diff's; fails_when=a present runner exiting 127 reads as missing-binary; why_new=the veto // is new (adversarial review); seam=none; no_claim=none test('a missing binary is the machine\'s only when the command\'s own binary is off PATH', () => { expect(commandBinary("(cd 'pkg' && UBI_OWNER=x node run.js a b)")).toBe('node'); expect(commandBinary('FOO=1 BAR=2 bun test x')).toBe('bun'); expect(commandBinary('')).toBeNull(); const env = { PATH: process.env.PATH }; expect(classifyCommandEnvironment('node scripts/x.js', '', 127, env)).toBeNull(); expect(classifyCommandEnvironment('node scripts/x.js', 'bash: vitest: command not found', 127, env)).toBeNull(); expect(classifyCommandEnvironment('gstack-fixture-runner-that-does-not-exist x', '', 127, env)).toMatchObject({ kind: 'missing-binary', proof: 'launch' }); expect(classifyCommandEnvironment('node scripts/x.js', 'Error: getaddrinfo ENOTFOUND registry.example.invalid', 1, env)).toMatchObject({ kind: 'network', proof: 'output' }); }); }); describe('environment failures are reported apart from diff failures', () => { // Value: protects=a runner whose binary is missing is PREGATE_ENVIRONMENT missing-binary (incomplete, cause environment) rather // than a lane failure that reads as the diff's fault; fails_when=the lane is PREGATE_LANE_FAILED with no cause (the behaviour // before this change); why_new=the code and cause are new; seam=none; no_claim=none test('a runner binary missing from PATH is PREGATE_ENVIRONMENT missing-binary, incomplete, cause environment', () => { const repo = fixture({ ...BASE_PREGATE, runner: 'gstack-fixture-runner-that-does-not-exist {files}' }); commit(repo, UTIL_EDIT); const r = pregate(repo, []); expect(r.status).toBe(EXIT.fail); expect(r.err).toContain('(PREGATE_ENVIRONMENT)'); expect(r.err).not.toContain('PREGATE_LANE_FAILED'); const c = check(repo, 'lanes'); expect([c.status, c.code, c.cause, c.environment.kind, c.environment.proof]).toEqual(['incomplete', 'PREGATE_ENVIRONMENT', 'environment', 'missing-binary', 'launch']); expect(c.environment.evidence).toContain('gstack-fixture-runner-that-does-not-exist'); expect(c.fix).toContain('the diff is not at fault'); expect(r.out).toContain('environment: missing-binary'); const verify = runBin(repo, 'gstack-pregate', ['verify']); expect(verify.status).toBe(EXIT.fail); expect(verify.err).toContain('(PREGATE_ENVIRONMENT)'); expect(verify.err).toContain('blocked by the environment (missing-binary), not by the diff'); }); // Value: protects=a runner that dies on a network error before reporting any test is environment/network, while a test that // failed and whose log also names a connection error stays a diff failure (its summary counted a failure); fails_when=either // direction is misread; why_new=the summary-first rule is new; seam=none; no_claim=none test('a network error with no test summary is environment; a counted test failure beside a network line is still the diff', () => { const net = fixture({ ...BASE_PREGATE, runner: 'node scripts/net.js {files}' }, { 'scripts/net.js': "console.error('Error: getaddrinfo ENOTFOUND registry.example.invalid');\nprocess.exit(1);\n" }); commit(net, UTIL_EDIT); pregate(net, []); const c = check(net, 'lanes'); expect([c.status, c.code, c.cause, c.environment.kind]).toEqual(['incomplete', 'PREGATE_ENVIRONMENT', 'environment', 'network']); const diff = fixture({ ...BASE_PREGATE, runner: 'node scripts/fail.js {files}' }, { 'scripts/fail.js': "console.log('✖ talks to the server\\n Error: connect ECONNREFUSED 127.0.0.1:3000\\nℹ tests 1\\nℹ pass 0\\nℹ fail 1');\nprocess.exit(1);\n" }); commit(diff, UTIL_EDIT); const r = pregate(diff, []); expect(r.err).toContain('(PREGATE_LANE_FAILED)'); const d = check(diff, 'lanes'); expect([d.status, d.code, d.cause]).toEqual(['fail', 'PREGATE_LANE_FAILED', 'diff']); expect(d.environment).toBeUndefined(); }); // Value: protects=a test in the diff cannot clear itself: a runner whose binary is present but exits 127 is PREGATE_LANE_FAILED // (diff), and a bun test that prints a network line and exits the process before the summary is at most an output-inferred // environment cause whose fix and verify message ask for confirmation elsewhere, never "the diff is not at fault"; // fails_when=exit 127 from a present runner reads as missing-binary, or the spoof's fix clears the diff; why_new=the binary veto // and the launch/output proof are new (adversarial review); seam=none; no_claim=a spoof with no summary cannot be told from a // real network failure by the output alone, which is exactly what the proof records test('a present runner that exits 127 is the diff\'s; a test that exits with a network line is only an output-inferred cause', () => { const exit127 = fixture({ ...BASE_PREGATE, runner: 'node scripts/x.js {files}' }, { 'scripts/x.js': 'process.exit(127);\n' }); commit(exit127, UTIL_EDIT); pregate(exit127, []); const c = check(exit127, 'lanes'); expect([c.status, c.code, c.cause, c.environment]).toEqual(['fail', 'PREGATE_LANE_FAILED', 'diff', undefined]); if (!Bun.which('bun')) return; const spoof = fixture({ ...BASE_PREGATE, runner: 'bun test {files}' }, { 'test/util.test.js': "const { test } = require('bun:test');\nrequire('../lib/util.js');\ntest('spoof', () => { console.log('Received: fetch failed'); process.exit(1); });\n" }); commit(spoof, UTIL_EDIT); pregate(spoof, []); const s = check(spoof, 'lanes'); expect([s.status, s.code, s.environment.kind, s.environment.proof]).toEqual(['incomplete', 'PREGATE_ENVIRONMENT', 'network', 'output']); expect(s.fix).not.toContain('the diff is not at fault'); expect(s.fix).toContain('confirm on another machine or with --remote'); expect(s.lines.join('\n')).toContain('environment: network — Received: fetch failed (inferred from output)'); const verify = runBin(spoof, 'gstack-pregate', ['verify']); expect(verify.err).toContain('lanes inferred from output alone'); }); // Value: protects=a remote gate is classified from its preamble only: quota text streamed from the tests past the first lines // is not the environment; fails_when=a test name with "quota" waives the gate; why_new=new (adversarial review); seam=none; no_claim=none test('a remote gate whose test stream names a quota past the preamble is PREGATE_INCOMPLETE, not environment', () => { const repo = fixture({ ...BASE_PREGATE, remote_gate: 'node scripts/gate.js' }, { 'scripts/gate.js': "for (let i = 0; i < 60; i++) console.log('ok test ' + i);\nconsole.log('✗ billing > enforces quota per user');\nprocess.exit(1);\n" }); commit(repo, UTIL_EDIT); pregate(repo, ['--remote', '--spend-cap', '1']); const c = check(repo, 'lanes'); expect([c.status, c.code, c.environment]).toEqual(['incomplete', 'PREGATE_INCOMPLETE', undefined]); }); // Value: protects=a remote gate refused by a quota (no PREGATE_REMOTE line) is environment/quota and the obligation stays // uncleared; fails_when=it is a generic PREGATE_INCOMPLETE; why_new=new; seam=none; no_claim=none test('a remote gate that hit a quota is PREGATE_ENVIRONMENT quota and leaves the obligation uncleared', () => { const repo = fixture({ ...BASE_PREGATE, remote_gate: 'node scripts/gate.js' }, { 'scripts/gate.js': "console.error('ubicloud: vcpu quota exceeded (528/512)');\nprocess.exit(1);\n" }); commit(repo, UTIL_EDIT); const r = pregate(repo, ['--remote', '--spend-cap', '1']); expect(r.status).toBe(EXIT.fail); expect(r.err).toContain('(PREGATE_ENVIRONMENT)'); const c = check(repo, 'lanes'); expect([c.status, c.cause, c.environment.kind]).toEqual(['incomplete', 'environment', 'quota']); expect(readOut(repo).requires_remote.find((o: any) => o.lane === 'pregate.json/remote_gate').receipt).toBeNull(); }); // Value: protects=guards classify the same way: a guard whose command is missing is environment, a guard that fails on its // own terms is the diff; fails_when=a missing guard binary is PREGATE_GUARD_FAILED; why_new=new; seam=none; no_claim=none test('a guard whose binary is missing is environment; a guard that fails is the diff', () => { // The red guard's output names a rate limit and a refused connection: a guard that ran and said no is the diff's whatever it printed. const repo = fixture({ ...BASE_PREGATE, guards: [{ id: 'absent', when: ['lib/**'], command: 'gstack-fixture-guard-that-does-not-exist --check' }, { id: 'red', when: ['lib/**'], command: 'node -e "console.log(\'FAIL api > retries after a rate limit (ECONNREFUSED fixture)\'); process.exit(2)"', fix: 'fix it' }] }); commit(repo, UTIL_EDIT); const r = pregate(repo, ['--only', 'guards']); expect(r.status).toBe(EXIT.fail); const g = check(repo, 'guards'); expect([g.status, g.code, g.cause]).toEqual(['fail', 'PREGATE_GUARD_FAILED', 'diff']); expect(g.lines.join('\n')).toContain('environment: missing-binary'); expect(g.lines.join('\n')).not.toContain('red environment:'); // A guard whose own binary is present but whose script exits 127 (a package script naming a missing tool) is the diff's. const present = fixture({ ...BASE_PREGATE, guards: [{ id: 'script', when: ['lib/**'], command: 'node -e "process.exit(127)"', fix: 'fix it' }] }); commit(present, UTIL_EDIT); pregate(present, ['--only', 'guards']); const p = check(present, 'guards'); expect([p.status, p.code, p.cause, p.environment]).toEqual(['fail', 'PREGATE_GUARD_FAILED', 'diff', undefined]); const only = fixture({ ...BASE_PREGATE, guards: [{ id: 'absent', when: ['lib/**'], command: 'gstack-fixture-guard-that-does-not-exist --check' }] }); commit(only, UTIL_EDIT); pregate(only, ['--only', 'guards']); const o = check(only, 'guards'); expect([o.status, o.code, o.cause, o.environment.kind]).toEqual(['incomplete', 'PREGATE_ENVIRONMENT', 'environment', 'missing-binary']); }); }); describe('verify: a mixed incomplete record stays PREGATE_INCOMPLETE and names the environment-blocked check', () => { test('one environment-caused check beside a plain incomplete one is not promoted to PREGATE_ENVIRONMENT', () => { const file: any = { schema_version: 1, tree: 'T', checks: [ { id: 'lanes', status: 'incomplete', cause: 'environment', environment: { kind: 'network', evidence: 'ENOTFOUND x' } }, { id: 'guards', status: 'incomplete', code: 'REPO_COMMANDS_NOT_ALLOWED' }, ] }; const v = verifyPregate('/nonexistent', file, { tree: 'T' } as any); expect(v.code).toBe('PREGATE_INCOMPLETE'); expect(v.message).toContain('(blocked by the environment: lanes network)'); const all: any = { ...file, checks: [file.checks[0]] }; expect(verifyPregate('/nonexistent', all, { tree: 'T' } as any).code).toBe('PREGATE_ENVIRONMENT'); }); });