/** * Lane D (GSTA-112, E5): `allowlist` in the pregate policy lets a repo accept * benign findings the `secrets` check would otherwise refuse (dummy per-trial * tokens in receipts, a run id read as a phone number, a known public key in * a rebuilt bundle: harness-evals #81/#83, GSTA-102, GSTA-106), by finding id * or category, path and a pattern anchored at the finding. The real value of a * credential-named environment variable (`env.value`, #3449) can never be * allowlisted: validation refuses the entry, and the matcher ignores one. */ import { afterEach, describe, expect, test } from 'bun:test'; import * as fs from 'node:fs'; import * as path from 'node:path'; import { EXIT } from '../lib/headless-artifacts'; import { applyAllowlist, type AllowlistFinding } from '../lib/pregate/allowlist'; import { NON_ALLOWLISTABLE, validateConfig } from '../lib/pregate/config'; import { CHECK_IDS } from '../lib/pregate/index'; import { BASE_PREGATE, cleanup, commit, makeFixtureRepo, runBin, type FixtureRepo } from './helpers/pregate-fixture'; const repos: FixtureRepo[] = []; const fixture = (pregate: Record | null) => { const r = makeFixtureRepo({ pregate }); repos.push(r); return r; }; afterEach(() => { while (repos.length) cleanup(repos.pop()!); }); const secrets = (repo: FixtureRepo, env: Record = {}) => runBin(repo, 'gstack-pregate', ['--base', 'main', '--only', 'secrets'], env); const row = (repo: FixtureRepo) => JSON.parse(fs.readFileSync(path.join(repo.work, '.gstack/tmp/pregate.json'), 'utf8')).checks.find((c: any) => c.id === 'secrets'); const codes = (c: unknown) => validateConfig(c, CHECK_IDS).map(e => `${e.code}:${e.path}`); const TRIAL = '{\n "trial": 1,\n "key": "sk-trial-9f8e7d6c-5b4a-3f2e-1d0c-7b6a5f4e3d2c"\n}\n'; const TRIAL_ENTRY = { finding: 'env.kv', paths: ['evals/**'], pattern: 'sk-trial-[0-9a-f-]+', reason: 'dummy per-trial meter ids in eval receipts' }; describe('allowlist: policy validation', () => { // Value: protects=the key's shape: finding id or non-secret category, paths and/or pattern, a reason; the env.value rule and // the secret wildcard are refused at validation; fails_when=an entry without a reason or scope is accepted, or env.value can be // named; why_new=the key is new; seam=none; no_claim=regex runtime cost test('accepts an id or category entry with a reason and a scope; refuses env.value, the secret wildcard, an empty scope and a bad regex', () => { expect(codes({ allowlist: [TRIAL_ENTRY] })).toEqual([]); expect(codes({ allowlist: [{ finding: 'pii', pattern: '\\d{11}', reason: 'GitHub run ids' }] })).toEqual([]); expect(codes({ allowlist: [{ finding: 'env.kv', paths: ['evals/**'] }] })).toEqual(['PREGATE_CONFIG_INVALID:allowlist[0].reason']); expect(codes({ allowlist: [{ finding: 'env.kv', reason: 'x' }] })).toEqual(['PREGATE_CONFIG_INVALID:allowlist[0]']); expect(codes({ allowlist: [{ finding: 'env.value', pattern: '.*', reason: 'never' }] })).toEqual(['PREGATE_CONFIG_INVALID:allowlist[0].finding']); expect(codes({ allowlist: [{ finding: 'secret', pattern: '.*', reason: 'never' }] })).toEqual(['PREGATE_CONFIG_INVALID:allowlist[0].finding']); expect(codes({ allowlist: [{ finding: 'env.kv', pattern: '(', reason: 'x' }] })).toEqual(['PREGATE_CONFIG_INVALID:allowlist[0].pattern']); expect(codes({ allowlist: [{ finding: 'env.kv', paths: ['../x'], reason: 'x' }] })).toEqual(['PREGATE_CONFIG_INVALID:allowlist[0].paths[0]']); expect(codes({ allowlist: { finding: 'env.kv' } })).toEqual(['PREGATE_CONFIG_INVALID:allowlist']); expect(codes({ allowlist: Array.from({ length: 65 }, () => TRIAL_ENTRY) })).toContain('PREGATE_CONFIG_INVALID:allowlist'); expect(codes({ allowlist: [{ ...TRIAL_ENTRY, extra: 1 }] })).toEqual(['PREGATE_CONFIG_INVALID:allowlist[0].extra']); expect(codes({ allowlist: [{ finding: 'env.kv', paths: [], reason: 'x' }] })).toEqual(['PREGATE_CONFIG_INVALID:allowlist[0].paths']); expect(codes({ allowlist: [{ finding: 'env.kv', pattern: 'a'.repeat(257), reason: 'x' }] })).toEqual(['PREGATE_CONFIG_INVALID:allowlist[0].pattern']); // A pattern entry never covers a finding the scanner gave no position; a paths-only entry does. const positionless: AllowlistFinding = { id: 'env.kv', category: 'secret', severity: 'MEDIUM', file: 'evals/x.json' }; const inert = applyAllowlist([positionless], [TRIAL_ENTRY]); expect(inert.kept).toHaveLength(1); expect(inert.untested.map(u => [u.index, u.finding.id])).toEqual([[0, 'env.kv']]); expect(applyAllowlist([positionless], [{ finding: 'env.kv', paths: ['evals/**'], reason: 'r' }]).allowlisted).toHaveLength(1); expect(NON_ALLOWLISTABLE).toContain('env.value'); // The scanner's own refusal (an oversize input fails closed) is never a benign finding: refused by validation, kept by the matcher. expect(codes({ allowlist: [{ finding: 'engine.input_too_large', paths: ['**'], reason: 'x' }] })).toEqual(['PREGATE_CONFIG_INVALID:allowlist[0].finding']); const refusal: AllowlistFinding = { id: 'engine.input_too_large', category: 'engine', severity: 'HIGH', file: 'big.json' }; expect(applyAllowlist([refusal], [{ finding: 'engine.input_too_large', paths: ['**'], reason: 'x' }]).kept).toHaveLength(1); }); }); describe('allowlist: the secrets check', () => { // Value: protects=a benign dummy token in the allowlisted path with the allowlisted shape passes and is named as allowlisted; // the same token outside the path, or a shape the pattern does not cover, still fails PREGATE_SECRETS; fails_when=the // allowlist is ignored (the pre-change behaviour: 1,800 dummy tokens failed harness-evals #81) or it silences a finding its // scope does not cover; why_new=the key is new; seam=none; no_claim=none test('a dummy trial token is allowlisted by path and pattern, and named; outside the scope it still fails', () => { const repo = fixture({ ...BASE_PREGATE, allowlist: [TRIAL_ENTRY] }); commit(repo, { 'evals/receipts/t1.json': TRIAL }); const ok = secrets(repo); expect(ok.status).toBe(EXIT.ok); expect(ok.err).not.toContain('PREGATE_SECRETS'); const r = row(repo); expect(r.status).toBe('pass'); expect(r.detail).toContain('1 allowlisted (env.kv)'); expect(r.lines.join('\n')).toContain('allowlisted env.kv (evals/receipts/t1.json, added line 3) by allowlist[0] env.kv: dummy per-trial meter ids in eval receipts'); expect(r.inputs.allowlist).toMatch(/^[0-9a-f]{64}$/); commit(repo, { 'notes/t2.json': TRIAL }); const outside = secrets(repo); expect(outside.status).toBe(EXIT.fail); expect(outside.err).toContain('(PREGATE_SECRETS)'); expect(row(repo).lines.join('\n')).toContain('notes/t2.json'); const shape = fixture({ ...BASE_PREGATE, allowlist: [{ ...TRIAL_ENTRY, pattern: 'sk-trial-[0-9]+' }] }); commit(shape, { 'evals/receipts/t1.json': TRIAL }); expect(secrets(shape).status).toBe(EXIT.fail); }); // Value: protects=a category entry covers an advisory pii hit (a run id after a generic word, which the engine's run-word exemption does not cover, read as a phone number) and the WARN count drops; // fails_when=category matching is id-only; why_new=new; seam=none; no_claim=none test('a category entry silences the advisory pii finding for a run id', () => { const repo = fixture({ ...BASE_PREGATE, allowlist: [{ finding: 'pii', pattern: '\\d{11}', reason: 'GitHub Actions run ids in reports' }] }); commit(repo, { 'docs/runs.md': 'job 40302077688 passed\n' }); expect(secrets(repo).status).toBe(EXIT.ok); const r = row(repo); expect([r.status, r.code]).toEqual(['pass', undefined]); expect(r.detail).toContain('1 allowlisted (pii.phone.e164)'); const control = fixture(BASE_PREGATE); commit(control, { 'docs/runs.md': 'job 40302077688 passed\n' }); secrets(control); expect([row(control).status, row(control).code]).toEqual(['warn', 'PREGATE_SECRETS']); expect(row(control).detail).toContain('pii.phone.e164'); }); // Value: protects=env.value is never allowlisted: a config naming it is refused before any check runs, and the matcher keeps an // env.value finding even when an entry's scope would match it; fails_when=either path lets a configured credential's value // through; why_new=the rule is new; seam=none; no_claim=none test('env.value survives every allowlist: the config is refused, and the matcher never drops it', () => { const value = 'q8Zt3vRm7Lk2Xn9Pw4Cs6Hd1Yf5Gb0Je'; const refused = fixture({ ...BASE_PREGATE, allowlist: [{ finding: 'env.value', pattern: '.*', reason: 'no' }] }); commit(refused, { 'lib/leak.txt': `token=${value}\n` }); const r = secrets(refused, { FIXTURE_API_TOKEN: value }); expect(r.status).toBe(EXIT.refused); expect(r.err).toContain('(PREGATE_CONFIG_INVALID)'); expect(r.err).toContain('allowlist[0].finding'); const findings: AllowlistFinding[] = [ { id: 'env.value', category: 'secret', severity: 'HIGH', file: 'lib/leak.txt', line: 1, col: 7, lineText: `token=${value}` }, { id: 'env.kv', category: 'secret', severity: 'MEDIUM', file: 'lib/leak.txt', line: 1, col: 7, lineText: `token=${value}` }, ]; const wide = [{ finding: 'env.value', pattern: '.*', reason: 'no' }, { finding: 'env.kv', pattern: '.*', reason: 'wide' }]; const applied = applyAllowlist(findings, wide); expect(applied.kept.map(f => f.id)).toEqual(['env.value']); expect(applied.allowlisted.map(a => a.finding.id)).toEqual(['env.kv']); const live = fixture({ ...BASE_PREGATE, allowlist: [{ finding: 'env.kv', pattern: '.*', reason: 'wide' }] }); commit(live, { 'lib/leak.txt': `token=${value}\n` }); const leak = secrets(live, { FIXTURE_API_TOKEN: value }); expect(leak.status).toBe(EXIT.fail); expect(row(live).lines.join('\n')).toContain('env.value'); }); });