mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-22 08:01:32 +01:00
A repo can now list published-data folders under a new `review_content_patterns` ship-policy key, so /ship leaves `.patch` diffs, `MANIFEST.sha256` and extension-less licences out of review, while source files there stay reviewed. Why: gstack-evals #352 is 13.8 MB, 13.6 MB of it published session data, and could never pass /ship's 2 MB review limit. Nothing to do until a repo adds the key to its base-branch policy; a pattern that keeps too many files now stops with `REVIEW_EXCLUDE_KEEP_TOO_LARGE` instead of being dropped. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> Review-Finding: p1/red-team#2 reader=red-team Review-Finding: p2full/red-team#1 reader=red-team
160 lines
11 KiB
TypeScript
160 lines
11 KiB
TypeScript
/**
|
|
* review_content_patterns and the kept-file bound (GST-3158, docs/ship-policy.md#review-content-patterns): a results PR shaped
|
|
* like gstack-evals #352 (1,300+ .patch files, extension-less licence texts and a MANIFEST.sha256 under one folder, beside
|
|
* ordinary code) excludes that folder only when the base policy names it under both keys; a pattern whose kept files pass the
|
|
* argument bound stops the plan with a named code instead of being dropped; and the secrets scan and the sealed guard still
|
|
* read the excluded files.
|
|
*/
|
|
import { afterEach, describe, expect, test } from 'bun:test';
|
|
import { spawnSync } from 'node:child_process';
|
|
import { EXIT } from '../lib/headless-artifacts';
|
|
import { REVIEW_CONTENT_KEY_VERSION } from '../lib/ship-policy';
|
|
import { BASE_PREGATE, cleanup, commit, makeFixtureRepo, runBin, type FixtureRepo } from './helpers/pregate-fixture';
|
|
|
|
const repos: FixtureRepo[] = [];
|
|
afterEach(() => { while (repos.length) cleanup(repos.pop()!); });
|
|
const DIR = 'results/h2h-2026-11/sessions';
|
|
const AWS = 'AKIA' + 'ZQ3XN7P2LM4RT8WV';
|
|
const policy = (exclude: string[], content?: string[]) => JSON.stringify({ review_exclude_paths: exclude, ...(content ? { review_content_patterns: content } : {}), min_gstack_version: REVIEW_CONTENT_KEY_VERSION }) + '\n';
|
|
|
|
/** A #352-shaped branch: `patches` .patch files, extension-less licence texts and a MANIFEST.sha256 under DIR, plus code and a test. */
|
|
function results(basePolicy: string | null, patches: number, extra: Record<string, string> = {}, pregate?: unknown): FixtureRepo {
|
|
const baseFiles: Record<string, string> = {};
|
|
if (basePolicy) baseFiles['.gstack/ship-policy.json'] = basePolicy;
|
|
if (pregate) baseFiles['.gstack/pregate.json'] = JSON.stringify(pregate, null, 2) + '\n';
|
|
const repo = makeFixtureRepo({ prefix: 'gstack-content-', baseFiles });
|
|
repos.push(repo);
|
|
const files: Record<string, string> = {
|
|
'lib/report.js': "exports.total = rows => rows.reduce((n, r) => n + r.cost, 0);\n",
|
|
'test/report.test.js': "const { total } = require('../lib/report');\nif (total([{ cost: 1 }]) !== 1) throw new Error('total');\n",
|
|
[`${DIR}/licences/MIT-task-repo`]: 'MIT License\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\n',
|
|
[`${DIR}/licences/Apache-2.0-task-repo`]: 'Apache License\nVersion 2.0, January 2004\n',
|
|
...extra,
|
|
};
|
|
const names: string[] = [];
|
|
for (let k = 0; k < patches; k++) {
|
|
const name = `${DIR}/task-${String(k).padStart(4, '0')}/arm-gstack.patch`;
|
|
names.push(name);
|
|
files[name] = `diff --git a/src/parse.ts b/src/parse.ts\n--- a/src/parse.ts\n+++ b/src/parse.ts\n@@ -1 +1 @@\n-export const limit = 1;\n+export const limit = ${k + 2};\n`;
|
|
}
|
|
files[`${DIR}/MANIFEST.sha256`] = names.map((n, k) => `${String(k).padStart(64, '0')} ${n.slice(DIR.length + 1)}`).join('\n') + '\n';
|
|
commit(repo, files);
|
|
return repo;
|
|
}
|
|
const plan = (repo: FixtureRepo) => runBin(repo, 'gstack-review-plan', ['--base', 'main', '--gate-tagged', '0']);
|
|
/** The paths a plan's DIFF_CMD reads, run as --name-only in the fixture. */
|
|
function diffCmdPaths(repo: FixtureRepo, text: string): string[] {
|
|
const cmd = text.split('\n').find(l => l.startsWith('DIFF_CMD: '))!.slice('DIFF_CMD: '.length).replace(/^\S+ /, '');
|
|
const named = cmd.replace(/^git diff/, 'git diff --name-only').replace('<diff-base>', '"$DIFF_BASE"');
|
|
const r = spawnSync('bash', ['-c', named], { cwd: repo.work, encoding: 'utf8', timeout: 60_000, env: { ...process.env, ...repo.env, DIFF_BASE: repo.git('merge-base', 'origin/main', 'HEAD') } });
|
|
expect({ cmd: named, status: r.status, err: r.stderr }).toMatchObject({ cmd: named, status: 0 });
|
|
return r.stdout.split('\n').filter(Boolean);
|
|
}
|
|
|
|
describe('review_content_patterns on a #352-shaped results PR', () => {
|
|
// Value: protects=a results PR that publishes raw session data (.patch, MANIFEST.sha256, extension-less licences) can be
|
|
// reviewed once its base policy names the folder under both keys, and its code stays in every reader's diff;
|
|
// fails_when=the folder is kept as code, dropped, or the code leaves the diff; why_new=GST-3158 (gstack-evals #352: 1,439
|
|
// files kept as code, patterns=0/5); seam=bin/gstack-review-plan and git running the printed DIFF_CMD; no_claim=a live /ship run
|
|
test('the folder is excluded as declared content and the code stays in review', () => {
|
|
const repo = results(policy([`${DIR}/**`], [`${DIR}/**`]), 1_320);
|
|
const r = plan(repo);
|
|
expect(r.err).not.toContain('REVIEW_EXCLUDE_KEEP_TOO_LARGE');
|
|
expect(r.status).toBe(EXIT.ok);
|
|
expect(r.out).toMatch(/^REVIEW_EXCLUDE: patterns=1\/1 excluded=1323 bytes=\d+ kept=0 content=declared:1323 /m);
|
|
expect(r.out).not.toContain('REVIEW_EXCLUDE_KEPT:');
|
|
const read = diffCmdPaths(repo, r.out);
|
|
expect(read).toContain('lib/report.js');
|
|
expect(read).toContain('test/report.test.js');
|
|
expect(read.filter(p => p.startsWith(`${DIR}/`))).toEqual([]);
|
|
}, 180_000);
|
|
|
|
// Value: protects=a pattern whose kept files pass the argument bound is never dropped quietly; fails_when=the plan prints
|
|
// patterns=0/1 and hands readers the whole folder; why_new=GST-3158 cause (b); seam=bin/gstack-review-plan; no_claim=Windows
|
|
test('without the content key the same folder stops the plan with REVIEW_EXCLUDE_KEEP_TOO_LARGE instead of dropping the pattern', () => {
|
|
const repo = results(policy([`${DIR}/**`]), 1_320, { [`${DIR}/summary.json`]: '{"tasks": 1320}\n' });
|
|
const r = plan(repo);
|
|
expect(r.status).toBe(EXIT.refused);
|
|
expect(r.out).not.toContain('REVIEW_EXCLUDE:');
|
|
expect(r.err).toMatch(new RegExp(`pattern ${DIR}/\\*\\* on origin/main@[0-9a-f]{12} keeps 1323 matched files in review \\(code:1323\\).*\\(more than 24000\\)`));
|
|
expect(r.err).toContain('review_content_patterns');
|
|
expect(r.err.trim()).toEndWith('nothing was planned (REVIEW_EXCLUDE_KEEP_TOO_LARGE)');
|
|
}, 180_000);
|
|
});
|
|
|
|
describe('review_content_patterns negative controls', () => {
|
|
// Value: protects=the content key only narrows what review_exclude_paths already names, and only the base's copy counts;
|
|
// fails_when=a content pattern excludes on its own or a branch excludes its own files; why_new=GST-3158;
|
|
// seam=bin/gstack-review-plan; no_claim=none
|
|
test('a content pattern with no matching exclude pattern excludes nothing', () => {
|
|
const repo = results(policy(['data/**'], [`${DIR}/**`]), 20, { 'data/rows.json': '[1]\n' });
|
|
const r = plan(repo);
|
|
expect(r.status).toBe(EXIT.ok);
|
|
expect(r.out).toMatch(/^REVIEW_EXCLUDE: patterns=1\/1 excluded=1 .* content=data:1 /m);
|
|
const read = diffCmdPaths(repo, r.out);
|
|
expect(read.filter(p => p.startsWith(`${DIR}/`)).length).toBe(23);
|
|
expect(read).not.toContain('data/rows.json');
|
|
}, 120_000);
|
|
|
|
test('the policy is read from the base: a branch that adds the content key still has the folder kept as code', () => {
|
|
const repo = results(policy([`${DIR}/**`]), 20, { [`${DIR}/summary.json`]: '{}\n', '.gstack/ship-policy.json': policy([`${DIR}/**`], [`${DIR}/**`]) });
|
|
const r = plan(repo);
|
|
expect(r.status).toBe(EXIT.ok);
|
|
expect(r.out).toMatch(/^REVIEW_EXCLUDE: patterns=1\/1 excluded=1 bytes=\d+ kept=23 content=data:1 /m);
|
|
expect(r.out).toContain(`REVIEW_EXCLUDE_KEPT: code ${DIR}/MANIFEST.sha256`);
|
|
const read = diffCmdPaths(repo, r.out);
|
|
expect(read).not.toContain(`${DIR}/summary.json`);
|
|
expect(r.out).toMatch(new RegExp(`^DIFF_KEEP: .*${DIR}/task-0000/arm-gstack\\.patch`, 'm'));
|
|
}, 120_000);
|
|
|
|
// Value: protects=a declaration reclassifies only names that are not code, so a branch cannot hide a source file inside a
|
|
// declared folder; fails_when=a .ts or .sh file under both keys leaves review as declared content; why_new=GST-3158 review
|
|
// (red-team p1#2); seam=bin/gstack-review-plan; no_claim=shebang detection in extension-less files
|
|
test('a source file inside a declared folder stays in review as code', () => {
|
|
const repo = results(policy([`${DIR}/**`], [`${DIR}/**`]), 3, { [`${DIR}/loader.ts`]: 'export const load = () => 1;\n', [`${DIR}/fetch.sh`]: 'curl -s example.invalid\n' });
|
|
const r = plan(repo);
|
|
expect(r.status).toBe(EXIT.ok);
|
|
expect(r.out).toMatch(/^REVIEW_EXCLUDE: patterns=1\/1 excluded=6 bytes=\d+ kept=2 content=declared:6 /m);
|
|
expect(r.out).toContain(`REVIEW_EXCLUDE_KEPT: code ${DIR}/loader.ts`);
|
|
expect(r.out).toContain(`REVIEW_EXCLUDE_KEPT: code ${DIR}/fetch.sh`);
|
|
}, 120_000);
|
|
|
|
// Value: protects=a NUL byte or an upper-case extension cannot move a source file in a declared folder out of review;
|
|
// fails_when=the binary arm or a case-sensitive test treats it as declared content; why_new=GST-3158 review (red-team p2#1);
|
|
// seam=bin/gstack-review-plan; no_claim=source languages outside SOURCE_EXT
|
|
test('a binary or upper-case source file inside a declared folder stays in review as code', () => {
|
|
const repo = results(policy([`${DIR}/**`], [`${DIR}/**`]), 3, { [`${DIR}/hidden.ts`]: 'export const y = 2;\n// \u0000\n', [`${DIR}/Loader.TS`]: 'export const load = () => 1;\n' });
|
|
const r = plan(repo);
|
|
expect(r.status).toBe(EXIT.ok);
|
|
expect(r.out).toMatch(/^REVIEW_EXCLUDE: patterns=1\/1 excluded=6 bytes=\d+ kept=2 content=declared:6 /m);
|
|
expect(r.out).toContain(`REVIEW_EXCLUDE_KEPT: code ${DIR}/hidden.ts`);
|
|
expect(r.out).toContain(`REVIEW_EXCLUDE_KEPT: code ${DIR}/Loader.TS`);
|
|
}, 120_000);
|
|
});
|
|
|
|
describe('excluded from review is not excluded from the guards', () => {
|
|
// Value: protects=a credential or a sealed answer inside declared content still stops the push; fails_when=the secrets scan
|
|
// or the sealed guard honours review_exclude_paths or review_content_patterns; why_new=GST-3158 (the guards must keep reading
|
|
// what review leaves out); seam=bin/gstack-pregate secrets and sealed; no_claim=the managed pre-push hook
|
|
test('the secrets scan and the sealed guard both read a .patch file the plan excludes', () => {
|
|
const leak = `${DIR}/task-0003/arm-plain.patch`;
|
|
const sealed = `${DIR}/task-0004/arm-plain.patch`;
|
|
const repo = results(policy([`${DIR}/**`], [`${DIR}/**`]), 20, {
|
|
[leak]: `diff --git a/ci.yml b/ci.yml\n+++ b/ci.yml\n@@ -0,0 +1 @@\n+ ${AWS}\n`,
|
|
[sealed]: 'diff --git a/notes.md b/notes.md\n+++ b/notes.md\n@@ -0,0 +1 @@\n+SEALED ANSWER: fund-b\n',
|
|
}, { ...BASE_PREGATE, sealed: { paths: ['heldout/**'], answer_markers: ['SEALED ANSWER:'] } });
|
|
const r = plan(repo);
|
|
expect(r.status).toBe(EXIT.ok);
|
|
expect(r.out).toMatch(/^REVIEW_EXCLUDE: patterns=1\/1 excluded=25 bytes=\d+ kept=0 /m);
|
|
expect(diffCmdPaths(repo, r.out)).not.toContain(leak);
|
|
const secrets = runBin(repo, 'gstack-pregate', ['--base', 'main', '--only', 'secrets', '--json']);
|
|
const check = JSON.parse(secrets.out).checks.find((c: { id: string }) => c.id === 'secrets');
|
|
expect(check.status).toBe('fail');
|
|
expect(JSON.stringify(check)).toContain(leak);
|
|
const guard = runBin(repo, 'gstack-pregate', ['sealed', '--base', 'main']);
|
|
expect(guard.status).toBe(1);
|
|
expect(guard.err).toMatch(new RegExp(`sealed\\.answer\\s+${sealed.replace(/[.]/g, '\\.')}:4`));
|
|
expect(guard.err).toContain('(SEALED_OUTPUT_REFUSED)');
|
|
}, 180_000);
|
|
});
|