Ship policy can mark published data as content, so /ship can review large data PRs (GST-3158) (#4511)

A repo can now list published-data folders under a new
`review_content_patterns` ship-policy key, so /ship leaves `.patch`
diffs, `MANIFEST.sha256` and extension-less licences out of review,
while source files there stay reviewed.

Why: gstack-evals #352 is 13.8 MB, 13.6 MB of it published session data,
and could never pass /ship's 2 MB review limit.

Nothing to do until a repo adds the key to its base-branch policy; a
pattern that keeps too many files now stops with
`REVIEW_EXCLUDE_KEEP_TOO_LARGE` instead of being dropped.

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
Review-Finding: p1/red-team#2 reader=red-team
Review-Finding: p2full/red-team#1 reader=red-team
This commit is contained in:
Garry Tanandcapy-ai[bot] authored and GitHub committed 2026-11-19 09:02:33 +00:00
1 parent c25390daa2
commit f13dbb0d4a
11 files changed
+340 -48

No files matched your search

+159
View File
@@ -0,0 +1,159 @@
/**
* review_content_patterns and the kept-file bound (GST-3158, docs/ship-policy.md#review-content-patterns): a results PR shaped
* like gstack-evals #352 (1,300+ .patch files, extension-less licence texts and a MANIFEST.sha256 under one folder, beside
* ordinary code) excludes that folder only when the base policy names it under both keys; a pattern whose kept files pass the
* argument bound stops the plan with a named code instead of being dropped; and the secrets scan and the sealed guard still
* read the excluded files.
*/
import { afterEach, describe, expect, test } from 'bun:test';
import { spawnSync } from 'node:child_process';
import { EXIT } from '../lib/headless-artifacts';
import { REVIEW_CONTENT_KEY_VERSION } from '../lib/ship-policy';
import { BASE_PREGATE, cleanup, commit, makeFixtureRepo, runBin, type FixtureRepo } from './helpers/pregate-fixture';
const repos: FixtureRepo[] = [];
afterEach(() => { while (repos.length) cleanup(repos.pop()!); });
const DIR = 'results/h2h-2026-11/sessions';
const AWS = 'AKIA' + 'ZQ3XN7P2LM4RT8WV';
const policy = (exclude: string[], content?: string[]) => JSON.stringify({ review_exclude_paths: exclude, ...(content ? { review_content_patterns: content } : {}), min_gstack_version: REVIEW_CONTENT_KEY_VERSION }) + '\n';
/** A #352-shaped branch: `patches` .patch files, extension-less licence texts and a MANIFEST.sha256 under DIR, plus code and a test. */
function results(basePolicy: string | null, patches: number, extra: Record<string, string> = {}, pregate?: unknown): FixtureRepo {
const baseFiles: Record<string, string> = {};
if (basePolicy) baseFiles['.gstack/ship-policy.json'] = basePolicy;
if (pregate) baseFiles['.gstack/pregate.json'] = JSON.stringify(pregate, null, 2) + '\n';
const repo = makeFixtureRepo({ prefix: 'gstack-content-', baseFiles });
repos.push(repo);
const files: Record<string, string> = {
'lib/report.js': "exports.total = rows => rows.reduce((n, r) => n + r.cost, 0);\n",
'test/report.test.js': "const { total } = require('../lib/report');\nif (total([{ cost: 1 }]) !== 1) throw new Error('total');\n",
[`${DIR}/licences/MIT-task-repo`]: 'MIT License\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\n',
[`${DIR}/licences/Apache-2.0-task-repo`]: 'Apache License\nVersion 2.0, January 2004\n',
...extra,
};
const names: string[] = [];
for (let k = 0; k < patches; k++) {
const name = `${DIR}/task-${String(k).padStart(4, '0')}/arm-gstack.patch`;
names.push(name);
files[name] = `diff --git a/src/parse.ts b/src/parse.ts\n--- a/src/parse.ts\n+++ b/src/parse.ts\n@@ -1 +1 @@\n-export const limit = 1;\n+export const limit = ${k + 2};\n`;
}
files[`${DIR}/MANIFEST.sha256`] = names.map((n, k) => `${String(k).padStart(64, '0')} ${n.slice(DIR.length + 1)}`).join('\n') + '\n';
commit(repo, files);
return repo;
}
const plan = (repo: FixtureRepo) => runBin(repo, 'gstack-review-plan', ['--base', 'main', '--gate-tagged', '0']);
/** The paths a plan's DIFF_CMD reads, run as --name-only in the fixture. */
function diffCmdPaths(repo: FixtureRepo, text: string): string[] {
const cmd = text.split('\n').find(l => l.startsWith('DIFF_CMD: '))!.slice('DIFF_CMD: '.length).replace(/^\S+ /, '');
const named = cmd.replace(/^git diff/, 'git diff --name-only').replace('<diff-base>', '"$DIFF_BASE"');
const r = spawnSync('bash', ['-c', named], { cwd: repo.work, encoding: 'utf8', timeout: 60_000, env: { ...process.env, ...repo.env, DIFF_BASE: repo.git('merge-base', 'origin/main', 'HEAD') } });
expect({ cmd: named, status: r.status, err: r.stderr }).toMatchObject({ cmd: named, status: 0 });
return r.stdout.split('\n').filter(Boolean);
}
describe('review_content_patterns on a #352-shaped results PR', () => {
// Value: protects=a results PR that publishes raw session data (.patch, MANIFEST.sha256, extension-less licences) can be
// reviewed once its base policy names the folder under both keys, and its code stays in every reader's diff;
// fails_when=the folder is kept as code, dropped, or the code leaves the diff; why_new=GST-3158 (gstack-evals #352: 1,439
// files kept as code, patterns=0/5); seam=bin/gstack-review-plan and git running the printed DIFF_CMD; no_claim=a live /ship run
test('the folder is excluded as declared content and the code stays in review', () => {
const repo = results(policy([`${DIR}/**`], [`${DIR}/**`]), 1_320);
const r = plan(repo);
expect(r.err).not.toContain('REVIEW_EXCLUDE_KEEP_TOO_LARGE');
expect(r.status).toBe(EXIT.ok);
expect(r.out).toMatch(/^REVIEW_EXCLUDE: patterns=1\/1 excluded=1323 bytes=\d+ kept=0 content=declared:1323 /m);
expect(r.out).not.toContain('REVIEW_EXCLUDE_KEPT:');
const read = diffCmdPaths(repo, r.out);
expect(read).toContain('lib/report.js');
expect(read).toContain('test/report.test.js');
expect(read.filter(p => p.startsWith(`${DIR}/`))).toEqual([]);
}, 180_000);
// Value: protects=a pattern whose kept files pass the argument bound is never dropped quietly; fails_when=the plan prints
// patterns=0/1 and hands readers the whole folder; why_new=GST-3158 cause (b); seam=bin/gstack-review-plan; no_claim=Windows
test('without the content key the same folder stops the plan with REVIEW_EXCLUDE_KEEP_TOO_LARGE instead of dropping the pattern', () => {
const repo = results(policy([`${DIR}/**`]), 1_320, { [`${DIR}/summary.json`]: '{"tasks": 1320}\n' });
const r = plan(repo);
expect(r.status).toBe(EXIT.refused);
expect(r.out).not.toContain('REVIEW_EXCLUDE:');
expect(r.err).toMatch(new RegExp(`pattern ${DIR}/\\*\\* on origin/main@[0-9a-f]{12} keeps 1323 matched files in review \\(code:1323\\).*\\(more than 24000\\)`));
expect(r.err).toContain('review_content_patterns');
expect(r.err.trim()).toEndWith('nothing was planned (REVIEW_EXCLUDE_KEEP_TOO_LARGE)');
}, 180_000);
});
describe('review_content_patterns negative controls', () => {
// Value: protects=the content key only narrows what review_exclude_paths already names, and only the base's copy counts;
// fails_when=a content pattern excludes on its own or a branch excludes its own files; why_new=GST-3158;
// seam=bin/gstack-review-plan; no_claim=none
test('a content pattern with no matching exclude pattern excludes nothing', () => {
const repo = results(policy(['data/**'], [`${DIR}/**`]), 20, { 'data/rows.json': '[1]\n' });
const r = plan(repo);
expect(r.status).toBe(EXIT.ok);
expect(r.out).toMatch(/^REVIEW_EXCLUDE: patterns=1\/1 excluded=1 .* content=data:1 /m);
const read = diffCmdPaths(repo, r.out);
expect(read.filter(p => p.startsWith(`${DIR}/`)).length).toBe(23);
expect(read).not.toContain('data/rows.json');
}, 120_000);
test('the policy is read from the base: a branch that adds the content key still has the folder kept as code', () => {
const repo = results(policy([`${DIR}/**`]), 20, { [`${DIR}/summary.json`]: '{}\n', '.gstack/ship-policy.json': policy([`${DIR}/**`], [`${DIR}/**`]) });
const r = plan(repo);
expect(r.status).toBe(EXIT.ok);
expect(r.out).toMatch(/^REVIEW_EXCLUDE: patterns=1\/1 excluded=1 bytes=\d+ kept=23 content=data:1 /m);
expect(r.out).toContain(`REVIEW_EXCLUDE_KEPT: code ${DIR}/MANIFEST.sha256`);
const read = diffCmdPaths(repo, r.out);
expect(read).not.toContain(`${DIR}/summary.json`);
expect(r.out).toMatch(new RegExp(`^DIFF_KEEP: .*${DIR}/task-0000/arm-gstack\\.patch`, 'm'));
}, 120_000);
// Value: protects=a declaration reclassifies only names that are not code, so a branch cannot hide a source file inside a
// declared folder; fails_when=a .ts or .sh file under both keys leaves review as declared content; why_new=GST-3158 review
// (red-team p1#2); seam=bin/gstack-review-plan; no_claim=shebang detection in extension-less files
test('a source file inside a declared folder stays in review as code', () => {
const repo = results(policy([`${DIR}/**`], [`${DIR}/**`]), 3, { [`${DIR}/loader.ts`]: 'export const load = () => 1;\n', [`${DIR}/fetch.sh`]: 'curl -s example.invalid\n' });
const r = plan(repo);
expect(r.status).toBe(EXIT.ok);
expect(r.out).toMatch(/^REVIEW_EXCLUDE: patterns=1\/1 excluded=6 bytes=\d+ kept=2 content=declared:6 /m);
expect(r.out).toContain(`REVIEW_EXCLUDE_KEPT: code ${DIR}/loader.ts`);
expect(r.out).toContain(`REVIEW_EXCLUDE_KEPT: code ${DIR}/fetch.sh`);
}, 120_000);
// Value: protects=a NUL byte or an upper-case extension cannot move a source file in a declared folder out of review;
// fails_when=the binary arm or a case-sensitive test treats it as declared content; why_new=GST-3158 review (red-team p2#1);
// seam=bin/gstack-review-plan; no_claim=source languages outside SOURCE_EXT
test('a binary or upper-case source file inside a declared folder stays in review as code', () => {
const repo = results(policy([`${DIR}/**`], [`${DIR}/**`]), 3, { [`${DIR}/hidden.ts`]: 'export const y = 2;\n// \u0000\n', [`${DIR}/Loader.TS`]: 'export const load = () => 1;\n' });
const r = plan(repo);
expect(r.status).toBe(EXIT.ok);
expect(r.out).toMatch(/^REVIEW_EXCLUDE: patterns=1\/1 excluded=6 bytes=\d+ kept=2 content=declared:6 /m);
expect(r.out).toContain(`REVIEW_EXCLUDE_KEPT: code ${DIR}/hidden.ts`);
expect(r.out).toContain(`REVIEW_EXCLUDE_KEPT: code ${DIR}/Loader.TS`);
}, 120_000);
});
describe('excluded from review is not excluded from the guards', () => {
// Value: protects=a credential or a sealed answer inside declared content still stops the push; fails_when=the secrets scan
// or the sealed guard honours review_exclude_paths or review_content_patterns; why_new=GST-3158 (the guards must keep reading
// what review leaves out); seam=bin/gstack-pregate secrets and sealed; no_claim=the managed pre-push hook
test('the secrets scan and the sealed guard both read a .patch file the plan excludes', () => {
const leak = `${DIR}/task-0003/arm-plain.patch`;
const sealed = `${DIR}/task-0004/arm-plain.patch`;
const repo = results(policy([`${DIR}/**`], [`${DIR}/**`]), 20, {
[leak]: `diff --git a/ci.yml b/ci.yml\n+++ b/ci.yml\n@@ -0,0 +1 @@\n+ ${AWS}\n`,
[sealed]: 'diff --git a/notes.md b/notes.md\n+++ b/notes.md\n@@ -0,0 +1 @@\n+SEALED ANSWER: fund-b\n',
}, { ...BASE_PREGATE, sealed: { paths: ['heldout/**'], answer_markers: ['SEALED ANSWER:'] } });
const r = plan(repo);
expect(r.status).toBe(EXIT.ok);
expect(r.out).toMatch(/^REVIEW_EXCLUDE: patterns=1\/1 excluded=25 bytes=\d+ kept=0 /m);
expect(diffCmdPaths(repo, r.out)).not.toContain(leak);
const secrets = runBin(repo, 'gstack-pregate', ['--base', 'main', '--only', 'secrets', '--json']);
const check = JSON.parse(secrets.out).checks.find((c: { id: string }) => c.id === 'secrets');
expect(check.status).toBe('fail');
expect(JSON.stringify(check)).toContain(leak);
const guard = runBin(repo, 'gstack-pregate', ['sealed', '--base', 'main']);
expect(guard.status).toBe(1);
expect(guard.err).toMatch(new RegExp(`sealed\\.answer\\s+${sealed.replace(/[.]/g, '\\.')}:4`));
expect(guard.err).toContain('(SEALED_OUTPUT_REFUSED)');
}, 180_000);
});
+4 -5
View File
@@ -64,7 +64,7 @@ describe('secret-like lines under review_exclude_paths', () => {
const p = JSON.parse(runBin(repo, 'gstack-review-plan', ['--base', 'main', '--gate-tagged', '0', '--json']).out);
expect(p.review_exclude.kept_files).toEqual(Object.keys(VALUE_LINES).sort().map(n => ({ path: `receipts/value/${n}.json`, reason: 'secret-line' })));
expect(p.review_exclude.excluded).toBe(1 + Object.keys(NAME_LINES).length);
expect(p.review_exclude.skipped).toEqual([]);
expect(p.review_exclude.applied).toEqual(['receipts/**']);
}, 120_000);
});
@@ -82,7 +82,6 @@ describe('the argument-byte bound', () => {
const baseSha = repo.git('rev-parse', 'origin/main');
const c = collectCommit(repo.work, baseSha, head);
const x = reviewExclude({ root: repo.work, baseSha, ref: 'origin/main', mergeBase: c.mergeBase!, head, perPath: c.perPath, added: c.added, classRoot: c.root ?? [], gateTagged: new Set(), context: 'test' });
if (x.state !== 'active') throw new Error(`exclusion is ${x.state}`);
return { repo, x, paths, mergeBase: c.mergeBase! };
}
@@ -93,7 +92,7 @@ describe('the argument-byte bound', () => {
// GIT_TRACE; no_claim=what an agent's shell does with a brief's keep command (those files still count)
test('excerpt-only kept files past the bound still apply the pattern, and their read is chunked under it', () => {
const { repo, x, paths, mergeBase } = excerptOnly(false);
expect(x.record.skipped).toEqual([]);
if (x.state !== 'active') throw new Error(`exclusion is ${x.state}`);
expect(x.record.applied).toEqual(['receipts/**']);
expect(x.excerpt.length).toBe(COUNT);
expect(paths.reduce((n, p) => n + Buffer.byteLength(`':(top,literal)${p}' `), 0)).toBeGreaterThan(KEEP_ARG_MAX_BYTES);
@@ -116,8 +115,8 @@ describe('the argument-byte bound', () => {
// bound; fails_when=the bound stops counting kept files that are not excerpted; why_new=GST-1967; seam=reviewExclude
test('kept files a reader runs as a command still count toward the bound', () => {
const { x } = excerptOnly(true);
expect(x.record.applied).toEqual([]);
expect(x.record.skipped).toMatchObject([{ pattern: 'receipts/**', kept: COUNT }]);
expect(x).toMatchObject({ state: 'refused', code: 'REVIEW_EXCLUDE_KEEP_TOO_LARGE' });
if (x.state === 'refused') expect(x.lines[0]).toContain(`keeps ${COUNT} matched files in review (secret-line:${COUNT})`);
}, 120_000);
});
+11 -8
View File
@@ -225,9 +225,9 @@ describe('the base policy read', () => {
describe('the kept-file command is bounded by argument bytes', () => {
// Value: protects=the DIFF_KEEP command stays under the Windows command line however many kept files a pattern has, and a
// disjoint small pattern still applies; fails_when=a pattern whose kept files pass KEEP_ARG_MAX_BYTES is applied, or the plan
// does not say why it was not; why_new=task 10; seam=bin/gstack-review-plan; no_claim=a real Windows launch
test('a pattern whose kept files pass the bound is not applied and the plan says why; a disjoint small pattern applies', () => {
// pattern past the bound is never dropped quietly; fails_when=a pattern whose kept files pass KEEP_ARG_MAX_BYTES is applied,
// or skipped while the plan goes on; why_new=task 10, GST-3158; seam=bin/gstack-review-plan; no_claim=a real Windows launch
test('a pattern whose kept files pass the bound stops the plan with REVIEW_EXCLUDE_KEEP_TOO_LARGE; without it the small pattern applies', () => {
const repo = makeFixtureRepo({ prefix: 'gstack-exclude-', baseFiles: { '.gstack/ship-policy.json': policyText(['big/**', 'small/**']) } }); repos.push(repo);
const long = 'x'.repeat(180);
for (let k = 0; k < 140; k++) {
@@ -235,8 +235,13 @@ describe('the kept-file command is bounded by argument bytes', () => {
fs.chmodSync(path.join(repo.work, `big/${long}-${k}.sh`), 0o755);
}
commit(repo, { 'big/data.json': '{}\n', 'small/data.json': '{}\n', 'lib/a.js': 'exports.a = 1;\n' });
const refused = plan(repo);
expect(refused.status).toBe(EXIT.refused);
expect(refused.out).not.toContain('REVIEW_EXCLUDE:');
expect(refused.err).toMatch(new RegExp(`pattern big/\\*\\* on origin/main@[0-9a-f]{12} keeps 140 matched files in review \\(executable:140\\).*\\(more than ${KEEP_ARG_MAX_BYTES}\\).*\\(REVIEW_EXCLUDE_KEEP_TOO_LARGE\\)`));
onBase(repo, { '.gstack/ship-policy.json': policyText(['small/**']) });
const text = plan(repo).out;
expect(text).toMatch(new RegExp(`REVIEW_EXCLUDE: patterns=1/2 .* skipped=big/\\*\\*:kept=140:bytes=\\d+ \\(kept files over ${KEEP_ARG_MAX_BYTES} argument bytes: pattern not applied\\)`));
expect(text).toMatch(/REVIEW_EXCLUDE: patterns=1\/1 /);
const cmd = text.split('\n').find(l => l.startsWith('DIFF_CMD: '))!;
expect(cmd).toContain("':(top,exclude,glob)small/**'");
expect(cmd).not.toContain('big/**');
@@ -248,16 +253,14 @@ describe('the review ceiling lines', () => {
const d = { bytes: 3_500_000, byDir: [['results', 3_000_000], ['src', 500_000]] as Array<[string, number]> };
// Value: protects=a REVIEW_DIFF_TOO_LARGE stop names the one fix that applies, with JSON a person can paste; fails_when=the
// fix ignores where the policy stands or prints a placeholder version; why_new=task 4; seam=tooLargeLine; no_claim=the bytes git reports
test('the fix branches on no key, head-only, a skipped pattern and still over, with pasteable JSON at the key version', () => {
test('the fix branches on no key, head-only and still over, with pasteable JSON at the key version', () => {
const json = (line: string) => JSON.parse(line.match(/land (\{.*?\}) in /)![1]!);
const none = tooLargeLine('ctx', 'batched', d, { state: 'none', base: 'main', source: 'origin/main@abc' }).line;
expect(json(none)).toEqual({ review_exclude_paths: ['results/**'], min_gstack_version: REVIEW_EXCLUDE_KEY_VERSION });
expect(none).toContain('then bring main into this branch and rerun once; if it is still over, split the PR');
const head = tooLargeLine('ctx', 'batched', d, { state: 'head-only', base: 'main', source: 'origin/main@abc', headPatterns: ['results/**'] }).line;
expect(head).toContain('this branch declares review_exclude_paths, but only the copy on main applies');
const skipped = tooLargeLine('ctx', 'batched', d, { state: 'active', base: 'main', source: 'origin/main@abc', record: { patterns: ['results/**'], skipped: [{ pattern: 'results/**', kept: 900, bytes: 40_000 }], excluded: 0 } }).line;
expect(skipped).toContain('pattern results/** was not applied: its 900 kept files need 40000 argument bytes');
const over = tooLargeLine('ctx', 'batched', d, { state: 'active', base: 'main', source: 'origin/main@abc', floor: '1.300.0.0', record: { patterns: ['receipts/**'], skipped: [], excluded: 12 } }).line;
const over = tooLargeLine('ctx', 'batched', d, { state: 'active', base: 'main', source: 'origin/main@abc', floor: '1.300.0.0', record: { patterns: ['receipts/**'], excluded: 12 } }).line;
expect(over).toContain('still over after excluding 12 files');
expect(json(over)).toEqual({ review_exclude_paths: ['receipts/**', 'results/**'], min_gstack_version: '1.300.0.0' });
expect(over).toEndWith('nothing was written (REVIEW_DIFF_TOO_LARGE)');