Merge oct8-yes-tools: repo redaction allowlist, opt-in sync removals, Windows setup.ps1

This commit is contained in:
garrytan committed 2026-10-08 15:52:12 +00:00
commit 035afd8555
11 files changed
+617 -11

No files matched your search

+153
View File
@@ -0,0 +1,153 @@
/**
* gstack-brain-sync removals (artifacts_sync_removals): off by default, a
* synced file gone from disk stays published; on, the drain publishes its
* removal, and a drain that would remove more than 20 files at once publishes
* none of them until `--publish-removals --yes`. Temp GSTACK_HOME, local bare
* remote; no network.
*/
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import { spawnSync } from 'child_process';
const ROOT = path.resolve(import.meta.dir, '..');
let home: string;
let remote: string;
function run(argv: string[]) {
const r = spawnSync(path.join(ROOT, 'bin', argv[0]), argv.slice(1), {
env: { ...process.env, HOME: home, GSTACK_HOME: home },
encoding: 'utf-8',
timeout: 60_000,
});
return { stdout: r.stdout || '', stderr: r.stderr || '', status: r.status ?? -1 };
}
const remoteFiles = () =>
spawnSync('git', ['--git-dir=' + remote, 'ls-tree', '-r', '--name-only', 'main'], { encoding: 'utf-8', timeout: 30_000 })
.stdout.split('\n').filter((p) => p.startsWith('projects/') || p.startsWith('transcripts/'));
const status = () => JSON.parse(fs.readFileSync(path.join(home, '.brain-sync-status.json'), 'utf-8'));
function write(p: string, body = '# doc\n') {
fs.mkdirSync(path.dirname(path.join(home, p)), { recursive: true });
fs.writeFileSync(path.join(home, p), body);
}
function syncAll(files: string[]) {
for (const f of files) write(f);
for (const f of files) run(['gstack-brain-enqueue', f]);
const r = run(['gstack-brain-sync', '--once']);
expect(r.status).toBe(0);
expect(remoteFiles()).toEqual(expect.arrayContaining(files));
}
const designs = (dir: string, n: number) => Array.from({ length: n }, (_, i) => `projects/${dir}/designs/d${i}.md`);
beforeEach(() => {
home = fs.mkdtempSync(path.join(os.tmpdir(), 'brain-removals-home-'));
remote = fs.mkdtempSync(path.join(os.tmpdir(), 'brain-removals-remote-'));
spawnSync('git', ['init', '--bare', '-q', '-b', 'main', remote], { timeout: 30_000 });
expect(run(['gstack-artifacts-init', '--remote', remote]).status).toBe(0);
run(['gstack-config', 'set', 'artifacts_sync_mode', 'full']);
});
afterEach(() => {
fs.rmSync(home, { recursive: true, force: true });
fs.rmSync(remote, { recursive: true, force: true });
});
describe('artifacts_sync_removals', () => {
test('off by default: a deleted synced file stays published', () => {
expect(run(['gstack-config', 'get', 'artifacts_sync_removals']).stdout.trim()).toBe('off');
syncAll(['projects/p/designs/a.md', 'projects/p/designs/b.md']);
fs.rmSync(path.join(home, 'projects/p/designs/a.md'));
run(['gstack-brain-sync', '--once']);
expect(remoteFiles()).toContain('projects/p/designs/a.md');
});
test('on: a slug migration removes the old copies with the queue empty or not', () => {
run(['gstack-config', 'set', 'artifacts_sync_removals', 'on']);
syncAll(['projects/acme/designs/x.md', 'projects/acme/designs/y.md']);
fs.renameSync(path.join(home, 'projects/acme'), path.join(home, 'projects/owner-acme'));
run(['gstack-brain-sync', '--discover-new']);
expect(run(['gstack-brain-sync', '--once']).status).toBe(0);
expect(remoteFiles().sort()).toEqual(['projects/owner-acme/designs/x.md', 'projects/owner-acme/designs/y.md']);
expect(status().message).toContain('published 2 removal(s)');
fs.rmSync(path.join(home, 'projects/owner-acme/designs/y.md'));
expect(run(['gstack-brain-sync', '--once']).status).toBe(0);
expect(remoteFiles()).toEqual(['projects/owner-acme/designs/x.md']);
expect(status().status).toBe('ok');
});
test('on: tracked files outside the allowlist and transcript pages are never removed', () => {
run(['gstack-config', 'set', 'artifacts_sync_removals', 'on']);
syncAll(['projects/p/designs/a.md']);
write('projects/p/notes.txt');
write('transcripts/run-1/s1.md');
spawnSync('git', ['-C', home, 'add', '-f', 'projects/p/notes.txt', 'transcripts/run-1/s1.md'], { timeout: 30_000 });
spawnSync('git', ['-C', home, '-c', 'user.email=u@example.com', '-c', 'user.name=u', 'commit', '-qm', 'manual'], { timeout: 30_000 });
spawnSync('git', ['-C', home, 'push', '-q', 'origin', 'HEAD'], { timeout: 30_000 });
for (const p of ['projects/p/notes.txt', 'transcripts/run-1/s1.md', 'projects/p/designs/a.md']) fs.rmSync(path.join(home, p));
run(['gstack-brain-sync', '--once']);
expect(remoteFiles().sort()).toEqual(['projects/p/notes.txt', 'transcripts/run-1/s1.md']);
});
test('on: a decision log goes only together with its active snapshot', () => {
run(['gstack-config', 'set', 'artifacts_sync_removals', 'on']);
const LOG = 'projects/p/decisions.jsonl';
const SNAP = 'projects/p/decisions.active.json';
write(SNAP, JSON.stringify([{ id: 'a', kind: 'decide', note: '' }]));
write(LOG, JSON.stringify({ id: 'a', kind: 'decide', decision: 'd a', scope: 'repo', date: '2026-10-01', source: 'user' }) + '\n');
run(['gstack-brain-enqueue', LOG]);
run(['gstack-brain-sync', '--once']);
expect(remoteFiles()).toEqual(expect.arrayContaining([LOG, SNAP]));
fs.rmSync(path.join(home, LOG));
run(['gstack-brain-sync', '--once']);
expect(remoteFiles()).toEqual(expect.arrayContaining([LOG, SNAP]));
fs.rmSync(path.join(home, SNAP));
run(['gstack-brain-sync', '--once']);
expect(remoteFiles()).toEqual([]);
});
test('on: more than 20 removals at once publishes none until --publish-removals --yes', () => {
run(['gstack-config', 'set', 'artifacts_sync_removals', 'on']);
const files = designs('big', 21);
syncAll(files);
fs.rmSync(path.join(home, 'projects/big'), { recursive: true });
const drain = run(['gstack-brain-sync', '--once']);
expect(drain.stderr).toContain('holding 21 removal(s), more than 20 at once');
expect(remoteFiles().sort()).toEqual([...files].sort());
expect(status().message).toContain('none of the removals were published');
const listed = run(['gstack-brain-sync', '--publish-removals']);
expect(listed.stdout).toContain('21 synced file(s) are gone from disk:');
expect(listed.stdout).toContain(' projects/big/designs/d0.md');
expect(remoteFiles()).toHaveLength(21);
const published = run(['gstack-brain-sync', '--publish-removals', '--yes']);
expect(published.status).toBe(0);
expect(published.stdout).toContain('published 21 removal(s)');
expect(remoteFiles()).toEqual([]);
});
test('--publish-removals --yes works with the setting off and only touches removals', () => {
syncAll(['projects/p/designs/a.md', 'projects/p/designs/b.md']);
fs.rmSync(path.join(home, 'projects/p/designs/a.md'));
write('projects/p/designs/b.md', '# changed, not yet synced\n');
const r = run(['gstack-brain-sync', '--publish-removals', '--yes']);
expect(r.status).toBe(0);
expect(remoteFiles()).toEqual(['projects/p/designs/b.md']);
const remoteB = spawnSync('git', ['--git-dir=' + remote, 'show', 'main:projects/p/designs/b.md'], { encoding: 'utf-8', timeout: 30_000 });
expect(remoteB.stdout).toBe('# doc\n');
expect(run(['gstack-brain-sync', '--publish-removals', '--yes']).stdout).toContain('no removals to publish');
});
test('config rejects values other than on and off', () => {
const r = run(['gstack-config', 'set', 'artifacts_sync_removals', 'yes']);
expect(r.stderr).toContain("artifacts_sync_removals 'yes' not recognized");
expect(run(['gstack-config', 'get', 'artifacts_sync_removals']).stdout.trim()).toBe('off');
});
});
+94
View File
@@ -191,6 +191,100 @@ describe("release version exemption (#2856)", () => {
});
});
describe("committed allowlist (.gstack-redact-allowlist)", () => {
const HOOK_ALLOWLIST = ".gstack-redact-allowlist";
const pushOf = (base: string, head: string) => `refs/heads/main ${head} refs/heads/main ${base}\n`;
const commitAllowlist = (content: string): string => {
return commit(HOOK_ALLOWLIST, content, "allowlist");
};
const DEV_URL = ["postgres://", "app:devpass", "@db:5432"].join("");
test("an exact committed span passes and the suppression is listed", () => {
const base = git(["rev-parse", "HEAD"]);
commit("config.txt", "key " + FAKE_AWS_KEY + "\n", "add key");
const head = commitAllowlist("# docs sample key\n" + FAKE_AWS_KEY + "\n");
const { code, stderr } = runHook(pushOf(base, head));
expect(code).toBe(0);
expect(stderr).not.toContain("BLOCKED");
expect(stderr).toContain(".gstack-redact-allowlist (1 entry) suppressed 2 finding(s) in this push:");
expect(stderr).toContain("HIGH aws.access_key config.txt:1");
expect(stderr).toContain("HIGH aws.access_key .gstack-redact-allowlist:2");
});
test("a different credential still blocks next to an allowlisted one", () => {
const base = git(["rev-parse", "HEAD"]);
const otherKey = ["AKIA", "FEDCBA0987654321"].join("");
commit("config.txt", "ok " + FAKE_AWS_KEY + "\nleak " + otherKey + "\n", "one benign one real");
const head = commitAllowlist(FAKE_AWS_KEY + "\n");
const { code, stderr } = runHook(pushOf(base, head));
expect(code).toBe(1);
expect(stderr).toContain("BLOCKED");
expect(stderr).toContain("suppressed 2 finding(s)");
const blocked = stderr.slice(stderr.indexOf("BLOCKED"));
expect(blocked.match(/HIGH {2}aws\.access_key/g)).toHaveLength(1);
});
test("a substring of the matched span does not suppress it", () => {
const base = git(["rev-parse", "HEAD"]);
commit("compose.txt", "DATABASE_URL=" + DEV_URL + "\n", "dev url");
const head = commitAllowlist("app:devpass\n");
const { code, stderr } = runHook(pushOf(base, head));
expect(code).toBe(1);
expect(stderr).toContain("db.url_with_password");
expect(stderr).toContain("suppressed 0 finding(s) in this push.");
});
test("the whole matched URL suppresses that URL only", () => {
const base = git(["rev-parse", "HEAD"]);
const prodUrl = ["postgres://", "app:devpass", "@prod.internal:5432"].join("");
commit("compose.txt", "DATABASE_URL=" + DEV_URL + "\nPROD_URL=" + prodUrl + "\n", "two urls");
const head = commitAllowlist(DEV_URL + "\n");
const { code, stderr } = runHook(pushOf(base, head));
expect(code).toBe(1);
expect(stderr).toContain("HIGH db.url_with_password compose.txt:1");
expect(stderr).toMatch(/BLOCKED[\s\S]*HIGH {2}db\.url_with_password/);
});
test("an uncommitted allowlist in the working tree suppresses nothing", () => {
const base = git(["rev-parse", "HEAD"]);
const head = commit("config.txt", "key " + FAKE_AWS_KEY + "\n", "add key");
fs.writeFileSync(path.join(repo, HOOK_ALLOWLIST), FAKE_AWS_KEY + "\n");
const { code, stderr } = runHook(pushOf(base, head));
expect(code).toBe(1);
expect(stderr).not.toContain("suppressed");
});
test("an oversized allowlist is ignored and the scan stays strict", () => {
const base = git(["rev-parse", "HEAD"]);
commit("config.txt", "key " + FAKE_AWS_KEY + "\n", "add key");
const head = commitAllowlist(FAKE_AWS_KEY + "\n" + "#".repeat(65 * 1024) + "\n");
const { code, stderr } = runHook(pushOf(base, head));
expect(code).toBe(1);
expect(stderr).toContain("is over 65536 bytes; ignoring it");
});
test("entries cover MEDIUM emails alongside gstack.redact.allowEmail", () => {
const base = git(["rev-parse", "HEAD"]);
git(["config", "--add", "gstack.redact.allowEmail", "carol@corp.io"]);
commit("notes.md", "contact bob@corp.io or carol@corp.io or dave@corp.io\n", "contacts");
const head = commitAllowlist(" bob@corp.io \n");
const { code, stderr } = runHook(pushOf(base, head));
expect(code).toBe(0);
expect(stderr).toContain("MEDIUM pii.email notes.md:1");
expect(stderr).toContain("suppressed 2 finding(s)");
expect(stderr).toContain("1 MEDIUM finding(s) in pushed diff");
});
test("the block message names the allowlist without one present", () => {
const base = git(["rev-parse", "HEAD"]);
const head = commit("config.txt", "key " + FAKE_AWS_KEY + "\n", "add key");
const { code, stderr } = runHook(pushOf(base, head));
expect(code).toBe(1);
expect(stderr).toContain("commit its exact matched text as a line of .gstack-redact-allowlist");
expect(stderr).not.toContain("suppressed");
});
});
describe("diff direction + special refs", () => {
test("only NEW content is scanned (remote..local), not pre-existing", () => {
// Put a secret in the FIRST commit (already on remote), then push a clean commit.
+75
View File
@@ -0,0 +1,75 @@
/**
* setup.ps1: the Windows entry point checks Git for Windows, Bun and Node.js,
* then runs ./setup in Git Bash with every argument passed as its own argv
* entry, never pasted into shell source. The behavioral cases copy the script
* next to a stub ./setup that prints its argv and run it under pwsh.
*/
import { describe, test, expect, afterEach } from 'bun:test';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import { spawnSync } from 'child_process';
const SCRIPT = path.resolve(import.meta.dir, '..', 'setup.ps1');
const SOURCE = fs.readFileSync(SCRIPT, 'utf-8');
const PWSH = Bun.which('pwsh');
const BASH = Bun.which('bash');
const dirs: string[] = [];
afterEach(() => {
for (const d of dirs.splice(0)) fs.rmSync(d, { recursive: true, force: true });
});
function stubCheckout(): string {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'setup-ps1-'));
dirs.push(dir);
fs.copyFileSync(SCRIPT, path.join(dir, 'setup.ps1'));
fs.writeFileSync(path.join(dir, 'setup'), '#!/usr/bin/env bash\nprintf \'cwd=%s\\n\' "$PWD"\nfor a in "$@"; do printf \'arg=[%s]\\n\' "$a"; done\nexit 7\n', { mode: 0o755 });
return dir;
}
function runPs1(dir: string, args: string[], env: Record<string, string | undefined> = {}) {
const r = spawnSync(PWSH!, ['-NoProfile', '-NonInteractive', '-File', path.join(dir, 'setup.ps1'), ...args], {
cwd: os.tmpdir(),
encoding: 'utf-8',
env: { ...process.env, ...(process.platform === 'win32' ? {} : { GSTACK_GIT_BASH: BASH! }), ...env },
timeout: 60_000,
});
return { code: r.status, out: (r.stdout || '') + (r.stderr || '') };
}
describe('setup.ps1', () => {
test('passes arguments as an array to a constant bash script', () => {
expect(SOURCE).toContain(`& $bash -c 'exec ./setup "$@"' setup @args`);
expect(SOURCE).not.toMatch(/-join|Invoke-Expression|\biex\b|git-bash\.exe/);
expect([...SOURCE].every((c) => c.charCodeAt(0) < 128)).toBe(true);
});
test.skipIf(!PWSH || !BASH)('forwards every argument literally from the script directory and keeps the exit code', () => {
const dir = stubCheckout();
const args = ['--host', 'codex', 'two words', '$(touch PWNED)', "it's", '"quoted"', ';echo injected', '`whoami`'];
const { code, out } = runPs1(dir, args);
expect(code).toBe(7);
const lines = out.split(/\r?\n/);
expect(lines.filter((l) => l.startsWith('arg=['))).toEqual(args.map((a) => `arg=[${a}]`));
expect(lines.find((l) => l.startsWith('cwd='))).toContain(path.basename(dir));
expect(out).not.toContain('injected\n');
expect(fs.existsSync(path.join(dir, 'PWNED'))).toBe(false);
expect(fs.existsSync(path.join(os.tmpdir(), 'PWNED'))).toBe(false);
});
test.skipIf(!PWSH || !BASH || process.platform === 'win32')('a missing prerequisite names its winget command and runs nothing', () => {
const dir = stubCheckout();
const bin = fs.mkdtempSync(path.join(os.tmpdir(), 'setup-ps1-bin-'));
dirs.push(bin);
for (const tool of ['git', 'node']) {
const real = Bun.which(tool);
if (real) fs.symlinkSync(real, path.join(bin, tool));
}
const { code, out } = runPs1(dir, ['--host', 'codex'], { PATH: bin });
expect(code).toBe(1);
expect(out).toContain('Bun: winget install --id Oven-sh.Bun -e');
expect(out).not.toContain('Git for Windows:');
expect(out).not.toContain('arg=[');
});
});