mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-20 23:23:26 +01:00
Merge oct8-yes-tools: repo redaction allowlist, opt-in sync removals, Windows setup.ps1
This commit is contained in:
commit
035afd8555
11 files changed
+617
-11
No files matched your search
@@ -0,0 +1,153 @@
|
||||
/**
|
||||
* gstack-brain-sync removals (artifacts_sync_removals): off by default, a
|
||||
* synced file gone from disk stays published; on, the drain publishes its
|
||||
* removal, and a drain that would remove more than 20 files at once publishes
|
||||
* none of them until `--publish-removals --yes`. Temp GSTACK_HOME, local bare
|
||||
* remote; no network.
|
||||
*/
|
||||
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { spawnSync } from 'child_process';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
|
||||
let home: string;
|
||||
let remote: string;
|
||||
|
||||
function run(argv: string[]) {
|
||||
const r = spawnSync(path.join(ROOT, 'bin', argv[0]), argv.slice(1), {
|
||||
env: { ...process.env, HOME: home, GSTACK_HOME: home },
|
||||
encoding: 'utf-8',
|
||||
timeout: 60_000,
|
||||
});
|
||||
return { stdout: r.stdout || '', stderr: r.stderr || '', status: r.status ?? -1 };
|
||||
}
|
||||
|
||||
const remoteFiles = () =>
|
||||
spawnSync('git', ['--git-dir=' + remote, 'ls-tree', '-r', '--name-only', 'main'], { encoding: 'utf-8', timeout: 30_000 })
|
||||
.stdout.split('\n').filter((p) => p.startsWith('projects/') || p.startsWith('transcripts/'));
|
||||
const status = () => JSON.parse(fs.readFileSync(path.join(home, '.brain-sync-status.json'), 'utf-8'));
|
||||
|
||||
function write(p: string, body = '# doc\n') {
|
||||
fs.mkdirSync(path.dirname(path.join(home, p)), { recursive: true });
|
||||
fs.writeFileSync(path.join(home, p), body);
|
||||
}
|
||||
|
||||
function syncAll(files: string[]) {
|
||||
for (const f of files) write(f);
|
||||
for (const f of files) run(['gstack-brain-enqueue', f]);
|
||||
const r = run(['gstack-brain-sync', '--once']);
|
||||
expect(r.status).toBe(0);
|
||||
expect(remoteFiles()).toEqual(expect.arrayContaining(files));
|
||||
}
|
||||
|
||||
const designs = (dir: string, n: number) => Array.from({ length: n }, (_, i) => `projects/${dir}/designs/d${i}.md`);
|
||||
|
||||
beforeEach(() => {
|
||||
home = fs.mkdtempSync(path.join(os.tmpdir(), 'brain-removals-home-'));
|
||||
remote = fs.mkdtempSync(path.join(os.tmpdir(), 'brain-removals-remote-'));
|
||||
spawnSync('git', ['init', '--bare', '-q', '-b', 'main', remote], { timeout: 30_000 });
|
||||
expect(run(['gstack-artifacts-init', '--remote', remote]).status).toBe(0);
|
||||
run(['gstack-config', 'set', 'artifacts_sync_mode', 'full']);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
fs.rmSync(home, { recursive: true, force: true });
|
||||
fs.rmSync(remote, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('artifacts_sync_removals', () => {
|
||||
test('off by default: a deleted synced file stays published', () => {
|
||||
expect(run(['gstack-config', 'get', 'artifacts_sync_removals']).stdout.trim()).toBe('off');
|
||||
syncAll(['projects/p/designs/a.md', 'projects/p/designs/b.md']);
|
||||
fs.rmSync(path.join(home, 'projects/p/designs/a.md'));
|
||||
run(['gstack-brain-sync', '--once']);
|
||||
expect(remoteFiles()).toContain('projects/p/designs/a.md');
|
||||
});
|
||||
|
||||
test('on: a slug migration removes the old copies with the queue empty or not', () => {
|
||||
run(['gstack-config', 'set', 'artifacts_sync_removals', 'on']);
|
||||
syncAll(['projects/acme/designs/x.md', 'projects/acme/designs/y.md']);
|
||||
fs.renameSync(path.join(home, 'projects/acme'), path.join(home, 'projects/owner-acme'));
|
||||
run(['gstack-brain-sync', '--discover-new']);
|
||||
expect(run(['gstack-brain-sync', '--once']).status).toBe(0);
|
||||
expect(remoteFiles().sort()).toEqual(['projects/owner-acme/designs/x.md', 'projects/owner-acme/designs/y.md']);
|
||||
expect(status().message).toContain('published 2 removal(s)');
|
||||
|
||||
fs.rmSync(path.join(home, 'projects/owner-acme/designs/y.md'));
|
||||
expect(run(['gstack-brain-sync', '--once']).status).toBe(0);
|
||||
expect(remoteFiles()).toEqual(['projects/owner-acme/designs/x.md']);
|
||||
expect(status().status).toBe('ok');
|
||||
});
|
||||
|
||||
test('on: tracked files outside the allowlist and transcript pages are never removed', () => {
|
||||
run(['gstack-config', 'set', 'artifacts_sync_removals', 'on']);
|
||||
syncAll(['projects/p/designs/a.md']);
|
||||
write('projects/p/notes.txt');
|
||||
write('transcripts/run-1/s1.md');
|
||||
spawnSync('git', ['-C', home, 'add', '-f', 'projects/p/notes.txt', 'transcripts/run-1/s1.md'], { timeout: 30_000 });
|
||||
spawnSync('git', ['-C', home, '-c', 'user.email=u@example.com', '-c', 'user.name=u', 'commit', '-qm', 'manual'], { timeout: 30_000 });
|
||||
spawnSync('git', ['-C', home, 'push', '-q', 'origin', 'HEAD'], { timeout: 30_000 });
|
||||
for (const p of ['projects/p/notes.txt', 'transcripts/run-1/s1.md', 'projects/p/designs/a.md']) fs.rmSync(path.join(home, p));
|
||||
run(['gstack-brain-sync', '--once']);
|
||||
expect(remoteFiles().sort()).toEqual(['projects/p/notes.txt', 'transcripts/run-1/s1.md']);
|
||||
});
|
||||
|
||||
test('on: a decision log goes only together with its active snapshot', () => {
|
||||
run(['gstack-config', 'set', 'artifacts_sync_removals', 'on']);
|
||||
const LOG = 'projects/p/decisions.jsonl';
|
||||
const SNAP = 'projects/p/decisions.active.json';
|
||||
write(SNAP, JSON.stringify([{ id: 'a', kind: 'decide', note: '' }]));
|
||||
write(LOG, JSON.stringify({ id: 'a', kind: 'decide', decision: 'd a', scope: 'repo', date: '2026-10-01', source: 'user' }) + '\n');
|
||||
run(['gstack-brain-enqueue', LOG]);
|
||||
run(['gstack-brain-sync', '--once']);
|
||||
expect(remoteFiles()).toEqual(expect.arrayContaining([LOG, SNAP]));
|
||||
fs.rmSync(path.join(home, LOG));
|
||||
run(['gstack-brain-sync', '--once']);
|
||||
expect(remoteFiles()).toEqual(expect.arrayContaining([LOG, SNAP]));
|
||||
fs.rmSync(path.join(home, SNAP));
|
||||
run(['gstack-brain-sync', '--once']);
|
||||
expect(remoteFiles()).toEqual([]);
|
||||
});
|
||||
|
||||
test('on: more than 20 removals at once publishes none until --publish-removals --yes', () => {
|
||||
run(['gstack-config', 'set', 'artifacts_sync_removals', 'on']);
|
||||
const files = designs('big', 21);
|
||||
syncAll(files);
|
||||
fs.rmSync(path.join(home, 'projects/big'), { recursive: true });
|
||||
const drain = run(['gstack-brain-sync', '--once']);
|
||||
expect(drain.stderr).toContain('holding 21 removal(s), more than 20 at once');
|
||||
expect(remoteFiles().sort()).toEqual([...files].sort());
|
||||
expect(status().message).toContain('none of the removals were published');
|
||||
|
||||
const listed = run(['gstack-brain-sync', '--publish-removals']);
|
||||
expect(listed.stdout).toContain('21 synced file(s) are gone from disk:');
|
||||
expect(listed.stdout).toContain(' projects/big/designs/d0.md');
|
||||
expect(remoteFiles()).toHaveLength(21);
|
||||
|
||||
const published = run(['gstack-brain-sync', '--publish-removals', '--yes']);
|
||||
expect(published.status).toBe(0);
|
||||
expect(published.stdout).toContain('published 21 removal(s)');
|
||||
expect(remoteFiles()).toEqual([]);
|
||||
});
|
||||
|
||||
test('--publish-removals --yes works with the setting off and only touches removals', () => {
|
||||
syncAll(['projects/p/designs/a.md', 'projects/p/designs/b.md']);
|
||||
fs.rmSync(path.join(home, 'projects/p/designs/a.md'));
|
||||
write('projects/p/designs/b.md', '# changed, not yet synced\n');
|
||||
const r = run(['gstack-brain-sync', '--publish-removals', '--yes']);
|
||||
expect(r.status).toBe(0);
|
||||
expect(remoteFiles()).toEqual(['projects/p/designs/b.md']);
|
||||
const remoteB = spawnSync('git', ['--git-dir=' + remote, 'show', 'main:projects/p/designs/b.md'], { encoding: 'utf-8', timeout: 30_000 });
|
||||
expect(remoteB.stdout).toBe('# doc\n');
|
||||
expect(run(['gstack-brain-sync', '--publish-removals', '--yes']).stdout).toContain('no removals to publish');
|
||||
});
|
||||
|
||||
test('config rejects values other than on and off', () => {
|
||||
const r = run(['gstack-config', 'set', 'artifacts_sync_removals', 'yes']);
|
||||
expect(r.stderr).toContain("artifacts_sync_removals 'yes' not recognized");
|
||||
expect(run(['gstack-config', 'get', 'artifacts_sync_removals']).stdout.trim()).toBe('off');
|
||||
});
|
||||
});
|
||||
@@ -191,6 +191,100 @@ describe("release version exemption (#2856)", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("committed allowlist (.gstack-redact-allowlist)", () => {
|
||||
const HOOK_ALLOWLIST = ".gstack-redact-allowlist";
|
||||
const pushOf = (base: string, head: string) => `refs/heads/main ${head} refs/heads/main ${base}\n`;
|
||||
const commitAllowlist = (content: string): string => {
|
||||
return commit(HOOK_ALLOWLIST, content, "allowlist");
|
||||
};
|
||||
const DEV_URL = ["postgres://", "app:devpass", "@db:5432"].join("");
|
||||
|
||||
test("an exact committed span passes and the suppression is listed", () => {
|
||||
const base = git(["rev-parse", "HEAD"]);
|
||||
commit("config.txt", "key " + FAKE_AWS_KEY + "\n", "add key");
|
||||
const head = commitAllowlist("# docs sample key\n" + FAKE_AWS_KEY + "\n");
|
||||
const { code, stderr } = runHook(pushOf(base, head));
|
||||
expect(code).toBe(0);
|
||||
expect(stderr).not.toContain("BLOCKED");
|
||||
expect(stderr).toContain(".gstack-redact-allowlist (1 entry) suppressed 2 finding(s) in this push:");
|
||||
expect(stderr).toContain("HIGH aws.access_key config.txt:1");
|
||||
expect(stderr).toContain("HIGH aws.access_key .gstack-redact-allowlist:2");
|
||||
});
|
||||
|
||||
test("a different credential still blocks next to an allowlisted one", () => {
|
||||
const base = git(["rev-parse", "HEAD"]);
|
||||
const otherKey = ["AKIA", "FEDCBA0987654321"].join("");
|
||||
commit("config.txt", "ok " + FAKE_AWS_KEY + "\nleak " + otherKey + "\n", "one benign one real");
|
||||
const head = commitAllowlist(FAKE_AWS_KEY + "\n");
|
||||
const { code, stderr } = runHook(pushOf(base, head));
|
||||
expect(code).toBe(1);
|
||||
expect(stderr).toContain("BLOCKED");
|
||||
expect(stderr).toContain("suppressed 2 finding(s)");
|
||||
const blocked = stderr.slice(stderr.indexOf("BLOCKED"));
|
||||
expect(blocked.match(/HIGH {2}aws\.access_key/g)).toHaveLength(1);
|
||||
});
|
||||
|
||||
test("a substring of the matched span does not suppress it", () => {
|
||||
const base = git(["rev-parse", "HEAD"]);
|
||||
commit("compose.txt", "DATABASE_URL=" + DEV_URL + "\n", "dev url");
|
||||
const head = commitAllowlist("app:devpass\n");
|
||||
const { code, stderr } = runHook(pushOf(base, head));
|
||||
expect(code).toBe(1);
|
||||
expect(stderr).toContain("db.url_with_password");
|
||||
expect(stderr).toContain("suppressed 0 finding(s) in this push.");
|
||||
});
|
||||
|
||||
test("the whole matched URL suppresses that URL only", () => {
|
||||
const base = git(["rev-parse", "HEAD"]);
|
||||
const prodUrl = ["postgres://", "app:devpass", "@prod.internal:5432"].join("");
|
||||
commit("compose.txt", "DATABASE_URL=" + DEV_URL + "\nPROD_URL=" + prodUrl + "\n", "two urls");
|
||||
const head = commitAllowlist(DEV_URL + "\n");
|
||||
const { code, stderr } = runHook(pushOf(base, head));
|
||||
expect(code).toBe(1);
|
||||
expect(stderr).toContain("HIGH db.url_with_password compose.txt:1");
|
||||
expect(stderr).toMatch(/BLOCKED[\s\S]*HIGH {2}db\.url_with_password/);
|
||||
});
|
||||
|
||||
test("an uncommitted allowlist in the working tree suppresses nothing", () => {
|
||||
const base = git(["rev-parse", "HEAD"]);
|
||||
const head = commit("config.txt", "key " + FAKE_AWS_KEY + "\n", "add key");
|
||||
fs.writeFileSync(path.join(repo, HOOK_ALLOWLIST), FAKE_AWS_KEY + "\n");
|
||||
const { code, stderr } = runHook(pushOf(base, head));
|
||||
expect(code).toBe(1);
|
||||
expect(stderr).not.toContain("suppressed");
|
||||
});
|
||||
|
||||
test("an oversized allowlist is ignored and the scan stays strict", () => {
|
||||
const base = git(["rev-parse", "HEAD"]);
|
||||
commit("config.txt", "key " + FAKE_AWS_KEY + "\n", "add key");
|
||||
const head = commitAllowlist(FAKE_AWS_KEY + "\n" + "#".repeat(65 * 1024) + "\n");
|
||||
const { code, stderr } = runHook(pushOf(base, head));
|
||||
expect(code).toBe(1);
|
||||
expect(stderr).toContain("is over 65536 bytes; ignoring it");
|
||||
});
|
||||
|
||||
test("entries cover MEDIUM emails alongside gstack.redact.allowEmail", () => {
|
||||
const base = git(["rev-parse", "HEAD"]);
|
||||
git(["config", "--add", "gstack.redact.allowEmail", "carol@corp.io"]);
|
||||
commit("notes.md", "contact bob@corp.io or carol@corp.io or dave@corp.io\n", "contacts");
|
||||
const head = commitAllowlist(" bob@corp.io \n");
|
||||
const { code, stderr } = runHook(pushOf(base, head));
|
||||
expect(code).toBe(0);
|
||||
expect(stderr).toContain("MEDIUM pii.email notes.md:1");
|
||||
expect(stderr).toContain("suppressed 2 finding(s)");
|
||||
expect(stderr).toContain("1 MEDIUM finding(s) in pushed diff");
|
||||
});
|
||||
|
||||
test("the block message names the allowlist without one present", () => {
|
||||
const base = git(["rev-parse", "HEAD"]);
|
||||
const head = commit("config.txt", "key " + FAKE_AWS_KEY + "\n", "add key");
|
||||
const { code, stderr } = runHook(pushOf(base, head));
|
||||
expect(code).toBe(1);
|
||||
expect(stderr).toContain("commit its exact matched text as a line of .gstack-redact-allowlist");
|
||||
expect(stderr).not.toContain("suppressed");
|
||||
});
|
||||
});
|
||||
|
||||
describe("diff direction + special refs", () => {
|
||||
test("only NEW content is scanned (remote..local), not pre-existing", () => {
|
||||
// Put a secret in the FIRST commit (already on remote), then push a clean commit.
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
/**
|
||||
* setup.ps1: the Windows entry point checks Git for Windows, Bun and Node.js,
|
||||
* then runs ./setup in Git Bash with every argument passed as its own argv
|
||||
* entry, never pasted into shell source. The behavioral cases copy the script
|
||||
* next to a stub ./setup that prints its argv and run it under pwsh.
|
||||
*/
|
||||
import { describe, test, expect, afterEach } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { spawnSync } from 'child_process';
|
||||
|
||||
const SCRIPT = path.resolve(import.meta.dir, '..', 'setup.ps1');
|
||||
const SOURCE = fs.readFileSync(SCRIPT, 'utf-8');
|
||||
const PWSH = Bun.which('pwsh');
|
||||
const BASH = Bun.which('bash');
|
||||
|
||||
const dirs: string[] = [];
|
||||
afterEach(() => {
|
||||
for (const d of dirs.splice(0)) fs.rmSync(d, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function stubCheckout(): string {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'setup-ps1-'));
|
||||
dirs.push(dir);
|
||||
fs.copyFileSync(SCRIPT, path.join(dir, 'setup.ps1'));
|
||||
fs.writeFileSync(path.join(dir, 'setup'), '#!/usr/bin/env bash\nprintf \'cwd=%s\\n\' "$PWD"\nfor a in "$@"; do printf \'arg=[%s]\\n\' "$a"; done\nexit 7\n', { mode: 0o755 });
|
||||
return dir;
|
||||
}
|
||||
|
||||
function runPs1(dir: string, args: string[], env: Record<string, string | undefined> = {}) {
|
||||
const r = spawnSync(PWSH!, ['-NoProfile', '-NonInteractive', '-File', path.join(dir, 'setup.ps1'), ...args], {
|
||||
cwd: os.tmpdir(),
|
||||
encoding: 'utf-8',
|
||||
env: { ...process.env, ...(process.platform === 'win32' ? {} : { GSTACK_GIT_BASH: BASH! }), ...env },
|
||||
timeout: 60_000,
|
||||
});
|
||||
return { code: r.status, out: (r.stdout || '') + (r.stderr || '') };
|
||||
}
|
||||
|
||||
describe('setup.ps1', () => {
|
||||
test('passes arguments as an array to a constant bash script', () => {
|
||||
expect(SOURCE).toContain(`& $bash -c 'exec ./setup "$@"' setup @args`);
|
||||
expect(SOURCE).not.toMatch(/-join|Invoke-Expression|\biex\b|git-bash\.exe/);
|
||||
expect([...SOURCE].every((c) => c.charCodeAt(0) < 128)).toBe(true);
|
||||
});
|
||||
|
||||
test.skipIf(!PWSH || !BASH)('forwards every argument literally from the script directory and keeps the exit code', () => {
|
||||
const dir = stubCheckout();
|
||||
const args = ['--host', 'codex', 'two words', '$(touch PWNED)', "it's", '"quoted"', ';echo injected', '`whoami`'];
|
||||
const { code, out } = runPs1(dir, args);
|
||||
expect(code).toBe(7);
|
||||
const lines = out.split(/\r?\n/);
|
||||
expect(lines.filter((l) => l.startsWith('arg=['))).toEqual(args.map((a) => `arg=[${a}]`));
|
||||
expect(lines.find((l) => l.startsWith('cwd='))).toContain(path.basename(dir));
|
||||
expect(out).not.toContain('injected\n');
|
||||
expect(fs.existsSync(path.join(dir, 'PWNED'))).toBe(false);
|
||||
expect(fs.existsSync(path.join(os.tmpdir(), 'PWNED'))).toBe(false);
|
||||
});
|
||||
|
||||
test.skipIf(!PWSH || !BASH || process.platform === 'win32')('a missing prerequisite names its winget command and runs nothing', () => {
|
||||
const dir = stubCheckout();
|
||||
const bin = fs.mkdtempSync(path.join(os.tmpdir(), 'setup-ps1-bin-'));
|
||||
dirs.push(bin);
|
||||
for (const tool of ['git', 'node']) {
|
||||
const real = Bun.which(tool);
|
||||
if (real) fs.symlinkSync(real, path.join(bin, tool));
|
||||
}
|
||||
const { code, out } = runPs1(dir, ['--host', 'codex'], { PATH: bin });
|
||||
expect(code).toBe(1);
|
||||
expect(out).toContain('Bun: winget install --id Oven-sh.Bun -e');
|
||||
expect(out).not.toContain('Git for Windows:');
|
||||
expect(out).not.toContain('arg=[');
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user