From 33ceff32bbc4856a5c2dfb4a56a531e82fe70032 Mon Sep 17 00:00:00 2001 From: garrytan Date: Thu, 8 Oct 2026 14:54:53 +0000 Subject: [PATCH 1/3] feat(redact-prepush): committed exact-span allowlist at .gstack-redact-allowlist The pre-push hook never fed the engine's allowlist option, so a reviewed, known-benign value (a public dev-compose URL, a documented sample key) warned or blocked every push, and the only ways out were a bypass or reshaping code until the detector stopped matching. The hook now reads the repo-root .gstack-redact-allowlist from each pushed commit (never the working tree), one exact matched span per line with # comments. Only a finding whose matched text equals an entry is suppressed: a password alone, a substring, or a different key still reports and still blocks, and engine.* unscanned findings are untouched. Every push that carries entries prints the entry count and each suppressed finding's rule and file line. Oversized files (over 64 KiB) are ignored. gstack.redact.allowEmail keeps working as before. Co-authored-by: Boyard Engels Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- bin/gstack-redact-prepush | 74 ++++++++++++++++++++++--- docs/CONTRIBUTOR_REFERENCE.md | 3 + docs/troubleshooting.md | 18 ++++++ test/redact-prepush-hook.test.ts | 94 ++++++++++++++++++++++++++++++++ 4 files changed, 181 insertions(+), 8 deletions(-) diff --git a/bin/gstack-redact-prepush b/bin/gstack-redact-prepush index 98e9b931c..31a87c97e 100755 --- a/bin/gstack-redact-prepush +++ b/bin/gstack-redact-prepush @@ -24,6 +24,10 @@ * - `pii.email` skips the pusher's own address, `gstack.redact.allowEmail` * entries, and addresses already in the destination's commit metadata * (see emailScanOptions). + * - `.gstack-redact-allowlist`, read from each pushed commit (never the + * working tree), lists exact matched spans reviewed as benign, one per + * line, `#` comments. Only a finding whose matched text equals an entry is + * suppressed; every push that carries entries lists what they suppressed. * - GSTACK_REDACT_PREPUSH=skip → log + exit 0 (escape valve). * * Installed/uninstalled via `gstack-redact install-prepush-hook` (see the @@ -549,11 +553,40 @@ function emailScanOptions(historyRevs: string[], trackingRemote: string | null): }; } +const ALLOWLIST_PATH = ".gstack-redact-allowlist"; +const ALLOWLIST_MAX_BYTES = 64 * 1024; + +/** + * Entries of the allowlist committed in `sha`. Read from the pushed commit so + * a suppression exists only as a reviewable commit. Absent, unreadable or + * oversized means no entries: the file can record a reviewed exception, never + * loosen the scan by accident. + */ +function committedAllowlist(sha: string): string[] { + const spec = `${sha}:${ALLOWLIST_PATH}`; + const size = Number(git(["cat-file", "-s", spec]).trim()); + if (!size) return []; + if (size > ALLOWLIST_MAX_BYTES) { + process.stderr.write( + `gstack-redact-prepush: ${ALLOWLIST_PATH} in ${sha.slice(0, 12)} is over ${ALLOWLIST_MAX_BYTES} bytes; ignoring it.\n`, + ); + return []; + } + return git(["cat-file", "blob", spec]).split("\n") + .map((l) => l.trim()) + .filter((l) => l && !l.startsWith("#")); +} + +/** File line (and commit) of a finding. Never the matched value. */ +function findingWhere(f: Finding, part: AddedFile): string { + const line = part.lines[f.line - 1]; + return `${part.file || "(unknown file)"}${line ? `:${line}` : ""}` + + (part.commit ? ` in commit ${part.commit.slice(0, 12)}` : ""); +} + /** One MEDIUM report line: rule id, file line (and commit), and the fix. Never the matched value. */ function mediumLine(f: Finding, part: AddedFile): string { - const line = part.lines[f.line - 1]; - const where = `${part.file || "(unknown file)"}${line ? `:${line}` : ""}` + - (part.commit ? ` in commit ${part.commit.slice(0, 12)}` : ""); + const where = findingWhere(f, part); const fix = f.id === "pii.email" ? "remove the address, or allow it: git config --add gstack.redact.allowEmail
" : "remove the value before it becomes public"; @@ -574,7 +607,8 @@ function main() { .filter(Boolean) .map((l) => l.split(/\s+/)); - const parts: AddedFile[] = []; + const parts: Array<{ part: AddedFile; allowlist: string[] }> = []; + const allowEntries = new Set(); const historyRevs = new Set(); let trackingRemote: string | null = null; @@ -610,7 +644,9 @@ function main() { ); process.exit(1); } - parts.push(...added.filter((part) => part.text.trim())); + const allowlist = committedAllowlist(localSha); + for (const entry of allowlist) allowEntries.add(entry); + for (const part of added) if (part.text.trim()) parts.push({ part, allowlist }); historyRevs.add(localSha); if (!ZERO.test(remoteSha) && objectExists(remoteSha)) historyRevs.add(remoteSha); else { @@ -628,9 +664,21 @@ function main() { // Sliced (see scanAddedLines) so a large-but-legitimate diff is actually // scanned rather than blocked unscanned on the engine's size cap. // Scanned per file so a pattern can use the file's type (#3048). - for (const part of parts) { + // An allowlisted span is found by scanning with and without the entries, + // so the report names exactly what they suppressed. + const allowed: string[] = []; + const findingKey = (f: Finding) => `${f.id}:${f.line}:${f.col}`; + for (const { part, allowlist } of parts) { const opts = { repoVisibility: "private" as const, ...emails.options, ...(part.file ? { sourcePath: part.file } : {}) }; - for (const f of scanAddedLines(part.text, opts)) { + const findings = scanAddedLines(part.text, opts); + const kept = allowlist.length + ? new Set(scanAddedLines(part.text, { ...opts, allowlist }).map(findingKey)) + : null; + for (const f of findings) { + if (kept && !kept.has(findingKey(f))) { + if (f.severity === "HIGH" || f.severity === "MEDIUM") allowed.push(` ${f.severity} ${f.id} ${findingWhere(f, part)}\n`); + continue; + } if (f.severity === "HIGH") allHigh.push(f); else if (f.severity === "MEDIUM") { mediums.push(mediumLine(f, part)); @@ -639,6 +687,14 @@ function main() { } } + if (allowEntries.size > 0) { + process.stderr.write( + `gstack-redact-prepush: ${ALLOWLIST_PATH} (${allowEntries.size} entr${allowEntries.size === 1 ? "y" : "ies"}) ` + + `suppressed ${allowed.length} finding(s) in this push${allowed.length ? ":" : "."}\n`, + ); + for (const line of allowed) process.stderr.write(line); + } + if (mediums.length > 0) { process.stderr.write( `gstack-redact-prepush: ${mediums.length} MEDIUM finding(s) in pushed diff (PII/internal). ` + @@ -673,7 +729,9 @@ function main() { process.stderr.write(` HIGH ${f.id} ${f.preview}\n`); } process.stderr.write( - "\nRotate the credential (a pushed secret is compromised) and remove it from the diff.\n", + "\nRotate the credential (a pushed secret is compromised) and remove it from the diff.\n" + + `If the value is a known-benign sample, commit its exact matched text as a line of ${ALLOWLIST_PATH}\n` + + "at the repo root instead of reshaping code to dodge the pattern; every push lists what that file suppresses.\n", ); } diff --git a/docs/CONTRIBUTOR_REFERENCE.md b/docs/CONTRIBUTOR_REFERENCE.md index 45a05a3f8..b6b89d996 100644 --- a/docs/CONTRIBUTOR_REFERENCE.md +++ b/docs/CONTRIBUTOR_REFERENCE.md @@ -134,6 +134,9 @@ determined leaker (a CHANGELOG line that does would fail a hostile screenshotter - **Config keys:** `redact_repo_visibility` (public|private|unknown, local-only override for repos gh/glab can't read), `redact_prepush_hook` (true|false). There is intentionally NO key to disable HIGH blocking. +- **Committed allowlist:** `.gstack-redact-allowlist` at the root of the pushed commit lists + exact matched spans reviewed as benign; the pre-push hook suppresses only a + finding whose span equals an entry and lists every suppression on each push. - **Audit:** the /spec semantic pass appends a content-free record (categories + body sha256, no spec text) to `~/.gstack/security/semantic-reviews.jsonl` (0600). diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index f71008f53..64c78c1bf 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -1120,6 +1120,24 @@ up deployed, so it still blocks. **Fix.** Use `postgres://postgres:postgres@localhost:5432/...` in local and CI config, or read the URL from an env var. If the credential is real, rotate it. Bypass once: `GSTACK_REDACT_PREPUSH=skip git push`. +If the URL is a reviewed, public dev-only value, list it in +[`.gstack-redact-allowlist`](#redact-allowlist). + + +### `.gstack-redact-allowlist ( entries) suppressed finding(s) in this push` + +**Meaning.** The pushed commit carries `.gstack-redact-allowlist` at the repo +root. Each line (after trimming; `#` starts a comment) is one exact matched +span: the whole `postgres://USER:PASSWORD@host:port` URL for +`db.url_with_password`, the key for a key pattern, the address for `pii.email`. A finding is suppressed only when its +matched text equals an entry, so a password alone, a substring, or a different +key still reports and still blocks. The hook reads the file from each pushed +commit, never the working tree, and ignores a file over 64 KiB. Every push that +carries entries prints this line and each suppressed finding's file and line. +It works alongside `gstack.redact.allowEmail`, which stays a local setting. + +**Fix.** Nothing, if every listed suppression is the reviewed value. Remove an +entry that no longer applies; rotate a credential that is real. ### `pii.ip_public` MEDIUM on a four-part version number diff --git a/test/redact-prepush-hook.test.ts b/test/redact-prepush-hook.test.ts index 36ced9365..66727b828 100644 --- a/test/redact-prepush-hook.test.ts +++ b/test/redact-prepush-hook.test.ts @@ -191,6 +191,100 @@ describe("release version exemption (#2856)", () => { }); }); +describe("committed allowlist (.gstack-redact-allowlist)", () => { + const HOOK_ALLOWLIST = ".gstack-redact-allowlist"; + const pushOf = (base: string, head: string) => `refs/heads/main ${head} refs/heads/main ${base}\n`; + const commitAllowlist = (content: string): string => { + return commit(HOOK_ALLOWLIST, content, "allowlist"); + }; + const DEV_URL = ["postgres://", "app:devpass", "@db:5432"].join(""); + + test("an exact committed span passes and the suppression is listed", () => { + const base = git(["rev-parse", "HEAD"]); + commit("config.txt", "key " + FAKE_AWS_KEY + "\n", "add key"); + const head = commitAllowlist("# docs sample key\n" + FAKE_AWS_KEY + "\n"); + const { code, stderr } = runHook(pushOf(base, head)); + expect(code).toBe(0); + expect(stderr).not.toContain("BLOCKED"); + expect(stderr).toContain(".gstack-redact-allowlist (1 entry) suppressed 2 finding(s) in this push:"); + expect(stderr).toContain("HIGH aws.access_key config.txt:1"); + expect(stderr).toContain("HIGH aws.access_key .gstack-redact-allowlist:2"); + }); + + test("a different credential still blocks next to an allowlisted one", () => { + const base = git(["rev-parse", "HEAD"]); + const otherKey = ["AKIA", "FEDCBA0987654321"].join(""); + commit("config.txt", "ok " + FAKE_AWS_KEY + "\nleak " + otherKey + "\n", "one benign one real"); + const head = commitAllowlist(FAKE_AWS_KEY + "\n"); + const { code, stderr } = runHook(pushOf(base, head)); + expect(code).toBe(1); + expect(stderr).toContain("BLOCKED"); + expect(stderr).toContain("suppressed 2 finding(s)"); + const blocked = stderr.slice(stderr.indexOf("BLOCKED")); + expect(blocked.match(/HIGH {2}aws\.access_key/g)).toHaveLength(1); + }); + + test("a substring of the matched span does not suppress it", () => { + const base = git(["rev-parse", "HEAD"]); + commit("compose.txt", "DATABASE_URL=" + DEV_URL + "\n", "dev url"); + const head = commitAllowlist("app:devpass\n"); + const { code, stderr } = runHook(pushOf(base, head)); + expect(code).toBe(1); + expect(stderr).toContain("db.url_with_password"); + expect(stderr).toContain("suppressed 0 finding(s) in this push."); + }); + + test("the whole matched URL suppresses that URL only", () => { + const base = git(["rev-parse", "HEAD"]); + const prodUrl = ["postgres://", "app:devpass", "@prod.internal:5432"].join(""); + commit("compose.txt", "DATABASE_URL=" + DEV_URL + "\nPROD_URL=" + prodUrl + "\n", "two urls"); + const head = commitAllowlist(DEV_URL + "\n"); + const { code, stderr } = runHook(pushOf(base, head)); + expect(code).toBe(1); + expect(stderr).toContain("HIGH db.url_with_password compose.txt:1"); + expect(stderr).toMatch(/BLOCKED[\s\S]*HIGH {2}db\.url_with_password/); + }); + + test("an uncommitted allowlist in the working tree suppresses nothing", () => { + const base = git(["rev-parse", "HEAD"]); + const head = commit("config.txt", "key " + FAKE_AWS_KEY + "\n", "add key"); + fs.writeFileSync(path.join(repo, HOOK_ALLOWLIST), FAKE_AWS_KEY + "\n"); + const { code, stderr } = runHook(pushOf(base, head)); + expect(code).toBe(1); + expect(stderr).not.toContain("suppressed"); + }); + + test("an oversized allowlist is ignored and the scan stays strict", () => { + const base = git(["rev-parse", "HEAD"]); + commit("config.txt", "key " + FAKE_AWS_KEY + "\n", "add key"); + const head = commitAllowlist(FAKE_AWS_KEY + "\n" + "#".repeat(65 * 1024) + "\n"); + const { code, stderr } = runHook(pushOf(base, head)); + expect(code).toBe(1); + expect(stderr).toContain("is over 65536 bytes; ignoring it"); + }); + + test("entries cover MEDIUM emails alongside gstack.redact.allowEmail", () => { + const base = git(["rev-parse", "HEAD"]); + git(["config", "--add", "gstack.redact.allowEmail", "carol@corp.io"]); + commit("notes.md", "contact bob@corp.io or carol@corp.io or dave@corp.io\n", "contacts"); + const head = commitAllowlist(" bob@corp.io \n"); + const { code, stderr } = runHook(pushOf(base, head)); + expect(code).toBe(0); + expect(stderr).toContain("MEDIUM pii.email notes.md:1"); + expect(stderr).toContain("suppressed 2 finding(s)"); + expect(stderr).toContain("1 MEDIUM finding(s) in pushed diff"); + }); + + test("the block message names the allowlist without one present", () => { + const base = git(["rev-parse", "HEAD"]); + const head = commit("config.txt", "key " + FAKE_AWS_KEY + "\n", "add key"); + const { code, stderr } = runHook(pushOf(base, head)); + expect(code).toBe(1); + expect(stderr).toContain("commit its exact matched text as a line of .gstack-redact-allowlist"); + expect(stderr).not.toContain("suppressed"); + }); +}); + describe("diff direction + special refs", () => { test("only NEW content is scanned (remote..local), not pre-existing", () => { // Put a secret in the FIRST commit (already on remote), then push a clean commit. From 671fb0577e0fbab7ee1a5c0b8a61969ee2006112 Mon Sep 17 00:00:00 2001 From: garrytan Date: Thu, 8 Oct 2026 15:27:22 +0000 Subject: [PATCH 2/3] feat(brain-sync): opt-in removal publishing with a bulk-removal valve gstack-brain-sync never published a removal: a deleted synced file, or the old directory after a project slug migration, stayed on the remote forever and gbrain indexed both copies. New setting artifacts_sync_removals (off by default). When on, each drain commits and pushes, receipted like every push, the removal of tracked files that are gone from disk and match the sync allowlist. Transcript pages are never removed here, and a decision log goes only together with its active snapshot. One valve: a drain that would remove more than 20 files at once publishes none and says so. `gstack-brain-sync --publish-removals` lists the pending removals and `--publish-removals --yes` publishes them on demand, whatever the setting. Co-authored-by: Jey2311 <5177215+Jey2311@users.noreply.github.com> --- bin/gstack-brain-sync | 119 +++++++++++++++++++++++- bin/gstack-config | 14 ++- docs/gbrain-sync.md | 22 +++++ test/brain-sync-removals.test.ts | 153 +++++++++++++++++++++++++++++++ 4 files changed, 305 insertions(+), 3 deletions(-) create mode 100644 test/brain-sync-removals.test.ts diff --git a/bin/gstack-brain-sync b/bin/gstack-brain-sync index bc7ac76d5..093fc2693 100755 --- a/bin/gstack-brain-sync +++ b/bin/gstack-brain-sync @@ -10,6 +10,9 @@ # gstack-brain-sync --drop-queue --yes clear queue without committing # gstack-brain-sync --discover-new scan allowlist dirs, enqueue changed files # gstack-brain-sync --attention print skill start's attention lines +# gstack-brain-sync --publish-removals [--yes] +# list (or, with --yes, commit and push) +# the removal of synced files gone from disk # # Invoked by the preamble at skill START and END boundaries. No persistent # daemon. Typical run <1s when queue empty; ~200-800ms with network push. @@ -18,6 +21,14 @@ # with every other gstack writer to ~/.gstack's git state. A drain that finds # it held skips; the next skill boundary catches up. # +# Removals: with artifacts_sync_removals=on, each drain also publishes the +# removal of synced files that are gone from disk (a deleted design doc, the +# old directory after a project slug migration), so the remote and the brain +# stop serving stale copies. A drain that would remove more than +# REMOVAL_VALVE files at once publishes none of them and says so: that many +# at once looks like a loss, not a cleanup. --publish-removals --yes publishes +# them on demand, whatever the setting. +# # A file the secret scan flags is held back on its own (with the rest of its # coupled group) and stays queued; everything else syncs. `--status` lists # each held file, the scanner rule that matched (never the matched text) and @@ -98,6 +109,8 @@ SELF_BIN="$SCRIPT_DIR/gstack-brain-sync" # independent. COUPLED_GROUPS='[{"name": "decisions", "members": ["projects/*/decisions.jsonl", "projects/*/decisions.active.json"], "check": "decisions-active"}]' +REMOVAL_VALVE=20 + # Fields every status write of this drain carries (see write_status). ST_HELD='[]' ST_DRAINABLE=0 @@ -911,6 +924,104 @@ hold_flagged_paths() { return 1 } +# Write to $1 the tracked files gone from disk that the sync may remove, one +# per line: allowlisted, not transcript pages (their consent guard owns those), +# and a coupled group only when every member is gone. +removal_candidates() { + git -C "$GSTACK_HOME" ls-files -d -z > "$1.gone" 2>/dev/null || { rm -f "$1.gone"; return 1; } + python3 - "$GSTACK_HOME" "$ALLOWLIST" "$COUPLED_GROUPS" "$1" <<'PYEOF' +import fnmatch, json, os, re, sys +home, allowlist_path, coupled, out = sys.argv[1:5] +try: + with open(allowlist_path) as f: + globs = [l.strip() for l in f if l.strip() and not l.lstrip().startswith("#")] +except FileNotFoundError: + globs = [] +with open(out + ".gone", "rb") as f: + gone = {p for p in f.read().decode("utf-8", "surrogateescape").split("\0") if p} +os.remove(out + ".gone") +def removable(p): + if p.split("/")[0] == "transcripts" or not any(fnmatch.fnmatchcase(p, g) for g in globs): + return False + for spec in json.loads(coupled): + for member in spec["members"]: + m = re.fullmatch(re.escape(member).replace(r"\*", "([^/]+)"), p) + if m: + return all(not os.path.exists(os.path.join(home, x.replace("*", m.group(1), 1))) for x in spec["members"]) + return True +with open(out, "w", newline="\n", encoding="utf-8", errors="surrogateescape") as f: + for p in sorted(gone): + if removable(p): + f.write(p + "\n") +PYEOF +} + +# Commit the removal of the paths listed in $1 and push it, receipted like +# every push. Returns 0 pushed, 1 commit failed, 2 committed but not pushed +# (the run-start detector retries it). +publish_removals() { + local list="$1" n + n=$(grep -c . "$list" 2>/dev/null || true) + git --literal-pathspecs -C "$GSTACK_HOME" -c user.email="gstack@localhost" -c user.name="gstack-brain-sync" \ + commit -q -m "sync: remove $n file(s) | $(date -u +%Y-%m-%dT%H:%M:%SZ)" --pathspec-from-file="$list" >/dev/null 2>&1 || return 1 + guard_unpublished_range || return 2 + GSTACK_HOME="$GSTACK_HOME" _receipted_git closed brain-sync "$(remote_host)" curated-memory-git-push "artifacts_sync_mode!=off" \ + bash -c 'GIT_TERMINAL_PROMPT=0 git -c http.lowSpeedLimit=1024 -c http.lowSpeedTime=30 -C "$1" push origin HEAD 2>/dev/null' _ "$GSTACK_HOME" || return 2 + date -u +%Y-%m-%dT%H:%M:%SZ > "$LAST_PUSH_FILE" +} + +# The drain's removal step (artifacts_sync_removals=on). Records the outcome +# in ST_NOTE; sets REMOVALS_PUSHED when the removals reached the remote. +sync_removals() { + local list n rc=0 + REMOVALS_PUSHED=0 + list=$(mktemp "${TMPDIR:-/tmp}/brain-sync-removals.XXXXXX") || return 0 + if ! removal_candidates "$list"; then + rm -f "$list" "$list.gone" + ST_NOTE="$ST_NOTE; could not list removed files; none published" + return 0 + fi + n=$(grep -c . "$list" 2>/dev/null || true) + if [ "$n" -gt "$REMOVAL_VALVE" ]; then + ST_NOTE="$ST_NOTE; $n synced file(s) are gone from disk, more than $REMOVAL_VALVE at once, so none of the removals were published (review: $SELF_BIN --publish-removals; publish: $SELF_BIN --publish-removals --yes)" + echo "BRAIN_SYNC: holding $n removal(s), more than $REMOVAL_VALVE at once; review with $SELF_BIN --publish-removals" >&2 + elif [ "$n" -gt 0 ]; then + publish_removals "$list" || rc=$? + case "$rc" in + 0) REMOVALS_PUSHED=1; ST_NOTE="$ST_NOTE; published $n removal(s)" ;; + 1) ST_NOTE="$ST_NOTE; committing $n removal(s) failed; the next sync retries" ;; + *) ST_NOTE="$ST_NOTE; $n removal(s) committed locally but not pushed" ;; + esac + fi + rm -f "$list" +} + +subcmd_publish_removals() { + local confirm="${1:-}" list n rc=0 + sync_active || { echo "BRAIN_SYNC: artifacts sync is not active; nothing to publish" >&2; exit 1; } + gstack_brain_lock_acquire "$GSTACK_HOME" try || { echo "BRAIN_SYNC: another sync is running; try again" >&2; exit 1; } + list=$(mktemp "${TMPDIR:-/tmp}/brain-sync-removals.XXXXXX") || exit 1 + trap 'rm -f "$list" "$list.gone"; gstack_brain_lock_release' EXIT INT TERM + removal_candidates "$list" || { echo "BRAIN_SYNC: could not list removed files" >&2; exit 1; } + n=$(grep -c . "$list" 2>/dev/null || true) + if [ "$n" -eq 0 ]; then + echo "no removals to publish" + exit 0 + fi + if [ "$confirm" != "--yes" ]; then + echo "$n synced file(s) are gone from disk:" + sed 's/^/ /' "$list" + echo "publish their removal: $SELF_BIN --publish-removals --yes" + exit 0 + fi + publish_removals "$list" || rc=$? + case "$rc" in + 0) echo "published $n removal(s)" ;; + 1) echo "BRAIN_SYNC: error: committing the removals failed" >&2; exit 1 ;; + *) echo "BRAIN_SYNC: $n removal(s) committed locally; the push failed and retries at skill start" >&2; exit 1 ;; + esac +} + subcmd_once() { if ! sync_active; then # Silent no-op when feature not initialized / disabled. @@ -962,6 +1073,11 @@ subcmd_once() { fi fi + # Removals go first, so a removal commit whose push fails is one more + # unpushed commit for the detector below. + REMOVALS_PUSHED=0 + [ "$("$CONFIG_BIN" get artifacts_sync_removals 2>/dev/null || echo off)" = "on" ] && sync_removals + # #2549 unpushed-commit detector: a prior drain may have COMMITTED but # failed to push (auth blip, offline). The data was never lost — it sits in # a local commit — but nothing re-pushed it until NEW changes arrived. @@ -1030,7 +1146,7 @@ subcmd_once() { write_status "push_failed" "$det_total local commit(s) are not pushed yet; the push retries at a skill start at most every 10 minutes (commits made outside gstack-brain-sync are never auto-pushed: push them with git -C $GSTACK_HOME push origin HEAD)" exit 0 fi - write_status "idle" "queue empty" + if [ "$REMOVALS_PUSHED" = 1 ]; then write_status "ok" "queue empty"; else write_status "idle" "queue empty"; fi exit 0 fi @@ -1472,6 +1588,7 @@ case "${1:-}" in --drop-queue) shift; subcmd_drop_queue "${1:-}" ;; --discover-new) subcmd_discover_new ;; --attention) subcmd_attention ;; + --publish-removals) shift; subcmd_publish_removals "${1:-}" ;; --help|-h) awk 'NR > 1 && /^# Heredoc delivery guard/ { exit } NR > 1' "$0" | sed 's/^# \{0,1\}//' ;; diff --git a/bin/gstack-config b/bin/gstack-config index 84c46c57f..bb7067d67 100755 --- a/bin/gstack-config +++ b/bin/gstack-config @@ -92,6 +92,11 @@ CONFIG_HEADER='# gstack configuration — edit freely, changes take effect on ne # # Set to true once the privacy gate has asked the user. # # Flip back to false to be re-prompted. # +# artifacts_sync_removals: off # on | off — on: each sync also publishes the +# # removal of synced files gone from disk; more than +# # 20 at once is held for gstack-brain-sync +# # --publish-removals. See docs/gbrain-sync.md. +# # ─── Transcript ingest (gbrain memory) ─────────────────────────────── # transcript_ingest_mode: off # recent | all | off | new@ — consent for # # /sync-gbrain to ingest Claude Code and Codex @@ -252,6 +257,7 @@ lookup_default() { cross_project_learnings) echo "" ;; # intentionally empty → unset triggers first-time prompt artifacts_sync_mode) echo "off" ;; artifacts_sync_mode_prompted) echo "false" ;; + artifacts_sync_removals) echo "off" ;; # on | off — publish removals of synced files gone from disk plan_tune_hooks) echo "prompt" ;; # prompt | yes | no — controls ./setup plan-tune hook install timeline_stop_hook) echo "yes" ;; # yes | no — controls ./setup timeline Stop hook registration (#2677) @@ -665,6 +671,10 @@ case "${1:-}" in echo "Warning: artifacts_sync_mode '$VALUE' not recognized. Valid values: off, artifacts-only, full. Using off." >&2 VALUE="off" fi + if [ "$KEY" = "artifacts_sync_removals" ] && [ "$VALUE" != "on" ] && [ "$VALUE" != "off" ]; then + echo "Warning: artifacts_sync_removals '$VALUE' not recognized. Valid values: on, off. Using off." >&2 + VALUE="off" + fi # redact_repo_visibility: a LOCAL override for repos gh/glab can't read (e.g. # self-hosted GitLab). It lives in ~/.gstack/config.yaml (never committed), so # it can't be used to weaken the gate repo-wide for other contributors. @@ -925,7 +935,7 @@ case "${1:-}" in for KEY in proactive routing_declined telemetry auto_upgrade update_check \ skill_prefix explain_level \ codex_reviews gstack_contributor skip_eng_review workspace_root \ - artifacts_sync_mode artifacts_sync_mode_prompted plan_tune_hooks \ + artifacts_sync_mode artifacts_sync_mode_prompted artifacts_sync_removals plan_tune_hooks \ timeline_stop_hook design_detector design_detector_install_prompted memorable_recall \ browse_extension_id ship_measure_rule_trials ship_measure_behavior_panels \ ship_measure_judge_outputs ship_measure_ask_per_trial_usd ship_measure_budget_usd \ @@ -950,7 +960,7 @@ case "${1:-}" in for KEY in proactive routing_declined telemetry auto_upgrade update_check \ skill_prefix explain_level \ codex_reviews gstack_contributor skip_eng_review workspace_root \ - artifacts_sync_mode artifacts_sync_mode_prompted plan_tune_hooks \ + artifacts_sync_mode artifacts_sync_mode_prompted artifacts_sync_removals plan_tune_hooks \ timeline_stop_hook design_detector design_detector_install_prompted memorable_recall \ browse_extension_id ship_measure_rule_trials ship_measure_behavior_panels \ ship_measure_judge_outputs ship_measure_ask_per_trial_usd ship_measure_budget_usd \ diff --git a/docs/gbrain-sync.md b/docs/gbrain-sync.md index 6e308d89e..3e4348784 100644 --- a/docs/gbrain-sync.md +++ b/docs/gbrain-sync.md @@ -114,6 +114,28 @@ gstack-config set artifacts_sync_mode full gstack-config set artifacts_sync_mode off ``` +## Removed and moved files + +By default the sync never publishes a removal: delete a synced design doc, +or move a project directory (a project slug migration does), and the old +path stays on the remote, where the brain keeps indexing it next to the new +copy. To publish removals too: + +```bash +gstack-config set artifacts_sync_removals on +``` + +Each sync then commits the removal of synced files that are gone from disk. +Transcript pages are left alone, and a decision log and its active snapshot +go only together. A sync that would remove more than 20 files at once +publishes none of them: that looks like a loss, not a cleanup. Review and +publish those yourself, whatever the setting: + +```bash +gstack-brain-sync --publish-removals # list them +gstack-brain-sync --publish-removals --yes # commit and push their removal +``` + ## Secret protection Every commit is scanned for credential-shaped content before it leaves diff --git a/test/brain-sync-removals.test.ts b/test/brain-sync-removals.test.ts new file mode 100644 index 000000000..66b457375 --- /dev/null +++ b/test/brain-sync-removals.test.ts @@ -0,0 +1,153 @@ +/** + * gstack-brain-sync removals (artifacts_sync_removals): off by default, a + * synced file gone from disk stays published; on, the drain publishes its + * removal, and a drain that would remove more than 20 files at once publishes + * none of them until `--publish-removals --yes`. Temp GSTACK_HOME, local bare + * remote; no network. + */ +import { describe, test, expect, beforeEach, afterEach } from 'bun:test'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { spawnSync } from 'child_process'; + +const ROOT = path.resolve(import.meta.dir, '..'); + +let home: string; +let remote: string; + +function run(argv: string[]) { + const r = spawnSync(path.join(ROOT, 'bin', argv[0]), argv.slice(1), { + env: { ...process.env, HOME: home, GSTACK_HOME: home }, + encoding: 'utf-8', + timeout: 60_000, + }); + return { stdout: r.stdout || '', stderr: r.stderr || '', status: r.status ?? -1 }; +} + +const remoteFiles = () => + spawnSync('git', ['--git-dir=' + remote, 'ls-tree', '-r', '--name-only', 'main'], { encoding: 'utf-8', timeout: 30_000 }) + .stdout.split('\n').filter((p) => p.startsWith('projects/') || p.startsWith('transcripts/')); +const status = () => JSON.parse(fs.readFileSync(path.join(home, '.brain-sync-status.json'), 'utf-8')); + +function write(p: string, body = '# doc\n') { + fs.mkdirSync(path.dirname(path.join(home, p)), { recursive: true }); + fs.writeFileSync(path.join(home, p), body); +} + +function syncAll(files: string[]) { + for (const f of files) write(f); + for (const f of files) run(['gstack-brain-enqueue', f]); + const r = run(['gstack-brain-sync', '--once']); + expect(r.status).toBe(0); + expect(remoteFiles()).toEqual(expect.arrayContaining(files)); +} + +const designs = (dir: string, n: number) => Array.from({ length: n }, (_, i) => `projects/${dir}/designs/d${i}.md`); + +beforeEach(() => { + home = fs.mkdtempSync(path.join(os.tmpdir(), 'brain-removals-home-')); + remote = fs.mkdtempSync(path.join(os.tmpdir(), 'brain-removals-remote-')); + spawnSync('git', ['init', '--bare', '-q', '-b', 'main', remote], { timeout: 30_000 }); + expect(run(['gstack-artifacts-init', '--remote', remote]).status).toBe(0); + run(['gstack-config', 'set', 'artifacts_sync_mode', 'full']); +}); + +afterEach(() => { + fs.rmSync(home, { recursive: true, force: true }); + fs.rmSync(remote, { recursive: true, force: true }); +}); + +describe('artifacts_sync_removals', () => { + test('off by default: a deleted synced file stays published', () => { + expect(run(['gstack-config', 'get', 'artifacts_sync_removals']).stdout.trim()).toBe('off'); + syncAll(['projects/p/designs/a.md', 'projects/p/designs/b.md']); + fs.rmSync(path.join(home, 'projects/p/designs/a.md')); + run(['gstack-brain-sync', '--once']); + expect(remoteFiles()).toContain('projects/p/designs/a.md'); + }); + + test('on: a slug migration removes the old copies with the queue empty or not', () => { + run(['gstack-config', 'set', 'artifacts_sync_removals', 'on']); + syncAll(['projects/acme/designs/x.md', 'projects/acme/designs/y.md']); + fs.renameSync(path.join(home, 'projects/acme'), path.join(home, 'projects/owner-acme')); + run(['gstack-brain-sync', '--discover-new']); + expect(run(['gstack-brain-sync', '--once']).status).toBe(0); + expect(remoteFiles().sort()).toEqual(['projects/owner-acme/designs/x.md', 'projects/owner-acme/designs/y.md']); + expect(status().message).toContain('published 2 removal(s)'); + + fs.rmSync(path.join(home, 'projects/owner-acme/designs/y.md')); + expect(run(['gstack-brain-sync', '--once']).status).toBe(0); + expect(remoteFiles()).toEqual(['projects/owner-acme/designs/x.md']); + expect(status().status).toBe('ok'); + }); + + test('on: tracked files outside the allowlist and transcript pages are never removed', () => { + run(['gstack-config', 'set', 'artifacts_sync_removals', 'on']); + syncAll(['projects/p/designs/a.md']); + write('projects/p/notes.txt'); + write('transcripts/run-1/s1.md'); + spawnSync('git', ['-C', home, 'add', '-f', 'projects/p/notes.txt', 'transcripts/run-1/s1.md'], { timeout: 30_000 }); + spawnSync('git', ['-C', home, '-c', 'user.email=u@example.com', '-c', 'user.name=u', 'commit', '-qm', 'manual'], { timeout: 30_000 }); + spawnSync('git', ['-C', home, 'push', '-q', 'origin', 'HEAD'], { timeout: 30_000 }); + for (const p of ['projects/p/notes.txt', 'transcripts/run-1/s1.md', 'projects/p/designs/a.md']) fs.rmSync(path.join(home, p)); + run(['gstack-brain-sync', '--once']); + expect(remoteFiles().sort()).toEqual(['projects/p/notes.txt', 'transcripts/run-1/s1.md']); + }); + + test('on: a decision log goes only together with its active snapshot', () => { + run(['gstack-config', 'set', 'artifacts_sync_removals', 'on']); + const LOG = 'projects/p/decisions.jsonl'; + const SNAP = 'projects/p/decisions.active.json'; + write(SNAP, JSON.stringify([{ id: 'a', kind: 'decide', note: '' }])); + write(LOG, JSON.stringify({ id: 'a', kind: 'decide', decision: 'd a', scope: 'repo', date: '2026-10-01', source: 'user' }) + '\n'); + run(['gstack-brain-enqueue', LOG]); + run(['gstack-brain-sync', '--once']); + expect(remoteFiles()).toEqual(expect.arrayContaining([LOG, SNAP])); + fs.rmSync(path.join(home, LOG)); + run(['gstack-brain-sync', '--once']); + expect(remoteFiles()).toEqual(expect.arrayContaining([LOG, SNAP])); + fs.rmSync(path.join(home, SNAP)); + run(['gstack-brain-sync', '--once']); + expect(remoteFiles()).toEqual([]); + }); + + test('on: more than 20 removals at once publishes none until --publish-removals --yes', () => { + run(['gstack-config', 'set', 'artifacts_sync_removals', 'on']); + const files = designs('big', 21); + syncAll(files); + fs.rmSync(path.join(home, 'projects/big'), { recursive: true }); + const drain = run(['gstack-brain-sync', '--once']); + expect(drain.stderr).toContain('holding 21 removal(s), more than 20 at once'); + expect(remoteFiles().sort()).toEqual([...files].sort()); + expect(status().message).toContain('none of the removals were published'); + + const listed = run(['gstack-brain-sync', '--publish-removals']); + expect(listed.stdout).toContain('21 synced file(s) are gone from disk:'); + expect(listed.stdout).toContain(' projects/big/designs/d0.md'); + expect(remoteFiles()).toHaveLength(21); + + const published = run(['gstack-brain-sync', '--publish-removals', '--yes']); + expect(published.status).toBe(0); + expect(published.stdout).toContain('published 21 removal(s)'); + expect(remoteFiles()).toEqual([]); + }); + + test('--publish-removals --yes works with the setting off and only touches removals', () => { + syncAll(['projects/p/designs/a.md', 'projects/p/designs/b.md']); + fs.rmSync(path.join(home, 'projects/p/designs/a.md')); + write('projects/p/designs/b.md', '# changed, not yet synced\n'); + const r = run(['gstack-brain-sync', '--publish-removals', '--yes']); + expect(r.status).toBe(0); + expect(remoteFiles()).toEqual(['projects/p/designs/b.md']); + const remoteB = spawnSync('git', ['--git-dir=' + remote, 'show', 'main:projects/p/designs/b.md'], { encoding: 'utf-8', timeout: 30_000 }); + expect(remoteB.stdout).toBe('# doc\n'); + expect(run(['gstack-brain-sync', '--publish-removals', '--yes']).stdout).toContain('no removals to publish'); + }); + + test('config rejects values other than on and off', () => { + const r = run(['gstack-config', 'set', 'artifacts_sync_removals', 'yes']); + expect(r.stderr).toContain("artifacts_sync_removals 'yes' not recognized"); + expect(run(['gstack-config', 'get', 'artifacts_sync_removals']).stdout.trim()).toBe('off'); + }); +}); From 86ce012c5e21b5cb4208ee17915994f7992f96f2 Mon Sep 17 00:00:00 2001 From: garrytan Date: Thu, 8 Oct 2026 15:27:22 +0000 Subject: [PATCH 3/3] feat(windows): setup.ps1 checks prerequisites and runs ./setup in Git Bash A PowerShell entry point for Windows users. It checks that Git for Windows, Bun and Node.js are on PATH (printing the winget command for each missing one and changing nothing), finds Git Bash from git.exe rather than a PATH bash that may be WSL's, and runs ./setup from the checkout with every argument passed as its own argv entry to a constant bash script. No argument is interpolated into shell source. ./setup itself is unchanged. Co-authored-by: Jim McKeeth --- README.md | 2 ++ setup.ps1 | 54 ++++++++++++++++++++++++++++++ test/setup-ps1.test.ts | 75 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 131 insertions(+) create mode 100644 setup.ps1 create mode 100644 test/setup-ps1.test.ts diff --git a/README.md b/README.md index e2c99f362..4de9e2f5d 100644 --- a/README.md +++ b/README.md @@ -708,6 +708,8 @@ types into that element; bare `browse type ` types into whatever has focus **Windows users:** gstack works on Windows 11 via Git Bash or WSL. Aside is macOS-only, so on Windows (and Linux) the browser skills, `/make-pdf`, and `/diagram` always use gstack's bundled browser. Node.js is required in addition to Bun — Bun has a known bug with Playwright's pipe transport on Windows ([bun#4253](https://github.com/oven-sh/bun/issues/4253)). The browse server automatically falls back to Node.js. Make sure both `bun` and `node` are on your PATH. Native `/cso` additionally requires Windows PowerShell and Visual Studio 2022 Build Tools with the Desktop development with C++ workload; setup leaves that skill explicitly unavailable when they are absent. +From PowerShell, `.\setup.ps1` (same arguments as `./setup`) checks that Git for Windows, Bun and Node.js are on PATH, prints the `winget` command for any that are missing, and otherwise runs `./setup` in Git Bash. + On Windows without Developer Mode (MSYS2 / Git Bash), `setup` falls back to file copies instead of symlinks because `ln -snf` produces frozen copies that don't refresh on `git pull`. **Re-run `cd ~/.claude/skills/gstack && ./setup` after every `git pull`** so your skill files match the repo. `setup` prints a one-line note reminding you. Unix and WSL keep symlinks and don't need the re-run. **Chromium install failed or hung during `./setup`?** The bundled browser is diff --git a/setup.ps1 b/setup.ps1 new file mode 100644 index 000000000..ab3b82386 --- /dev/null +++ b/setup.ps1 @@ -0,0 +1,54 @@ +#Requires -Version 5.1 +<# +.SYNOPSIS + Windows entry point for gstack's ./setup. + +.DESCRIPTION + Checks that Git for Windows, Bun and Node.js are on PATH, then runs ./setup + from this directory in Git Bash. Every argument goes to ./setup unchanged, + as separate arguments; none is pasted into shell source. + + .\setup.ps1 + .\setup.ps1 --host codex + + Set GSTACK_GIT_BASH to a bash.exe to use instead of Git for Windows' own. +#> +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +$missing = @( + @{ Cmd = 'git'; Name = 'Git for Windows'; Id = 'Git.Git' }, + @{ Cmd = 'bun'; Name = 'Bun'; Id = 'Oven-sh.Bun' }, + @{ Cmd = 'node'; Name = 'Node.js'; Id = 'OpenJS.NodeJS.LTS' } +) | Where-Object { -not (Get-Command $_.Cmd -CommandType Application -ErrorAction SilentlyContinue) } +if ($missing) { + Write-Host 'gstack setup needs these on PATH first:' + foreach ($tool in $missing) { Write-Host (' {0}: winget install --id {1} -e' -f $tool.Name, $tool.Id) } + Write-Host 'Install them, open a new PowerShell window and run .\setup.ps1 again. Nothing was installed or changed.' + exit 1 +} + +# Git Bash, found from git.exe (Git\cmd, Git\bin or Git\mingw64\bin). Never a +# bare `bash` from PATH: on Windows that is often WSL's, with a different home. +$bash = $env:GSTACK_GIT_BASH +if (-not $bash) { + $dir = Split-Path (Get-Command git -CommandType Application | Select-Object -First 1).Source + for ($i = 0; $i -lt 3 -and $dir; $i++) { + $candidate = Join-Path $dir 'bin\bash.exe' + if (Test-Path -LiteralPath $candidate -PathType Leaf) { $bash = $candidate; break } + $dir = Split-Path $dir + } +} +if (-not $bash -or -not (Test-Path -LiteralPath $bash -PathType Leaf)) { + Write-Host 'Git Bash (bash.exe) was not found next to git. Reinstall Git for Windows, or set GSTACK_GIT_BASH to its bash.exe.' + exit 1 +} + +Push-Location -LiteralPath $PSScriptRoot +try { + & $bash -c 'exec ./setup "$@"' setup @args + $code = $LASTEXITCODE +} finally { + Pop-Location +} +exit $code diff --git a/test/setup-ps1.test.ts b/test/setup-ps1.test.ts new file mode 100644 index 000000000..abd1beff6 --- /dev/null +++ b/test/setup-ps1.test.ts @@ -0,0 +1,75 @@ +/** + * setup.ps1: the Windows entry point checks Git for Windows, Bun and Node.js, + * then runs ./setup in Git Bash with every argument passed as its own argv + * entry, never pasted into shell source. The behavioral cases copy the script + * next to a stub ./setup that prints its argv and run it under pwsh. + */ +import { describe, test, expect, afterEach } from 'bun:test'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { spawnSync } from 'child_process'; + +const SCRIPT = path.resolve(import.meta.dir, '..', 'setup.ps1'); +const SOURCE = fs.readFileSync(SCRIPT, 'utf-8'); +const PWSH = Bun.which('pwsh'); +const BASH = Bun.which('bash'); + +const dirs: string[] = []; +afterEach(() => { + for (const d of dirs.splice(0)) fs.rmSync(d, { recursive: true, force: true }); +}); + +function stubCheckout(): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'setup-ps1-')); + dirs.push(dir); + fs.copyFileSync(SCRIPT, path.join(dir, 'setup.ps1')); + fs.writeFileSync(path.join(dir, 'setup'), '#!/usr/bin/env bash\nprintf \'cwd=%s\\n\' "$PWD"\nfor a in "$@"; do printf \'arg=[%s]\\n\' "$a"; done\nexit 7\n', { mode: 0o755 }); + return dir; +} + +function runPs1(dir: string, args: string[], env: Record = {}) { + const r = spawnSync(PWSH!, ['-NoProfile', '-NonInteractive', '-File', path.join(dir, 'setup.ps1'), ...args], { + cwd: os.tmpdir(), + encoding: 'utf-8', + env: { ...process.env, ...(process.platform === 'win32' ? {} : { GSTACK_GIT_BASH: BASH! }), ...env }, + timeout: 60_000, + }); + return { code: r.status, out: (r.stdout || '') + (r.stderr || '') }; +} + +describe('setup.ps1', () => { + test('passes arguments as an array to a constant bash script', () => { + expect(SOURCE).toContain(`& $bash -c 'exec ./setup "$@"' setup @args`); + expect(SOURCE).not.toMatch(/-join|Invoke-Expression|\biex\b|git-bash\.exe/); + expect([...SOURCE].every((c) => c.charCodeAt(0) < 128)).toBe(true); + }); + + test.skipIf(!PWSH || !BASH)('forwards every argument literally from the script directory and keeps the exit code', () => { + const dir = stubCheckout(); + const args = ['--host', 'codex', 'two words', '$(touch PWNED)', "it's", '"quoted"', ';echo injected', '`whoami`']; + const { code, out } = runPs1(dir, args); + expect(code).toBe(7); + const lines = out.split(/\r?\n/); + expect(lines.filter((l) => l.startsWith('arg=['))).toEqual(args.map((a) => `arg=[${a}]`)); + expect(lines.find((l) => l.startsWith('cwd='))).toContain(path.basename(dir)); + expect(out).not.toContain('injected\n'); + expect(fs.existsSync(path.join(dir, 'PWNED'))).toBe(false); + expect(fs.existsSync(path.join(os.tmpdir(), 'PWNED'))).toBe(false); + }); + + test.skipIf(!PWSH || !BASH || process.platform === 'win32')('a missing prerequisite names its winget command and runs nothing', () => { + const dir = stubCheckout(); + const bin = fs.mkdtempSync(path.join(os.tmpdir(), 'setup-ps1-bin-')); + dirs.push(bin); + for (const tool of ['git', 'node']) { + const real = Bun.which(tool); + if (real) fs.symlinkSync(real, path.join(bin, tool)); + } + const { code, out } = runPs1(dir, ['--host', 'codex'], { PATH: bin }); + expect(code).toBe(1); + expect(out).toContain('Bun: winget install --id Oven-sh.Bun -e'); + expect(out).not.toContain('Git for Windows:'); + expect(out).not.toContain('arg=['); + }); +});