/sync-gbrain reports failed pushes and imports instead of OK, and one bad queued path no longer blocks artifacts sync (#4455)

/sync-gbrain now says when an artifacts push failed or memory pages
failed to import, instead of printing OK and "pushed", and one malformed
queued path (from a skill started where the project slug was empty) no
longer stops artifacts sync for every project.

Why: people trusted those rows and lost track of unsynced work; fixes
#4445, #4418 and #4417.

Nothing to do after upgrading: a stuck `projects//timeline.jsonl` record
is dropped and reported on the next sync.

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
Review-Finding: p1/red-team#1 reader=red-team
Review-Finding: p1/red-team#2 reader=red-team
Review-Finding: p1/specialist-batched#1 reader=testing
Review-Finding: p1/specialist-batched#2 reader=testing
Review-Finding: p1/specialist-batched#3 reader=testing
This commit is contained in:
Garry Tanandcapy-ai[bot] authored and GitHub committed 2026-11-17 18:34:52 +00:00
1 parent 88da124dbc
commit bc64f5e210
11 files changed
+451 -58

No files matched your search

+13
View File
@@ -15,6 +15,9 @@
# - ~/.gstack/.git doesn't exist (feature not initialized)
# - <file-path> matches a line in ~/.gstack/.brain-skip.txt
#
# Refuses (exit 1, nothing queued) a <file-path> with an empty, "." or ".."
# component, such as projects//timeline.jsonl from an empty project slug.
#
# Env:
# GSTACK_HOME — override ~/.gstack state directory (aligns with writers).
# Tests use GSTACK_HOME=/tmp/test-$$ for isolation.
@@ -32,6 +35,16 @@ set -uo pipefail
FILE="${1:-}"
[ -z "$FILE" ] && exit 0
# A path with an empty, "." or ".." component (projects//timeline.jsonl from an
# empty project slug, #4445) names a file git stages under another spelling,
# and the drain refuses that batch. Refuse it here instead of spooling it.
case "/$FILE/" in
*//*|*/./*|*/../*)
echo "gstack-brain-enqueue: refusing malformed path '$FILE' (empty, '.' or '..' component; an empty project slug?); nothing queued" >&2
exit 1
;;
esac
# Plan B1: an unattended session never spools for cross-machine sync, whatever
# the inherited mode; every writer (timeline, decision, learnings, review log)
# is covered here, at the one entrypoint, instead of per caller.
+40 -14
View File
@@ -193,6 +193,23 @@ with open(path, "w") as f:
PYEOF
}
# Status for a drain that pushed nothing. While local commits are still not on
# origin, push_failed replaces ok and idle: writing those instead erased a
# failed push one run later, so a push that never recovered went silent
# (#4418). held and collision keep their code (their attention line names the
# files to fix) and carry the unpushed count in the message.
# $1 code, $2 message, $3 unpushed local commits.
write_drain_status() {
local unpushed="$3 local commit(s) are not pushed yet; the push retries at a skill start at most every 10 minutes (commits made outside gstack-brain-sync are never auto-pushed: push them with 'git -C $GSTACK_HOME push origin HEAD')"
if [ "${3:-0}" -eq 0 ]; then
write_status "$1" "$2"
elif [ "$1" = held ] || [ "$1" = collision ]; then
write_status "$1" "$2; $unpushed"
else
write_status "push_failed" "$unpushed; this drain: $2"
fi
}
# Read config; return 0 if sync active, 1 otherwise.
sync_active() {
if [ ! -d "$GSTACK_HOME/.git" ]; then
@@ -464,8 +481,11 @@ for p in sorted(queue_paths):
if p in skip_lines or any(m in skip_lines for m in members):
classified["dropped"]["skipped"].append(p)
continue
# Must be under GSTACK_HOME root. Reject absolute + reject ../ escape.
if p.startswith("/") or ".." in p.split("/"):
# Must be under GSTACK_HOME root and spelled the way git stages it: an
# absolute path, a ../ escape, or an empty or "." component (#4445:
# projects//timeline.jsonl from an empty slug, staged by git as
# projects/timeline.jsonl, made the index check refuse the whole batch).
if any(c in ("", ".", "..") for c in p.split("/")):
classified["dropped"]["invalid"].append(p)
continue
# Must match at least one allowlist glob.
@@ -677,7 +697,8 @@ finalize_queue() {
# Fail-open by design (a failed finalize self-corrects next run: re-stage →
# nothing-to-commit), but say so — a silent failure here would let the
# subsequent "ok/idle" status claim a drain that did not happen.
python3 - "$QUEUE_DIR" "$snapshot_file" "$class_file" "$paths_file" "$GSTACK_HOME/.brain-sync-drops.json" <<'PYEOF' || echo "BRAIN_SYNC: warning: queue finalize failed — entries retained; next run re-drains" >&2
local finalized=1
python3 - "$QUEUE_DIR" "$snapshot_file" "$class_file" "$paths_file" "$GSTACK_HOME/.brain-sync-drops.json" <<'PYEOF' || { finalized=0; echo "BRAIN_SYNC: warning: queue finalize failed — entries retained; next run re-drains" >&2; }
import json, os, sys, time
spool_dir, snapshot_file, class_file, paths_file, drops_file = sys.argv[1:6]
@@ -762,6 +783,16 @@ if dropped:
json.dump({"ts": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
"dropped": classified.get("dropped", {})}, f)
PYEOF
[ "$finalized" = 1 ] || return 0
# #4445: report malformed records once finalize has dropped them; an early
# exit before finalize keeps them queued and must not claim the drop.
local malformed
malformed=$(python3 -c 'import json, sys; print(len((json.load(open(sys.argv[1])).get("dropped") or {}).get("invalid") or []))' "$class_file" 2>/dev/null || echo 0)
case "$malformed" in ''|*[!0-9]*) malformed=0 ;; esac
if [ "$malformed" -gt 0 ]; then
ST_NOTE="$ST_NOTE; dropped $malformed queued record(s) with a malformed path (absolute, or an empty, '.' or '..' component), listed in $GSTACK_HOME/.brain-sync-drops.json"
echo "BRAIN_SYNC: dropped $malformed queued record(s) with a malformed path; see $GSTACK_HOME/.brain-sync-drops.json" >&2
fi
}
# Human-readable classification counts for status messages.
@@ -1316,14 +1347,9 @@ subcmd_once() {
# the queue is empty.) The lock-release trap installed at acquisition
# covers this exit.
if ! spool_has_records && [ ! -s "$QUEUE" ] && [ ! -s "$QUEUE.migrating" ]; then
# Stranded local commits that this run did not push keep the status at
# push_failed: an "idle" here used to erase the failure one boundary later,
# so a push that never recovered went silent.
if [ "$det_total" -gt 0 ]; then
write_status "push_failed" "$det_total local commit(s) are not pushed yet; the push retries at a skill start at most every 10 minutes (commits made outside gstack-brain-sync are never auto-pushed: push them with git -C $GSTACK_HOME push origin HEAD)"
exit 0
fi
if [ "$REMOVALS_PUSHED" = 1 ]; then write_status "ok" "queue empty"; else write_status "idle" "queue empty"; fi
local empty_code=idle
[ "$REMOVALS_PUSHED" = 1 ] && empty_code=ok
write_drain_status "$empty_code" "queue empty" "$det_total"
exit 0
fi
@@ -1362,7 +1388,7 @@ subcmd_once() {
local summary idle_code=idle
summary=$(queue_summary "$class_file")
[ "$ST_CASE_HELD" -gt 0 ] && idle_code=collision
write_status "$idle_code" "no stageable changes${summary:+ ($summary)}"
write_drain_status "$idle_code" "no stageable changes${summary:+ ($summary)}" "$det_total"
exit 0
fi
local n_stageable
@@ -1419,9 +1445,9 @@ subcmd_once() {
if [ "$cached_rc" -eq 0 ]; then
finalize_queue "$snapshot_file" "$class_file" "$paths_file"
if [ "$ok_code" != "ok" ]; then
write_status "$ok_code" "nothing else to sync"
write_drain_status "$ok_code" "nothing else to sync" "$det_total"
else
write_status "idle" "queue drained but no new changes to commit"
write_drain_status "idle" "queue drained but no new changes to commit" "$det_total"
fi
exit 0
fi
+93 -34
View File
@@ -29,10 +29,10 @@
* than building a gstack-side daemon.
*/
import { existsSync, statSync, mkdirSync, writeFileSync, readFileSync, unlinkSync, renameSync, realpathSync, readdirSync, appendFileSync } from "fs";
import { existsSync, statSync, mkdirSync, mkdtempSync, writeFileSync, readFileSync, rmSync, unlinkSync, renameSync, realpathSync, readdirSync, appendFileSync } from "fs";
import { join, dirname, basename, isAbsolute } from "path";
import { execSync, spawnSync } from "child_process";
import { homedir, hostname } from "os";
import { homedir, hostname, tmpdir } from "os";
import { createHash } from "crypto";
import "../lib/conductor-env-shim";
@@ -262,32 +262,88 @@ function isArtifactsSourceRow(r: { local_path?: string }, env?: NodeJS.ProcessEn
}
/**
* A5 (#2670): after a successful artifacts push, report what gbrain actually
* holds. A git push is not indexing; a maintained artifacts source with zero
* pages is the one provably broken state and fails the stage.
* A5 (#2670): after an artifacts drain, report what gbrain actually holds. A
* git push is not indexing; a maintained artifacts source with zero pages is
* the one provably broken state and fails the stage. `lead` says what the
* drain did.
*/
export function brainSyncIndexVerdict(env: NodeJS.ProcessEnv = process.env): { ok: boolean; summary: string } {
export function brainSyncIndexVerdict(
env: NodeJS.ProcessEnv = process.env,
lead = "curated artifacts pushed to git",
): { ok: boolean; summary: string } {
const raw = execGbrainJson(["sources", "list", "--json"], { baseEnv: env, timeout: 10_000 });
if (raw === null) {
return { ok: true, summary: "curated artifacts pushed to git; gbrain page count unavailable (gbrain sources list failed)" };
return { ok: true, summary: `${lead}; gbrain page count unavailable (gbrain sources list failed)` };
}
const explicit = env.GSTACK_BRAIN_SOURCE_ID;
const row = parseSourcesList(raw).find((r) => (explicit ? r.id === explicit : isArtifactsSourceRow(r, env)));
if (!row?.id) {
return { ok: true, summary: "curated artifacts pushed to git (no gbrain artifacts source on this machine, so indexing is not checked here)" };
return { ok: true, summary: `${lead} (no gbrain artifacts source on this machine, so indexing is not checked here)` };
}
if (typeof row.page_count !== "number") {
return { ok: true, summary: `curated artifacts pushed to git; gbrain source ${row.id} did not report a page count` };
return { ok: true, summary: `${lead}; gbrain source ${row.id} did not report a page count` };
}
if (row.page_count === 0) {
return {
ok: false,
summary:
`curated artifacts pushed to git, but gbrain source ${row.id} has 0 indexed pages. ` +
`Fix: gbrain sync --source ${row.id}, then re-run /sync-gbrain`,
summary: `${lead}, but gbrain source ${row.id} has 0 indexed pages. Fix: gbrain sync --source ${row.id}, then re-run /sync-gbrain`,
};
}
return { ok: true, summary: `curated artifacts pushed; gbrain source ${row.id} has ${row.page_count} pages` };
return { ok: true, summary: `${lead}; gbrain source ${row.id} has ${row.page_count} pages` };
}
/**
* The brain-sync row (#4418). `gstack-brain-sync --once` exits 0 when its push
* fails (the commit is kept and retried), so the row comes from the status the
* drain wrote, and only a drain whose status is ok says "pushed". `drain` is
* null when this run wrote no status (sync off, or another drain held the lock).
*/
export function brainSyncStageVerdict(
drain: { status?: unknown; message?: unknown } | null,
env: NodeJS.ProcessEnv = process.env,
): { ok: boolean; warn?: boolean; summary: string } {
if (!drain) return brainSyncIndexVerdict(env, "no artifacts drain ran (artifacts sync is off, or another sync held the lock), so nothing was pushed");
const status = typeof drain.status === "string" ? drain.status : "unknown";
const message = (typeof drain.message === "string" ? drain.message : "").slice(0, 400);
const detail = `${status}: ${message || "no message"}; details: gstack-brain-sync --status`;
if (status === "ok") return brainSyncIndexVerdict(env);
if (status === "idle") return brainSyncIndexVerdict(env, `nothing new to push (${message || "idle"})`);
if (status === "held" || status === "collision") {
const v = brainSyncIndexVerdict(env, `artifacts not all pushed (${detail})`);
return { ...v, warn: v.ok };
}
return { ok: false, summary: `artifacts not pushed (${detail})` };
}
/**
* The memory row (#4417). gstack-memory-ingest exits 0 when single pages fail
* (they are retried next run), so the exit code alone printed OK with every
* page refused. The counts decide: a failed page makes the row WARN, and a run
* that imported nothing while pages failed is ERR. A source file that could not
* be parsed is never retried as an import, so it is a note, not a failure.
*/
export function memoryStageVerdict(
status: number | null,
stderr: string,
counts: { written?: unknown; failed?: unknown; parse_failed?: unknown } | null,
): { ok: boolean; warn?: boolean; summary: string } {
const memLines = stderr.split("\n").filter((l) => l.includes("[memory-ingest]"));
const errLine = memLines.find((l) => l.includes("[memory-ingest] ERR"));
const summary = (errLine || memLines.slice(-1)[0] || "ingest pass complete")
.replace(/^.*\[memory-ingest\]\s*/, "")
.replace(/^ERR:\s*/, "");
if (status !== 0) {
return { ok: false, summary: `${summary}${status === null ? " (killed by signal / timeout)" : ` (exit ${status})`}` };
}
const parseFailed = typeof counts?.parse_failed === "number" ? counts.parse_failed : 0;
const failed = Math.max(0, (typeof counts?.failed === "number" ? counts.failed : 0) - parseFailed);
const unparsed = parseFailed > 0 ? `; ${parseFailed} source file(s) could not be parsed and were skipped` : "";
if (failed === 0) return { ok: true, summary: summary + unparsed };
const written = typeof counts?.written === "number" ? counts.written : 0;
const first = memLines.find((l) => l.includes("[memory-ingest] FAILED "));
const cause = first ? `; first: ${first.replace(/^.*\[memory-ingest\] FAILED\s*/, "").slice(0, 240)}` : "";
const line = `${written} imported, ${failed} failed (retried next run; each failure: gstack-memory-ingest --incremental)${cause}${unparsed}`;
return written === 0 ? { ok: false, summary: line } : { ok: true, warn: true, summary: line };
}
export interface MemorySourceSelection {
@@ -1634,36 +1690,27 @@ function runMemoryIngest(args: CliArgs, previous: SyncState, consent: Transcript
process.env.GSTACK_SYNC_MEMORY_TIMEOUT_MS,
"GSTACK_SYNC_MEMORY_TIMEOUT_MS",
);
// D6: the child's [memory-ingest] stderr lines carry the summary and any
// system error; its counts (GSTACK_INGEST_RESULT_FILE) decide whether pages
// failed, since per-page failures exit 0 (#4417).
const resultFile = join(mkdtempSync(join(tmpdir(), "gstack-ingest-result-")), "result.json");
childEnv.GSTACK_INGEST_RESULT_FILE = resultFile;
const result = spawnSync(process.execPath, ingestArgs, {
encoding: "utf-8",
timeout: memoryTimeoutMs,
env: childEnv,
});
let counts: { written?: unknown; failed?: unknown } | null = null;
try {
counts = JSON.parse(readFileSync(resultFile, "utf-8"));
} catch {}
rmSync(dirname(resultFile), { recursive: true, force: true });
// D6: parse [memory-ingest] lines from the child's stderr. ERR-prefixed
// lines indicate a system-level failure (gbrain crashed or CLI missing)
// and the child exits non-zero. Per-file failures are summarized in the
// last non-ERR [memory-ingest] line but do NOT make the verdict ERR.
const stderrLines = (result.stderr || "").split("\n");
const memLines = stderrLines.filter((l) => l.includes("[memory-ingest]"));
const errLine = memLines.find((l) => l.includes("[memory-ingest] ERR"));
const lastMemLine = memLines.slice(-1)[0];
const rawSummary = errLine || lastMemLine || "ingest pass complete";
// Strip the "[memory-ingest] " prefix and any leading "ERR: " for cleaner
// verdict output. The orchestrator's own formatStage will prefix with OK/ERR.
const summary = rawSummary
.replace(/^.*\[memory-ingest\]\s*/, "")
.replace(/^ERR:\s*/, "");
const ok = result.status === 0;
return {
name: "memory",
ran: true,
ok,
duration_ms: Date.now() - t0,
summary: ok
? summary
: `${summary}${result.status === null ? " (killed by signal / timeout)" : ` (exit ${result.status})`}`,
...memoryStageVerdict(result.status, result.stderr || "", counts),
memory_sources: selected,
transcript_consent: consentRecord,
};
@@ -1708,12 +1755,24 @@ function runBrainSyncPush(args: CliArgs): StageResult {
: ["ignore", "inherit", "inherit"];
spawnSync(discover.cmd, discover.argv, { stdio, timeout: 60 * 1000, shell: discover.shell });
const statusPath = join(GSTACK_HOME, ".brain-sync-status.json");
const statusStamp = () => (existsSync(statusPath) ? `${statSync(statusPath).mtimeMs}:${readFileSync(statusPath, "utf-8")}` : null);
const before = statusStamp();
const result = spawnSync(once.cmd, once.argv, { stdio, timeout: 60 * 1000, shell: once.shell });
if (result.status !== 0) {
return { name: "brain-sync", ran: true, ok: false, duration_ms: Date.now() - t0, summary: `gstack-brain-sync exited ${result.status}` };
}
return { name: "brain-sync", ran: true, duration_ms: Date.now() - t0, ...brainSyncIndexVerdict() };
let drain: { status?: unknown; message?: unknown } | null = null;
const after = statusStamp();
if (after !== null && after !== before) {
try {
drain = JSON.parse(readFileSync(statusPath, "utf-8"));
} catch {
drain = { status: "unknown", message: `${statusPath} is unreadable` };
}
}
return { name: "brain-sync", ran: true, duration_ms: Date.now() - t0, ...brainSyncStageVerdict(drain) };
}
/**
+15 -1
View File
@@ -195,6 +195,8 @@ interface BulkResult {
/** #2392: transcripts skipped because their remote's trust tier is `deny`. */
skipped_policy_deny: number;
failed: number;
/** Source files that could not be parsed, counted in failed; not import failures. */
parse_failed?: number;
duration_ms: number;
partial_pages: number;
/** A1: pages gbrain refused or that were not found after import (retried next run). */
@@ -2134,6 +2136,7 @@ async function ingestPassScoped(args: CliArgs, state: IngestState, scope: Transc
skipped_policy_readonly: prep.skippedPolicyReadonly,
skipped_policy_deny: prep.skippedPolicyDeny,
failed: prep.parseFailed + prep.prepared.length,
parse_failed: prep.parseFailed,
duration_ms: Date.now() - t0,
partial_pages: prep.partialPages,
system_error: prep.policyError,
@@ -2159,6 +2162,7 @@ async function ingestPassScoped(args: CliArgs, state: IngestState, scope: Transc
skipped_policy_readonly: prep.skippedPolicyReadonly,
skipped_policy_deny: prep.skippedPolicyDeny,
failed: prep.parseFailed,
parse_failed: prep.parseFailed,
duration_ms: Date.now() - t0,
partial_pages: prep.partialPages,
};
@@ -2180,6 +2184,7 @@ async function ingestPassScoped(args: CliArgs, state: IngestState, scope: Transc
skipped_policy_readonly: prep.skippedPolicyReadonly,
skipped_policy_deny: prep.skippedPolicyDeny,
failed: prep.parseFailed,
parse_failed: prep.parseFailed,
duration_ms: Date.now() - t0,
partial_pages: prep.partialPages,
};
@@ -2198,6 +2203,7 @@ async function ingestPassScoped(args: CliArgs, state: IngestState, scope: Transc
skipped_policy_readonly: prep.skippedPolicyReadonly,
skipped_policy_deny: prep.skippedPolicyDeny,
failed: prep.parseFailed + prep.prepared.length,
parse_failed: prep.parseFailed,
duration_ms: Date.now() - t0,
partial_pages: prep.partialPages,
system_error: msg,
@@ -2262,6 +2268,7 @@ async function ingestPassScoped(args: CliArgs, state: IngestState, scope: Transc
written: 0,
skipped_secret: prep.skippedSecret + (r.scannerFailed ? 1 : 0),
failed: prep.parseFailed + prep.prepared.length,
parse_failed: prep.parseFailed,
duration_ms: Date.now() - t0,
system_error: r.error,
};
@@ -2287,7 +2294,7 @@ async function ingestPassScoped(args: CliArgs, state: IngestState, scope: Transc
const msg = `EGRESS_RECEIPT_FAILED: ${(err as Error).message} — ingest refused`;
console.error(`[memory-ingest] ERR: ${msg}`);
if (resumed && !args.scanSecrets) cleanupStagingDir(resumed.stagingDir);
return { ...base, state_untouched: true, written: 0, failed: prep.parseFailed + prep.prepared.length, duration_ms: Date.now() - t0, system_error: msg };
return { ...base, state_untouched: true, written: 0, failed: prep.parseFailed + prep.prepared.length, parse_failed: prep.parseFailed, duration_ms: Date.now() - t0, system_error: msg };
}
const run: RunContext = {
@@ -2343,6 +2350,7 @@ async function ingestPassScoped(args: CliArgs, state: IngestState, scope: Transc
...base,
written: run.written,
failed: run.failed + prep.parseFailed,
parse_failed: prep.parseFailed,
refused: run.refused,
unverified: check.unverified,
quarantined: run.quarantined,
@@ -3236,6 +3244,12 @@ async function runLocked(args: CliArgs): Promise<number> {
const t0 = Date.now();
const result = await ingestPass(args, state);
// /sync-gbrain reads the counts here: per-page failures exit 0 (#4417).
if (process.env.GSTACK_INGEST_RESULT_FILE) {
try {
writeFileSync(process.env.GSTACK_INGEST_RESULT_FILE, JSON.stringify({ written: result.written, failed: result.failed, parse_failed: result.parse_failed ?? 0 }));
} catch {}
}
if (!result.state_untouched) {
// A pending reconcile (set by the upgrade migration or a schema migration)
// gets one bounded pass per ingest run until it completes.