mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-31 22:14:25 +01:00
A9: --no-write is a dry run and leaves ingest state untouched
--no-write stamped every prepared page ingested_at and saved state, so a later real run skipped those pages forever. It now prepares and reports pages without importing or changing the state file. Tests that used GSTACK_MEMORY_INGEST_NO_WRITE / --no-write to seed state move to an explicit seam (a fake gbrain that accepts every staged page), because the --no-write contract deliberately changes.
This commit is contained in:
1 parent
ff5fbd09ee
commit
7e2edf2a3a
4 files changed
+85
-60
No files matched your search
+11
-25
@@ -261,8 +261,8 @@ Options:
|
||||
transcript_ingest_mode is recent (90 days) or all
|
||||
(all history). A list naming transcript overrides it.
|
||||
--limit <N> Stop after N pages written (smoke testing).
|
||||
--no-write Skip gbrain put calls (still updates state file).
|
||||
Used by tests + dry runs without actual ingest.
|
||||
--no-write Dry run: prepare pages and report counts, but import
|
||||
nothing and leave the state file untouched.
|
||||
--scan-secrets Opt-in gitleaks scan of outgoing rendered pages, including
|
||||
resumed staging. Findings and incomplete scans block
|
||||
writes and remain retryable. Off by default.
|
||||
@@ -2052,31 +2052,17 @@ async function ingestPass(args: CliArgs): Promise<BulkResult> {
|
||||
}
|
||||
|
||||
if (args.noWrite) {
|
||||
// --no-write: skip the gbrain import call but still record state for
|
||||
// prepared pages (treat them as ingested for dedup purposes). Matches
|
||||
// the prior contract from --help: "Skip gbrain put calls (still
|
||||
// updates state file)".
|
||||
const nowIso = new Date().toISOString();
|
||||
for (const p of prep.prepared) {
|
||||
try {
|
||||
const fingerprint = sourceFingerprintForStamp(p);
|
||||
if (!fingerprint) continue;
|
||||
state.sessions[p.source_path] = {
|
||||
...fingerprint,
|
||||
ingested_at: nowIso,
|
||||
page_slug: p.page_slug,
|
||||
partial: p.partial,
|
||||
};
|
||||
written++;
|
||||
} catch {
|
||||
// best-effort state record
|
||||
}
|
||||
// --no-write is a dry run: it prepares pages (so parse, attribution,
|
||||
// policy and scan results are reported) but neither imports nor touches
|
||||
// the state file. Stamping here marked every prepared page ingested, so
|
||||
// the next real run skipped them forever (A9).
|
||||
if (!args.quiet) {
|
||||
console.error(
|
||||
`[memory-ingest] --no-write: ${prep.prepared.length} page(s) would be imported; nothing was imported and the state file was not changed.`,
|
||||
);
|
||||
}
|
||||
state.last_full_walk = new Date().toISOString();
|
||||
state.last_writer = "gstack-memory-ingest";
|
||||
saveState(state);
|
||||
return {
|
||||
written,
|
||||
written: 0,
|
||||
skipped_secret: prep.skippedSecret,
|
||||
skipped_dedup: prep.skippedDedup,
|
||||
skipped_unattributed: prep.skippedUnattributed,
|
||||
|
||||
@@ -156,8 +156,11 @@ Saved pages determine the expected import count; only pages matching the
|
||||
current rendered source can advance its ingest state. Incomplete or mismatched
|
||||
staging remains available for recovery.
|
||||
|
||||
With scanning requested, fresh, resumed, persistent, and `--no-write` passes
|
||||
stamp only the source snapshot used to render the page, and only while its
|
||||
`--no-write` is a dry run: it prepares and counts pages but imports nothing
|
||||
and never changes the ingest state, so a later real run still imports every
|
||||
eligible page.
|
||||
|
||||
With scanning requested, fresh, resumed and persistent passes stamp only the source snapshot used to render the page, and only while its
|
||||
hash and modification time are unchanged. Incremental checks verify the hash
|
||||
even when the timestamp matches. Scanned pages are fully written in private
|
||||
staging before atomic promotion, so partial writes never become outgoing pages.
|
||||
|
||||
@@ -399,22 +399,40 @@ if (process.env.LIMIT_STAGE_WRITES === '1') {
|
||||
});
|
||||
}
|
||||
|
||||
for (const mode of ["no-write", "remote-http"]) {
|
||||
it(`does not stamp an append during the ${mode} scan`, () => {
|
||||
scanner("clean");
|
||||
const path = source();
|
||||
if (mode === "remote-http") writeFileSync(join(home, ".claude.json"), JSON.stringify({ mcpServers: { gbrain: { type: "http", url: "http://fixture.invalid/mcp" } } }));
|
||||
env.APPEND_DURING_SCAN = path;
|
||||
env.APPEND_RECORD = appendRecord();
|
||||
const args = mode === "no-write" ? ["--scan-secrets", "--no-write"] : ["--scan-secrets"];
|
||||
expect(run(args).status).toBe(0);
|
||||
expect(sessions()[path]).toBeUndefined();
|
||||
expect(imported()).toEqual([]);
|
||||
delete env.APPEND_DURING_SCAN;
|
||||
expect(run(args).status).toBe(0);
|
||||
expect(sessions()[path]).toBeDefined();
|
||||
});
|
||||
}
|
||||
it("does not stamp an append during the remote-http scan", () => {
|
||||
scanner("clean");
|
||||
const path = source();
|
||||
writeFileSync(join(home, ".claude.json"), JSON.stringify({ mcpServers: { gbrain: { type: "http", url: "http://fixture.invalid/mcp" } } }));
|
||||
env.APPEND_DURING_SCAN = path;
|
||||
env.APPEND_RECORD = appendRecord();
|
||||
expect(run(["--scan-secrets"]).status).toBe(0);
|
||||
expect(sessions()[path]).toBeUndefined();
|
||||
expect(imported()).toEqual([]);
|
||||
delete env.APPEND_DURING_SCAN;
|
||||
expect(run(["--scan-secrets"]).status).toBe(0);
|
||||
expect(sessions()[path]).toBeDefined();
|
||||
});
|
||||
|
||||
it("--no-write leaves state untouched, so the next real run imports every eligible page (A9)", () => {
|
||||
scanner("clean");
|
||||
const first = source("first conversation");
|
||||
const second = source("second conversation");
|
||||
const statePath = join(env.GSTACK_HOME, ".transcript-ingest-state.json");
|
||||
const dry = run(["--no-write"]);
|
||||
expect(dry.status).toBe(0);
|
||||
expect(existsSync(statePath)).toBe(false);
|
||||
expect(dry.stderr).toContain("--no-write: 2 page(s) would be imported");
|
||||
expect(imported()).toEqual([]);
|
||||
expect(run(["--no-write", "--scan-secrets"]).status).toBe(0);
|
||||
expect(existsSync(statePath)).toBe(false);
|
||||
expect(run([]).status).toBe(0);
|
||||
expect(imported()).toHaveLength(2);
|
||||
expect(Object.keys(sessions()).sort()).toEqual([first, second].sort());
|
||||
const before = readFileSync(statePath, "utf8");
|
||||
source("third conversation");
|
||||
expect(run(["--no-write"]).status).toBe(0);
|
||||
expect(readFileSync(statePath, "utf8")).toBe(before);
|
||||
});
|
||||
|
||||
for (const remote of [false, true]) {
|
||||
it(`never stamps a page that failed to stage (remote-http: ${remote})`, () => {
|
||||
@@ -599,15 +617,16 @@ process.stdout.write(execFileSync(process.execPath, ['-e', 'process.stdout.write
|
||||
expect(existsSync(join(home, "scanner-late"))).toBe(false);
|
||||
});
|
||||
|
||||
it("does not stamp --no-write pages that could not pass the requested scan", () => {
|
||||
it("does not stamp or import pages that could not pass the requested scan", () => {
|
||||
scanner("error");
|
||||
const path = source();
|
||||
run(["--scan-secrets", "--no-write"]);
|
||||
run(["--scan-secrets"]);
|
||||
expect(sessions()[path]).toBeUndefined();
|
||||
scanner("clean");
|
||||
expect(run(["--scan-secrets", "--no-write"]).status).toBe(0);
|
||||
expect(sessions()[path]).toBeDefined();
|
||||
expect(imported()).toEqual([]);
|
||||
scanner("clean");
|
||||
expect(run(["--scan-secrets"]).status).toBe(0);
|
||||
expect(sessions()[path]).toBeDefined();
|
||||
expect(imported()).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("accepts a complete clean report exactly at the 16 MiB ceiling", () => {
|
||||
|
||||
@@ -25,7 +25,7 @@
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from "bun:test";
|
||||
import { mkdtempSync, writeFileSync, readFileSync, existsSync, rmSync, mkdirSync, statSync } from "fs";
|
||||
import { mkdtempSync, writeFileSync, readFileSync, existsSync, rmSync, mkdirSync, statSync, chmodSync } from "fs";
|
||||
import { tmpdir } from "os";
|
||||
import { join } from "path";
|
||||
import { spawnSync } from "child_process";
|
||||
@@ -87,6 +87,23 @@ function setupFixture(home: string): { gstackHome: string; counts: Record<string
|
||||
};
|
||||
}
|
||||
|
||||
// Test seam for writes (A9): --no-write no longer stamps state, so runs that
|
||||
// need a populated state file import through a fake gbrain that accepts every
|
||||
// staged page.
|
||||
function fakeGbrainEnv(home: string, gstackHome: string): Record<string, string> {
|
||||
const bin = join(home, "fake-bin");
|
||||
mkdirSync(bin, { recursive: true });
|
||||
writeFileSync(join(bin, "gbrain"), `#!/usr/bin/env bash
|
||||
case "\${1:-}" in
|
||||
--help) echo " import <dir>" ;;
|
||||
import) n=$(find "$2" -name '*.md' | wc -l | tr -d ' '); echo "{\\"status\\":\\"success\\",\\"imported\\":$n,\\"skipped\\":0,\\"errors\\":0,\\"total_files\\":$n}" ;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
`);
|
||||
chmodSync(join(bin, "gbrain"), 0o755);
|
||||
return { HOME: home, GSTACK_HOME: gstackHome, PATH: `${bin}:${process.env.PATH || ""}` };
|
||||
}
|
||||
|
||||
function seedTranscriptConsent(gstackHome: string | undefined): void {
|
||||
if (!gstackHome) return;
|
||||
mkdirSync(gstackHome, { recursive: true });
|
||||
@@ -111,7 +128,7 @@ describe("V1 memory ingest pipeline E2E", () => {
|
||||
it("--probe accounts for all 9 fixture files: 7 attributable + 2 unattributed transcripts skipped (#2394)", () => {
|
||||
const home = makeFixtureHome();
|
||||
const { gstackHome, counts } = setupFixture(home);
|
||||
const env = { HOME: home, GSTACK_HOME: gstackHome, GSTACK_MEMORY_INGEST_NO_WRITE: "1" };
|
||||
const env = fakeGbrainEnv(home, gstackHome);
|
||||
|
||||
const r = runBun(INGEST, ["--probe"], env);
|
||||
expect(r.exitCode).toBe(0);
|
||||
@@ -135,7 +152,7 @@ describe("V1 memory ingest pipeline E2E", () => {
|
||||
it("--probe --include-unattributed counts all 9 fixture files, transcripts included", () => {
|
||||
const home = makeFixtureHome();
|
||||
const { gstackHome, counts } = setupFixture(home);
|
||||
const env = { HOME: home, GSTACK_HOME: gstackHome, GSTACK_MEMORY_INGEST_NO_WRITE: "1" };
|
||||
const env = fakeGbrainEnv(home, gstackHome);
|
||||
|
||||
const r = runBun(INGEST, ["--probe", "--include-unattributed"], env);
|
||||
expect(r.exitCode).toBe(0);
|
||||
@@ -150,7 +167,7 @@ describe("V1 memory ingest pipeline E2E", () => {
|
||||
it("--incremental writes a state file with schema_version: 1 + last_writer", () => {
|
||||
const home = makeFixtureHome();
|
||||
const { gstackHome } = setupFixture(home);
|
||||
const env = { HOME: home, GSTACK_HOME: gstackHome, GSTACK_MEMORY_INGEST_NO_WRITE: "1" };
|
||||
const env = fakeGbrainEnv(home, gstackHome);
|
||||
|
||||
runBun(INGEST, ["--incremental", "--quiet"], env);
|
||||
|
||||
@@ -167,7 +184,7 @@ describe("V1 memory ingest pipeline E2E", () => {
|
||||
it("--incremental is idempotent — re-run reports 0 changes", () => {
|
||||
const home = makeFixtureHome();
|
||||
const { gstackHome } = setupFixture(home);
|
||||
const env = { HOME: home, GSTACK_HOME: gstackHome, GSTACK_MEMORY_INGEST_NO_WRITE: "1" };
|
||||
const env = fakeGbrainEnv(home, gstackHome);
|
||||
|
||||
// First run
|
||||
runBun(INGEST, ["--incremental", "--quiet"], env);
|
||||
@@ -184,7 +201,7 @@ describe("V1 memory ingest pipeline E2E", () => {
|
||||
it("--probe shows new vs unchanged distinction after first --incremental", () => {
|
||||
const home = makeFixtureHome();
|
||||
const { gstackHome } = setupFixture(home);
|
||||
const env = { HOME: home, GSTACK_HOME: gstackHome, GSTACK_MEMORY_INGEST_NO_WRITE: "1" };
|
||||
const env = fakeGbrainEnv(home, gstackHome);
|
||||
|
||||
// First, write some state by running --incremental quietly
|
||||
runBun(INGEST, ["--incremental", "--quiet"], env);
|
||||
@@ -208,7 +225,7 @@ describe("V1 /gbrain-sync orchestrator E2E", () => {
|
||||
it("--dry-run with all stages enabled previews 3 stages", () => {
|
||||
const home = makeFixtureHome();
|
||||
const { gstackHome } = setupFixture(home);
|
||||
const env = { HOME: home, GSTACK_HOME: gstackHome, GSTACK_MEMORY_INGEST_NO_WRITE: "1" };
|
||||
const env = fakeGbrainEnv(home, gstackHome);
|
||||
|
||||
const r = runBun(SYNC, ["--dry-run"], env);
|
||||
expect(r.exitCode).toBe(0);
|
||||
@@ -225,7 +242,7 @@ describe("V1 /gbrain-sync orchestrator E2E", () => {
|
||||
it("--no-code --no-brain-sync --incremental runs only memory ingest, writes sync state", () => {
|
||||
const home = makeFixtureHome();
|
||||
const { gstackHome } = setupFixture(home);
|
||||
const env = { HOME: home, GSTACK_HOME: gstackHome, GSTACK_MEMORY_INGEST_NO_WRITE: "1" };
|
||||
const env = fakeGbrainEnv(home, gstackHome);
|
||||
|
||||
const r = runBun(SYNC, ["--incremental", "--no-code", "--no-brain-sync", "--quiet"], env);
|
||||
expect([0, 1]).toContain(r.exitCode); // memory stage may fail if gbrain CLI is missing; both ok
|
||||
@@ -250,7 +267,7 @@ describe("V1 retrieval surface — real V1 manifest dispatch", () => {
|
||||
it("loads office-hours/SKILL.md manifest and dispatches 4 queries", () => {
|
||||
const home = makeFixtureHome();
|
||||
const { gstackHome } = setupFixture(home);
|
||||
const env = { HOME: home, GSTACK_HOME: gstackHome, GSTACK_MEMORY_INGEST_NO_WRITE: "1" };
|
||||
const env = { HOME: home, GSTACK_HOME: gstackHome };
|
||||
|
||||
const skillFile = join(REPO_ROOT, "office-hours", "SKILL.md");
|
||||
expect(existsSync(skillFile)).toBe(true);
|
||||
@@ -271,7 +288,7 @@ describe("V1 retrieval surface — real V1 manifest dispatch", () => {
|
||||
it("renders datamark envelope around every loaded section (Section 1D + D12)", () => {
|
||||
const home = makeFixtureHome();
|
||||
const { gstackHome } = setupFixture(home);
|
||||
const env = { HOME: home, GSTACK_HOME: gstackHome, GSTACK_MEMORY_INGEST_NO_WRITE: "1" };
|
||||
const env = { HOME: home, GSTACK_HOME: gstackHome };
|
||||
|
||||
const skillFile = join(REPO_ROOT, "office-hours", "SKILL.md");
|
||||
const r = runBun(CONTEXT, ["--skill-file", skillFile, "--repo", "test-repo"], env);
|
||||
@@ -292,7 +309,7 @@ describe("V1 retrieval surface — real V1 manifest dispatch", () => {
|
||||
it("Layer 1 fallback when no skill specified — default 3-section manifest", () => {
|
||||
const home = makeFixtureHome();
|
||||
const { gstackHome } = setupFixture(home);
|
||||
const env = { HOME: home, GSTACK_HOME: gstackHome, GSTACK_MEMORY_INGEST_NO_WRITE: "1" };
|
||||
const env = { HOME: home, GSTACK_HOME: gstackHome };
|
||||
|
||||
const r = runBun(CONTEXT, ["--repo", "test-repo", "--explain", "--quiet"], env);
|
||||
expect(r.exitCode).toBe(0);
|
||||
@@ -305,7 +322,7 @@ describe("V1 retrieval surface — real V1 manifest dispatch", () => {
|
||||
it("plan-ceo-review/SKILL.md manifest also dispatches correctly (regression for V1 manifest authoring)", () => {
|
||||
const home = makeFixtureHome();
|
||||
const { gstackHome } = setupFixture(home);
|
||||
const env = { HOME: home, GSTACK_HOME: gstackHome, GSTACK_MEMORY_INGEST_NO_WRITE: "1" };
|
||||
const env = { HOME: home, GSTACK_HOME: gstackHome };
|
||||
|
||||
const skillFile = join(REPO_ROOT, "plan-ceo-review", "SKILL.md");
|
||||
expect(existsSync(skillFile)).toBe(true);
|
||||
|
||||
Reference in new issue
Block a user