diff --git a/autoplan/DRIVER.md b/autoplan/DRIVER.md index 54a7e660e..48cad0438 100644 --- a/autoplan/DRIVER.md +++ b/autoplan/DRIVER.md @@ -74,7 +74,7 @@ line yourself; gstack never writes the plan. - `gstack-autoplan status|resume --out ` read the manifest (PENDING: lines name open attempts); take ownership after a crash (EXECUTION_UNKNOWN lists unfinished attempts). - `gstack-autoplan resume --out --from ` - restore a hash-verified checkpoint (GSTACK_AUTOPLAN_CHECKPOINT_DIR) into an empty directory and take ownership. + restore the newest valid checkpoint generation (GSTACK_AUTOPLAN_CHECKPOINT_DIR) into an empty directory and take ownership; `checkpoints --from ` lists them. ## Reviewer result format @@ -128,7 +128,7 @@ is the completion check. Copy the whole directory, never single files. - `GATE_OWNER_REPLY_MISMATCH`: relay exactly what the owner chose (or their reply itself, e.g. `--reply rec`); ask the owner when their words and the tokens disagree - `PLAN_AMEND_PENDING`: edit the plan, then `gstack-autoplan revise --out --plan --incorporates `; settle a packaging or ordering change with `gstack-autoplan revise --out --settle `; or drop one with `gstack-autoplan revise --out --withdraw ` - `AMENDMENT_NOT_SETTLEABLE`: edit the plan, then `gstack-autoplan revise --out --plan --incorporates ` (add `--settle ` to settle packaging/ordering changes in the same call); settle alone only on the plan the run already reviews -- `CHECKPOINT_INVALID`: restore from an intact copy of `//` (its `restore.md` names the command); nothing was written to --out +- `CHECKPOINT_INVALID`: run `gstack-autoplan checkpoints --from ` to see each generation's hash status and restore an intact one; nothing was written to --out - `GSTACK_VERSION_CHANGED`: nothing to do for an older run (journaled forward migrations ran); a run written by a newer gstack needs that gstack: upgrade with bin/gstack-upgrade ## Coordinator contract (inherited by every lane) diff --git a/bin/gstack-autoplan b/bin/gstack-autoplan index 29aa65289..becab9f1d 100755 --- a/bin/gstack-autoplan +++ b/bin/gstack-autoplan @@ -16,6 +16,7 @@ * gstack-autoplan status --out [--json] query: the manifest, no GSTACK_RESULT line * gstack-autoplan export --out [--json] rewrite plan.md, review-record.md, findings/tasks.jsonl and run.json hashes * gstack-autoplan release --out --attempt drop a reservation for an attempt that never ran + * gstack-autoplan checkpoints --from list checkpoint generations, newest first, with hash ok|bad * gstack-autoplan contract print the coordinator contract every lane prompt inherits (plan E2) * * Logic: lib/autoplan-run.ts (next/submit/resume), lib/autoplan-gate.ts (close, gate, answer), @@ -27,7 +28,7 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; import { amendmentSteps, answerGate, recommendGate } from '../lib/autoplan-gate'; -import { checkpointRun, restoreCheckpoint } from '../lib/autoplan-checkpoint'; +import { checkpointLines, checkpointRun, restoreCheckpoint } from '../lib/autoplan-checkpoint'; import { admit, touchInstallLease } from '../lib/autoplan-admission'; import { exportRun } from '../lib/autoplan-export'; import { migrateRun } from '../lib/autoplan-migrate'; @@ -69,7 +70,9 @@ DRIVER: ${driverFile()} recommend --out --gate-rev --item --option --basis "" [--by ] set one item's recommendation before relay (provenance recorded, p1 follows); re-render and relay afterwards resume --out [--from ] [--json] --from restores a hash-verified checkpoint into an empty --out first - (GSTACK_AUTOPLAN_CHECKPOINT_DIR set: every command leaving the run at rest writes // and prints CHECKPOINT:) + (a collection or root restores its newest valid generation; GSTACK_AUTOPLAN_CHECKPOINT_DIR set: every command + leaving the run at rest publishes a generation under // and prints CHECKPOINT: on stderr) + checkpoints --from list checkpoint generations, newest first: run, gate_rev, time, hash ok|bad, current status --out [--json] export --out release --out --attempt contract print lib/coordinator-contract.ts (the block every lane and reviewer prompt inherits word for word) @@ -124,12 +127,21 @@ function mutating(out: string, json: boolean, fn: () => number): number { if (notices.length) (json ? process.stderr : process.stdout).write(notices.join('\n') + '\n'); const code = fn(); if (admission.deviation && fs.existsSync(path.join(out, FILES.run))) { const state = readState(out); if (record(state)) writeState(state); } - const checkpoint = code === EXIT.ok ? checkpointRun(out) : undefined; - if (checkpoint) process.stderr.write(checkpoint + '\n'); + if (code === EXIT.ok) checkpointQuietly(out); return code; }); } +/** A checkpoint never changes a committed command's outcome: a failure is a stderr warning, the exit code stands. */ +function checkpointQuietly(out: string): void { + try { + const line = checkpointRun(out); + if (line) process.stderr.write(line + '\n'); + } catch (e: any) { + process.stderr.write(`CHECKPOINT: failed for ${out}: ${e instanceof RunError ? `${e.detail} (${e.code})` : e?.message ?? e}; the command's result stands, the previous checkpoint stays current\n`); + } +} + function emit(json: boolean, lines: string[], payload: Record): void { if (json) process.stdout.write(JSON.stringify(payload) + '\n'); else process.stdout.write(lines.join('\n') + '\n'); @@ -234,6 +246,12 @@ async function main(argv: string[]): Promise { const json = rest.includes('--json'); if (!cmd || cmd === '--help' || cmd === '-h' || cmd === 'help') { process.stdout.write(HELP + '\n'); return cmd ? EXIT.ok : EXIT.usage; } if (cmd === 'contract') { process.stdout.write(COORDINATOR_CONTRACT + '\n'); return EXIT.ok; } + if (cmd === 'checkpoints') { + const from = flag(rest, '--from'); + if (!from) return usage('checkpoints needs --from '); + try { process.stdout.write(checkpointLines(from).join('\n') + '\n'); return EXIT.ok; } + catch (e: any) { if (!(e instanceof RunError)) throw e; process.stderr.write(resultLine('gstack-autoplan checkpoints', e.code, e.detail) + '\n'); return EXIT.refused; } + } const outFlag = flag(rest, '--out'); if (!outFlag) return usage(`${cmd} needs --out `); const out = path.resolve(outFlag); @@ -252,8 +270,7 @@ async function main(argv: string[]): Promise { const restored = from !== undefined ? restoreCheckpoint(from, out) : []; return withRunLock(out, () => { const r = resumeRun(out); - const checkpoint = checkpointRun(out); - if (checkpoint) process.stderr.write(checkpoint + '\n'); + checkpointQuietly(out); const lines = [...restored, ...r.lines, resultLineFor(r.state.status, out)]; emit(json, lines, { run: r.state.run, status: r.state.status, execution_unknown: r.unknown, rebound: r.rebound, out }); return EXIT.ok; diff --git a/bin/gstack-doctor-components.sh b/bin/gstack-doctor-components.sh index cdff473ed..334148c1f 100644 --- a/bin/gstack-doctor-components.sh +++ b/bin/gstack-doctor-components.sh @@ -192,6 +192,7 @@ gstack_result_anchor() { AMENDMENT_NOT_SETTLEABLE) printf '%s' 'amendment-not-settleable' ;; GATE_REC_STALE) printf '%s' 'gate-rec-stale' ;; CHECKPOINT_INVALID) printf '%s' 'checkpoint-invalid' ;; + CHECKPOINT_OVERLAP) printf '%s' 'checkpoint-overlap' ;; CHECKPOINT_TARGET_EXISTS) printf '%s' 'checkpoint-target-exists' ;; OWNER_REPLY_REQUIRED) printf '%s' 'owner-reply-required' ;; GATE_OWNER_REPLY_MISMATCH) printf '%s' 'gate-owner-reply-mismatch' ;; diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 7c3bda6d8..8bc4ce592 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -2841,16 +2841,32 @@ tokens are never refused. ### `CHECKPOINT_INVALID` -**Meaning.** `gstack-autoplan resume --out --from ` refused -the checkpoint: `checkpoint.json` is unreadable or not a schema_version 1 -checkpoint, or a listed file is missing, is not a regular file, escapes the -checkpoint directory, or does not match its recorded sha256. Nothing was -written to `--out`. +**Meaning.** `gstack-autoplan resume --out --from ` found no +generation it can restore: a `checkpoint.json` is unreadable or not a +schema_version 1 checkpoint, a listed file is missing, is not a regular file, +escapes the generation (an `..` path or a symlinked directory), or does not +match its recorded sha256; or the staged run does not pass +`gstack-artifact validate`. `--from` naming a run's collection restores the +newest generation that verifies, skipping corrupt newer ones; a root holding +several runs must name one. Nothing was written to `--out`. -**Fix.** Restore from an intact copy. With `GSTACK_AUTOPLAN_CHECKPOINT_DIR` -set, every command that leaves the run at rest rewrites -`//` (its `restore.md` names the exact command); a copy that was -edited or partially synced fails here by design. +**Fix.** `gstack-autoplan checkpoints --from ` lists every generation +with `hash=ok` or `hash=bad ()`; restore an intact one by passing its +directory. With `GSTACK_AUTOPLAN_CHECKPOINT_DIR` set, every command that +leaves the run at rest publishes a new generation and keeps the previous one. + + +### `CHECKPOINT_OVERLAP` + +**Meaning.** The checkpoint directory for this run and the run directory (or, +on restore, the checkpoint and `--out` or its staging directory) are the same +directory or one contains the other, after resolving symlinks. Copying one +into the other would replace or delete the live run, so nothing was touched. +A checkpoint refusal after a committed command prints `CHECKPOINT: failed …` +on stderr and leaves that command's exit code alone. + +**Fix.** Point `GSTACK_AUTOPLAN_CHECKPOINT_DIR` (or `--from` / `--out`) at a +directory outside the run, for example the private project-user Drive folder. ### `CHECKPOINT_TARGET_EXISTS` diff --git a/docs/unattended.md b/docs/unattended.md index bf5919c9f..906257591 100644 --- a/docs/unattended.md +++ b/docs/unattended.md @@ -239,8 +239,14 @@ the plan. consumption steps above apply unchanged. `$AP status --out ` and `resume` work after the run directory moves (paths are re-rooted; journal paths are run-relative). With `GSTACK_AUTOPLAN_CHECKPOINT_DIR` set, a run - at rest is copied, hash-listed, to `//`; `$AP resume --out --from /` verifies and restores it (`CHECKPOINT_INVALID`). + at rest publishes an immutable, hash-listed generation under + `//generations/` and switches `current.json` to it (the previous + generation is kept; a stale copy never replaces a newer one; a directory + overlapping the run is refused with `CHECKPOINT_OVERLAP`; a failure is a + `CHECKPOINT: failed` warning that leaves the command's exit code alone). + `$AP checkpoints --from ` lists generations with their hash status, and + `$AP resume --out --from /` restores the newest valid + one after validating the staged run (`CHECKPOINT_INVALID`). Concurrency and recovery: every state write is a locked read-modify-write (`run.lock`; a second writer sees `RUN_LOCKED`), attempts are journaled to diff --git a/lib/autoplan-checkpoint.ts b/lib/autoplan-checkpoint.ts index 687c4a28c..dff1b3ad0 100644 --- a/lib/autoplan-checkpoint.ts +++ b/lib/autoplan-checkpoint.ts @@ -2,26 +2,71 @@ * autoplan-checkpoint — an opt-in durable copy of a run directory (next-wave * B12). With GSTACK_AUTOPLAN_CHECKPOINT_DIR set, every mutating command that * leaves the run at rest (gate_pending, amend_pending, complete, incomplete) - * publishes `//`: `checkpoint.json` (every file's sha256), the run - * files under `run/`, the owner's plan file under `plan-input/` when it lives - * outside the run, and `restore.md`, the human index. Publication swaps a - * fully written sibling in, so a reader sees the old copy or the new one. - * `restoreCheckpoint` verifies containment and every hash before it publishes - * the run directory with one rename; the caller then resumes and owns it. + * publishes one immutable generation under `//generations//`: + * `checkpoint.json` (every file's sha256), the run files under `run/`, the + * owner's plan file under `plan-input/` when it lives outside the run, and + * `restore.md`. `current.json` then switches to it with one rename; the + * previous generation stays until the switch commits, so an interrupted + * publish leaves the last checkpoint current. A generation never replaces a + * newer one (higher run revision). Publication and restore hold the + * collection's `.lock`; restore also holds `.lock`. + * + * Restore picks the named generation, or the newest generation that verifies + * when `--from` names the collection (or a root holding one run). It refuses + * overlapping source and target paths (symlink aliases included), any listed + * file whose real path leaves the generation, and any hash mismatch; it + * validates the staged run (gstack-artifact's validateRun) before publishing + * it to `--out` with one rename, so a refusal leaves `--out` absent. */ import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; -import { sha256File } from './headless-artifacts'; +import { sha256File, validateRun } from './headless-artifacts'; import { FILES, RunError, journal, now, readState, type RunState } from './autoplan-state'; const MANIFEST = 'checkpoint.json'; +const CURRENT = 'current.json'; +const GENERATIONS = 'generations'; const AT_REST = ['gate_pending', 'amend_pending', 'complete', 'incomplete']; +const FOREIGN_LOCK_STALE_MS = 10 * 60_000; interface CheckpointManifest { schema_version: 1; run: string; status: string; gate_rev: number; revision: number; at: string; host: string; gstack_version?: string; files: Record; plan_input?: { original: string; file: string; sha256: string }; } +export interface Generation { dir: string; manifest?: CheckpointManifest; problem?: string } + +const inside = (root: string, p: string) => { const rel = path.relative(root, p); return rel !== '' && !rel.startsWith('..') && !path.isAbsolute(rel); }; +const overlaps = (a: string, b: string) => a === b || inside(a, b) || inside(b, a); +/** The real path of `p`, resolving symlinks through its deepest existing ancestor. */ +function canonical(p: string): string { + let cur = path.resolve(p); + const rest: string[] = []; + while (!fs.existsSync(cur) && path.dirname(cur) !== cur) { rest.unshift(path.basename(cur)); cur = path.dirname(cur); } + return path.join(fs.realpathSync(cur), ...rest); +} +function refuseOverlap(what: string, a: string, b: string): void { + const ca = canonical(a), cb = canonical(b); + if (overlaps(ca, cb)) throw new RunError('CHECKPOINT_OVERLAP', `${what}: ${a} (${ca}) and ${b} (${cb}) are the same directory or one contains the other`, 3); +} + +function exclusive(lock: string, fn: () => T): T { + fs.mkdirSync(path.dirname(lock), { recursive: true }); + const mine = JSON.stringify({ pid: process.pid, host: os.hostname(), at: now() }); + for (let i = 0; ; i++) { + try { fs.writeFileSync(lock, mine, { flag: 'wx' }); break; } catch (e: any) { + if (e.code !== 'EEXIST') throw e; + let owner: { pid?: number; host?: string; at?: string } = {}; + try { owner = JSON.parse(fs.readFileSync(lock, 'utf8')); } catch { /* being written */ } + let alive = true; + if (owner.host === os.hostname() && typeof owner.pid === 'number') { try { process.kill(owner.pid, 0); } catch (k: any) { alive = k.code === 'EPERM'; } } + else if (owner.host && owner.at) alive = Date.now() - Date.parse(owner.at) < FOREIGN_LOCK_STALE_MS; + if (alive || i > 3) throw new RunError('RUN_LOCKED', `${lock} is held by pid ${owner.pid ?? '?'} on ${owner.host ?? '?'} since ${owner.at ?? '?'}`, 3); + fs.rmSync(lock, { force: true }); + } + } + try { return fn(); } finally { try { if (fs.readFileSync(lock, 'utf8') === mine) fs.unlinkSync(lock); } catch { /* gone */ } } +} function runFiles(dir: string, rel = ''): string[] { return fs.readdirSync(path.join(dir, rel), { withFileTypes: true }).flatMap(e => { @@ -30,86 +75,154 @@ function runFiles(dir: string, rel = ''): string[] { return e.isFile() && r !== FILES.lock && !r.endsWith('.tmp') ? [r] : []; }); } - const copy = (from: string, to: string) => { fs.mkdirSync(path.dirname(to), { recursive: true }); fs.copyFileSync(from, to); }; -const inside = (root: string, p: string) => !path.relative(root, p).startsWith('..') && !path.isAbsolute(path.relative(root, p)); +const writeAtomic = (file: string, text: string) => { const tmp = `${file}.${process.pid}.tmp`; fs.writeFileSync(tmp, text); fs.renameSync(tmp, file); }; -function restoreIndex(m: CheckpointManifest, target: string): string { +function restoreIndex(m: CheckpointManifest, collection: string): string { return [`# autoplan checkpoint: ${m.run}`, '', - `Written ${m.at} on ${m.host} at status=${m.status}, gate_rev=${m.gate_rev}; ${Object.keys(m.files).length} files, each listed with its sha256 in checkpoint.json.`, '', - 'Restore on any machine (the target directory must not hold a run):', '', - ` gstack-autoplan resume --out --from ${target}`, '', - 'resume verifies every hash, publishes the run directory, takes ownership and prints the next step.', - ...(m.plan_input ? ['', `The owner's plan file was ${m.plan_input.original}; a copy is in ${m.plan_input.file} and is restored into the run directory when the original is missing.`] : []), ''].join('\n'); + `Current generation written ${m.at} on ${m.host} at status=${m.status}, gate_rev=${m.gate_rev}, revision=${m.revision}; ${Object.keys(m.files).length} files, each listed with its sha256 in generations//checkpoint.json.`, '', + 'List the generations and their hash status, then restore the newest valid one on any machine (the target directory must not hold a run):', '', + ` gstack-autoplan checkpoints --from ${collection}`, + ` gstack-autoplan resume --out --from ${collection}`, '', + 'resume verifies every hash and the staged run, publishes the run directory, takes ownership and prints the next step.', + ...(m.plan_input ? ['', `The owner's plan file was ${m.plan_input.original}; a copy is in plan-input/ and is restored into the run directory when the original is missing.`] : []), ''].join('\n'); } -/** Publish the checkpoint when the env opts in and the run is at rest; returns the CHECKPOINT line, or nothing. */ +/** The newest run revision any generation in the collection records (unreadable generations count as nothing). */ +function newestRevision(collection: string): number { + const dir = path.join(collection, GENERATIONS); + if (!fs.existsSync(dir)) return -1; + return Math.max(-1, ...fs.readdirSync(dir).map(g => { try { return Number(JSON.parse(fs.readFileSync(path.join(dir, g, MANIFEST), 'utf8')).revision); } catch { return -1; } }).filter(Number.isFinite)); +} + +/** Publish a generation when the env opts in and the run is at rest; returns the CHECKPOINT line, or nothing. */ export function checkpointRun(out: string, env: NodeJS.ProcessEnv = process.env): string | undefined { const root = env.GSTACK_AUTOPLAN_CHECKPOINT_DIR; if (!root || !fs.existsSync(path.join(out, FILES.run))) return undefined; const state = readState(out); if (!AT_REST.includes(state.status)) return undefined; - const target = path.join(path.resolve(root), state.run); - const tmp = `${target}.${process.pid}.tmp`; - fs.rmSync(tmp, { recursive: true, force: true }); - const files: Record = {}; - for (const rel of runFiles(state.out)) { copy(path.join(state.out, rel), path.join(tmp, 'run', rel)); files[rel] = sha256File(path.join(tmp, 'run', rel)); } - const input = state.plan_input && !inside(state.out, state.plan_input) && fs.existsSync(state.plan_input) ? state.plan_input : undefined; - const planFile = input ? `plan-input/${path.basename(input)}` : undefined; - if (input) copy(input, path.join(tmp, planFile!)); - const m: CheckpointManifest = { schema_version: 1, run: state.run, status: state.status, gate_rev: state.gate_rev ?? 0, revision: state.revision, at: now(), host: os.hostname(), - ...(state.gstack_version ? { gstack_version: state.gstack_version } : {}), files, ...(input ? { plan_input: { original: input, file: planFile!, sha256: sha256File(path.join(tmp, planFile!)) } } : {}) }; - fs.writeFileSync(path.join(tmp, 'restore.md'), restoreIndex(m, target)); - fs.writeFileSync(path.join(tmp, MANIFEST), JSON.stringify(m, null, 2) + '\n'); - const old = `${target}.${process.pid}.old`; - if (fs.existsSync(target)) fs.renameSync(target, old); - fs.renameSync(tmp, target); - fs.rmSync(old, { recursive: true, force: true }); - return `CHECKPOINT: ${target} status=${m.status} gate_rev=${m.gate_rev} files=${Object.keys(files).length}`; + const collection = path.join(path.resolve(root), state.run); + refuseOverlap('the checkpoint directory overlaps the run directory', collection, state.out); + return exclusive(path.join(collection, '.lock'), () => { + const newest = newestRevision(collection); + if (newest > state.revision) return `CHECKPOINT: skipped ${collection}: it holds a newer checkpoint (revision ${newest} > ${state.revision}); nothing replaced`; + const name = `g${String(state.revision).padStart(6, '0')}-${now().replace(/[-:.TZ]/g, '')}-${process.pid}`; + const tmp = path.join(collection, `.tmp-${name}`); + const files: Record = {}; + for (const rel of runFiles(state.out)) { copy(path.join(state.out, rel), path.join(tmp, 'run', rel)); files[rel] = sha256File(path.join(tmp, 'run', rel)); } + const input = state.plan_input && !inside(state.out, path.resolve(state.plan_input)) && fs.existsSync(state.plan_input) ? state.plan_input : undefined; + const planFile = input ? `plan-input/${path.basename(input)}` : undefined; + if (input) copy(input, path.join(tmp, planFile!)); + const m: CheckpointManifest = { schema_version: 1, run: state.run, status: state.status, gate_rev: state.gate_rev ?? 0, revision: state.revision, at: now(), host: os.hostname(), + ...(state.gstack_version ? { gstack_version: state.gstack_version } : {}), files, ...(input ? { plan_input: { original: input, file: planFile!, sha256: sha256File(path.join(tmp, planFile!)) } } : {}) }; + fs.writeFileSync(path.join(tmp, 'restore.md'), restoreIndex(m, collection)); + fs.writeFileSync(path.join(tmp, MANIFEST), JSON.stringify(m, null, 2) + '\n'); + fs.mkdirSync(path.join(collection, GENERATIONS), { recursive: true }); + fs.renameSync(tmp, path.join(collection, GENERATIONS, name)); + if (env.GSTACK_AUTOPLAN_FAULT === 'checkpoint-before-pointer') throw new Error('fault injected: checkpoint interrupted before switching current.json (GSTACK_AUTOPLAN_FAULT, tests only)'); + let previous: string | undefined; + try { previous = JSON.parse(fs.readFileSync(path.join(collection, CURRENT), 'utf8')).generation; } catch { /* first generation */ } + writeAtomic(path.join(collection, CURRENT), JSON.stringify({ generation: name, run: m.run, status: m.status, gate_rev: m.gate_rev, revision: m.revision, at: m.at }, null, 2) + '\n'); + writeAtomic(path.join(collection, 'restore.md'), restoreIndex(m, collection)); + for (const g of fs.readdirSync(path.join(collection, GENERATIONS))) if (g !== name && g !== previous) fs.rmSync(path.join(collection, GENERATIONS, g), { recursive: true, force: true }); + for (const e of fs.readdirSync(collection)) if (e.startsWith('.tmp-')) fs.rmSync(path.join(collection, e), { recursive: true, force: true }); + return `CHECKPOINT: ${path.join(collection, GENERATIONS, name)} status=${m.status} gate_rev=${m.gate_rev} revision=${m.revision} files=${Object.keys(files).length}`; + }); } -/** Verify a checkpoint and publish it as the run directory `out`. Nothing is written to `out` unless every hash matches. */ +/** Verify one generation without throwing: manifest shape, containment by real path, regular files, hashes. */ +export function verifyGeneration(dir: string): Generation { + let m: CheckpointManifest; + try { m = JSON.parse(fs.readFileSync(path.join(dir, MANIFEST), 'utf8')); } catch (e) { return { dir, problem: `no readable ${MANIFEST} (${(e as Error).message})` }; } + if (m.schema_version !== 1 || typeof m.run !== 'string' || !m.files || typeof m.files !== 'object' || !m.files[FILES.run]) return { dir, manifest: m, problem: `${MANIFEST} is not a schema_version 1 checkpoint with run.json listed` }; + const check = (rel: string, sha: string, base: string): string | undefined => { + if (typeof rel !== 'string' || path.isAbsolute(rel) || rel.split(/[\\/]/).includes('..')) return `${rel} escapes the checkpoint`; + const file = path.join(base, rel); + let st: fs.Stats; + try { st = fs.lstatSync(file); } catch { return `${rel} is listed but missing`; } + if (!st.isFile()) return `${rel} is not a regular file`; + if (!inside(fs.realpathSync(base), fs.realpathSync(file))) return `${rel} escapes the checkpoint through a symlinked directory`; + if (sha256File(file) !== sha) return `${rel} does not match its sha256 in ${MANIFEST}`; + return undefined; + }; + for (const [rel, sha] of Object.entries(m.files)) { const problem = check(rel, sha, path.join(dir, 'run')); if (problem) return { dir, manifest: m, problem }; } + if (m.plan_input) { const problem = check(m.plan_input.file, m.plan_input.sha256, dir); if (problem) return { dir, manifest: m, problem }; } + return { dir, manifest: m }; +} + +/** Every generation `from` names: one generation, a run's collection, or a root of collections; newest first. */ +export function listGenerations(from: string): Generation[] { + const src = path.resolve(from); + if (fs.existsSync(path.join(src, MANIFEST))) return [verifyGeneration(src)]; + const collections = fs.existsSync(path.join(src, GENERATIONS)) ? [src] + : fs.existsSync(src) ? fs.readdirSync(src).map(e => path.join(src, e)).filter(c => fs.existsSync(path.join(c, GENERATIONS))) : []; + const order = (g: Generation) => [g.manifest?.revision ?? -1, Date.parse(g.manifest?.at ?? '') || 0] as const; + return collections.flatMap(c => fs.readdirSync(path.join(c, GENERATIONS)).map(g => verifyGeneration(path.join(c, GENERATIONS, g)))) + .sort((a, b) => order(b)[0] - order(a)[0] || order(b)[1] - order(a)[1]); +} + +/** `gstack-autoplan checkpoints --from `: one line per generation, newest first. */ +export function checkpointLines(from: string): string[] { + const gens = listGenerations(from); + if (!gens.length) throw new RunError('CHECKPOINT_INVALID', `${path.resolve(from)}: no checkpoint generations found`, 3); + return gens.map(g => { + const m = g.manifest; + const collection = path.dirname(path.dirname(g.dir)); + let current = false; + try { current = JSON.parse(fs.readFileSync(path.join(collection, CURRENT), 'utf8')).generation === path.basename(g.dir); } catch { /* no pointer */ } + return `CHECKPOINT: ${g.dir} run=${m?.run ?? '?'} gate_rev=${m?.gate_rev ?? '?'} status=${m?.status ?? '?'} revision=${m?.revision ?? '?'} at=${m?.at ?? '?'} hash=${g.problem ? `bad (${g.problem})` : 'ok'}${current ? ' current' : ''}`; + }); +} + +/** The generation to restore: the one named, or the newest that verifies in a collection or a root holding one run. */ +function chooseGeneration(from: string): Generation { + const gens = listGenerations(from); + const runs = [...new Set(gens.map(g => g.manifest?.run).filter(Boolean))]; + if (runs.length > 1) throw new RunError('CHECKPOINT_INVALID', `${path.resolve(from)} holds checkpoints of ${runs.length} runs (${runs.join(', ')}); pass --from /`, 3); + const valid = gens.find(g => !g.problem); + if (!valid) throw new RunError('CHECKPOINT_INVALID', `${path.resolve(from)}: ${gens.length ? gens.map(g => `${path.basename(g.dir)}: ${g.problem}`).join('; ') : 'no checkpoint generations found'}`, 3); + return valid; +} + +/** Verify a checkpoint and publish it as the run directory `out`. Nothing is written to `out` unless every check passes. */ export function restoreCheckpoint(from: string, out: string): string[] { const src = path.resolve(from); - const bad = (detail: string): never => { throw new RunError('CHECKPOINT_INVALID', `${src}: ${detail}`, 3); }; - let m: CheckpointManifest; - try { m = JSON.parse(fs.readFileSync(path.join(src, MANIFEST), 'utf8')); } catch (e) { return bad(`no readable ${MANIFEST} (${(e as Error).message})`); } - if (m.schema_version !== 1 || typeof m.run !== 'string' || !m.files || typeof m.files !== 'object' || !m.files[FILES.run]) bad(`${MANIFEST} is not a schema_version 1 checkpoint with run.json listed`); const target = path.resolve(out); - if (fs.existsSync(target) && fs.readdirSync(target).length) throw new RunError('CHECKPOINT_TARGET_EXISTS', `${target} is not empty`, 3); - const verify = (rel: string, sha: string, base: string) => { - const file = path.resolve(base, rel); - if (path.isAbsolute(rel) || rel.split(/[\\/]/).includes('..') || !inside(base, file)) bad(`${rel} escapes the checkpoint`); - let st: fs.Stats; - try { st = fs.lstatSync(file); } catch { return bad(`${rel} is listed but missing`); } - if (!st.isFile()) bad(`${rel} is not a regular file`); - if (sha256File(file) !== sha) bad(`${rel} does not match its sha256 in ${MANIFEST}`); - return file; - }; const staging = `${target}.restore-${process.pid}`; - fs.rmSync(staging, { recursive: true, force: true }); - try { - for (const [rel, sha] of Object.entries(m.files)) copy(verify(rel, sha, path.join(src, 'run')), path.join(staging, rel)); - for (const [rel, sha] of Object.entries(m.files)) verify(rel, sha, staging); - const lines: string[] = []; - if (m.plan_input) { - const copyOf = verify(m.plan_input.file, m.plan_input.sha256, src); - if (!fs.existsSync(m.plan_input.original)) { - const restored = path.join(staging, m.plan_input.file); - copy(copyOf, restored); - const state = JSON.parse(fs.readFileSync(path.join(staging, FILES.run), 'utf8')) as RunState; - state.plan_input = path.join(target, m.plan_input.file); - fs.writeFileSync(path.join(staging, FILES.run), JSON.stringify(state, null, 2) + '\n'); - lines.push(`PLAN_INPUT: ${m.plan_input.original} is missing; restored the checkpoint copy to ${state.plan_input}`); - } else if (sha256File(m.plan_input.original) !== m.plan_input.sha256) lines.push(`PLAN_INPUT: ${m.plan_input.original} differs from the checkpoint copy; kept the file on disk`); - } - if (fs.existsSync(target)) fs.rmdirSync(target); - fs.mkdirSync(path.dirname(target), { recursive: true }); - fs.renameSync(staging, target); - const state = readState(target); - journal(target, { event: 'restored', from: src, checkpoint_at: m.at, checkpoint_host: m.host, host: os.hostname(), files: Object.keys(m.files).length }); - return [`RESTORED: run=${state.run} from=${src} status=${m.status} gate_rev=${m.gate_rev} files=${Object.keys(m.files).length} (written ${m.at} on ${m.host})`, ...lines]; - } finally { + refuseOverlap('the checkpoint and --out', src, target); + refuseOverlap('the checkpoint and the restore staging directory', src, staging); + const collectionLock = fs.existsSync(path.join(src, MANIFEST)) ? path.join(path.dirname(path.dirname(src)), '.lock') : path.join(src, '.lock'); + return exclusive(`${target}.lock`, () => exclusive(collectionLock, () => { + if (fs.existsSync(target) && fs.readdirSync(target).length) throw new RunError('CHECKPOINT_TARGET_EXISTS', `${target} is not empty`, 3); + const gen = chooseGeneration(src); + const m = gen.manifest!; + const skipped = listGenerations(src).filter(g => g.problem && (g.manifest?.revision ?? -1) >= m.revision).map(g => `CHECKPOINT: skipped ${g.dir} (${g.problem})`); fs.rmSync(staging, { recursive: true, force: true }); - } + try { + for (const rel of Object.keys(m.files)) copy(path.join(gen.dir, 'run', rel), path.join(staging, rel)); + for (const [rel, sha] of Object.entries(m.files)) if (sha256File(path.join(staging, rel)) !== sha) throw new RunError('CHECKPOINT_INVALID', `${gen.dir}: ${rel} changed while it was copied`, 3); + const lines: string[] = []; + const runFile = path.join(staging, FILES.run); + let state: RunState; + try { state = JSON.parse(fs.readFileSync(runFile, 'utf8')); } catch (e) { throw new RunError('CHECKPOINT_INVALID', `${gen.dir}: run.json does not parse (${(e as Error).message})`, 3); } + if (state.run !== m.run || typeof state.out !== 'string' || !state.phases || typeof state.phases !== 'object') throw new RunError('CHECKPOINT_INVALID', `${gen.dir}: run.json is not the run state of ${m.run}`, 3); + if (m.plan_input && !fs.existsSync(m.plan_input.original)) { + copy(path.join(gen.dir, m.plan_input.file), path.join(staging, m.plan_input.file)); + state.plan_input = path.join(state.out, m.plan_input.file); + if (!fs.existsSync(state.plan_source)) state.plan_source = state.plan_input; + fs.writeFileSync(runFile, JSON.stringify(state, null, 2) + '\n'); + lines.push(`PLAN_INPUT: ${m.plan_input.original} is missing; restored the checkpoint copy to ${path.join(target, m.plan_input.file)}`); + } else if (m.plan_input && sha256File(m.plan_input.original) !== m.plan_input.sha256) lines.push(`PLAN_INPUT: ${m.plan_input.original} differs from the checkpoint copy; kept the file on disk`); + const v = validateRun(runFile); + if (!v.valid) throw new RunError('CHECKPOINT_INVALID', `${gen.dir}: the staged run does not validate: ${v.errors.slice(0, 3).map(e => `${e.code} ${e.path}: ${e.message}`).join('; ')}`, 3); + if (fs.existsSync(target)) fs.rmdirSync(target); + fs.mkdirSync(path.dirname(target), { recursive: true }); + fs.renameSync(staging, target); + readState(target); + journal(target, { event: 'restored', from: gen.dir, checkpoint_at: m.at, checkpoint_host: m.host, revision: m.revision, host: os.hostname(), files: Object.keys(m.files).length }); + return [...skipped, `RESTORED: run=${m.run} from=${gen.dir} status=${m.status} gate_rev=${m.gate_rev} revision=${m.revision} files=${Object.keys(m.files).length} (written ${m.at} on ${m.host})`, ...lines]; + } finally { + fs.rmSync(staging, { recursive: true, force: true }); + } + })); } diff --git a/lib/autoplan-state.ts b/lib/autoplan-state.ts index 2d1d1ef9d..b7e48ca44 100644 --- a/lib/autoplan-state.ts +++ b/lib/autoplan-state.ts @@ -83,7 +83,7 @@ export function readState(out: string): RunState { state.out = path.resolve(out); if (previous && previous !== state.out) { const reroot = (p: string | undefined) => (p && p.startsWith(previous + path.sep) ? path.join(state.out, path.relative(previous, p)) : p); - state.active_plan = reroot(state.active_plan)!; state.restore = reroot(state.restore)!; + state.active_plan = reroot(state.active_plan)!; state.restore = reroot(state.restore)!; state.plan_source = reroot(state.plan_source)!; state.plan_input = reroot(state.plan_input); for (const ps of Object.values(state.phases)) { ps.snapshot_dir = reroot(ps.snapshot_dir); ps.snapshot_path = reroot(ps.snapshot_path); ps.methodology = reroot(ps.methodology); if (ps.prompts) for (const k of Object.keys(ps.prompts) as Array) ps.prompts[k] = reroot(ps.prompts[k])!; diff --git a/lib/result-codes.ts b/lib/result-codes.ts index d2b12e11a..00da74272 100644 --- a/lib/result-codes.ts +++ b/lib/result-codes.ts @@ -543,8 +543,13 @@ export const RESULT_CODES = { }, CHECKPOINT_INVALID: { anchor: 'checkpoint-invalid', - summary: 'the autoplan checkpoint cannot be restored: its manifest is unreadable, a listed file is missing, escapes the checkpoint or does not match its sha256', - fix: 'restore from an intact copy of `//` (its `restore.md` names the command); nothing was written to --out', + summary: 'no autoplan checkpoint generation can be restored: a manifest is unreadable, a listed file is missing, escapes the generation (by path or symlink) or does not match its sha256, or the staged run does not validate', + fix: 'run `gstack-autoplan checkpoints --from ` to see each generation\'s hash status and restore an intact one; nothing was written to --out', + }, + CHECKPOINT_OVERLAP: { + anchor: 'checkpoint-overlap', + summary: 'the checkpoint directory and the run directory (or the restore target) are the same directory or one contains the other, symlinks resolved', + fix: 'point GSTACK_AUTOPLAN_CHECKPOINT_DIR (or --from / --out) at a directory outside the run; nothing was moved or deleted', }, CHECKPOINT_TARGET_EXISTS: { anchor: 'checkpoint-target-exists', diff --git a/scripts/resolvers/autoplan-driver.ts b/scripts/resolvers/autoplan-driver.ts index 75690fba4..867a80ca9 100644 --- a/scripts/resolvers/autoplan-driver.ts +++ b/scripts/resolvers/autoplan-driver.ts @@ -29,7 +29,7 @@ export const AUTOPLAN_DRIVER_COMMANDS: Array<{ cmd: string; does: string }> = [ { cmd: 'gstack-autoplan revise --out --withdraw ', does: 'drop a pending amendment instead of applying it' }, { cmd: 'gstack-autoplan export --out ', does: 'write the artifact set; then `gstack-artifact validate /run.json`' }, { cmd: 'gstack-autoplan status|resume --out ', does: 'read the manifest (PENDING: lines name open attempts); take ownership after a crash (EXECUTION_UNKNOWN lists unfinished attempts)' }, - { cmd: 'gstack-autoplan resume --out --from ', does: 'restore a hash-verified checkpoint (GSTACK_AUTOPLAN_CHECKPOINT_DIR) into an empty directory and take ownership' }, + { cmd: 'gstack-autoplan resume --out --from ', does: 'restore the newest valid checkpoint generation (GSTACK_AUTOPLAN_CHECKPOINT_DIR) into an empty directory and take ownership; `checkpoints --from ` lists them' }, ]; /** The reply grammar `gstack-gate` parses, one form per line. */ diff --git a/test/autoplan-checkpoint.test.ts b/test/autoplan-checkpoint.test.ts index 264b1a1c4..72aac6ab3 100644 --- a/test/autoplan-checkpoint.test.ts +++ b/test/autoplan-checkpoint.test.ts @@ -1,90 +1,183 @@ /** - * Durable checkpoint (next-wave B12): with GSTACK_AUTOPLAN_CHECKPOINT_DIR set, - * a run at rest is copied, hash-listed, to //; journal paths are - * run-relative, so `resume --from` restores the run on a new directory (even - * with the owner's plan file gone), takes ownership, and the run answers and - * exports a directory `gstack-artifact validate` accepts. A tampered, - * escaping or misplaced restore writes nothing. + * Durable checkpoint (next-wave B12, integration fixes): with + * GSTACK_AUTOPLAN_CHECKPOINT_DIR set, a run at rest publishes an immutable, + * hash-listed generation under //generations/ and switches + * current.json to it; journal paths are run-relative, so `resume --from` + * restores the run on a new directory (even with the owner's plan file gone, + * twice over), takes ownership, and the run answers, revises and exports a + * directory `gstack-artifact validate` accepts. A checkpoint never replaces + * or deletes the live run, never replaces a newer generation, and a failure + * after a committed command is a warning. Restore picks the newest valid + * generation and refuses tampered, escaping (by path or symlink), invalid or + * overlapping input, writing nothing. */ import { afterAll, describe, expect, test } from 'bun:test'; import { spawnSync } from 'node:child_process'; import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; -import { ROOT, autoplan, env, journal, readRun, resultOf, review, runPhase, startSmallRun } from './helpers/autoplan-run-fixture'; +import { ROOT, SMALL_PLAN, autoplan, env, journal, readRun, resultOf, review, runPhase, startSmallRun } from './helpers/autoplan-run-fixture'; const TMP = fs.mkdtempSync(path.join(os.tmpdir(), 'autoplan-checkpoint-')); afterAll(() => fs.rmSync(TMP, { recursive: true, force: true })); const validate = (out: string) => spawnSync('bun', [path.join(ROOT, 'bin', 'gstack-artifact'), 'validate', path.join(out, 'run.json')], { cwd: ROOT, encoding: 'utf8', timeout: 60_000 }); +const esc = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +const current = (collection: string) => JSON.parse(fs.readFileSync(path.join(collection, 'current.json'), 'utf8')); +const generations = (collection: string) => fs.readdirSync(path.join(collection, 'generations')).sort(); -function checkpointed(name: string): { dir: string; out: string; cp: string; run: string; e: Record } { +/** A small run brought to gate_pending with the checkpoint env on for the last submit. */ +function checkpointed(name: string): { dir: string; out: string; root: string; collection: string; run: string; e: Record } { const dir = path.join(TMP, name); - const cpRoot = path.join(dir, 'drive'); - const e = env({ GSTACK_AUTOPLAN_CHECKPOINT_DIR: cpRoot }); + const root = path.join(dir, 'drive'); + const e = env({ GSTACK_AUTOPLAN_CHECKPOINT_DIR: root }); const out = startSmallRun(dir); runPhase(out, 'ceo'); - const next = autoplan(['next', '--out', out], { env: e }); - expect(next.stderr).not.toContain('CHECKPOINT:'); + expect(autoplan(['next', '--out', out], { env: e }).stderr).not.toContain('CHECKPOINT:'); for (const voice of ['native', 'outside'] as const) review(path.join(out, `eng-${voice}-prompt.md`)); - const run = readRun(out); expect(autoplan(['submit', '--out', out, '--phase', 'eng', '--voice', 'native', '--result', resultOf(out, 'eng', 'native'), '--model', 'anthropic/claude-opus-4-7'], { env: e }).status).toBe(0); const gated = autoplan(['submit', '--out', out, '--phase', 'eng', '--voice', 'outside', '--result', resultOf(out, 'eng', 'outside'), '--model', 'openai/gpt-6-astra'], { env: e }); expect(gated.stdout).toContain('status=gate_pending'); - const cp = path.join(cpRoot, run.run); - expect(gated.stderr).toMatch(new RegExp(`^CHECKPOINT: ${cp.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')} status=gate_pending gate_rev=1 files=\\d+$`, 'm')); - return { dir, out, cp, run: run.run, e }; + const run = readRun(out).run; + const collection = path.join(root, run); + expect(gated.stderr).toMatch(new RegExp(`^CHECKPOINT: ${esc(path.join(collection, 'generations'))}/g\\d{6}-\\S+ status=gate_pending gate_rev=1 revision=\\d+ files=\\d+$`, 'm')); + return { dir, out, root, collection, run, e }; } describe('checkpoint and restore', () => { - test('write at gate_pending, delete the run and the plan file, restore, resume, answer and export: validate passes', () => { + test('write at gate_pending, delete the run and the plan file, restore twice, answer, revise and export: validate passes', () => { const c = checkpointed('roundtrip'); - const manifest = JSON.parse(fs.readFileSync(path.join(c.cp, 'checkpoint.json'), 'utf8')); - expect(manifest).toMatchObject({ schema_version: 1, run: c.run, status: 'gate_pending', gate_rev: 1 }); + const [gen] = generations(c.collection); + expect(current(c.collection)).toMatchObject({ generation: gen, run: c.run, status: 'gate_pending', gate_rev: 1 }); + const manifest = JSON.parse(fs.readFileSync(path.join(c.collection, 'generations', gen!, 'checkpoint.json'), 'utf8')); expect(Object.keys(manifest.files)).toEqual(expect.arrayContaining(['run.json', 'attempts.jsonl', 'gate.json', 'decisions.jsonl'])); expect(Object.keys(manifest.files)).not.toContain('run.lock'); - expect(fs.readFileSync(path.join(c.cp, 'restore.md'), 'utf8')).toContain(`gstack-autoplan resume --out --from ${c.cp}`); + expect(fs.readFileSync(path.join(c.collection, 'restore.md'), 'utf8')).toContain(`gstack-autoplan resume --out --from ${c.collection}`); for (const r of journal(c.out)) for (const k of ['prompt', 'result', 'staging']) if (typeof r[k] === 'string') expect(path.isAbsolute(r[k])).toBe(false); fs.rmSync(c.out, { recursive: true, force: true }); fs.rmSync(path.join(c.dir, 'plan.md')); - const moved = path.join(c.dir, 'elsewhere', 'run'); - const resumed = autoplan(['resume', '--out', moved, '--from', c.cp], { env: c.e }); + const first = path.join(c.dir, 'first', 'run'); + const resumed = autoplan(['resume', '--out', first, '--from', c.collection], { env: c.e }); expect(resumed.status, resumed.stderr).toBe(0); - expect(resumed.stdout).toMatch(new RegExp(`^RESTORED: run=${c.run} from=.* status=gate_pending gate_rev=1 files=\\d+ `, 'm')); - expect(resumed.stdout).toMatch(/^PLAN_INPUT: .*plan\.md is missing; restored the checkpoint copy to .*elsewhere\/run\/plan-input\/plan\.md$/m); - expect(resumed.stdout).toContain(`GSTACK_RESULT: skill=autoplan status=gate_pending run=${moved}`); - expect(journal(moved).find(r => r.event === 'restored')).toMatchObject({ host: os.hostname() }); - const state = readRun(moved); - expect(state.plan_input).toBe(path.join(moved, 'plan-input', 'plan.md')); - expect(state.active_plan).toBe(path.join(moved, 'plan-active.md')); - const answered = autoplan(['answer', '--out', moved, '--gate-rev', '1', '--reply', 'as-is', '--by', 'human'], { env: c.e }); + expect(resumed.stdout).toMatch(new RegExp(`^RESTORED: run=${c.run} from=.* status=gate_pending gate_rev=1 revision=\\d+ files=\\d+ `, 'm')); + expect(resumed.stdout).toMatch(/^PLAN_INPUT: .*plan\.md is missing; restored the checkpoint copy to .*first\/run\/plan-input\/plan\.md$/m); + expect(resumed.stdout).toContain(`GSTACK_RESULT: skill=autoplan status=gate_pending run=${first}`); + expect(journal(first).find(r => r.event === 'restored')).toMatchObject({ host: os.hostname() }); + expect(resumed.stderr).toMatch(/^CHECKPOINT: .* status=gate_pending /m); + fs.rmSync(first, { recursive: true, force: true }); + const second = path.join(c.dir, 'second', 'run'); + expect(autoplan(['resume', '--out', second, '--from', c.collection]).status).toBe(0); + const state = readRun(second); + expect(state.plan_input).toBe(path.join(second, 'plan-input', 'plan.md')); + expect(state.plan_source.startsWith(second + path.sep)).toBe(true); + expect(state.active_plan).toBe(path.join(second, 'plan-active.md')); + const answered = autoplan(['answer', '--out', second, '--gate-rev', '1', '--reply', 'p1d', '--by', 'human'], { env: c.e }); expect(answered.status, answered.stderr).toBe(0); - expect(answered.stdout).toContain('status=complete'); - expect(answered.stderr).toMatch(/^CHECKPOINT: .* status=complete gate_rev=1 /m); - expect(autoplan(['export', '--out', moved]).status).toBe(0); - const v = validate(moved); + fs.writeFileSync(state.plan_input, SMALL_PLAN.replace('Tests cover', 'Ship behind RETRY_BACKOFF=1. Tests cover')); + const revised = autoplan(['revise', '--out', second, '--plan', state.plan_input]); + expect(revised.status, revised.stderr).toBe(0); + runPhase(second, 'ceo'); + runPhase(second, 'eng'); + expect(autoplan(['answer', '--out', second, '--gate-rev', '2', '--reply', 'as-is', '--by', 'human']).stdout).toContain('status=complete'); + expect(autoplan(['export', '--out', second]).status).toBe(0); + const v = validate(second); expect(v.status, v.stdout + v.stderr).toBe(0); }); - test('a tampered file, an escaping path and a non-empty target are refused and write nothing', () => { + test('`checkpoints` lists every generation; a corrupt newest is skipped for the newest valid; an interrupted publish keeps the previous current', () => { + const c = checkpointed('generations'); + const [older] = generations(c.collection); + const faulted = autoplan(['export', '--out', c.out], { env: { ...c.e, GSTACK_AUTOPLAN_FAULT: 'checkpoint-before-pointer' } }); + expect(faulted.status, faulted.stderr).toBe(0); + expect(faulted.stderr).toMatch(/^CHECKPOINT: failed for .*fault injected: checkpoint interrupted before switching current\.json.*; the command's result stands, the previous checkpoint stays current$/m); + expect(current(c.collection).generation).toBe(older); + const newer = generations(c.collection).find(g => g !== older)!; + const listed = autoplan(['checkpoints', '--from', c.root]); + expect(listed.status, listed.stderr).toBe(0); + expect(listed.stdout.trim().split('\n')).toEqual([ + expect.stringMatching(new RegExp(`^CHECKPOINT: \\S+/${newer} run=${c.run} gate_rev=1 status=gate_pending revision=\\d+ at=\\S+ hash=ok$`)), + expect.stringMatching(new RegExp(`^CHECKPOINT: \\S+/${older} run=${c.run} gate_rev=1 status=gate_pending revision=\\d+ at=\\S+ hash=ok current$`)), + ]); + fs.appendFileSync(path.join(c.collection, 'generations', newer, 'run', 'gate.json'), ' '); + expect(autoplan(['checkpoints', '--from', c.collection]).stdout).toMatch(new RegExp(`${newer} .* hash=bad \\(gate\\.json does not match its sha256 in checkpoint\\.json\\)$`, 'm')); + const target = path.join(c.dir, 'restored'); + const explicit = autoplan(['resume', '--out', target, '--from', path.join(c.collection, 'generations', newer)]); + expect(explicit.status).toBe(3); + expect(explicit.stderr).toMatch(/gate\.json does not match its sha256.*\(CHECKPOINT_INVALID\)/); + expect(fs.existsSync(target)).toBe(false); + const newest = autoplan(['resume', '--out', target, '--from', c.collection]); + expect(newest.status, newest.stderr).toBe(0); + expect(newest.stdout).toMatch(new RegExp(`^CHECKPOINT: skipped \\S+/${newer} \\(gate\\.json does not match`, 'm')); + expect(newest.stdout).toMatch(new RegExp(`^RESTORED: run=${c.run} from=\\S+/${older} `, 'm')); + }); + + test('a stale copy never replaces a newer generation', () => { + const c = checkpointed('newer'); + const stale = path.join(c.dir, 'stale'); + fs.cpSync(c.out, stale, { recursive: true }); + for (let i = 0; i < 3; i++) expect(autoplan(['export', '--out', c.out], { env: c.e }).status).toBe(0); + const before = current(c.collection); + const r = autoplan(['export', '--out', stale], { env: c.e }); + expect(r.status, r.stderr).toBe(0); + expect(r.stderr).toMatch(/^CHECKPOINT: skipped .*: it holds a newer checkpoint \(revision \d+ > \d+\); nothing replaced$/m); + expect(current(c.collection)).toEqual(before); + }); + + test('a checkpoint directory that is, contains or aliases the live run is refused before anything moves; the command still succeeds', () => { + const c = checkpointed('overlap'); + const root = path.join(c.dir, 'root'); + fs.mkdirSync(root); + const live = path.join(root, c.run); + fs.renameSync(c.out, live); + const same = autoplan(['export', '--out', live], { env: env({ GSTACK_AUTOPLAN_CHECKPOINT_DIR: root }) }); + expect(same.status, same.stderr).toBe(0); + expect(same.stderr).toMatch(/^CHECKPOINT: failed for .*\(CHECKPOINT_OVERLAP\); the command's result stands/m); + expect(readRun(live).run).toBe(c.run); + const inner = autoplan(['export', '--out', live], { env: env({ GSTACK_AUTOPLAN_CHECKPOINT_DIR: path.join(live, 'ckpt') }) }); + expect(inner.stderr).toMatch(/\(CHECKPOINT_OVERLAP\)/); + expect(fs.existsSync(path.join(live, 'ckpt'))).toBe(false); + const aliasRoot = path.join(c.dir, 'alias'); + fs.mkdirSync(aliasRoot); + fs.symlinkSync(live, path.join(aliasRoot, c.run)); + const alias = autoplan(['export', '--out', live], { env: env({ GSTACK_AUTOPLAN_CHECKPOINT_DIR: aliasRoot }) }); + expect(alias.stderr).toMatch(/\(CHECKPOINT_OVERLAP\)/); + expect(readRun(live).run).toBe(c.run); + expect(fs.existsSync(path.join(live, 'run.json'))).toBe(true); + const restoreInto = autoplan(['resume', '--out', path.join(c.collection, 'inside'), '--from', c.collection]); + expect(restoreInto.status).toBe(3); + expect(restoreInto.stderr).toMatch(/\(CHECKPOINT_OVERLAP\)/); + }); + + test('tampered, escaping, symlink-escaping and non-validating checkpoints and a non-empty target are refused and write nothing', () => { const c = checkpointed('refusals'); + const gen = path.join(c.collection, 'generations', generations(c.collection)[0]!); + const target = path.join(c.dir, 'restored'); + const refused = (pattern: RegExp, code = 'CHECKPOINT_INVALID') => { + const r = autoplan(['resume', '--out', target, '--from', gen]); + expect(r.status).toBe(3); + expect(r.stderr).toMatch(new RegExp(`${pattern.source}.*\\(${code}\\)`)); + expect(fs.existsSync(target)).toBe(false); + }; const nonEmpty = path.join(c.dir, 'busy'); fs.mkdirSync(nonEmpty, { recursive: true }); fs.writeFileSync(path.join(nonEmpty, 'keep.txt'), 'x'); - const busy = autoplan(['resume', '--out', nonEmpty, '--from', c.cp]); - expect(busy.status).toBe(3); + const busy = autoplan(['resume', '--out', nonEmpty, '--from', gen]); expect(busy.stderr).toMatch(/\(CHECKPOINT_TARGET_EXISTS\)/); expect(fs.readdirSync(nonEmpty)).toEqual(['keep.txt']); - fs.appendFileSync(path.join(c.cp, 'run', 'gate.json'), ' '); - const target = path.join(c.dir, 'restored'); - const tampered = autoplan(['resume', '--out', target, '--from', c.cp]); - expect(tampered.status).toBe(3); - expect(tampered.stderr).toMatch(/gate\.json does not match its sha256.*\(CHECKPOINT_INVALID\)/); - expect(fs.existsSync(target)).toBe(false); - const m = JSON.parse(fs.readFileSync(path.join(c.cp, 'checkpoint.json'), 'utf8')); - fs.writeFileSync(path.join(c.cp, 'checkpoint.json'), JSON.stringify({ ...m, files: { ...m.files, '../../escape.md': m.files['run.json'] } })); - const escaping = autoplan(['resume', '--out', target, '--from', c.cp]); - expect(escaping.status).toBe(3); - expect(escaping.stderr).toMatch(/escapes the checkpoint.*\(CHECKPOINT_INVALID\)/); - expect(fs.existsSync(target)).toBe(false); + const manifestFile = path.join(gen, 'checkpoint.json'); + const m = JSON.parse(fs.readFileSync(manifestFile, 'utf8')); + const snapDir = Object.keys(m.files).find(f => f.includes('/'))!.split('/')[0]!; + const outsideCopy = path.join(c.dir, 'outside-copy'); + fs.cpSync(path.join(gen, 'run', snapDir), outsideCopy, { recursive: true }); + fs.rmSync(path.join(gen, 'run', snapDir), { recursive: true }); + fs.symlinkSync(outsideCopy, path.join(gen, 'run', snapDir)); + refused(/escapes the checkpoint through a symlinked directory/); + fs.rmSync(path.join(gen, 'run', snapDir)); + fs.cpSync(outsideCopy, path.join(gen, 'run', snapDir), { recursive: true }); + fs.writeFileSync(manifestFile, JSON.stringify({ ...m, files: { ...m.files, '../../escape.md': m.files['run.json'] } })); + refused(/escapes the checkpoint/); + const gateFile = path.join(gen, 'run', 'gate.json'); + fs.appendFileSync(gateFile, '\n'); + fs.writeFileSync(manifestFile, JSON.stringify({ ...m, files: { ...m.files, 'gate.json': require('node:crypto').createHash('sha256').update(fs.readFileSync(gateFile)).digest('hex') } })); + refused(/the staged run does not validate: ARTIFACT_STALE gate\.json/); }); });