diff --git a/bin/gstack-artifact b/bin/gstack-artifact index 0161e1120..a8f7d5a0b 100755 --- a/bin/gstack-artifact +++ b/bin/gstack-artifact @@ -2,7 +2,7 @@ /** * gstack-artifact — validate, print and acknowledge the unattended artifact contract. * - * gstack-artifact validate [--json] completion check for a run directory + * gstack-artifact validate [--json] completion check for a run directory gstack-autoplan wrote * gstack-artifact validate --public [--private ] [--json] * a public run copy: provenance, recorded machine roots * gstack-artifact validate --as [--run ] [--json] @@ -17,12 +17,13 @@ import * as path from 'node:path'; import { ARTIFACT_NAMES, EXIT, ackRun, exitTableHelp, jsonSchema, readJsonl, renderUrgentBlock, - validateFile, validateRun, type ArtifactName, type ValidationError, + validateFile, type ArtifactName, type ValidationError, } from '../lib/headless-artifacts'; import { resultLine } from '../lib/result-codes'; import { resolveStateRoot } from '../lib/state-root'; import { validatePublic } from '../lib/autoplan-public'; import { validateShadow } from '../lib/autoplan-shadow-reader'; +import { validateAutoplanRun } from '../lib/autoplan-writer-check'; const HELP = `gstack-artifact — the unattended artifact contract (docs/unattended.md) @@ -88,7 +89,7 @@ function main(argv: string[]): number { else { printErrors(errors); process.stdout.write(errors.length ? `ARTIFACT_INVALID: ${errors.length} error(s)\n` : `ARTIFACT_VALID: ${as} ${path.basename(file)}\n`); } return errors.length ? EXIT.fail : EXIT.ok; } - const run = validateRun(file); + const run = validateAutoplanRun(file); const shadow = validateShadow(run.runDir); const result = { ...run, valid: run.valid && shadow.length === 0, errors: [...run.errors, ...shadow] }; if (json) { @@ -106,7 +107,7 @@ function main(argv: string[]): number { const file = rest.find(a => !a.startsWith('--') && a !== flag(rest, '--consumer')); const consumer = flag(rest, '--consumer'); if (!file || !consumer) return usage('ack needs --consumer '); - const validation = validateRun(file); + const validation = validateAutoplanRun(file); if (!validation.valid) { printErrors(validation.errors); process.stderr.write('gstack-artifact ack: refusing to acknowledge an invalid run; nothing written\n'); diff --git a/bin/gstack-doctor-components.sh b/bin/gstack-doctor-components.sh index beff2e417..468ca5e49 100644 --- a/bin/gstack-doctor-components.sh +++ b/bin/gstack-doctor-components.sh @@ -443,6 +443,8 @@ gstack_result_anchor() { OFFICE_HOURS_PROBE_REFUSED) printf '%s' 'office-hours-probe-refused' ;; TASKS_SUPERSEDE_REFUSED) printf '%s' 'tasks-supersede-refused' ;; REALITY_RECEIPT_UNRESOLVED) printf '%s' 'reality-receipt-unresolved' ;; + ARTIFACT_NOT_AUTOPLAN_RUN) printf '%s' 'artifact-not-autoplan-run' ;; + ARTIFACT_JOURNAL_MISMATCH) printf '%s' 'artifact-journal-mismatch' ;; GUARD_ENFORCED) printf '%s' 'guard-enforced' ;; CEO_ONLY_CONFLICT) printf '%s' 'ceo-only-conflict' ;; CEO_ONLY_AFTER_INIT) printf '%s' 'ceo-only-after-init' ;; diff --git a/changelog.d/artifact-validate-refuses-hand-built-runs.md b/changelog.d/artifact-validate-refuses-hand-built-runs.md new file mode 100644 index 000000000..b6fa64e12 --- /dev/null +++ b/changelog.d/artifact-validate-refuses-hand-built-runs.md @@ -0,0 +1 @@ +- **`gstack-artifact validate` no longer passes a run.json that gstack-autoplan never wrote.** A session that skipped `gstack-autoplan next` and built run.json with a script, copying the reference fixture, got `ARTIFACT_VALID`. The validator now also checks that the run came from gstack-autoplan: run.json must carry `host: gstack-autoplan`, `plan_input` and the other fields it writes on every run, and must agree with its `attempts.jsonl` journal, its bound review outputs and its checkpoint. A run that fails says `ARTIFACT_NOT_AUTOPLAN_RUN` or `ARTIFACT_JOURNAL_MISMATCH`, and `gstack-artifact ack` refuses it too. Public copies (`validate --public`) are unchanged. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index e0b6ae21d..ff1fe95c1 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -3511,6 +3511,38 @@ record does not state that publication order was unverified. **Fix.** Start the run through `gstack-skill-start` (it prints the line) and keep the line in the review record. + +### `ARTIFACT_NOT_AUTOPLAN_RUN` + +**Meaning.** `run.json` parses and matches its schema, but gstack-autoplan did +not write it: `host` is not `gstack-autoplan`, a field gstack-autoplan writes +on every run is missing (`plan_input`, `plan_source`, `active_plan`, +`restore`, `out`, `created_at`, `revision`, `attempt_counter`, +`outside_enabled`, `phase_order`, `history`, `scope`), or the run directory +has no `attempts.jsonl` journal. A run.json written by hand or by a script, +for example one copied from `test/fixtures/multi-agent-wave/reference-run`, +is refused here even when every structural check passes. + +**Fix.** Drive the run with `gstack-autoplan next`, `submit`, `answer` and +`export`, then validate the directory it wrote. A run written by a gstack +older than these fields is refused too; rerun it with this gstack. + + +### `ARTIFACT_JOURNAL_MISMATCH` + +**Meaning.** The run directory disagrees with its own record of what happened: +the journal has no `init` row for this run, a reviewer attempt in `run.json` +was never journaled as dispatched, a completed voice's output or hash differs +from its journaled `submitted` row, a bound review output changed after +submit, or the run's checkpoint (when `GSTACK_AUTOPLAN_CHECKPOINT_DIR` or the +Drive opt-in names one) is ahead of `run.json`, disagrees on status or gate +revision at the same revision, or holds a journal the live one no longer begins with. + +**Fix.** Treat the run as edited after the fact. Restore it from its +checkpoint (`gstack-autoplan resume --out --from `) or +rerun it with gstack-autoplan; never edit `run.json` or `attempts.jsonl` by +hand. + ### `autoplan guard: not enforced by this host; publication order is unverified (GUARD_NOT_INSTALLED)` diff --git a/docs/unattended.md b/docs/unattended.md index 4d9bb7f93..a5849f9cb 100644 --- a/docs/unattended.md +++ b/docs/unattended.md @@ -424,8 +424,14 @@ checks its prompt. terminal outcome, every listed artifact exists inside the run directory with the recorded hash, every JSONL file parses row by row against its schema with run-bound unique ids, task dependencies and finding references resolve - without cycles, `counts` agree with the files, and (unattended) the guard - line is present. Otherwise it exits `1` with one error line per problem and + without cycles, `counts` agree with the files, (unattended) the guard + line is present, and gstack-autoplan wrote the run: `run.json` carries + the fields it writes on every run (`host: gstack-autoplan`, `plan_input` + and the rest, `ARTIFACT_NOT_AUTOPLAN_RUN`) and agrees with its + `attempts.jsonl` journal, its bound review outputs and its checkpoint + (`ARTIFACT_JOURNAL_MISMATCH`). `gstack-artifact ack` runs the same + check. A public copy (`validate --public`) carries no journal and is + checked against its private run's digests instead. Otherwise it exits `1` with one error line per problem and `ARTIFACT_INVALID: error(s)`; `--json` returns the same as `{ valid, run, status, run_dir, errors[] }`. A run that was interrupted shows up here as `ARTIFACT_RUN_INTERRUPTED`, never as a partial success. @@ -1015,8 +1021,13 @@ line, on an ephemeral state root. ## Reference run -`test/fixtures/multi-agent-wave/reference-run/` is a complete unattended -`/autoplan` run (the one that produced this wave's plan) and validates clean. +`test/fixtures/multi-agent-wave/reference-run/` is the hand-written run +that produced this wave's plan, from before `gstack-autoplan` existed. It +pins the structural contract (`validateRun` passes it), but +`gstack-artifact validate` refuses it with `ARTIFACT_NOT_AUTOPLAN_RUN`: it +has no `plan_input` and no journal, because no gstack-autoplan wrote it. Copy +a run gstack-autoplan wrote, never this one, when you need a run that +validates. `test/fixtures/multi-agent-wave/INCIDENTS.md` records what went wrong while it was produced by hand and which validator rule now catches each incident. diff --git a/lib/autoplan-checkpoint.ts b/lib/autoplan-checkpoint.ts index 141144b86..37228fb98 100644 --- a/lib/autoplan-checkpoint.ts +++ b/lib/autoplan-checkpoint.ts @@ -130,7 +130,7 @@ export function checkpointNotice(env: NodeJS.ProcessEnv, repo?: string): string /** A checkpoint beyond the at-rest ones: after a bound voice, after a phase transition, and the notice at the first next. */ export type CheckpointMoment = 'bound' | 'transition' | 'init'; -const repoOf = (state: RunState) => { const input = state.plan_input ?? state.plan_source; const root = input ? repoRoot(path.dirname(input)) : undefined; return root ? path.basename(root) : undefined; }; +export const repoOf = (state: Pick) => { const input = state.plan_input ?? state.plan_source; const root = input ? repoRoot(path.dirname(input)) : undefined; return root ? path.basename(root) : undefined; }; /** * Publish a generation when a checkpoint directory is set (or Drive opted in) and the run is at rest, or at a diff --git a/lib/autoplan-writer-check.ts b/lib/autoplan-writer-check.ts new file mode 100644 index 000000000..5b96bfc74 --- /dev/null +++ b/lib/autoplan-writer-check.ts @@ -0,0 +1,94 @@ +/** + * Provenance for `gstack-artifact validate ` and `ack` (GST-3090). + * validateRun (lib/headless-artifacts.ts) says a run directory is well formed; + * this says gstack-autoplan wrote it. gstack-autoplan is the only writer of + * run.json (docs/unattended.md), so a run.json that parses against its schema + * gets this check too. A public copy does not: it carries no journal and is + * bound to its private run by provenance digests (lib/autoplan-public.ts). + * + * ARTIFACT_NOT_AUTOPLAN_RUN host is not gstack-autoplan, a field initRun always writes is missing, or attempts.jsonl is absent + * ARTIFACT_JOURNAL_MISMATCH attempts.jsonl, a bound review output or the run's checkpoint disagrees with run.json + */ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { checkpointRoot, listGenerations, repoOf } from './autoplan-checkpoint'; +import { FILES, readJournal, type RunState } from './autoplan-state'; +import { sha256File, validateRun, type ValidationError, type ValidationResult } from './headless-artifacts'; + +/** What initRun (lib/autoplan-run.ts) writes on every run, whatever its kind; gstack_version is absent on runs before 1.91.96. */ +const WRITER_FIELDS = [ + ['plan_input', 'string'], ['plan_source', 'string'], ['active_plan', 'string'], ['restore', 'string'], ['out', 'string'], + ['created_at', 'string'], ['revision', 'number'], ['attempt_counter', 'number'], ['outside_enabled', 'boolean'], + ['phase_order', 'array'], ['history', 'array'], ['scope', 'object'], +] as const; +const STRUCTURE = ['ARTIFACT_INVALID_JSON', 'ARTIFACT_UNSUPPORTED_VERSION', 'ARTIFACT_SCHEMA']; +const kindOf = (x: unknown) => (Array.isArray(x) ? 'array' : x === null ? 'null' : typeof x); + +/** validateRun plus the writer check, for a run directory a consumer treats as evidence. */ +export function validateAutoplanRun(runJson: string, env: NodeJS.ProcessEnv = process.env): ValidationResult { + const result = validateRun(runJson); + if (!result.manifest || result.errors.some(e => STRUCTURE.includes(e.code))) return result; + const errors = [...result.errors, ...checkWriter(result.manifest as unknown as RunState, result.runDir, env)]; + return { ...result, errors, valid: errors.length === 0 }; +} + +export function checkWriter(m: RunState, runDir: string, env: NodeJS.ProcessEnv = process.env): ValidationError[] { + const out: ValidationError[] = []; + const notWritten = (p: string, message: string) => out.push({ code: 'ARTIFACT_NOT_AUTOPLAN_RUN', path: p, message }); + const mismatch = (p: string, message: string) => out.push({ code: 'ARTIFACT_JOURNAL_MISMATCH', path: p, message }); + if (m.host !== 'gstack-autoplan') notWritten('run.json.host', `host is ${JSON.stringify(m.host ?? null)}; gstack-autoplan writes "gstack-autoplan"`); + for (const [field, kind] of WRITER_FIELDS) { + const got = kindOf((m as unknown as Record)[field]); + if (got !== kind) notWritten(`run.json.${field}`, got === 'undefined' ? `missing; gstack-autoplan writes it on every run` : `expected ${kind}, got ${got}`); + } + if (!fs.existsSync(path.join(runDir, FILES.journal))) { + notWritten(FILES.journal, 'no journal; gstack-autoplan appends one from the first next'); + return out; + } + const rows = readJournal(runDir); + if (!rows.some(r => r.event === 'init' && r.run === m.run)) mismatch(FILES.journal, `no init row for run ${m.run}`); + for (const [phase, ps] of Object.entries(m.phases ?? {})) { + for (const voice of ['native', 'outside'] as const) { + const v = ps?.[voice]; + if (!v) continue; + for (const a of v.attempts ?? []) { + if (!rows.some(r => r.event === 'dispatched' && r.attempt === a.attempt && r.phase === phase && r.voice === voice)) mismatch(FILES.journal, `${phase} ${voice} attempt ${a.attempt} was never journaled as dispatched`); + } + if (v.status !== 'completed') continue; + if (typeof v.attempt !== 'string' || typeof v.result_sha256 !== 'string') { mismatch(`run.json.phases.${phase}.${voice}`, 'completed without the attempt and result_sha256 submit records'); continue; } + const sub = rows.find(r => r.event === 'submitted' && r.attempt === v.attempt); + if (!sub || sub.phase !== phase || sub.voice !== voice || sub.result_sha256 !== v.result_sha256 || (v.output && sub.result !== v.output)) { + mismatch(FILES.journal, `${phase} ${voice}: run.json binds ${v.output ?? '?'} (${v.result_sha256.slice(0, 12)}) to ${v.attempt}; the journal's submitted row ${sub ? `says ${sub.phase} ${sub.voice} ${sub.result} (${String(sub.result_sha256).slice(0, 12)})` : 'is missing'}`); + continue; + } + const bound = v.output ? path.resolve(runDir, v.output) : ''; + if (bound.startsWith(path.resolve(runDir) + path.sep) && fs.existsSync(bound) && sha256File(bound) !== v.result_sha256) mismatch(v.output!, `changed after submit: sha256 ${sha256File(bound).slice(0, 12)} is not the submitted ${v.result_sha256.slice(0, 12)}`); + } + } + checkCheckpoint(m, runDir, rows, env, mismatch); + return out; +} + +/** + * The run's own checkpoint is never ahead of run.json, agrees on status and gate at the same revision, and holds a + * journal that is a prefix of the live one (append-only). Its own: the generation it was restored from, else the + * newest valid one written from this run directory (its run.json names the same `out`); a copy restored elsewhere + * forks the run and checkpoints into the same collection, so its generations say nothing about this directory. + */ +function checkCheckpoint(m: RunState, runDir: string, rows: Array>, env: NodeJS.ProcessEnv, mismatch: (p: string, message: string) => void): void { + const root = checkpointRoot(env, repoOf(m)); + const collection = root ? path.join(path.resolve(root), m.run) : ''; + if (!collection || !fs.existsSync(collection)) return; + const restored = rows.filter(r => r.event === 'restored').pop(); + const valid = listGenerations(collection).filter(g => g.manifest && !g.problem); + const writtenFrom = (dir: string) => { try { return JSON.parse(fs.readFileSync(path.join(dir, 'run', FILES.run), 'utf8')).out; } catch { return undefined; } }; + const ref = restored ? valid.find(g => path.resolve(g.dir) === path.resolve(String(restored.from))) : valid.find(g => writtenFrom(g.dir) === m.out); + if (!ref) return; + const g = ref.manifest!; + if (g.revision > m.revision) mismatch(ref.dir, `this checkpoint is at revision ${g.revision}, ahead of run.json's ${m.revision}`); + else if (g.revision === m.revision && (g.status !== m.status || g.gate_rev !== (m.gate_rev ?? 0))) mismatch(ref.dir, `revision ${g.revision} was checkpointed as status=${g.status} gate_rev=${g.gate_rev}; run.json says status=${m.status} gate_rev=${m.gate_rev ?? 0}`); + const saved = path.join(ref.dir, 'run', FILES.journal); + if (fs.existsSync(saved) && !fs.readFileSync(path.join(runDir, FILES.journal)).subarray(0, fs.statSync(saved).size).equals(fs.readFileSync(saved))) { + mismatch(ref.dir, `attempts.jsonl no longer begins with the journal checkpointed at revision ${g.revision}: an earlier row was changed or removed`); + } +} diff --git a/lib/result-codes-autoplan.ts b/lib/result-codes-autoplan.ts index fdab4981b..78d5dfcdf 100644 --- a/lib/result-codes-autoplan.ts +++ b/lib/result-codes-autoplan.ts @@ -2,7 +2,8 @@ * Result codes for the /autoplan publication guard line (GUARD_ENFORCED, printed by * gstack-skill-start; lib/autoplan-guard-status.ts), the CEO-only driver run (`gstack-autoplan next --ceo-only`, * lib/autoplan-ceo-only.ts) and its shadow-mode reader (`gstack-autoplan shadow`, - * lib/autoplan-shadow-reader.ts). Kept beside lib/result-codes.ts, which spreads + * lib/autoplan-shadow-reader.ts), and the writer check `gstack-artifact validate` runs on a run.json + * (lib/autoplan-writer-check.ts). Kept beside lib/result-codes.ts, which spreads * this table into RESULT_CODES; the same rules apply: one stable * docs/troubleshooting.md anchor, a summary and a fix clause each. The run is * documented in docs/unattended.md#ceo-only-runs. @@ -10,6 +11,16 @@ import type { ResultCode } from './result-codes'; export const AUTOPLAN_RESULT_CODES = { + ARTIFACT_NOT_AUTOPLAN_RUN: { + anchor: 'artifact-not-autoplan-run', + summary: 'run.json lacks what gstack-autoplan writes on every run (its host, plan_input and the other run-state fields) or has no attempts.jsonl journal', + fix: 'drive the run with gstack-autoplan next/submit/answer/export; a run.json written by hand or by a script is not evidence of a review', + }, + ARTIFACT_JOURNAL_MISMATCH: { + anchor: 'artifact-journal-mismatch', + summary: 'attempts.jsonl, a bound review output or the run’s checkpoint disagrees with run.json', + fix: 'treat the run as edited after the fact; restore it from a checkpoint (gstack-autoplan resume --from) or rerun it with gstack-autoplan', + }, GUARD_ENFORCED: { anchor: 'guard-enforced', summary: 'Claude Code runs the autoplan publication hook, so a phase Read or reviewer dispatch is denied until the previous phase report is published', diff --git a/test/artifact-writer-check.test.ts b/test/artifact-writer-check.test.ts new file mode 100644 index 000000000..cf810bfcf --- /dev/null +++ b/test/artifact-writer-check.test.ts @@ -0,0 +1,146 @@ +/** + * GST-3090: `gstack-artifact validate` refuses a run.json that gstack-autoplan + * never wrote. A day-one eval session hand-built run.json from the reference + * fixture (no plan_input, no journal) and got ARTIFACT_VALID. A run written by + * the CLI still validates; a tampered journal, review output or checkpoint is + * refused. + */ +import { afterAll, describe, expect, test } from 'bun:test'; +import { spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { ROOT, autoplan, env, recCarrying, runPhase, startSmallRun } from './helpers/autoplan-run-fixture'; + +const BIN = path.join(ROOT, 'bin', 'gstack-artifact'); +const REFERENCE = path.join(ROOT, 'test', 'fixtures', 'multi-agent-wave', 'reference-run'); +const TMP = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-writer-check-')); +afterAll(() => fs.rmSync(TMP, { recursive: true, force: true })); + +function validate(runJson: string, extra: Record = {}): { code: number | null; valid: boolean; codes: string[]; paths: string[] } { + const r = spawnSync(BIN, ['validate', runJson, '--json'], { encoding: 'utf8', timeout: 60_000, env: env(extra) }); + const parsed = JSON.parse(r.stdout); + return { code: r.status, valid: parsed.valid, codes: parsed.errors.map((e: any) => e.code), paths: parsed.errors.map((e: any) => e.path) }; +} + +/** One complete ceo → eng run driven only through gstack-autoplan, then exported; each test copies it. */ +let built: string | undefined; +function realRun(): string { + if (built) return built; + const out = startSmallRun(path.join(TMP, 'built')); + runPhase(out, 'ceo'); + runPhase(out, 'eng'); + const answer = autoplan(['answer', '--by', 'human', '--out', out, '--gate-rev', '1', '--reply', recCarrying(out)]); + if (answer.status !== 0) throw new Error(`answer failed: ${answer.stderr}\n${answer.stdout}`); + const exported = autoplan(['export', '--out', out]); + if (exported.status !== 0) throw new Error(`export failed: ${exported.stderr}\n${exported.stdout}`); + return (built = out); +} +let n = 0; +function copyOf(src: string): string { + const dir = path.join(TMP, `copy${++n}`, 'run'); + fs.cpSync(src, dir, { recursive: true }); + return dir; +} +const readJson = (file: string) => JSON.parse(fs.readFileSync(file, 'utf8')); +const editJournal = (dir: string, edit: (rows: any[]) => any[]) => { + const file = path.join(dir, 'attempts.jsonl'); + const rows = fs.readFileSync(file, 'utf8').split('\n').filter(Boolean).map(l => JSON.parse(l)); + fs.writeFileSync(file, edit(rows).map(r => JSON.stringify(r)).join('\n') + '\n'); +}; + +describe('gstack-artifact validate refuses a run gstack-autoplan never wrote', () => { + test('the eval shape: a hand-built run.json with no plan_input and no journal is refused', () => { + const forged = copyOf(REFERENCE); + const v = validate(path.join(forged, 'run.json')); + expect(v.code).toBe(1); + expect(v.valid).toBe(false); + expect(v.codes).toContain('ARTIFACT_NOT_AUTOPLAN_RUN'); + expect(v.paths).toContain('run.json.plan_input'); + expect(v.paths).toContain('attempts.jsonl'); + }); + + test('a real run with plan_input removed, or with its journal gone, is refused', () => { + const noInput = copyOf(realRun()); + const m = readJson(path.join(noInput, 'run.json')); + delete m.plan_input; + fs.writeFileSync(path.join(noInput, 'run.json'), JSON.stringify(m, null, 2) + '\n'); + const a = validate(path.join(noInput, 'run.json')); + expect(a.codes).toEqual(['ARTIFACT_NOT_AUTOPLAN_RUN']); + expect(a.paths).toEqual(['run.json.plan_input']); + + const noJournal = copyOf(realRun()); + fs.rmSync(path.join(noJournal, 'attempts.jsonl')); + const b = validate(path.join(noJournal, 'run.json')); + expect(b.codes).toEqual(['ARTIFACT_NOT_AUTOPLAN_RUN']); + expect(b.paths).toEqual(['attempts.jsonl']); + }); + + test('control: a run written by gstack-autoplan in a temp dir validates', () => { + const v = validate(path.join(copyOf(realRun()), 'run.json')); + expect(v.codes).toEqual([]); + expect(v.valid).toBe(true); + expect(v.code).toBe(0); + }); +}); + +describe('the journal must match run.json', () => { + test('a submitted row whose result hash was changed is refused', () => { + const dir = copyOf(realRun()); + editJournal(dir, rows => rows.map(r => (r.event === 'submitted' && r.phase === 'eng' && r.voice === 'native' ? { ...r, result_sha256: 'f'.repeat(64) } : r))); + const v = validate(path.join(dir, 'run.json')); + expect(v.codes).toEqual(['ARTIFACT_JOURNAL_MISMATCH']); + expect(v.paths).toEqual(['attempts.jsonl']); + }); + + test('a journal with no init row for this run, or a dispatch row that names another attempt, is refused', () => { + const noInit = copyOf(realRun()); + editJournal(noInit, rows => rows.filter(r => r.event !== 'init')); + expect(validate(path.join(noInit, 'run.json')).codes).toEqual(['ARTIFACT_JOURNAL_MISMATCH']); + + const renamed = copyOf(realRun()); + editJournal(renamed, rows => rows.map(r => (r.event === 'dispatched' && r.phase === 'ceo' && r.voice === 'outside' ? { ...r, attempt: `${r.attempt}x` } : r))); + expect(validate(path.join(renamed, 'run.json')).codes).toContain('ARTIFACT_JOURNAL_MISMATCH'); + }); + + test('a bound review output edited after submit is refused, even with its artifact hash rebound', () => { + const dir = copyOf(realRun()); + const m = readJson(path.join(dir, 'run.json')); + const output = m.phases.eng.outside.output; + fs.chmodSync(path.join(dir, output), 0o644); + fs.appendFileSync(path.join(dir, output), '\nNo findings.\n'); + m.artifacts[output].sha256 = createHash('sha256').update(fs.readFileSync(path.join(dir, output))).digest('hex'); + fs.writeFileSync(path.join(dir, 'run.json'), JSON.stringify(m, null, 2) + '\n'); + const v = validate(path.join(dir, 'run.json')); + expect(v.codes).toEqual(['ARTIFACT_JOURNAL_MISMATCH']); + expect(v.paths).toEqual([output]); + }); +}); + +describe('a checkpoint must match run.json', () => { + test('a journal rewritten after its checkpoint is refused; the untouched run validates', () => { + const dir = copyOf(realRun()); + const checkpoints = path.join(TMP, 'checkpoints'); + const ck = { GSTACK_AUTOPLAN_CHECKPOINT_DIR: checkpoints }; + const exported = autoplan(['export', '--out', dir], { env: env(ck) }); + expect(exported.status, exported.stderr + exported.stdout).toBe(0); + expect(exported.stderr).toMatch(/^CHECKPOINT: \S+ status=complete/m); + expect(validate(path.join(dir, 'run.json'), ck).codes).toEqual([]); + + editJournal(dir, rows => rows.map(r => (r.event === 'plan_reality' ? { ...r, verdict: 'pass' } : r))); + const v = validate(path.join(dir, 'run.json'), ck); + expect(v.codes).toEqual(['ARTIFACT_JOURNAL_MISMATCH']); + expect(v.paths[0]).toContain(checkpoints); + }); + + test('a run.json older than its own checkpoint is refused', () => { + const dir = copyOf(realRun()); + const ck = { GSTACK_AUTOPLAN_CHECKPOINT_DIR: path.join(TMP, 'checkpoints-rollback') }; + expect(autoplan(['export', '--out', dir], { env: env(ck) }).status).toBe(0); + const m = readJson(path.join(dir, 'run.json')); + m.revision -= 1; + fs.writeFileSync(path.join(dir, 'run.json'), JSON.stringify(m, null, 2) + '\n'); + expect(validate(path.join(dir, 'run.json'), ck).codes).toEqual(['ARTIFACT_JOURNAL_MISMATCH']); + }); +}); diff --git a/test/fixtures/multi-agent-wave/INCIDENTS.md b/test/fixtures/multi-agent-wave/INCIDENTS.md index a2bfb1ff9..b986caed5 100644 --- a/test/fixtures/multi-agent-wave/INCIDENTS.md +++ b/test/fixtures/multi-agent-wave/INCIDENTS.md @@ -51,3 +51,4 @@ Later PRs of the wave append their rows; a row is never removed. | p2-10-draft-left-standing: `draft direction stands until the owner decides` survived the owner's answer | E3 `gstack-owner-brief close` | plan with three draft lines, one still pending | `test/owner-brief-contributor-banned.test.ts` "close rewrites" | | p2-10-outside-pr-as-code: an outside PR was reviewed as code first and credited without carried-over evidence | E3 `gstack-contributor-mode` | explicit `maintainers`, `read` vs `write` permission, `gh` offline | `test/owner-brief-contributor-banned.test.ts` "contributor mode" | | p2-10-banned-term: a renamed product's old name landed in README while the glossary was allowed | E3 `gstack-banned-terms` | diff touching README, `docs/migration/**`, the glossary and the term file itself | `test/owner-brief-contributor-banned.test.ts` "added lines are scanned" | +| gst-3090-hand-built-run: a day-one eval session wrote run.json with a script modeled on `reference-run/` (no `plan_input`, no journal) and `gstack-artifact validate` printed ARTIFACT_VALID | GST-3090 | `reference-run/` copied as a run; a real gstack-autoplan run with `plan_input` or `attempts.jsonl` removed, a journal row or bound output edited, a run.json behind its checkpoint | `test/artifact-writer-check.test.ts` | diff --git a/test/fixtures/multi-agent-wave/reference-run/run.json b/test/fixtures/multi-agent-wave/reference-run/run.json index 41987ab08..aa959bc32 100644 --- a/test/fixtures/multi-agent-wave/reference-run/run.json +++ b/test/fixtures/multi-agent-wave/reference-run/run.json @@ -1,7 +1,7 @@ { "schema_version": 1, "run": "maw-20261010", - "generator": "hand-written by the autoplan run; `gstack-artifact validate` (B0) does not exist yet", + "generator": "hand-written by the autoplan run before gstack-autoplan existed; a structural fixture that `gstack-artifact validate` refuses (ARTIFACT_NOT_AUTOPLAN_RUN)", "status": "complete", "base": "91bbd9e", "host": "capy", diff --git a/test/headless-artifacts.test.ts b/test/headless-artifacts.test.ts index e9f91652a..2c433e82f 100644 --- a/test/headless-artifacts.test.ts +++ b/test/headless-artifacts.test.ts @@ -2,7 +2,9 @@ * B0: the unattended artifact contract. Pins the wire contract (schema * versions, ids, hashes, the GSTACK_RESULT grammar, exit codes, reason codes) * on the plan branch's own hand-written run (test/fixtures/multi-agent-wave/ - * reference-run) and on mutations of it, never on prose. + * reference-run) and on mutations of it, never on prose. That run is a + * structural fixture: gstack-autoplan never wrote it, so the CLI refuses it + * (test/artifact-writer-check.test.ts). */ import { describe, expect, test, beforeEach, afterEach } from 'bun:test'; import { spawnSync } from 'node:child_process'; @@ -15,6 +17,7 @@ import { } from '../lib/headless-artifacts'; import { RESULT_CODES } from '../lib/result-codes'; import { GUARD_ENFORCED_LINE } from '../lib/autoplan-guard-status'; +import { autoplan, recCarrying, runPhase, startSmallRun } from './helpers/autoplan-run-fixture'; const ROOT = path.resolve(import.meta.dir, '..'); const BIN = path.join(ROOT, 'bin', 'gstack-artifact'); @@ -45,20 +48,23 @@ const codes = (file: string) => validateRun(file).errors.map(e => e.code); const sha = (file: string) => require('node:crypto').createHash('sha256').update(fs.readFileSync(file)).digest('hex'); describe('reference run (the plan branch artifacts)', () => { - test('validates: the hand-written v1 shapes are the contract', () => { + test('the hand-written v1 shapes pass the structural contract; the CLI refuses them as a run gstack-autoplan never wrote (GST-3090)', () => { const result = validateRun(path.join(REFERENCE, 'run.json')); expect(result.errors).toEqual([]); expect(result.valid).toBe(true); expect(result.manifest!.run).toBe('maw-20261010'); const r = run(['validate', path.join(REFERENCE, 'run.json')]); - expect(r.code).toBe(EXIT.ok); - expect(r.out.trim()).toBe('ARTIFACT_VALID: run=maw-20261010 status=complete'); + expect(r.code).toBe(EXIT.fail); + expect(r.err).toContain('(ARTIFACT_NOT_AUTOPLAN_RUN)'); + expect(r.out.trim()).toMatch(/^ARTIFACT_INVALID: \d+ error\(s\)$/); }); test('--json: JSON only on stdout, {valid, errors:[{code,path,message}]}', () => { const r = run(['validate', path.join(REFERENCE, 'run.json'), '--json']); const parsed = JSON.parse(r.out); - expect(parsed).toMatchObject({ valid: true, run: 'maw-20261010', status: 'complete', errors: [] }); + expect(parsed).toMatchObject({ valid: false, run: 'maw-20261010', status: 'complete' }); + expect(parsed.errors.length).toBeGreaterThan(0); + for (const e of parsed.errors) expect(Object.keys(e).sort()).toEqual(['code', 'message', 'path']); expect(r.err).toBe(''); }); @@ -283,11 +289,17 @@ describe('ack', () => { }); test('bin: records through the state root, says so when the root is ephemeral, refuses an invalid run', () => { - const file = copyRun(); + const out = startSmallRun(path.join(tmp, 'real')); + runPhase(out, 'ceo'); + runPhase(out, 'eng'); + expect(autoplan(['answer', '--by', 'human', '--out', out, '--gate-rev', '1', '--reply', recCarrying(out)]).status).toBe(0); + expect(autoplan(['export', '--out', out]).status).toBe(0); + const file = path.join(out, 'run.json'); + const id = JSON.parse(fs.readFileSync(file, 'utf8')).run; const state = path.join(tmp, 'state'); const r = run(['ack', file, '--consumer', 'thread-a'], { GSTACK_STATE_ROOT: state }); expect(r.code).toBe(EXIT.ok); - expect(r.out.trim()).toBe('ARTIFACT_ACK: run=maw-20261010 consumer=thread-a recorded analytics=appended'); + expect(r.out.trim()).toBe(`ARTIFACT_ACK: run=${id} consumer=thread-a recorded analytics=appended`); expect(fs.existsSync(path.join(state, 'analytics', 'autoplan-timing.jsonl'))).toBe(true); const eph = run(['ack', file, '--consumer', 'thread-b', '--json'], { GSTACK_STATE_ROOT: path.join(tmp, 'state2'), GSTACK_EPHEMERAL: '1' }); expect(eph.code).toBe(EXIT.ok); @@ -299,6 +311,11 @@ describe('ack', () => { expect(refused.code).toBe(EXIT.refused); expect(refused.err).toContain('nothing written'); expect(JSON.parse(fs.readFileSync(invalid, 'utf8')).consumed_by).toBeUndefined(); + const handWritten = copyRun(); + const forged = run(['ack', handWritten, '--consumer', 'thread-d'], { GSTACK_STATE_ROOT: state }); + expect(forged.code).toBe(EXIT.refused); + expect(forged.err).toContain('(ARTIFACT_NOT_AUTOPLAN_RUN)'); + expect(JSON.parse(fs.readFileSync(handWritten, 'utf8')).consumed_by).toBeUndefined(); }); });