gstack-artifact validate now refuses a run.json that gstack-autoplan never wrote (GST-3090) (#4494)

`gstack-artifact validate` and `ack` now refuse a run.json that
gstack-autoplan never wrote, so a hand-built run can no longer pass as a
finished review.

Why: a session skipped `gstack-autoplan next`, built run.json with a
script copied from the reference fixture, and got ARTIFACT_VALID.

Nothing to do unless you validate old or hand-made run directories:
rerun those through gstack-autoplan.

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
Garry Tanandcapy-ai[bot] authored and GitHub committed 2026-11-18 16:45:06 +00:00
1 parent 45f879b534
commit 2767d229fb
12 files changed
+334 -18

No files matched your search

+146
View File
@@ -0,0 +1,146 @@
/**
* GST-3090: `gstack-artifact validate` refuses a run.json that gstack-autoplan
* never wrote. A day-one eval session hand-built run.json from the reference
* fixture (no plan_input, no journal) and got ARTIFACT_VALID. A run written by
* the CLI still validates; a tampered journal, review output or checkpoint is
* refused.
*/
import { afterAll, describe, expect, test } from 'bun:test';
import { spawnSync } from 'node:child_process';
import { createHash } from 'node:crypto';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { ROOT, autoplan, env, recCarrying, runPhase, startSmallRun } from './helpers/autoplan-run-fixture';
const BIN = path.join(ROOT, 'bin', 'gstack-artifact');
const REFERENCE = path.join(ROOT, 'test', 'fixtures', 'multi-agent-wave', 'reference-run');
const TMP = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-writer-check-'));
afterAll(() => fs.rmSync(TMP, { recursive: true, force: true }));
function validate(runJson: string, extra: Record<string, string> = {}): { code: number | null; valid: boolean; codes: string[]; paths: string[] } {
const r = spawnSync(BIN, ['validate', runJson, '--json'], { encoding: 'utf8', timeout: 60_000, env: env(extra) });
const parsed = JSON.parse(r.stdout);
return { code: r.status, valid: parsed.valid, codes: parsed.errors.map((e: any) => e.code), paths: parsed.errors.map((e: any) => e.path) };
}
/** One complete ceo → eng run driven only through gstack-autoplan, then exported; each test copies it. */
let built: string | undefined;
function realRun(): string {
if (built) return built;
const out = startSmallRun(path.join(TMP, 'built'));
runPhase(out, 'ceo');
runPhase(out, 'eng');
const answer = autoplan(['answer', '--by', 'human', '--out', out, '--gate-rev', '1', '--reply', recCarrying(out)]);
if (answer.status !== 0) throw new Error(`answer failed: ${answer.stderr}\n${answer.stdout}`);
const exported = autoplan(['export', '--out', out]);
if (exported.status !== 0) throw new Error(`export failed: ${exported.stderr}\n${exported.stdout}`);
return (built = out);
}
let n = 0;
function copyOf(src: string): string {
const dir = path.join(TMP, `copy${++n}`, 'run');
fs.cpSync(src, dir, { recursive: true });
return dir;
}
const readJson = (file: string) => JSON.parse(fs.readFileSync(file, 'utf8'));
const editJournal = (dir: string, edit: (rows: any[]) => any[]) => {
const file = path.join(dir, 'attempts.jsonl');
const rows = fs.readFileSync(file, 'utf8').split('\n').filter(Boolean).map(l => JSON.parse(l));
fs.writeFileSync(file, edit(rows).map(r => JSON.stringify(r)).join('\n') + '\n');
};
describe('gstack-artifact validate refuses a run gstack-autoplan never wrote', () => {
test('the eval shape: a hand-built run.json with no plan_input and no journal is refused', () => {
const forged = copyOf(REFERENCE);
const v = validate(path.join(forged, 'run.json'));
expect(v.code).toBe(1);
expect(v.valid).toBe(false);
expect(v.codes).toContain('ARTIFACT_NOT_AUTOPLAN_RUN');
expect(v.paths).toContain('run.json.plan_input');
expect(v.paths).toContain('attempts.jsonl');
});
test('a real run with plan_input removed, or with its journal gone, is refused', () => {
const noInput = copyOf(realRun());
const m = readJson(path.join(noInput, 'run.json'));
delete m.plan_input;
fs.writeFileSync(path.join(noInput, 'run.json'), JSON.stringify(m, null, 2) + '\n');
const a = validate(path.join(noInput, 'run.json'));
expect(a.codes).toEqual(['ARTIFACT_NOT_AUTOPLAN_RUN']);
expect(a.paths).toEqual(['run.json.plan_input']);
const noJournal = copyOf(realRun());
fs.rmSync(path.join(noJournal, 'attempts.jsonl'));
const b = validate(path.join(noJournal, 'run.json'));
expect(b.codes).toEqual(['ARTIFACT_NOT_AUTOPLAN_RUN']);
expect(b.paths).toEqual(['attempts.jsonl']);
});
test('control: a run written by gstack-autoplan in a temp dir validates', () => {
const v = validate(path.join(copyOf(realRun()), 'run.json'));
expect(v.codes).toEqual([]);
expect(v.valid).toBe(true);
expect(v.code).toBe(0);
});
});
describe('the journal must match run.json', () => {
test('a submitted row whose result hash was changed is refused', () => {
const dir = copyOf(realRun());
editJournal(dir, rows => rows.map(r => (r.event === 'submitted' && r.phase === 'eng' && r.voice === 'native' ? { ...r, result_sha256: 'f'.repeat(64) } : r)));
const v = validate(path.join(dir, 'run.json'));
expect(v.codes).toEqual(['ARTIFACT_JOURNAL_MISMATCH']);
expect(v.paths).toEqual(['attempts.jsonl']);
});
test('a journal with no init row for this run, or a dispatch row that names another attempt, is refused', () => {
const noInit = copyOf(realRun());
editJournal(noInit, rows => rows.filter(r => r.event !== 'init'));
expect(validate(path.join(noInit, 'run.json')).codes).toEqual(['ARTIFACT_JOURNAL_MISMATCH']);
const renamed = copyOf(realRun());
editJournal(renamed, rows => rows.map(r => (r.event === 'dispatched' && r.phase === 'ceo' && r.voice === 'outside' ? { ...r, attempt: `${r.attempt}x` } : r)));
expect(validate(path.join(renamed, 'run.json')).codes).toContain('ARTIFACT_JOURNAL_MISMATCH');
});
test('a bound review output edited after submit is refused, even with its artifact hash rebound', () => {
const dir = copyOf(realRun());
const m = readJson(path.join(dir, 'run.json'));
const output = m.phases.eng.outside.output;
fs.chmodSync(path.join(dir, output), 0o644);
fs.appendFileSync(path.join(dir, output), '\nNo findings.\n');
m.artifacts[output].sha256 = createHash('sha256').update(fs.readFileSync(path.join(dir, output))).digest('hex');
fs.writeFileSync(path.join(dir, 'run.json'), JSON.stringify(m, null, 2) + '\n');
const v = validate(path.join(dir, 'run.json'));
expect(v.codes).toEqual(['ARTIFACT_JOURNAL_MISMATCH']);
expect(v.paths).toEqual([output]);
});
});
describe('a checkpoint must match run.json', () => {
test('a journal rewritten after its checkpoint is refused; the untouched run validates', () => {
const dir = copyOf(realRun());
const checkpoints = path.join(TMP, 'checkpoints');
const ck = { GSTACK_AUTOPLAN_CHECKPOINT_DIR: checkpoints };
const exported = autoplan(['export', '--out', dir], { env: env(ck) });
expect(exported.status, exported.stderr + exported.stdout).toBe(0);
expect(exported.stderr).toMatch(/^CHECKPOINT: \S+ status=complete/m);
expect(validate(path.join(dir, 'run.json'), ck).codes).toEqual([]);
editJournal(dir, rows => rows.map(r => (r.event === 'plan_reality' ? { ...r, verdict: 'pass' } : r)));
const v = validate(path.join(dir, 'run.json'), ck);
expect(v.codes).toEqual(['ARTIFACT_JOURNAL_MISMATCH']);
expect(v.paths[0]).toContain(checkpoints);
});
test('a run.json older than its own checkpoint is refused', () => {
const dir = copyOf(realRun());
const ck = { GSTACK_AUTOPLAN_CHECKPOINT_DIR: path.join(TMP, 'checkpoints-rollback') };
expect(autoplan(['export', '--out', dir], { env: env(ck) }).status).toBe(0);
const m = readJson(path.join(dir, 'run.json'));
m.revision -= 1;
fs.writeFileSync(path.join(dir, 'run.json'), JSON.stringify(m, null, 2) + '\n');
expect(validate(path.join(dir, 'run.json'), ck).codes).toEqual(['ARTIFACT_JOURNAL_MISMATCH']);
});
});
+1
View File
@@ -51,3 +51,4 @@ Later PRs of the wave append their rows; a row is never removed.
| p2-10-draft-left-standing: `draft direction stands until the owner decides` survived the owner's answer | E3 `gstack-owner-brief close` | plan with three draft lines, one still pending | `test/owner-brief-contributor-banned.test.ts` "close rewrites" |
| p2-10-outside-pr-as-code: an outside PR was reviewed as code first and credited without carried-over evidence | E3 `gstack-contributor-mode` | explicit `maintainers`, `read` vs `write` permission, `gh` offline | `test/owner-brief-contributor-banned.test.ts` "contributor mode" |
| p2-10-banned-term: a renamed product's old name landed in README while the glossary was allowed | E3 `gstack-banned-terms` | diff touching README, `docs/migration/**`, the glossary and the term file itself | `test/owner-brief-contributor-banned.test.ts` "added lines are scanned" |
| gst-3090-hand-built-run: a day-one eval session wrote run.json with a script modeled on `reference-run/` (no `plan_input`, no journal) and `gstack-artifact validate` printed ARTIFACT_VALID | GST-3090 | `reference-run/` copied as a run; a real gstack-autoplan run with `plan_input` or `attempts.jsonl` removed, a journal row or bound output edited, a run.json behind its checkpoint | `test/artifact-writer-check.test.ts` |
+1 -1
View File
@@ -1,7 +1,7 @@
{
"schema_version": 1,
"run": "maw-20261010",
"generator": "hand-written by the autoplan run; `gstack-artifact validate` (B0) does not exist yet",
"generator": "hand-written by the autoplan run before gstack-autoplan existed; a structural fixture that `gstack-artifact validate` refuses (ARTIFACT_NOT_AUTOPLAN_RUN)",
"status": "complete",
"base": "91bbd9e",
"host": "capy",
+24 -7
View File
@@ -2,7 +2,9 @@
* B0: the unattended artifact contract. Pins the wire contract (schema
* versions, ids, hashes, the GSTACK_RESULT grammar, exit codes, reason codes)
* on the plan branch's own hand-written run (test/fixtures/multi-agent-wave/
* reference-run) and on mutations of it, never on prose.
* reference-run) and on mutations of it, never on prose. That run is a
* structural fixture: gstack-autoplan never wrote it, so the CLI refuses it
* (test/artifact-writer-check.test.ts).
*/
import { describe, expect, test, beforeEach, afterEach } from 'bun:test';
import { spawnSync } from 'node:child_process';
@@ -15,6 +17,7 @@ import {
} from '../lib/headless-artifacts';
import { RESULT_CODES } from '../lib/result-codes';
import { GUARD_ENFORCED_LINE } from '../lib/autoplan-guard-status';
import { autoplan, recCarrying, runPhase, startSmallRun } from './helpers/autoplan-run-fixture';
const ROOT = path.resolve(import.meta.dir, '..');
const BIN = path.join(ROOT, 'bin', 'gstack-artifact');
@@ -45,20 +48,23 @@ const codes = (file: string) => validateRun(file).errors.map(e => e.code);
const sha = (file: string) => require('node:crypto').createHash('sha256').update(fs.readFileSync(file)).digest('hex');
describe('reference run (the plan branch artifacts)', () => {
test('validates: the hand-written v1 shapes are the contract', () => {
test('the hand-written v1 shapes pass the structural contract; the CLI refuses them as a run gstack-autoplan never wrote (GST-3090)', () => {
const result = validateRun(path.join(REFERENCE, 'run.json'));
expect(result.errors).toEqual([]);
expect(result.valid).toBe(true);
expect(result.manifest!.run).toBe('maw-20261010');
const r = run(['validate', path.join(REFERENCE, 'run.json')]);
expect(r.code).toBe(EXIT.ok);
expect(r.out.trim()).toBe('ARTIFACT_VALID: run=maw-20261010 status=complete');
expect(r.code).toBe(EXIT.fail);
expect(r.err).toContain('(ARTIFACT_NOT_AUTOPLAN_RUN)');
expect(r.out.trim()).toMatch(/^ARTIFACT_INVALID: \d+ error\(s\)$/);
});
test('--json: JSON only on stdout, {valid, errors:[{code,path,message}]}', () => {
const r = run(['validate', path.join(REFERENCE, 'run.json'), '--json']);
const parsed = JSON.parse(r.out);
expect(parsed).toMatchObject({ valid: true, run: 'maw-20261010', status: 'complete', errors: [] });
expect(parsed).toMatchObject({ valid: false, run: 'maw-20261010', status: 'complete' });
expect(parsed.errors.length).toBeGreaterThan(0);
for (const e of parsed.errors) expect(Object.keys(e).sort()).toEqual(['code', 'message', 'path']);
expect(r.err).toBe('');
});
@@ -283,11 +289,17 @@ describe('ack', () => {
});
test('bin: records through the state root, says so when the root is ephemeral, refuses an invalid run', () => {
const file = copyRun();
const out = startSmallRun(path.join(tmp, 'real'));
runPhase(out, 'ceo');
runPhase(out, 'eng');
expect(autoplan(['answer', '--by', 'human', '--out', out, '--gate-rev', '1', '--reply', recCarrying(out)]).status).toBe(0);
expect(autoplan(['export', '--out', out]).status).toBe(0);
const file = path.join(out, 'run.json');
const id = JSON.parse(fs.readFileSync(file, 'utf8')).run;
const state = path.join(tmp, 'state');
const r = run(['ack', file, '--consumer', 'thread-a'], { GSTACK_STATE_ROOT: state });
expect(r.code).toBe(EXIT.ok);
expect(r.out.trim()).toBe('ARTIFACT_ACK: run=maw-20261010 consumer=thread-a recorded analytics=appended');
expect(r.out.trim()).toBe(`ARTIFACT_ACK: run=${id} consumer=thread-a recorded analytics=appended`);
expect(fs.existsSync(path.join(state, 'analytics', 'autoplan-timing.jsonl'))).toBe(true);
const eph = run(['ack', file, '--consumer', 'thread-b', '--json'], { GSTACK_STATE_ROOT: path.join(tmp, 'state2'), GSTACK_EPHEMERAL: '1' });
expect(eph.code).toBe(EXIT.ok);
@@ -299,6 +311,11 @@ describe('ack', () => {
expect(refused.code).toBe(EXIT.refused);
expect(refused.err).toContain('nothing written');
expect(JSON.parse(fs.readFileSync(invalid, 'utf8')).consumed_by).toBeUndefined();
const handWritten = copyRun();
const forged = run(['ack', handWritten, '--consumer', 'thread-d'], { GSTACK_STATE_ROOT: state });
expect(forged.code).toBe(EXIT.refused);
expect(forged.err).toContain('(ARTIFACT_NOT_AUTOPLAN_RUN)');
expect(JSON.parse(fs.readFileSync(handWritten, 'utf8')).consumed_by).toBeUndefined();
});
});