mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-18 22:24:17 +01:00
gstack-artifact validate now refuses a run.json that gstack-autoplan never wrote (GST-3090) (#4494)
`gstack-artifact validate` and `ack` now refuse a run.json that gstack-autoplan never wrote, so a hand-built run can no longer pass as a finished review. Why: a session skipped `gstack-autoplan next`, built run.json with a script copied from the reference fixture, and got ARTIFACT_VALID. Nothing to do unless you validate old or hand-made run directories: rerun those through gstack-autoplan. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
45f879b534
commit
2767d229fb
12 files changed
+334
-18
No files matched your search
+5
-4
@@ -2,7 +2,7 @@
|
||||
/**
|
||||
* gstack-artifact — validate, print and acknowledge the unattended artifact contract.
|
||||
*
|
||||
* gstack-artifact validate <run.json> [--json] completion check for a run directory
|
||||
* gstack-artifact validate <run.json> [--json] completion check for a run directory gstack-autoplan wrote
|
||||
* gstack-artifact validate --public <dir> [--private <run.json>] [--json]
|
||||
* a public run copy: provenance, recorded machine roots
|
||||
* gstack-artifact validate <file> --as <name> [--run <id>] [--json]
|
||||
@@ -17,12 +17,13 @@
|
||||
import * as path from 'node:path';
|
||||
import {
|
||||
ARTIFACT_NAMES, EXIT, ackRun, exitTableHelp, jsonSchema, readJsonl, renderUrgentBlock,
|
||||
validateFile, validateRun, type ArtifactName, type ValidationError,
|
||||
validateFile, type ArtifactName, type ValidationError,
|
||||
} from '../lib/headless-artifacts';
|
||||
import { resultLine } from '../lib/result-codes';
|
||||
import { resolveStateRoot } from '../lib/state-root';
|
||||
import { validatePublic } from '../lib/autoplan-public';
|
||||
import { validateShadow } from '../lib/autoplan-shadow-reader';
|
||||
import { validateAutoplanRun } from '../lib/autoplan-writer-check';
|
||||
|
||||
const HELP = `gstack-artifact — the unattended artifact contract (docs/unattended.md)
|
||||
|
||||
@@ -88,7 +89,7 @@ function main(argv: string[]): number {
|
||||
else { printErrors(errors); process.stdout.write(errors.length ? `ARTIFACT_INVALID: ${errors.length} error(s)\n` : `ARTIFACT_VALID: ${as} ${path.basename(file)}\n`); }
|
||||
return errors.length ? EXIT.fail : EXIT.ok;
|
||||
}
|
||||
const run = validateRun(file);
|
||||
const run = validateAutoplanRun(file);
|
||||
const shadow = validateShadow(run.runDir);
|
||||
const result = { ...run, valid: run.valid && shadow.length === 0, errors: [...run.errors, ...shadow] };
|
||||
if (json) {
|
||||
@@ -106,7 +107,7 @@ function main(argv: string[]): number {
|
||||
const file = rest.find(a => !a.startsWith('--') && a !== flag(rest, '--consumer'));
|
||||
const consumer = flag(rest, '--consumer');
|
||||
if (!file || !consumer) return usage('ack needs <run.json> --consumer <id>');
|
||||
const validation = validateRun(file);
|
||||
const validation = validateAutoplanRun(file);
|
||||
if (!validation.valid) {
|
||||
printErrors(validation.errors);
|
||||
process.stderr.write('gstack-artifact ack: refusing to acknowledge an invalid run; nothing written\n');
|
||||
|
||||
@@ -443,6 +443,8 @@ gstack_result_anchor() {
|
||||
OFFICE_HOURS_PROBE_REFUSED) printf '%s' 'office-hours-probe-refused' ;;
|
||||
TASKS_SUPERSEDE_REFUSED) printf '%s' 'tasks-supersede-refused' ;;
|
||||
REALITY_RECEIPT_UNRESOLVED) printf '%s' 'reality-receipt-unresolved' ;;
|
||||
ARTIFACT_NOT_AUTOPLAN_RUN) printf '%s' 'artifact-not-autoplan-run' ;;
|
||||
ARTIFACT_JOURNAL_MISMATCH) printf '%s' 'artifact-journal-mismatch' ;;
|
||||
GUARD_ENFORCED) printf '%s' 'guard-enforced' ;;
|
||||
CEO_ONLY_CONFLICT) printf '%s' 'ceo-only-conflict' ;;
|
||||
CEO_ONLY_AFTER_INIT) printf '%s' 'ceo-only-after-init' ;;
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
- **`gstack-artifact validate` no longer passes a run.json that gstack-autoplan never wrote.** A session that skipped `gstack-autoplan next` and built run.json with a script, copying the reference fixture, got `ARTIFACT_VALID`. The validator now also checks that the run came from gstack-autoplan: run.json must carry `host: gstack-autoplan`, `plan_input` and the other fields it writes on every run, and must agree with its `attempts.jsonl` journal, its bound review outputs and its checkpoint. A run that fails says `ARTIFACT_NOT_AUTOPLAN_RUN` or `ARTIFACT_JOURNAL_MISMATCH`, and `gstack-artifact ack` refuses it too. Public copies (`validate --public`) are unchanged.
|
||||
@@ -3511,6 +3511,38 @@ record does not state that publication order was unverified.
|
||||
**Fix.** Start the run through `gstack-skill-start` (it prints the line) and
|
||||
keep the line in the review record.
|
||||
|
||||
<a id="artifact-not-autoplan-run"></a>
|
||||
### `ARTIFACT_NOT_AUTOPLAN_RUN`
|
||||
|
||||
**Meaning.** `run.json` parses and matches its schema, but gstack-autoplan did
|
||||
not write it: `host` is not `gstack-autoplan`, a field gstack-autoplan writes
|
||||
on every run is missing (`plan_input`, `plan_source`, `active_plan`,
|
||||
`restore`, `out`, `created_at`, `revision`, `attempt_counter`,
|
||||
`outside_enabled`, `phase_order`, `history`, `scope`), or the run directory
|
||||
has no `attempts.jsonl` journal. A run.json written by hand or by a script,
|
||||
for example one copied from `test/fixtures/multi-agent-wave/reference-run`,
|
||||
is refused here even when every structural check passes.
|
||||
|
||||
**Fix.** Drive the run with `gstack-autoplan next`, `submit`, `answer` and
|
||||
`export`, then validate the directory it wrote. A run written by a gstack
|
||||
older than these fields is refused too; rerun it with this gstack.
|
||||
|
||||
<a id="artifact-journal-mismatch"></a>
|
||||
### `ARTIFACT_JOURNAL_MISMATCH`
|
||||
|
||||
**Meaning.** The run directory disagrees with its own record of what happened:
|
||||
the journal has no `init` row for this run, a reviewer attempt in `run.json`
|
||||
was never journaled as dispatched, a completed voice's output or hash differs
|
||||
from its journaled `submitted` row, a bound review output changed after
|
||||
submit, or the run's checkpoint (when `GSTACK_AUTOPLAN_CHECKPOINT_DIR` or the
|
||||
Drive opt-in names one) is ahead of `run.json`, disagrees on status or gate
|
||||
revision at the same revision, or holds a journal the live one no longer begins with.
|
||||
|
||||
**Fix.** Treat the run as edited after the fact. Restore it from its
|
||||
checkpoint (`gstack-autoplan resume --out <new dir> --from <checkpoint>`) or
|
||||
rerun it with gstack-autoplan; never edit `run.json` or `attempts.jsonl` by
|
||||
hand.
|
||||
|
||||
<a id="guard-not-installed"></a>
|
||||
### `autoplan guard: not enforced by this host; publication order is unverified (GUARD_NOT_INSTALLED)`
|
||||
|
||||
|
||||
+15
-4
@@ -424,8 +424,14 @@ checks its prompt.
|
||||
terminal outcome, every listed artifact exists inside the run directory with
|
||||
the recorded hash, every JSONL file parses row by row against its schema with
|
||||
run-bound unique ids, task dependencies and finding references resolve
|
||||
without cycles, `counts` agree with the files, and (unattended) the guard
|
||||
line is present. Otherwise it exits `1` with one error line per problem and
|
||||
without cycles, `counts` agree with the files, (unattended) the guard
|
||||
line is present, and gstack-autoplan wrote the run: `run.json` carries
|
||||
the fields it writes on every run (`host: gstack-autoplan`, `plan_input`
|
||||
and the rest, `ARTIFACT_NOT_AUTOPLAN_RUN`) and agrees with its
|
||||
`attempts.jsonl` journal, its bound review outputs and its checkpoint
|
||||
(`ARTIFACT_JOURNAL_MISMATCH`). `gstack-artifact ack` runs the same
|
||||
check. A public copy (`validate --public`) carries no journal and is
|
||||
checked against its private run's digests instead. Otherwise it exits `1` with one error line per problem and
|
||||
`ARTIFACT_INVALID: <n> error(s)`; `--json` returns the same as
|
||||
`{ valid, run, status, run_dir, errors[] }`. A run that was interrupted shows up here as
|
||||
`ARTIFACT_RUN_INTERRUPTED`, never as a partial success.
|
||||
@@ -1015,8 +1021,13 @@ line, on an ephemeral state root.
|
||||
|
||||
## Reference run
|
||||
|
||||
`test/fixtures/multi-agent-wave/reference-run/` is a complete unattended
|
||||
`/autoplan` run (the one that produced this wave's plan) and validates clean.
|
||||
`test/fixtures/multi-agent-wave/reference-run/` is the hand-written run
|
||||
that produced this wave's plan, from before `gstack-autoplan` existed. It
|
||||
pins the structural contract (`validateRun` passes it), but
|
||||
`gstack-artifact validate` refuses it with `ARTIFACT_NOT_AUTOPLAN_RUN`: it
|
||||
has no `plan_input` and no journal, because no gstack-autoplan wrote it. Copy
|
||||
a run gstack-autoplan wrote, never this one, when you need a run that
|
||||
validates.
|
||||
`test/fixtures/multi-agent-wave/INCIDENTS.md` records what went wrong while it
|
||||
was produced by hand and which validator rule now catches each incident.
|
||||
|
||||
|
||||
@@ -130,7 +130,7 @@ export function checkpointNotice(env: NodeJS.ProcessEnv, repo?: string): string
|
||||
|
||||
/** A checkpoint beyond the at-rest ones: after a bound voice, after a phase transition, and the notice at the first next. */
|
||||
export type CheckpointMoment = 'bound' | 'transition' | 'init';
|
||||
const repoOf = (state: RunState) => { const input = state.plan_input ?? state.plan_source; const root = input ? repoRoot(path.dirname(input)) : undefined; return root ? path.basename(root) : undefined; };
|
||||
export const repoOf = (state: Pick<RunState, 'plan_input' | 'plan_source'>) => { const input = state.plan_input ?? state.plan_source; const root = input ? repoRoot(path.dirname(input)) : undefined; return root ? path.basename(root) : undefined; };
|
||||
|
||||
/**
|
||||
* Publish a generation when a checkpoint directory is set (or Drive opted in) and the run is at rest, or at a
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
/**
|
||||
* Provenance for `gstack-artifact validate <run.json>` and `ack` (GST-3090).
|
||||
* validateRun (lib/headless-artifacts.ts) says a run directory is well formed;
|
||||
* this says gstack-autoplan wrote it. gstack-autoplan is the only writer of
|
||||
* run.json (docs/unattended.md), so a run.json that parses against its schema
|
||||
* gets this check too. A public copy does not: it carries no journal and is
|
||||
* bound to its private run by provenance digests (lib/autoplan-public.ts).
|
||||
*
|
||||
* ARTIFACT_NOT_AUTOPLAN_RUN host is not gstack-autoplan, a field initRun always writes is missing, or attempts.jsonl is absent
|
||||
* ARTIFACT_JOURNAL_MISMATCH attempts.jsonl, a bound review output or the run's checkpoint disagrees with run.json
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { checkpointRoot, listGenerations, repoOf } from './autoplan-checkpoint';
|
||||
import { FILES, readJournal, type RunState } from './autoplan-state';
|
||||
import { sha256File, validateRun, type ValidationError, type ValidationResult } from './headless-artifacts';
|
||||
|
||||
/** What initRun (lib/autoplan-run.ts) writes on every run, whatever its kind; gstack_version is absent on runs before 1.91.96. */
|
||||
const WRITER_FIELDS = [
|
||||
['plan_input', 'string'], ['plan_source', 'string'], ['active_plan', 'string'], ['restore', 'string'], ['out', 'string'],
|
||||
['created_at', 'string'], ['revision', 'number'], ['attempt_counter', 'number'], ['outside_enabled', 'boolean'],
|
||||
['phase_order', 'array'], ['history', 'array'], ['scope', 'object'],
|
||||
] as const;
|
||||
const STRUCTURE = ['ARTIFACT_INVALID_JSON', 'ARTIFACT_UNSUPPORTED_VERSION', 'ARTIFACT_SCHEMA'];
|
||||
const kindOf = (x: unknown) => (Array.isArray(x) ? 'array' : x === null ? 'null' : typeof x);
|
||||
|
||||
/** validateRun plus the writer check, for a run directory a consumer treats as evidence. */
|
||||
export function validateAutoplanRun(runJson: string, env: NodeJS.ProcessEnv = process.env): ValidationResult {
|
||||
const result = validateRun(runJson);
|
||||
if (!result.manifest || result.errors.some(e => STRUCTURE.includes(e.code))) return result;
|
||||
const errors = [...result.errors, ...checkWriter(result.manifest as unknown as RunState, result.runDir, env)];
|
||||
return { ...result, errors, valid: errors.length === 0 };
|
||||
}
|
||||
|
||||
export function checkWriter(m: RunState, runDir: string, env: NodeJS.ProcessEnv = process.env): ValidationError[] {
|
||||
const out: ValidationError[] = [];
|
||||
const notWritten = (p: string, message: string) => out.push({ code: 'ARTIFACT_NOT_AUTOPLAN_RUN', path: p, message });
|
||||
const mismatch = (p: string, message: string) => out.push({ code: 'ARTIFACT_JOURNAL_MISMATCH', path: p, message });
|
||||
if (m.host !== 'gstack-autoplan') notWritten('run.json.host', `host is ${JSON.stringify(m.host ?? null)}; gstack-autoplan writes "gstack-autoplan"`);
|
||||
for (const [field, kind] of WRITER_FIELDS) {
|
||||
const got = kindOf((m as unknown as Record<string, unknown>)[field]);
|
||||
if (got !== kind) notWritten(`run.json.${field}`, got === 'undefined' ? `missing; gstack-autoplan writes it on every run` : `expected ${kind}, got ${got}`);
|
||||
}
|
||||
if (!fs.existsSync(path.join(runDir, FILES.journal))) {
|
||||
notWritten(FILES.journal, 'no journal; gstack-autoplan appends one from the first next');
|
||||
return out;
|
||||
}
|
||||
const rows = readJournal(runDir);
|
||||
if (!rows.some(r => r.event === 'init' && r.run === m.run)) mismatch(FILES.journal, `no init row for run ${m.run}`);
|
||||
for (const [phase, ps] of Object.entries(m.phases ?? {})) {
|
||||
for (const voice of ['native', 'outside'] as const) {
|
||||
const v = ps?.[voice];
|
||||
if (!v) continue;
|
||||
for (const a of v.attempts ?? []) {
|
||||
if (!rows.some(r => r.event === 'dispatched' && r.attempt === a.attempt && r.phase === phase && r.voice === voice)) mismatch(FILES.journal, `${phase} ${voice} attempt ${a.attempt} was never journaled as dispatched`);
|
||||
}
|
||||
if (v.status !== 'completed') continue;
|
||||
if (typeof v.attempt !== 'string' || typeof v.result_sha256 !== 'string') { mismatch(`run.json.phases.${phase}.${voice}`, 'completed without the attempt and result_sha256 submit records'); continue; }
|
||||
const sub = rows.find(r => r.event === 'submitted' && r.attempt === v.attempt);
|
||||
if (!sub || sub.phase !== phase || sub.voice !== voice || sub.result_sha256 !== v.result_sha256 || (v.output && sub.result !== v.output)) {
|
||||
mismatch(FILES.journal, `${phase} ${voice}: run.json binds ${v.output ?? '?'} (${v.result_sha256.slice(0, 12)}) to ${v.attempt}; the journal's submitted row ${sub ? `says ${sub.phase} ${sub.voice} ${sub.result} (${String(sub.result_sha256).slice(0, 12)})` : 'is missing'}`);
|
||||
continue;
|
||||
}
|
||||
const bound = v.output ? path.resolve(runDir, v.output) : '';
|
||||
if (bound.startsWith(path.resolve(runDir) + path.sep) && fs.existsSync(bound) && sha256File(bound) !== v.result_sha256) mismatch(v.output!, `changed after submit: sha256 ${sha256File(bound).slice(0, 12)} is not the submitted ${v.result_sha256.slice(0, 12)}`);
|
||||
}
|
||||
}
|
||||
checkCheckpoint(m, runDir, rows, env, mismatch);
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* The run's own checkpoint is never ahead of run.json, agrees on status and gate at the same revision, and holds a
|
||||
* journal that is a prefix of the live one (append-only). Its own: the generation it was restored from, else the
|
||||
* newest valid one written from this run directory (its run.json names the same `out`); a copy restored elsewhere
|
||||
* forks the run and checkpoints into the same collection, so its generations say nothing about this directory.
|
||||
*/
|
||||
function checkCheckpoint(m: RunState, runDir: string, rows: Array<Record<string, any>>, env: NodeJS.ProcessEnv, mismatch: (p: string, message: string) => void): void {
|
||||
const root = checkpointRoot(env, repoOf(m));
|
||||
const collection = root ? path.join(path.resolve(root), m.run) : '';
|
||||
if (!collection || !fs.existsSync(collection)) return;
|
||||
const restored = rows.filter(r => r.event === 'restored').pop();
|
||||
const valid = listGenerations(collection).filter(g => g.manifest && !g.problem);
|
||||
const writtenFrom = (dir: string) => { try { return JSON.parse(fs.readFileSync(path.join(dir, 'run', FILES.run), 'utf8')).out; } catch { return undefined; } };
|
||||
const ref = restored ? valid.find(g => path.resolve(g.dir) === path.resolve(String(restored.from))) : valid.find(g => writtenFrom(g.dir) === m.out);
|
||||
if (!ref) return;
|
||||
const g = ref.manifest!;
|
||||
if (g.revision > m.revision) mismatch(ref.dir, `this checkpoint is at revision ${g.revision}, ahead of run.json's ${m.revision}`);
|
||||
else if (g.revision === m.revision && (g.status !== m.status || g.gate_rev !== (m.gate_rev ?? 0))) mismatch(ref.dir, `revision ${g.revision} was checkpointed as status=${g.status} gate_rev=${g.gate_rev}; run.json says status=${m.status} gate_rev=${m.gate_rev ?? 0}`);
|
||||
const saved = path.join(ref.dir, 'run', FILES.journal);
|
||||
if (fs.existsSync(saved) && !fs.readFileSync(path.join(runDir, FILES.journal)).subarray(0, fs.statSync(saved).size).equals(fs.readFileSync(saved))) {
|
||||
mismatch(ref.dir, `attempts.jsonl no longer begins with the journal checkpointed at revision ${g.revision}: an earlier row was changed or removed`);
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,8 @@
|
||||
* Result codes for the /autoplan publication guard line (GUARD_ENFORCED, printed by
|
||||
* gstack-skill-start; lib/autoplan-guard-status.ts), the CEO-only driver run (`gstack-autoplan next --ceo-only`,
|
||||
* lib/autoplan-ceo-only.ts) and its shadow-mode reader (`gstack-autoplan shadow`,
|
||||
* lib/autoplan-shadow-reader.ts). Kept beside lib/result-codes.ts, which spreads
|
||||
* lib/autoplan-shadow-reader.ts), and the writer check `gstack-artifact validate` runs on a run.json
|
||||
* (lib/autoplan-writer-check.ts). Kept beside lib/result-codes.ts, which spreads
|
||||
* this table into RESULT_CODES; the same rules apply: one stable
|
||||
* docs/troubleshooting.md anchor, a summary and a fix clause each. The run is
|
||||
* documented in docs/unattended.md#ceo-only-runs.
|
||||
@@ -10,6 +11,16 @@
|
||||
import type { ResultCode } from './result-codes';
|
||||
|
||||
export const AUTOPLAN_RESULT_CODES = {
|
||||
ARTIFACT_NOT_AUTOPLAN_RUN: {
|
||||
anchor: 'artifact-not-autoplan-run',
|
||||
summary: 'run.json lacks what gstack-autoplan writes on every run (its host, plan_input and the other run-state fields) or has no attempts.jsonl journal',
|
||||
fix: 'drive the run with gstack-autoplan next/submit/answer/export; a run.json written by hand or by a script is not evidence of a review',
|
||||
},
|
||||
ARTIFACT_JOURNAL_MISMATCH: {
|
||||
anchor: 'artifact-journal-mismatch',
|
||||
summary: 'attempts.jsonl, a bound review output or the run’s checkpoint disagrees with run.json',
|
||||
fix: 'treat the run as edited after the fact; restore it from a checkpoint (gstack-autoplan resume --from) or rerun it with gstack-autoplan',
|
||||
},
|
||||
GUARD_ENFORCED: {
|
||||
anchor: 'guard-enforced',
|
||||
summary: 'Claude Code runs the autoplan publication hook, so a phase Read or reviewer dispatch is denied until the previous phase report is published',
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
/**
|
||||
* GST-3090: `gstack-artifact validate` refuses a run.json that gstack-autoplan
|
||||
* never wrote. A day-one eval session hand-built run.json from the reference
|
||||
* fixture (no plan_input, no journal) and got ARTIFACT_VALID. A run written by
|
||||
* the CLI still validates; a tampered journal, review output or checkpoint is
|
||||
* refused.
|
||||
*/
|
||||
import { afterAll, describe, expect, test } from 'bun:test';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { createHash } from 'node:crypto';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { ROOT, autoplan, env, recCarrying, runPhase, startSmallRun } from './helpers/autoplan-run-fixture';
|
||||
|
||||
const BIN = path.join(ROOT, 'bin', 'gstack-artifact');
|
||||
const REFERENCE = path.join(ROOT, 'test', 'fixtures', 'multi-agent-wave', 'reference-run');
|
||||
const TMP = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-writer-check-'));
|
||||
afterAll(() => fs.rmSync(TMP, { recursive: true, force: true }));
|
||||
|
||||
function validate(runJson: string, extra: Record<string, string> = {}): { code: number | null; valid: boolean; codes: string[]; paths: string[] } {
|
||||
const r = spawnSync(BIN, ['validate', runJson, '--json'], { encoding: 'utf8', timeout: 60_000, env: env(extra) });
|
||||
const parsed = JSON.parse(r.stdout);
|
||||
return { code: r.status, valid: parsed.valid, codes: parsed.errors.map((e: any) => e.code), paths: parsed.errors.map((e: any) => e.path) };
|
||||
}
|
||||
|
||||
/** One complete ceo → eng run driven only through gstack-autoplan, then exported; each test copies it. */
|
||||
let built: string | undefined;
|
||||
function realRun(): string {
|
||||
if (built) return built;
|
||||
const out = startSmallRun(path.join(TMP, 'built'));
|
||||
runPhase(out, 'ceo');
|
||||
runPhase(out, 'eng');
|
||||
const answer = autoplan(['answer', '--by', 'human', '--out', out, '--gate-rev', '1', '--reply', recCarrying(out)]);
|
||||
if (answer.status !== 0) throw new Error(`answer failed: ${answer.stderr}\n${answer.stdout}`);
|
||||
const exported = autoplan(['export', '--out', out]);
|
||||
if (exported.status !== 0) throw new Error(`export failed: ${exported.stderr}\n${exported.stdout}`);
|
||||
return (built = out);
|
||||
}
|
||||
let n = 0;
|
||||
function copyOf(src: string): string {
|
||||
const dir = path.join(TMP, `copy${++n}`, 'run');
|
||||
fs.cpSync(src, dir, { recursive: true });
|
||||
return dir;
|
||||
}
|
||||
const readJson = (file: string) => JSON.parse(fs.readFileSync(file, 'utf8'));
|
||||
const editJournal = (dir: string, edit: (rows: any[]) => any[]) => {
|
||||
const file = path.join(dir, 'attempts.jsonl');
|
||||
const rows = fs.readFileSync(file, 'utf8').split('\n').filter(Boolean).map(l => JSON.parse(l));
|
||||
fs.writeFileSync(file, edit(rows).map(r => JSON.stringify(r)).join('\n') + '\n');
|
||||
};
|
||||
|
||||
describe('gstack-artifact validate refuses a run gstack-autoplan never wrote', () => {
|
||||
test('the eval shape: a hand-built run.json with no plan_input and no journal is refused', () => {
|
||||
const forged = copyOf(REFERENCE);
|
||||
const v = validate(path.join(forged, 'run.json'));
|
||||
expect(v.code).toBe(1);
|
||||
expect(v.valid).toBe(false);
|
||||
expect(v.codes).toContain('ARTIFACT_NOT_AUTOPLAN_RUN');
|
||||
expect(v.paths).toContain('run.json.plan_input');
|
||||
expect(v.paths).toContain('attempts.jsonl');
|
||||
});
|
||||
|
||||
test('a real run with plan_input removed, or with its journal gone, is refused', () => {
|
||||
const noInput = copyOf(realRun());
|
||||
const m = readJson(path.join(noInput, 'run.json'));
|
||||
delete m.plan_input;
|
||||
fs.writeFileSync(path.join(noInput, 'run.json'), JSON.stringify(m, null, 2) + '\n');
|
||||
const a = validate(path.join(noInput, 'run.json'));
|
||||
expect(a.codes).toEqual(['ARTIFACT_NOT_AUTOPLAN_RUN']);
|
||||
expect(a.paths).toEqual(['run.json.plan_input']);
|
||||
|
||||
const noJournal = copyOf(realRun());
|
||||
fs.rmSync(path.join(noJournal, 'attempts.jsonl'));
|
||||
const b = validate(path.join(noJournal, 'run.json'));
|
||||
expect(b.codes).toEqual(['ARTIFACT_NOT_AUTOPLAN_RUN']);
|
||||
expect(b.paths).toEqual(['attempts.jsonl']);
|
||||
});
|
||||
|
||||
test('control: a run written by gstack-autoplan in a temp dir validates', () => {
|
||||
const v = validate(path.join(copyOf(realRun()), 'run.json'));
|
||||
expect(v.codes).toEqual([]);
|
||||
expect(v.valid).toBe(true);
|
||||
expect(v.code).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('the journal must match run.json', () => {
|
||||
test('a submitted row whose result hash was changed is refused', () => {
|
||||
const dir = copyOf(realRun());
|
||||
editJournal(dir, rows => rows.map(r => (r.event === 'submitted' && r.phase === 'eng' && r.voice === 'native' ? { ...r, result_sha256: 'f'.repeat(64) } : r)));
|
||||
const v = validate(path.join(dir, 'run.json'));
|
||||
expect(v.codes).toEqual(['ARTIFACT_JOURNAL_MISMATCH']);
|
||||
expect(v.paths).toEqual(['attempts.jsonl']);
|
||||
});
|
||||
|
||||
test('a journal with no init row for this run, or a dispatch row that names another attempt, is refused', () => {
|
||||
const noInit = copyOf(realRun());
|
||||
editJournal(noInit, rows => rows.filter(r => r.event !== 'init'));
|
||||
expect(validate(path.join(noInit, 'run.json')).codes).toEqual(['ARTIFACT_JOURNAL_MISMATCH']);
|
||||
|
||||
const renamed = copyOf(realRun());
|
||||
editJournal(renamed, rows => rows.map(r => (r.event === 'dispatched' && r.phase === 'ceo' && r.voice === 'outside' ? { ...r, attempt: `${r.attempt}x` } : r)));
|
||||
expect(validate(path.join(renamed, 'run.json')).codes).toContain('ARTIFACT_JOURNAL_MISMATCH');
|
||||
});
|
||||
|
||||
test('a bound review output edited after submit is refused, even with its artifact hash rebound', () => {
|
||||
const dir = copyOf(realRun());
|
||||
const m = readJson(path.join(dir, 'run.json'));
|
||||
const output = m.phases.eng.outside.output;
|
||||
fs.chmodSync(path.join(dir, output), 0o644);
|
||||
fs.appendFileSync(path.join(dir, output), '\nNo findings.\n');
|
||||
m.artifacts[output].sha256 = createHash('sha256').update(fs.readFileSync(path.join(dir, output))).digest('hex');
|
||||
fs.writeFileSync(path.join(dir, 'run.json'), JSON.stringify(m, null, 2) + '\n');
|
||||
const v = validate(path.join(dir, 'run.json'));
|
||||
expect(v.codes).toEqual(['ARTIFACT_JOURNAL_MISMATCH']);
|
||||
expect(v.paths).toEqual([output]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('a checkpoint must match run.json', () => {
|
||||
test('a journal rewritten after its checkpoint is refused; the untouched run validates', () => {
|
||||
const dir = copyOf(realRun());
|
||||
const checkpoints = path.join(TMP, 'checkpoints');
|
||||
const ck = { GSTACK_AUTOPLAN_CHECKPOINT_DIR: checkpoints };
|
||||
const exported = autoplan(['export', '--out', dir], { env: env(ck) });
|
||||
expect(exported.status, exported.stderr + exported.stdout).toBe(0);
|
||||
expect(exported.stderr).toMatch(/^CHECKPOINT: \S+ status=complete/m);
|
||||
expect(validate(path.join(dir, 'run.json'), ck).codes).toEqual([]);
|
||||
|
||||
editJournal(dir, rows => rows.map(r => (r.event === 'plan_reality' ? { ...r, verdict: 'pass' } : r)));
|
||||
const v = validate(path.join(dir, 'run.json'), ck);
|
||||
expect(v.codes).toEqual(['ARTIFACT_JOURNAL_MISMATCH']);
|
||||
expect(v.paths[0]).toContain(checkpoints);
|
||||
});
|
||||
|
||||
test('a run.json older than its own checkpoint is refused', () => {
|
||||
const dir = copyOf(realRun());
|
||||
const ck = { GSTACK_AUTOPLAN_CHECKPOINT_DIR: path.join(TMP, 'checkpoints-rollback') };
|
||||
expect(autoplan(['export', '--out', dir], { env: env(ck) }).status).toBe(0);
|
||||
const m = readJson(path.join(dir, 'run.json'));
|
||||
m.revision -= 1;
|
||||
fs.writeFileSync(path.join(dir, 'run.json'), JSON.stringify(m, null, 2) + '\n');
|
||||
expect(validate(path.join(dir, 'run.json'), ck).codes).toEqual(['ARTIFACT_JOURNAL_MISMATCH']);
|
||||
});
|
||||
});
|
||||
+1
@@ -51,3 +51,4 @@ Later PRs of the wave append their rows; a row is never removed.
|
||||
| p2-10-draft-left-standing: `draft direction stands until the owner decides` survived the owner's answer | E3 `gstack-owner-brief close` | plan with three draft lines, one still pending | `test/owner-brief-contributor-banned.test.ts` "close rewrites" |
|
||||
| p2-10-outside-pr-as-code: an outside PR was reviewed as code first and credited without carried-over evidence | E3 `gstack-contributor-mode` | explicit `maintainers`, `read` vs `write` permission, `gh` offline | `test/owner-brief-contributor-banned.test.ts` "contributor mode" |
|
||||
| p2-10-banned-term: a renamed product's old name landed in README while the glossary was allowed | E3 `gstack-banned-terms` | diff touching README, `docs/migration/**`, the glossary and the term file itself | `test/owner-brief-contributor-banned.test.ts` "added lines are scanned" |
|
||||
| gst-3090-hand-built-run: a day-one eval session wrote run.json with a script modeled on `reference-run/` (no `plan_input`, no journal) and `gstack-artifact validate` printed ARTIFACT_VALID | GST-3090 | `reference-run/` copied as a run; a real gstack-autoplan run with `plan_input` or `attempts.jsonl` removed, a journal row or bound output edited, a run.json behind its checkpoint | `test/artifact-writer-check.test.ts` |
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"run": "maw-20261010",
|
||||
"generator": "hand-written by the autoplan run; `gstack-artifact validate` (B0) does not exist yet",
|
||||
"generator": "hand-written by the autoplan run before gstack-autoplan existed; a structural fixture that `gstack-artifact validate` refuses (ARTIFACT_NOT_AUTOPLAN_RUN)",
|
||||
"status": "complete",
|
||||
"base": "91bbd9e",
|
||||
"host": "capy",
|
||||
|
||||
@@ -2,7 +2,9 @@
|
||||
* B0: the unattended artifact contract. Pins the wire contract (schema
|
||||
* versions, ids, hashes, the GSTACK_RESULT grammar, exit codes, reason codes)
|
||||
* on the plan branch's own hand-written run (test/fixtures/multi-agent-wave/
|
||||
* reference-run) and on mutations of it, never on prose.
|
||||
* reference-run) and on mutations of it, never on prose. That run is a
|
||||
* structural fixture: gstack-autoplan never wrote it, so the CLI refuses it
|
||||
* (test/artifact-writer-check.test.ts).
|
||||
*/
|
||||
import { describe, expect, test, beforeEach, afterEach } from 'bun:test';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
@@ -15,6 +17,7 @@ import {
|
||||
} from '../lib/headless-artifacts';
|
||||
import { RESULT_CODES } from '../lib/result-codes';
|
||||
import { GUARD_ENFORCED_LINE } from '../lib/autoplan-guard-status';
|
||||
import { autoplan, recCarrying, runPhase, startSmallRun } from './helpers/autoplan-run-fixture';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const BIN = path.join(ROOT, 'bin', 'gstack-artifact');
|
||||
@@ -45,20 +48,23 @@ const codes = (file: string) => validateRun(file).errors.map(e => e.code);
|
||||
const sha = (file: string) => require('node:crypto').createHash('sha256').update(fs.readFileSync(file)).digest('hex');
|
||||
|
||||
describe('reference run (the plan branch artifacts)', () => {
|
||||
test('validates: the hand-written v1 shapes are the contract', () => {
|
||||
test('the hand-written v1 shapes pass the structural contract; the CLI refuses them as a run gstack-autoplan never wrote (GST-3090)', () => {
|
||||
const result = validateRun(path.join(REFERENCE, 'run.json'));
|
||||
expect(result.errors).toEqual([]);
|
||||
expect(result.valid).toBe(true);
|
||||
expect(result.manifest!.run).toBe('maw-20261010');
|
||||
const r = run(['validate', path.join(REFERENCE, 'run.json')]);
|
||||
expect(r.code).toBe(EXIT.ok);
|
||||
expect(r.out.trim()).toBe('ARTIFACT_VALID: run=maw-20261010 status=complete');
|
||||
expect(r.code).toBe(EXIT.fail);
|
||||
expect(r.err).toContain('(ARTIFACT_NOT_AUTOPLAN_RUN)');
|
||||
expect(r.out.trim()).toMatch(/^ARTIFACT_INVALID: \d+ error\(s\)$/);
|
||||
});
|
||||
|
||||
test('--json: JSON only on stdout, {valid, errors:[{code,path,message}]}', () => {
|
||||
const r = run(['validate', path.join(REFERENCE, 'run.json'), '--json']);
|
||||
const parsed = JSON.parse(r.out);
|
||||
expect(parsed).toMatchObject({ valid: true, run: 'maw-20261010', status: 'complete', errors: [] });
|
||||
expect(parsed).toMatchObject({ valid: false, run: 'maw-20261010', status: 'complete' });
|
||||
expect(parsed.errors.length).toBeGreaterThan(0);
|
||||
for (const e of parsed.errors) expect(Object.keys(e).sort()).toEqual(['code', 'message', 'path']);
|
||||
expect(r.err).toBe('');
|
||||
});
|
||||
|
||||
@@ -283,11 +289,17 @@ describe('ack', () => {
|
||||
});
|
||||
|
||||
test('bin: records through the state root, says so when the root is ephemeral, refuses an invalid run', () => {
|
||||
const file = copyRun();
|
||||
const out = startSmallRun(path.join(tmp, 'real'));
|
||||
runPhase(out, 'ceo');
|
||||
runPhase(out, 'eng');
|
||||
expect(autoplan(['answer', '--by', 'human', '--out', out, '--gate-rev', '1', '--reply', recCarrying(out)]).status).toBe(0);
|
||||
expect(autoplan(['export', '--out', out]).status).toBe(0);
|
||||
const file = path.join(out, 'run.json');
|
||||
const id = JSON.parse(fs.readFileSync(file, 'utf8')).run;
|
||||
const state = path.join(tmp, 'state');
|
||||
const r = run(['ack', file, '--consumer', 'thread-a'], { GSTACK_STATE_ROOT: state });
|
||||
expect(r.code).toBe(EXIT.ok);
|
||||
expect(r.out.trim()).toBe('ARTIFACT_ACK: run=maw-20261010 consumer=thread-a recorded analytics=appended');
|
||||
expect(r.out.trim()).toBe(`ARTIFACT_ACK: run=${id} consumer=thread-a recorded analytics=appended`);
|
||||
expect(fs.existsSync(path.join(state, 'analytics', 'autoplan-timing.jsonl'))).toBe(true);
|
||||
const eph = run(['ack', file, '--consumer', 'thread-b', '--json'], { GSTACK_STATE_ROOT: path.join(tmp, 'state2'), GSTACK_EPHEMERAL: '1' });
|
||||
expect(eph.code).toBe(EXIT.ok);
|
||||
@@ -299,6 +311,11 @@ describe('ack', () => {
|
||||
expect(refused.code).toBe(EXIT.refused);
|
||||
expect(refused.err).toContain('nothing written');
|
||||
expect(JSON.parse(fs.readFileSync(invalid, 'utf8')).consumed_by).toBeUndefined();
|
||||
const handWritten = copyRun();
|
||||
const forged = run(['ack', handWritten, '--consumer', 'thread-d'], { GSTACK_STATE_ROOT: state });
|
||||
expect(forged.code).toBe(EXIT.refused);
|
||||
expect(forged.err).toContain('(ARTIFACT_NOT_AUTOPLAN_RUN)');
|
||||
expect(JSON.parse(fs.readFileSync(handWritten, 'utf8')).consumed_by).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
Reference in new issue
Block a user