mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-09 14:38:59 +02:00
feat(evidence): verification-evidence ledger mechanizes /ship's IRON LAW
New bin/gstack-evidence: a transparent wrapper that records every verification
run as {ts, label, command, cmd_sha256, exit, duration_s, commit, tree, dirty,
wtree, log_path} in ~/.gstack/projects/<slug>/<branch>-evidence.jsonl, plus a
read-only `check` that grades FRESH/STALE/MISSING per label. "Tests passed"
now binds to the exact working-tree content it ran on (bin/gstack-wtree
fingerprint), so evidence recorded on uncommitted code stays FRESH after the
exact tested content is committed — the /ship Step 5 -> Step 16 case — while
an untracked new source file or any content change invalidates it.
Check semantics: every named label's latest record must be green, within
--max-age, matching --expect-cmd's hash when given, and fingerprint-identical
(or diff confined to --allow-paths — mechanizing Step 16's existing "CHANGELOG
edits don't count" carve-out). No --any mode: a green lane can never mask a
red sibling. Any git failure inside check (gc'd tree object, not a repo)
degrades to STALE/MISSING, never an error into the calling skill flow.
Transparency invariant (load-bearing, test-pinned): the child's exit code is
ALWAYS the wrapper's exit code; ledger/log/redact failures are stderr
warnings. Logs are per-run (0600, exclusive-open, 2MB truncation marker,
30-day opportunistic prune) — no more shared /tmp collisions between
concurrent ships. Command strings are redact-scanned before recording (HIGH
credential -> stored redacted). Machine-local by design: neither ledger nor
logs brain-sync.
Wired: ship Step 5 lanes run wrapped (per-lane labels), ship Step 16 and
land-and-deploy 3.5b check the ledger first and cite FRESH evidence instead of
re-running; a failed CHECK never blocks (run live), a failed RUN does.
test/evidence.test.ts: 21 tests incl. the keystone dirty-record -> commit ->
FRESH case.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
8e4a4f7c3d
commit
4836f0d1e3
Executable
+396
@@ -0,0 +1,396 @@
|
||||
#!/usr/bin/env bun
|
||||
/**
|
||||
* gstack-evidence — verification-evidence ledger: the mechanical arm of /ship's
|
||||
* IRON LAW ("no completion claims without fresh verification evidence").
|
||||
*
|
||||
* gstack-evidence run --label <L> -- <cmd...>
|
||||
* gstack-evidence check [--label <L> [--expect-cmd <exact string>]]... | --all
|
||||
* [--max-age <hours>] [--allow-paths <csv>]
|
||||
*
|
||||
* `run` is a TRANSPARENT wrapper: it streams the child's output through
|
||||
* unchanged, tees it to a 0600 log (2MB cap with a truncation marker), and
|
||||
* appends {ts, label, command, cmd_sha256, exit, duration_s, commit, tree,
|
||||
* dirty, wtree, log_path} to ~/.gstack/projects/<slug>/<branch>-evidence.jsonl.
|
||||
*
|
||||
* TRANSPARENCY INVARIANT (load-bearing): the child's exit code is ALWAYS the
|
||||
* wrapper's exit code. Every bookkeeping failure — ledger append, log dir,
|
||||
* non-git context, redact scan — is a stderr warning, never a failure. The
|
||||
* wrapper must never turn green tests red.
|
||||
*
|
||||
* Freshness binds to `wtree`, the working-tree content fingerprint from
|
||||
* bin/gstack-wtree: evidence recorded on uncommitted code stays FRESH after
|
||||
* the exact tested content is committed, and an untracked new source file
|
||||
* invalidates it. `cmd_sha256` = sha256 of the exact command string, no
|
||||
* normalization — the same convention as bin/gstack-verify-gate (which hashes
|
||||
* for TRUST; this ledger hashes for FRESHNESS).
|
||||
*
|
||||
* MACHINE-LOCAL by design: neither the ledger nor the logs are brain-synced.
|
||||
* A synced record citing an unsynced log would grade FRESH on a machine where
|
||||
* the log doesn't exist.
|
||||
*
|
||||
* `check` is read-only and never throws into the calling skill flow: any git
|
||||
* failure (gc'd stored tree, not a repo) degrades to STALE/MISSING. Call sites
|
||||
* must name expected labels explicitly — `--all` checks only labels that exist
|
||||
* in the ledger; it cannot prove that an expected lane ever ran.
|
||||
*/
|
||||
|
||||
import { mkdirSync, openSync, writeSync, closeSync, readdirSync, statSync, unlinkSync, chmodSync } from "fs";
|
||||
import { join, dirname } from "path";
|
||||
import { spawnSync } from "child_process";
|
||||
import { appendJsonl, readJsonl } from "../lib/jsonl-store";
|
||||
import { resolveSlug } from "../lib/bin-context";
|
||||
import { scan, applyRedactions } from "../lib/redact-engine";
|
||||
|
||||
const BIN_DIR = dirname(Bun.fileURLToPath(import.meta.url));
|
||||
const LOG_MAX_BYTES = 2 * 1024 * 1024;
|
||||
const LOG_PRUNE_DAYS = 30;
|
||||
|
||||
interface EvidenceRecord {
|
||||
ts: string;
|
||||
label: string;
|
||||
command: string;
|
||||
cmd_sha256: string;
|
||||
exit: number;
|
||||
duration_s: number;
|
||||
commit?: string;
|
||||
tree?: string;
|
||||
dirty?: boolean;
|
||||
wtree?: string;
|
||||
log_path?: string;
|
||||
redacted?: boolean;
|
||||
}
|
||||
|
||||
function warn(msg: string): void {
|
||||
console.error(`gstack-evidence: warning: ${msg}`);
|
||||
}
|
||||
|
||||
function sha256(text: string): string {
|
||||
const h = new Bun.CryptoHasher("sha256");
|
||||
h.update(text);
|
||||
return h.digest("hex");
|
||||
}
|
||||
|
||||
function git(args: string[]): string | undefined {
|
||||
try {
|
||||
const r = spawnSync("git", args, { encoding: "utf-8", timeout: 15000 });
|
||||
if (r.status !== 0) return undefined;
|
||||
const out = (r.stdout || "").trim();
|
||||
return out || undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function currentWtree(): string | undefined {
|
||||
try {
|
||||
const r = spawnSync(join(BIN_DIR, "gstack-wtree"), { encoding: "utf-8", timeout: 30000 });
|
||||
if (r.status !== 0) return undefined;
|
||||
const out = (r.stdout || "").trim();
|
||||
return /^[0-9a-f]{40}$/.test(out) ? out : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function ledgerPath(): { dir: string; file: string; logsDir: string } {
|
||||
const home = process.env.GSTACK_HOME || join(process.env.HOME || "~", ".gstack");
|
||||
const slug = resolveSlug(join(BIN_DIR, "gstack-slug"));
|
||||
// Same branch→filename sanitization as reviews.jsonl (gstack-slug's BRANCH).
|
||||
const slugOut = spawnSync(join(BIN_DIR, "gstack-slug"), { encoding: "utf-8" });
|
||||
const bm = (slugOut.stdout || "").match(/^BRANCH=(.+)$/m);
|
||||
const branch = bm ? bm[1].trim() : "no-branch";
|
||||
const dir = join(home, "projects", slug);
|
||||
return { dir, file: join(dir, `${branch}-evidence.jsonl`), logsDir: join(dir, "logs") };
|
||||
}
|
||||
|
||||
/** Redact-engine pass over the command string. HIGH finding → store redacted. */
|
||||
function safeCommandForRecord(command: string): { command: string; redacted: boolean } {
|
||||
try {
|
||||
const { findings } = scan(command);
|
||||
const high = findings.filter((f) => f.tier === "HIGH");
|
||||
if (high.length === 0) return { command, redacted: false };
|
||||
const redactedBody = applyRedactions(command, findings.map((f) => f.id)).body;
|
||||
const still = scan(redactedBody).findings.some((f) => f.tier === "HIGH");
|
||||
return { command: still ? "<redacted: HIGH credential in command>" : redactedBody, redacted: true };
|
||||
} catch {
|
||||
return { command, redacted: false };
|
||||
}
|
||||
}
|
||||
|
||||
/** Opportunistic prune of logs older than LOG_PRUNE_DAYS. Best-effort. */
|
||||
function pruneOldLogs(logsDir: string): void {
|
||||
try {
|
||||
const cutoff = Date.now() - LOG_PRUNE_DAYS * 24 * 3600 * 1000;
|
||||
for (const name of readdirSync(logsDir)) {
|
||||
const p = join(logsDir, name);
|
||||
try {
|
||||
if (statSync(p).mtimeMs < cutoff) unlinkSync(p);
|
||||
} catch {}
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
|
||||
/** Exclusive-open a collision-safe log file. Returns undefined on failure. */
|
||||
function openLog(logsDir: string, label: string, cmdSha: string): { fd: number; path: string } | undefined {
|
||||
try {
|
||||
mkdirSync(logsDir, { recursive: true });
|
||||
pruneOldLogs(logsDir);
|
||||
const ts = new Date().toISOString().replace(/[:.]/g, "-");
|
||||
const base = `${ts}-${label}-${process.pid}-${cmdSha.slice(0, 8)}`;
|
||||
for (let i = 0; i < 3; i++) {
|
||||
const p = join(logsDir, i === 0 ? `${base}.log` : `${base}-${i}.log`);
|
||||
try {
|
||||
const fd = openSync(p, "ax", 0o600);
|
||||
return { fd, path: p };
|
||||
} catch {}
|
||||
}
|
||||
} catch (e: any) {
|
||||
warn(`log setup failed (${e?.message ?? e}) — running unlogged`);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
async function cmdRun(argv: string[]): Promise<number> {
|
||||
let label = "default";
|
||||
const li = argv.indexOf("--label");
|
||||
const sep = argv.indexOf("--");
|
||||
if (li >= 0 && li + 1 < argv.length && (sep < 0 || li < sep)) label = argv[li + 1];
|
||||
if (sep < 0 || sep + 1 >= argv.length) {
|
||||
console.error("usage: gstack-evidence run --label <L> -- <cmd...>");
|
||||
return 2;
|
||||
}
|
||||
const cmdArgv = argv.slice(sep + 1);
|
||||
// Compound/piped commands pass as ONE string via bash -c; a multi-token argv
|
||||
// runs directly. The hashed command string is exact, no normalization.
|
||||
const commandString = cmdArgv.length === 1 ? cmdArgv[0] : cmdArgv.join(" ");
|
||||
const spawnArgv = cmdArgv.length === 1 ? ["bash", "-c", cmdArgv[0]] : cmdArgv;
|
||||
const cmdSha = sha256(commandString);
|
||||
label = label.replace(/[^a-zA-Z0-9._-]/g, "_");
|
||||
|
||||
// Bookkeeping context — every piece is optional; failures only warn.
|
||||
let paths: ReturnType<typeof ledgerPath> | undefined;
|
||||
try {
|
||||
paths = ledgerPath();
|
||||
mkdirSync(paths.dir, { recursive: true });
|
||||
} catch (e: any) {
|
||||
warn(`ledger setup failed (${e?.message ?? e}) — result will not be recorded`);
|
||||
}
|
||||
const log = paths ? openLog(paths.logsDir, label, cmdSha) : undefined;
|
||||
|
||||
const started = Date.now();
|
||||
let exitCode: number;
|
||||
let proc: ReturnType<typeof Bun.spawn> | undefined;
|
||||
try {
|
||||
proc = Bun.spawn(spawnArgv, { stdin: "inherit", stdout: "pipe", stderr: "pipe" });
|
||||
} catch (e: any) {
|
||||
// Spawn failure (ENOENT on argv-direct form): record exit 127, propagate 127.
|
||||
exitCode = 127;
|
||||
warn(`spawn failed: ${e?.message ?? e}`);
|
||||
record(paths, log?.path, label, commandString, cmdSha, exitCode, started);
|
||||
return exitCode;
|
||||
}
|
||||
|
||||
// Stream-tee: forward chunks as they arrive (never buffer — E2E logs are MBs).
|
||||
let logBytes = 0;
|
||||
let truncated = false;
|
||||
const teeToLog = (chunk: Uint8Array) => {
|
||||
if (!log || truncated) return;
|
||||
try {
|
||||
if (logBytes + chunk.byteLength > LOG_MAX_BYTES) {
|
||||
const room = LOG_MAX_BYTES - logBytes;
|
||||
if (room > 0) writeSync(log.fd, chunk.subarray(0, room));
|
||||
writeSync(log.fd, Buffer.from("\n\n[gstack-evidence: log truncated at 2MB — output continued on console]\n"));
|
||||
truncated = true;
|
||||
} else {
|
||||
writeSync(log.fd, chunk);
|
||||
logBytes += chunk.byteLength;
|
||||
}
|
||||
} catch {
|
||||
truncated = true; // stop teeing on any write failure; console stream continues
|
||||
}
|
||||
};
|
||||
const pump = async (stream: ReadableStream<Uint8Array> | undefined, out: NodeJS.WriteStream) => {
|
||||
if (!stream) return;
|
||||
for await (const chunk of stream) {
|
||||
out.write(chunk);
|
||||
teeToLog(chunk);
|
||||
}
|
||||
};
|
||||
try {
|
||||
await Promise.all([pump(proc.stdout as any, process.stdout), pump(proc.stderr as any, process.stderr)]);
|
||||
exitCode = await proc.exited;
|
||||
if (exitCode === null || exitCode === undefined) exitCode = 1;
|
||||
} catch (e: any) {
|
||||
warn(`stream error: ${e?.message ?? e}`);
|
||||
try {
|
||||
exitCode = await proc.exited;
|
||||
} catch {
|
||||
exitCode = 1;
|
||||
}
|
||||
} finally {
|
||||
if (log) {
|
||||
try {
|
||||
closeSync(log.fd);
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
|
||||
record(paths, log?.path, label, commandString, cmdSha, exitCode, started);
|
||||
return exitCode;
|
||||
}
|
||||
|
||||
function record(
|
||||
paths: { dir: string; file: string } | undefined,
|
||||
logPath: string | undefined,
|
||||
label: string,
|
||||
commandString: string,
|
||||
cmdSha: string,
|
||||
exitCode: number,
|
||||
startedMs: number,
|
||||
): void {
|
||||
if (!paths) return;
|
||||
try {
|
||||
const { command, redacted } = safeCommandForRecord(commandString);
|
||||
const rec: EvidenceRecord = {
|
||||
ts: new Date().toISOString(),
|
||||
label,
|
||||
command,
|
||||
cmd_sha256: cmdSha,
|
||||
exit: exitCode,
|
||||
duration_s: Math.round((Date.now() - startedMs) / 100) / 10,
|
||||
};
|
||||
if (redacted) rec.redacted = true;
|
||||
const commit = git(["rev-parse", "HEAD"]);
|
||||
if (commit) {
|
||||
rec.commit = commit;
|
||||
rec.tree = git(["rev-parse", "HEAD^{tree}"]);
|
||||
rec.dirty = (git(["status", "--porcelain", "-uno"]) ?? "") !== "";
|
||||
rec.wtree = currentWtree();
|
||||
}
|
||||
if (logPath) rec.log_path = logPath;
|
||||
appendJsonl(paths.file, rec, { mode: 0o600 });
|
||||
try {
|
||||
chmodSync(paths.file, 0o600);
|
||||
} catch {}
|
||||
// Summary line on stderr so calling agents get the exit + log path even
|
||||
// when the lane ran backgrounded. Never on stdout (stays transparent).
|
||||
console.error(`gstack-evidence: recorded label=${label} exit=${exitCode} log=${logPath ?? "-"}`);
|
||||
} catch (e: any) {
|
||||
warn(`ledger append failed (${e?.message ?? e}) — the command result stands`);
|
||||
}
|
||||
}
|
||||
|
||||
function cmdCheck(argv: string[]): number {
|
||||
// Parse: repeated --label, each optionally followed (anywhere later) by its
|
||||
// own --expect-cmd; pairing is positional — an --expect-cmd binds to the most
|
||||
// recent --label before it.
|
||||
const wanted: { label: string; expectCmd?: string }[] = [];
|
||||
let all = false;
|
||||
let maxAgeHours: number | undefined;
|
||||
let allowPaths: string[] = [];
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i];
|
||||
if (a === "--label") wanted.push({ label: argv[++i] ?? "" });
|
||||
else if (a === "--expect-cmd") {
|
||||
if (wanted.length === 0) {
|
||||
console.error("gstack-evidence: --expect-cmd requires a preceding --label");
|
||||
return 2;
|
||||
}
|
||||
wanted[wanted.length - 1].expectCmd = argv[++i] ?? "";
|
||||
} else if (a === "--all") all = true;
|
||||
else if (a === "--max-age") maxAgeHours = Number(argv[++i]);
|
||||
else if (a === "--allow-paths") allowPaths = (argv[++i] ?? "").split(",").map((s) => s.trim()).filter(Boolean);
|
||||
}
|
||||
if (!all && wanted.length === 0) {
|
||||
console.error("usage: gstack-evidence check [--label <L> [--expect-cmd <s>]]... | --all [--max-age <hrs>] [--allow-paths <csv>]");
|
||||
return 2;
|
||||
}
|
||||
|
||||
let records: EvidenceRecord[] = [];
|
||||
try {
|
||||
records = readJsonl<EvidenceRecord>(ledgerPath().file);
|
||||
} catch {
|
||||
records = [];
|
||||
}
|
||||
|
||||
const labels = all
|
||||
? [...new Set(records.map((r) => r.label))].map((label) => ({ label, expectCmd: undefined as string | undefined }))
|
||||
: wanted;
|
||||
if (all && labels.length === 0) {
|
||||
console.log("EVIDENCE: MISSING (ledger empty — no labels recorded)");
|
||||
return 1;
|
||||
}
|
||||
|
||||
const wtreeNow = currentWtree();
|
||||
let allFresh = true;
|
||||
for (const { label, expectCmd } of labels) {
|
||||
const latest = [...records].reverse().find((r) => r.label === label);
|
||||
if (!latest) {
|
||||
console.log(`EVIDENCE: MISSING label=${label}`);
|
||||
allFresh = false;
|
||||
continue;
|
||||
}
|
||||
const detail = `label=${label} exit=${latest.exit} ts=${latest.ts}${latest.log_path ? ` log=${latest.log_path}` : ""}`;
|
||||
let verdict: "FRESH" | "STALE" = "FRESH";
|
||||
let reason = "";
|
||||
if (latest.exit !== 0) {
|
||||
verdict = "STALE";
|
||||
reason = "recorded run failed";
|
||||
} else if (maxAgeHours !== undefined && Number.isFinite(maxAgeHours)) {
|
||||
const ageMs = Date.now() - Date.parse(latest.ts);
|
||||
if (!(ageMs >= 0 && ageMs <= maxAgeHours * 3600 * 1000)) {
|
||||
verdict = "STALE";
|
||||
reason = `older than ${maxAgeHours}h`;
|
||||
}
|
||||
}
|
||||
if (verdict === "FRESH" && expectCmd !== undefined && sha256(expectCmd) !== latest.cmd_sha256) {
|
||||
verdict = "STALE";
|
||||
reason = "command changed (cmd_sha256 mismatch)";
|
||||
}
|
||||
if (verdict === "FRESH") {
|
||||
// Content binding: identical working-tree fingerprint, or a diff confined
|
||||
// to the allow-list. Any git failure (gc'd tree, not a repo) → STALE —
|
||||
// never an error into the calling flow.
|
||||
if (!latest.wtree || !wtreeNow) {
|
||||
verdict = "STALE";
|
||||
reason = !latest.wtree ? "record has no content fingerprint" : "current fingerprint unavailable";
|
||||
} else if (latest.wtree !== wtreeNow) {
|
||||
const diff = git(["diff", "--name-only", latest.wtree, wtreeNow]);
|
||||
if (diff === undefined) {
|
||||
verdict = "STALE";
|
||||
reason = "content changed (fingerprint diff unavailable)";
|
||||
} else {
|
||||
const changed = diff.split("\n").map((s) => s.trim()).filter(Boolean);
|
||||
const outside = changed.filter((f) => !allowPaths.some((a) => f === a || f.startsWith(a.replace(/\/$/, "") + "/")));
|
||||
if (changed.length === 0 || outside.length === 0) {
|
||||
reason = changed.length ? `diff confined to allow-paths (${changed.length} file(s))` : "";
|
||||
} else {
|
||||
verdict = "STALE";
|
||||
reason = `content changed: ${outside.slice(0, 5).join(", ")}${outside.length > 5 ? ", ..." : ""}`;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
console.log(`EVIDENCE: ${verdict} ${detail}${reason ? ` reason=${reason}` : ""}`);
|
||||
if (verdict !== "FRESH") allFresh = false;
|
||||
}
|
||||
return allFresh ? 0 : 1;
|
||||
}
|
||||
|
||||
const [, , sub, ...rest] = process.argv;
|
||||
try {
|
||||
if (sub === "run") {
|
||||
process.exit(await cmdRun(rest));
|
||||
} else if (sub === "check") {
|
||||
process.exit(cmdCheck(rest));
|
||||
} else {
|
||||
console.error("usage: gstack-evidence run|check ...");
|
||||
process.exit(2);
|
||||
}
|
||||
} catch (e: any) {
|
||||
// Never let the wrapper's own failure look like a command failure in a way
|
||||
// that breaks a skill flow: `run` propagates the child's code from inside
|
||||
// cmdRun; reaching here means bookkeeping blew up outside it.
|
||||
warn(`unexpected error: ${e?.message ?? e}`);
|
||||
process.exit(sub === "check" ? 1 : 1);
|
||||
}
|
||||
@@ -1369,16 +1369,29 @@ and tell the user: "I found and fixed a few issues during the review. The fixes
|
||||
|
||||
### 3.5b: Test results
|
||||
|
||||
**Free tests — run them now:**
|
||||
**Free tests — cite fresh evidence or run them now:**
|
||||
|
||||
Read CLAUDE.md to find the project's test command. If not specified, use `bun test`.
|
||||
Run the test command and capture the exit code and output.
|
||||
Check the evidence ledger first:
|
||||
|
||||
```bash
|
||||
bun test 2>&1 | tail -10
|
||||
~/.claude/skills/gstack/bin/gstack-evidence check --label tests --max-age 24
|
||||
```
|
||||
|
||||
If tests fail: **BLOCKER.** Cannot merge with failing tests.
|
||||
If it prints FRESH (exit 0), a green run is on record for THIS exact
|
||||
working-tree content (fingerprint-bound, so a rebase or an identical-content
|
||||
commit doesn't invalidate it) — cite the evidence line (exit, ts, log path)
|
||||
instead of re-running.
|
||||
|
||||
Otherwise (STALE/MISSING, or you want a live run anyway): read CLAUDE.md to
|
||||
find the project's test command (default `bun test`) and run it wrapped, so
|
||||
the fresh result is recorded:
|
||||
|
||||
```bash
|
||||
~/.claude/skills/gstack/bin/gstack-evidence run --label tests -- 'bun test 2>&1'
|
||||
```
|
||||
|
||||
If tests fail: **BLOCKER.** Cannot merge with failing tests. (A failed evidence
|
||||
CHECK is never a blocker — it just means run live; a failed RUN is.)
|
||||
|
||||
**E2E tests — check recent results:**
|
||||
|
||||
|
||||
@@ -465,16 +465,29 @@ and tell the user: "I found and fixed a few issues during the review. The fixes
|
||||
|
||||
### 3.5b: Test results
|
||||
|
||||
**Free tests — run them now:**
|
||||
**Free tests — cite fresh evidence or run them now:**
|
||||
|
||||
Read CLAUDE.md to find the project's test command. If not specified, use `bun test`.
|
||||
Run the test command and capture the exit code and output.
|
||||
Check the evidence ledger first:
|
||||
|
||||
```bash
|
||||
bun test 2>&1 | tail -10
|
||||
~/.claude/skills/gstack/bin/gstack-evidence check --label tests --max-age 24
|
||||
```
|
||||
|
||||
If tests fail: **BLOCKER.** Cannot merge with failing tests.
|
||||
If it prints FRESH (exit 0), a green run is on record for THIS exact
|
||||
working-tree content (fingerprint-bound, so a rebase or an identical-content
|
||||
commit doesn't invalidate it) — cite the evidence line (exit, ts, log path)
|
||||
instead of re-running.
|
||||
|
||||
Otherwise (STALE/MISSING, or you want a live run anyway): read CLAUDE.md to
|
||||
find the project's test command (default `bun test`) and run it wrapped, so
|
||||
the fresh result is recorded:
|
||||
|
||||
```bash
|
||||
~/.claude/skills/gstack/bin/gstack-evidence run --label tests -- 'bun test 2>&1'
|
||||
```
|
||||
|
||||
If tests fail: **BLOCKER.** Cannot merge with failing tests. (A failed evidence
|
||||
CHECK is never a blocker — it just means run live; a failed RUN is.)
|
||||
|
||||
**E2E tests — check recent results:**
|
||||
|
||||
|
||||
+16
-1
@@ -1280,9 +1280,24 @@ EOF
|
||||
|
||||
**IRON LAW: NO COMPLETION CLAIMS WITHOUT FRESH VERIFICATION EVIDENCE.**
|
||||
|
||||
The evidence ledger is the mechanical arm of this law. Check it FIRST:
|
||||
|
||||
```bash
|
||||
~/.claude/skills/gstack/bin/gstack-evidence check --label tests --label vitest --max-age 24 --allow-paths CHANGELOG.md,VERSION,package.json
|
||||
```
|
||||
|
||||
- **Every line FRESH (exit 0):** the recorded runs were green and the working-tree
|
||||
content is identical to what was tested, modulo the allow-listed release files
|
||||
(this mechanizes the "CHANGELOG edits don't count" rule — VERSION/CHANGELOG
|
||||
commits between Step 5 and here don't invalidate the run). Cite the evidence
|
||||
lines (label, exit, ts, log path) as the verification evidence and continue.
|
||||
- **Any STALE/MISSING (exit non-zero):** run live, wrapped, so the fresh run is
|
||||
recorded: `~/.claude/skills/gstack/bin/gstack-evidence run --label <lane> -- '<command>'`.
|
||||
The check is an advisory guardrail — a failed CHECK never blocks; a failed RUN does.
|
||||
|
||||
Before pushing, re-verify if code changed during Steps 4-6:
|
||||
|
||||
1. **Test verification:** If ANY code changed after Step 5's test run (fixes from review findings, CHANGELOG edits don't count), re-run the test suite. Paste fresh output. Stale output from Step 5 is NOT acceptable.
|
||||
1. **Test verification:** If ANY code changed after Step 5's test run (fixes from review findings, CHANGELOG edits don't count), re-run the test suite. The evidence check above IS this rule, mechanized — trust FRESH, re-run on STALE. Paste fresh output when you re-run. Stale output from Step 5 with changed content is NOT acceptable.
|
||||
|
||||
2. **Build verification:** If the project has a build step, run it. Paste output.
|
||||
|
||||
|
||||
+16
-1
@@ -375,9 +375,24 @@ EOF
|
||||
|
||||
**IRON LAW: NO COMPLETION CLAIMS WITHOUT FRESH VERIFICATION EVIDENCE.**
|
||||
|
||||
The evidence ledger is the mechanical arm of this law. Check it FIRST:
|
||||
|
||||
```bash
|
||||
~/.claude/skills/gstack/bin/gstack-evidence check --label tests --label vitest --max-age 24 --allow-paths CHANGELOG.md,VERSION,package.json
|
||||
```
|
||||
|
||||
- **Every line FRESH (exit 0):** the recorded runs were green and the working-tree
|
||||
content is identical to what was tested, modulo the allow-listed release files
|
||||
(this mechanizes the "CHANGELOG edits don't count" rule — VERSION/CHANGELOG
|
||||
commits between Step 5 and here don't invalidate the run). Cite the evidence
|
||||
lines (label, exit, ts, log path) as the verification evidence and continue.
|
||||
- **Any STALE/MISSING (exit non-zero):** run live, wrapped, so the fresh run is
|
||||
recorded: `~/.claude/skills/gstack/bin/gstack-evidence run --label <lane> -- '<command>'`.
|
||||
The check is an advisory guardrail — a failed CHECK never blocks; a failed RUN does.
|
||||
|
||||
Before pushing, re-verify if code changed during Steps 4-6:
|
||||
|
||||
1. **Test verification:** If ANY code changed after Step 5's test run (fixes from review findings, CHANGELOG edits don't count), re-run the test suite. Paste fresh output. Stale output from Step 5 is NOT acceptable.
|
||||
1. **Test verification:** If ANY code changed after Step 5's test run (fixes from review findings, CHANGELOG edits don't count), re-run the test suite. The evidence check above IS this rule, mechanized — trust FRESH, re-run on STALE. Paste fresh output when you re-run. Stale output from Step 5 with changed content is NOT acceptable.
|
||||
|
||||
2. **Build verification:** If the project has a build step, run it. Paste output.
|
||||
|
||||
|
||||
+11
-4
@@ -195,15 +195,22 @@ Only commit if there are changes. Stage all bootstrap files (config, test direct
|
||||
`db:test:prepare` internally, which loads the schema into the correct lane database.
|
||||
Running bare test migrations without INSTANCE hits an orphan DB and corrupts structure.sql.
|
||||
|
||||
Run both test suites in parallel:
|
||||
Run both test suites in parallel, each wrapped in the evidence ledger. The
|
||||
wrapper is transparent (streams output live, exit code passes through) and
|
||||
records `{command, exit, working-tree fingerprint, log path}` to
|
||||
`~/.gstack/projects/<slug>/<branch>-evidence.jsonl` — Step 16 cites this
|
||||
record instead of re-running when the content hasn't changed:
|
||||
|
||||
```bash
|
||||
bin/test-lane 2>&1 | tee /tmp/ship_tests.txt &
|
||||
npm run test 2>&1 | tee /tmp/ship_vitest.txt &
|
||||
~/.claude/skills/gstack/bin/gstack-evidence run --label tests -- 'bin/test-lane 2>&1' &
|
||||
~/.claude/skills/gstack/bin/gstack-evidence run --label vitest -- 'npm run test 2>&1' &
|
||||
wait
|
||||
```
|
||||
|
||||
After both complete, read the output files and check pass/fail.
|
||||
After both complete, check the `gstack-evidence: recorded label=... exit=...
|
||||
log=...` summary lines — each carries the lane's exit code and a per-run log
|
||||
file (no shared /tmp collisions between concurrent ships). Read the log files
|
||||
for failure detail.
|
||||
|
||||
**If any test fails:** Do NOT immediately stop. Apply the Test Failure Ownership Triage:
|
||||
|
||||
|
||||
@@ -10,15 +10,22 @@
|
||||
`db:test:prepare` internally, which loads the schema into the correct lane database.
|
||||
Running bare test migrations without INSTANCE hits an orphan DB and corrupts structure.sql.
|
||||
|
||||
Run both test suites in parallel:
|
||||
Run both test suites in parallel, each wrapped in the evidence ledger. The
|
||||
wrapper is transparent (streams output live, exit code passes through) and
|
||||
records `{command, exit, working-tree fingerprint, log path}` to
|
||||
`~/.gstack/projects/<slug>/<branch>-evidence.jsonl` — Step 16 cites this
|
||||
record instead of re-running when the content hasn't changed:
|
||||
|
||||
```bash
|
||||
bin/test-lane 2>&1 | tee /tmp/ship_tests.txt &
|
||||
npm run test 2>&1 | tee /tmp/ship_vitest.txt &
|
||||
~/.claude/skills/gstack/bin/gstack-evidence run --label tests -- 'bin/test-lane 2>&1' &
|
||||
~/.claude/skills/gstack/bin/gstack-evidence run --label vitest -- 'npm run test 2>&1' &
|
||||
wait
|
||||
```
|
||||
|
||||
After both complete, read the output files and check pass/fail.
|
||||
After both complete, check the `gstack-evidence: recorded label=... exit=...
|
||||
log=...` summary lines — each carries the lane's exit code and a per-run log
|
||||
file (no shared /tmp collisions between concurrent ships). Read the log files
|
||||
for failure detail.
|
||||
|
||||
**If any test fails:** Do NOT immediately stop. Apply the Test Failure Ownership Triage:
|
||||
|
||||
|
||||
@@ -0,0 +1,286 @@
|
||||
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
|
||||
import { execSync, spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const EVIDENCE = path.join(ROOT, 'bin', 'gstack-evidence');
|
||||
|
||||
let gstackHome: string;
|
||||
let repoDir: string;
|
||||
|
||||
function git(args: string) {
|
||||
execSync(`git -c user.email=t@test -c user.name=t ${args}`, { cwd: repoDir, encoding: 'utf-8', timeout: 10000 });
|
||||
}
|
||||
|
||||
function run(args: string[], opts: { cwd?: string } = {}): { status: number; stdout: string; stderr: string } {
|
||||
const r = spawnSync(EVIDENCE, args, {
|
||||
cwd: opts.cwd ?? repoDir,
|
||||
env: { ...process.env, GSTACK_HOME: gstackHome },
|
||||
encoding: 'utf-8',
|
||||
timeout: 60000,
|
||||
maxBuffer: 16 * 1024 * 1024, // the truncation test streams 3MB through the wrapper
|
||||
});
|
||||
return { status: r.status ?? 1, stdout: r.stdout ?? '', stderr: r.stderr ?? '' };
|
||||
}
|
||||
|
||||
function ledgerFile(): string {
|
||||
const found: string[] = [];
|
||||
const walk = (d: string) => {
|
||||
if (!fs.existsSync(d)) return;
|
||||
for (const e of fs.readdirSync(d, { withFileTypes: true })) {
|
||||
const p = path.join(d, e.name);
|
||||
if (e.isDirectory()) walk(p);
|
||||
else if (e.name.endsWith('-evidence.jsonl')) found.push(p);
|
||||
}
|
||||
};
|
||||
walk(path.join(gstackHome, 'projects'));
|
||||
expect(found.length).toBeGreaterThan(0);
|
||||
return found[0];
|
||||
}
|
||||
|
||||
function records(): any[] {
|
||||
return fs
|
||||
.readFileSync(ledgerFile(), 'utf-8')
|
||||
.trim()
|
||||
.split('\n')
|
||||
.map((l) => JSON.parse(l));
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
gstackHome = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-evidence-home-'));
|
||||
repoDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-evidence-repo-'));
|
||||
git('init -q -b main');
|
||||
fs.writeFileSync(path.join(repoDir, 'src.txt'), 'v1\n');
|
||||
fs.writeFileSync(path.join(repoDir, '.gitignore'), 'scratch.txt\n');
|
||||
git('add src.txt .gitignore');
|
||||
git('commit -q -m init');
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
fs.rmSync(gstackHome, { recursive: true, force: true });
|
||||
fs.rmSync(repoDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('gstack-evidence run', () => {
|
||||
test('records a complete evidence record and propagates exit 0', () => {
|
||||
const r = run(['run', '--label', 'tests', '--', 'echo ok']);
|
||||
expect(r.status).toBe(0);
|
||||
expect(r.stdout).toContain('ok');
|
||||
expect(r.stderr).toContain('recorded label=tests exit=0');
|
||||
const rec = records().pop();
|
||||
expect(rec.label).toBe('tests');
|
||||
expect(rec.command).toBe('echo ok');
|
||||
expect(rec.cmd_sha256).toMatch(/^[0-9a-f]{64}$/);
|
||||
expect(rec.exit).toBe(0);
|
||||
expect(typeof rec.duration_s).toBe('number');
|
||||
expect(rec.commit).toMatch(/^[0-9a-f]{40}$/);
|
||||
expect(rec.tree).toMatch(/^[0-9a-f]{40}$/);
|
||||
expect(rec.wtree).toMatch(/^[0-9a-f]{40}$/);
|
||||
expect(typeof rec.dirty).toBe('boolean');
|
||||
expect(fs.existsSync(rec.log_path)).toBe(true);
|
||||
expect(fs.readFileSync(rec.log_path, 'utf-8')).toContain('ok');
|
||||
});
|
||||
|
||||
test('propagates a failing exit code and records it', () => {
|
||||
const r = run(['run', '--label', 'tests', '--', 'exit 3']);
|
||||
expect(r.status).toBe(3);
|
||||
expect(records().pop().exit).toBe(3);
|
||||
});
|
||||
|
||||
test('spawn failure (ENOENT, argv-direct form) records and propagates 127', () => {
|
||||
const r = run(['run', '--label', 'tests', '--', '/nonexistent-gstack-binary', 'arg']);
|
||||
expect(r.status).toBe(127);
|
||||
expect(records().pop().exit).toBe(127);
|
||||
});
|
||||
|
||||
test('TRANSPARENCY: ledger failure never breaks the command (append-failure injection)', () => {
|
||||
// Point GSTACK_HOME somewhere mkdir cannot succeed.
|
||||
const r = spawnSync(EVIDENCE, ['run', '--label', 'tests', '--', 'echo still-ran'], {
|
||||
cwd: repoDir,
|
||||
env: { ...process.env, GSTACK_HOME: '/dev/null/nope' },
|
||||
encoding: 'utf-8',
|
||||
timeout: 60000,
|
||||
});
|
||||
expect(r.status).toBe(0);
|
||||
expect(r.stdout).toContain('still-ran');
|
||||
expect(r.stderr).toContain('warning');
|
||||
});
|
||||
|
||||
test('ledger and log files are 0600', () => {
|
||||
run(['run', '--label', 'tests', '--', 'echo ok']);
|
||||
const rec = records().pop();
|
||||
expect(fs.statSync(ledgerFile()).mode & 0o777).toBe(0o600);
|
||||
expect(fs.statSync(rec.log_path).mode & 0o777).toBe(0o600);
|
||||
});
|
||||
|
||||
test('two rapid runs get distinct log files (collision-safe exclusive open)', () => {
|
||||
run(['run', '--label', 'tests', '--', 'echo one']);
|
||||
run(['run', '--label', 'tests', '--', 'echo two']);
|
||||
const [a, b] = records().slice(-2);
|
||||
expect(a.log_path).not.toBe(b.log_path);
|
||||
});
|
||||
|
||||
test('log truncates at 2MB with a marker; exit code unaffected', () => {
|
||||
const r = run(['run', '--label', 'big', '--', 'head -c 3000000 /dev/zero | tr "\\0" a']);
|
||||
expect(r.status).toBe(0);
|
||||
const rec = records().pop();
|
||||
const size = fs.statSync(rec.log_path).size;
|
||||
expect(size).toBeLessThanOrEqual(2 * 1024 * 1024 + 200);
|
||||
expect(fs.readFileSync(rec.log_path, 'utf-8')).toContain('log truncated at 2MB');
|
||||
});
|
||||
|
||||
test('logs older than 30 days are pruned opportunistically', () => {
|
||||
run(['run', '--label', 'tests', '--', 'echo ok']);
|
||||
const logsDir = path.dirname(records().pop().log_path);
|
||||
const oldLog = path.join(logsDir, 'ancient.log');
|
||||
fs.writeFileSync(oldLog, 'old');
|
||||
const past = new Date(Date.now() - 40 * 24 * 3600 * 1000);
|
||||
fs.utimesSync(oldLog, past, past);
|
||||
run(['run', '--label', 'tests', '--', 'echo again']);
|
||||
expect(fs.existsSync(oldLog)).toBe(false);
|
||||
});
|
||||
|
||||
test('works as a backgrounded job (ship Step 5 lanes run with & wait)', () => {
|
||||
execSync(`bash -c '"${EVIDENCE}" run --label bg -- "echo backgrounded" & wait'`, {
|
||||
cwd: repoDir,
|
||||
env: { ...process.env, GSTACK_HOME: gstackHome },
|
||||
encoding: 'utf-8',
|
||||
timeout: 60000,
|
||||
});
|
||||
const rec = records().pop();
|
||||
expect(rec.label).toBe('bg');
|
||||
expect(rec.exit).toBe(0);
|
||||
});
|
||||
|
||||
test('a HIGH credential in the command is stored redacted', () => {
|
||||
const r = run(['run', '--label', 'sec', '--', 'echo ghp_A8bC2dE4fG6hI8jK0lM2nO4pQ6rS8tU0vW2x deploy']);
|
||||
expect(r.status).toBe(0);
|
||||
const rec = records().pop();
|
||||
expect(rec.command).not.toContain('ghp_A8bC2dE4fG6hI8jK0lM2nO4pQ6rS8tU0vW2x');
|
||||
expect(rec.redacted).toBe(true);
|
||||
// The hash still binds to the ORIGINAL exact string (freshness key).
|
||||
expect(rec.cmd_sha256).toMatch(/^[0-9a-f]{64}$/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('gstack-evidence check', () => {
|
||||
test('KEYSTONE: evidence recorded on a dirty tree stays FRESH after committing the exact tested content', () => {
|
||||
// Dirty the tree (this is /ship Step 5: tests run on uncommitted code).
|
||||
fs.writeFileSync(path.join(repoDir, 'src.txt'), 'v2-tested\n');
|
||||
expect(run(['run', '--label', 'tests', '--', 'echo green']).status).toBe(0);
|
||||
expect(records().pop().dirty).toBe(true);
|
||||
|
||||
// Step 15: commit the exact same content. HEAD tree changes; working-tree
|
||||
// content does not.
|
||||
git('commit -q -am ship');
|
||||
|
||||
const chk = run(['check', '--label', 'tests']);
|
||||
expect(chk.status).toBe(0);
|
||||
expect(chk.stdout).toContain('EVIDENCE: FRESH');
|
||||
});
|
||||
|
||||
test('a content change after the run grades STALE', () => {
|
||||
expect(run(['run', '--label', 'tests', '--', 'echo green']).status).toBe(0);
|
||||
fs.writeFileSync(path.join(repoDir, 'src.txt'), 'changed-after-tests\n');
|
||||
const chk = run(['check', '--label', 'tests']);
|
||||
expect(chk.status).toBe(1);
|
||||
expect(chk.stdout).toContain('EVIDENCE: STALE');
|
||||
});
|
||||
|
||||
test('an untracked NEW source file grades STALE; gitignored scratch stays FRESH', () => {
|
||||
expect(run(['run', '--label', 'tests', '--', 'echo green']).status).toBe(0);
|
||||
|
||||
fs.writeFileSync(path.join(repoDir, 'scratch.txt'), 'conductor noise\n');
|
||||
expect(run(['check', '--label', 'tests']).status).toBe(0);
|
||||
|
||||
fs.writeFileSync(path.join(repoDir, 'brand-new.ts'), 'export {}\n');
|
||||
const chk = run(['check', '--label', 'tests']);
|
||||
expect(chk.status).toBe(1);
|
||||
expect(chk.stdout).toContain('STALE');
|
||||
});
|
||||
|
||||
test('allow-paths carve-out: a CHANGELOG-only change stays FRESH with --allow-paths', () => {
|
||||
expect(run(['run', '--label', 'tests', '--', 'echo green']).status).toBe(0);
|
||||
fs.writeFileSync(path.join(repoDir, 'CHANGELOG.md'), '## v1\n');
|
||||
git('add CHANGELOG.md');
|
||||
git('commit -q -m changelog');
|
||||
|
||||
const without = run(['check', '--label', 'tests']);
|
||||
expect(without.status).toBe(1);
|
||||
|
||||
const withAllow = run(['check', '--label', 'tests', '--allow-paths', 'CHANGELOG.md,VERSION,package.json']);
|
||||
expect(withAllow.status).toBe(0);
|
||||
expect(withAllow.stdout).toContain('FRESH');
|
||||
|
||||
// A source change is NOT rescued by the allow-list.
|
||||
fs.writeFileSync(path.join(repoDir, 'src.txt'), 'v3\n');
|
||||
expect(run(['check', '--label', 'tests', '--allow-paths', 'CHANGELOG.md']).status).toBe(1);
|
||||
});
|
||||
|
||||
test('--expect-cmd binds the label to the exact command string', () => {
|
||||
expect(run(['run', '--label', 'tests', '--', 'echo green']).status).toBe(0);
|
||||
expect(run(['check', '--label', 'tests', '--expect-cmd', 'echo green']).status).toBe(0);
|
||||
const mismatch = run(['check', '--label', 'tests', '--expect-cmd', 'echo cheaper-command']);
|
||||
expect(mismatch.status).toBe(1);
|
||||
expect(mismatch.stdout).toContain('cmd_sha256 mismatch');
|
||||
});
|
||||
|
||||
test('a recorded FAILING run is never FRESH', () => {
|
||||
run(['run', '--label', 'tests', '--', 'exit 1']);
|
||||
const chk = run(['check', '--label', 'tests']);
|
||||
expect(chk.status).toBe(1);
|
||||
expect(chk.stdout).toContain('recorded run failed');
|
||||
});
|
||||
|
||||
test('--max-age expires old records', () => {
|
||||
expect(run(['run', '--label', 'tests', '--', 'echo green']).status).toBe(0);
|
||||
const file = ledgerFile();
|
||||
const rec = JSON.parse(fs.readFileSync(file, 'utf-8').trim());
|
||||
rec.ts = new Date(Date.now() - 48 * 3600 * 1000).toISOString();
|
||||
fs.writeFileSync(file, JSON.stringify(rec) + '\n');
|
||||
const chk = run(['check', '--label', 'tests', '--max-age', '24']);
|
||||
expect(chk.status).toBe(1);
|
||||
expect(chk.stdout).toContain('older than 24h');
|
||||
});
|
||||
|
||||
test('a gc-d / fabricated stored fingerprint degrades to STALE, never a crash', () => {
|
||||
expect(run(['run', '--label', 'tests', '--', 'echo green']).status).toBe(0);
|
||||
const file = ledgerFile();
|
||||
const rec = JSON.parse(fs.readFileSync(file, 'utf-8').trim());
|
||||
rec.wtree = 'deadbeefdeadbeefdeadbeefdeadbeefdeadbeef';
|
||||
fs.writeFileSync(file, JSON.stringify(rec) + '\n');
|
||||
const chk = run(['check', '--label', 'tests']);
|
||||
expect(chk.status).toBe(1);
|
||||
expect(chk.stdout).toContain('STALE');
|
||||
});
|
||||
|
||||
test('a green lane never masks a red sibling: every named label must be FRESH', () => {
|
||||
expect(run(['run', '--label', 'tests', '--', 'echo green']).status).toBe(0);
|
||||
run(['run', '--label', 'vitest', '--', 'exit 1']);
|
||||
const chk = run(['check', '--label', 'tests', '--label', 'vitest']);
|
||||
expect(chk.status).toBe(1);
|
||||
expect(chk.stdout).toContain('EVIDENCE: FRESH label=tests');
|
||||
expect(chk.stdout).toContain('EVIDENCE: STALE label=vitest');
|
||||
});
|
||||
|
||||
test('MISSING for a label that never ran (explicit labels prove expected lanes)', () => {
|
||||
expect(run(['run', '--label', 'tests', '--', 'echo green']).status).toBe(0);
|
||||
const chk = run(['check', '--label', 'tests', '--label', 'never-ran']);
|
||||
expect(chk.status).toBe(1);
|
||||
expect(chk.stdout).toContain('MISSING label=never-ran');
|
||||
});
|
||||
|
||||
test('check never errors outside a git repo — degrades to STALE', () => {
|
||||
expect(run(['run', '--label', 'tests', '--', 'echo green']).status).toBe(0);
|
||||
const nonGit = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-evidence-nongit-'));
|
||||
try {
|
||||
const chk = run(['check', '--label', 'tests'], { cwd: nonGit });
|
||||
expect([0, 1]).toContain(chk.status); // different slug → MISSING; the point is: no crash
|
||||
expect(chk.status).toBe(1);
|
||||
} finally {
|
||||
fs.rmSync(nonGit, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user