diff --git a/bin/gstack-evidence b/bin/gstack-evidence new file mode 100755 index 000000000..5bff726a4 --- /dev/null +++ b/bin/gstack-evidence @@ -0,0 +1,396 @@ +#!/usr/bin/env bun +/** + * gstack-evidence — verification-evidence ledger: the mechanical arm of /ship's + * IRON LAW ("no completion claims without fresh verification evidence"). + * + * gstack-evidence run --label -- + * gstack-evidence check [--label [--expect-cmd ]]... | --all + * [--max-age ] [--allow-paths ] + * + * `run` is a TRANSPARENT wrapper: it streams the child's output through + * unchanged, tees it to a 0600 log (2MB cap with a truncation marker), and + * appends {ts, label, command, cmd_sha256, exit, duration_s, commit, tree, + * dirty, wtree, log_path} to ~/.gstack/projects//-evidence.jsonl. + * + * TRANSPARENCY INVARIANT (load-bearing): the child's exit code is ALWAYS the + * wrapper's exit code. Every bookkeeping failure — ledger append, log dir, + * non-git context, redact scan — is a stderr warning, never a failure. The + * wrapper must never turn green tests red. + * + * Freshness binds to `wtree`, the working-tree content fingerprint from + * bin/gstack-wtree: evidence recorded on uncommitted code stays FRESH after + * the exact tested content is committed, and an untracked new source file + * invalidates it. `cmd_sha256` = sha256 of the exact command string, no + * normalization — the same convention as bin/gstack-verify-gate (which hashes + * for TRUST; this ledger hashes for FRESHNESS). + * + * MACHINE-LOCAL by design: neither the ledger nor the logs are brain-synced. + * A synced record citing an unsynced log would grade FRESH on a machine where + * the log doesn't exist. + * + * `check` is read-only and never throws into the calling skill flow: any git + * failure (gc'd stored tree, not a repo) degrades to STALE/MISSING. Call sites + * must name expected labels explicitly — `--all` checks only labels that exist + * in the ledger; it cannot prove that an expected lane ever ran. + */ + +import { mkdirSync, openSync, writeSync, closeSync, readdirSync, statSync, unlinkSync, chmodSync } from "fs"; +import { join, dirname } from "path"; +import { spawnSync } from "child_process"; +import { appendJsonl, readJsonl } from "../lib/jsonl-store"; +import { resolveSlug } from "../lib/bin-context"; +import { scan, applyRedactions } from "../lib/redact-engine"; + +const BIN_DIR = dirname(Bun.fileURLToPath(import.meta.url)); +const LOG_MAX_BYTES = 2 * 1024 * 1024; +const LOG_PRUNE_DAYS = 30; + +interface EvidenceRecord { + ts: string; + label: string; + command: string; + cmd_sha256: string; + exit: number; + duration_s: number; + commit?: string; + tree?: string; + dirty?: boolean; + wtree?: string; + log_path?: string; + redacted?: boolean; +} + +function warn(msg: string): void { + console.error(`gstack-evidence: warning: ${msg}`); +} + +function sha256(text: string): string { + const h = new Bun.CryptoHasher("sha256"); + h.update(text); + return h.digest("hex"); +} + +function git(args: string[]): string | undefined { + try { + const r = spawnSync("git", args, { encoding: "utf-8", timeout: 15000 }); + if (r.status !== 0) return undefined; + const out = (r.stdout || "").trim(); + return out || undefined; + } catch { + return undefined; + } +} + +function currentWtree(): string | undefined { + try { + const r = spawnSync(join(BIN_DIR, "gstack-wtree"), { encoding: "utf-8", timeout: 30000 }); + if (r.status !== 0) return undefined; + const out = (r.stdout || "").trim(); + return /^[0-9a-f]{40}$/.test(out) ? out : undefined; + } catch { + return undefined; + } +} + +function ledgerPath(): { dir: string; file: string; logsDir: string } { + const home = process.env.GSTACK_HOME || join(process.env.HOME || "~", ".gstack"); + const slug = resolveSlug(join(BIN_DIR, "gstack-slug")); + // Same branch→filename sanitization as reviews.jsonl (gstack-slug's BRANCH). + const slugOut = spawnSync(join(BIN_DIR, "gstack-slug"), { encoding: "utf-8" }); + const bm = (slugOut.stdout || "").match(/^BRANCH=(.+)$/m); + const branch = bm ? bm[1].trim() : "no-branch"; + const dir = join(home, "projects", slug); + return { dir, file: join(dir, `${branch}-evidence.jsonl`), logsDir: join(dir, "logs") }; +} + +/** Redact-engine pass over the command string. HIGH finding → store redacted. */ +function safeCommandForRecord(command: string): { command: string; redacted: boolean } { + try { + const { findings } = scan(command); + const high = findings.filter((f) => f.tier === "HIGH"); + if (high.length === 0) return { command, redacted: false }; + const redactedBody = applyRedactions(command, findings.map((f) => f.id)).body; + const still = scan(redactedBody).findings.some((f) => f.tier === "HIGH"); + return { command: still ? "" : redactedBody, redacted: true }; + } catch { + return { command, redacted: false }; + } +} + +/** Opportunistic prune of logs older than LOG_PRUNE_DAYS. Best-effort. */ +function pruneOldLogs(logsDir: string): void { + try { + const cutoff = Date.now() - LOG_PRUNE_DAYS * 24 * 3600 * 1000; + for (const name of readdirSync(logsDir)) { + const p = join(logsDir, name); + try { + if (statSync(p).mtimeMs < cutoff) unlinkSync(p); + } catch {} + } + } catch {} +} + +/** Exclusive-open a collision-safe log file. Returns undefined on failure. */ +function openLog(logsDir: string, label: string, cmdSha: string): { fd: number; path: string } | undefined { + try { + mkdirSync(logsDir, { recursive: true }); + pruneOldLogs(logsDir); + const ts = new Date().toISOString().replace(/[:.]/g, "-"); + const base = `${ts}-${label}-${process.pid}-${cmdSha.slice(0, 8)}`; + for (let i = 0; i < 3; i++) { + const p = join(logsDir, i === 0 ? `${base}.log` : `${base}-${i}.log`); + try { + const fd = openSync(p, "ax", 0o600); + return { fd, path: p }; + } catch {} + } + } catch (e: any) { + warn(`log setup failed (${e?.message ?? e}) — running unlogged`); + } + return undefined; +} + +async function cmdRun(argv: string[]): Promise { + let label = "default"; + const li = argv.indexOf("--label"); + const sep = argv.indexOf("--"); + if (li >= 0 && li + 1 < argv.length && (sep < 0 || li < sep)) label = argv[li + 1]; + if (sep < 0 || sep + 1 >= argv.length) { + console.error("usage: gstack-evidence run --label -- "); + return 2; + } + const cmdArgv = argv.slice(sep + 1); + // Compound/piped commands pass as ONE string via bash -c; a multi-token argv + // runs directly. The hashed command string is exact, no normalization. + const commandString = cmdArgv.length === 1 ? cmdArgv[0] : cmdArgv.join(" "); + const spawnArgv = cmdArgv.length === 1 ? ["bash", "-c", cmdArgv[0]] : cmdArgv; + const cmdSha = sha256(commandString); + label = label.replace(/[^a-zA-Z0-9._-]/g, "_"); + + // Bookkeeping context — every piece is optional; failures only warn. + let paths: ReturnType | undefined; + try { + paths = ledgerPath(); + mkdirSync(paths.dir, { recursive: true }); + } catch (e: any) { + warn(`ledger setup failed (${e?.message ?? e}) — result will not be recorded`); + } + const log = paths ? openLog(paths.logsDir, label, cmdSha) : undefined; + + const started = Date.now(); + let exitCode: number; + let proc: ReturnType | undefined; + try { + proc = Bun.spawn(spawnArgv, { stdin: "inherit", stdout: "pipe", stderr: "pipe" }); + } catch (e: any) { + // Spawn failure (ENOENT on argv-direct form): record exit 127, propagate 127. + exitCode = 127; + warn(`spawn failed: ${e?.message ?? e}`); + record(paths, log?.path, label, commandString, cmdSha, exitCode, started); + return exitCode; + } + + // Stream-tee: forward chunks as they arrive (never buffer — E2E logs are MBs). + let logBytes = 0; + let truncated = false; + const teeToLog = (chunk: Uint8Array) => { + if (!log || truncated) return; + try { + if (logBytes + chunk.byteLength > LOG_MAX_BYTES) { + const room = LOG_MAX_BYTES - logBytes; + if (room > 0) writeSync(log.fd, chunk.subarray(0, room)); + writeSync(log.fd, Buffer.from("\n\n[gstack-evidence: log truncated at 2MB — output continued on console]\n")); + truncated = true; + } else { + writeSync(log.fd, chunk); + logBytes += chunk.byteLength; + } + } catch { + truncated = true; // stop teeing on any write failure; console stream continues + } + }; + const pump = async (stream: ReadableStream | undefined, out: NodeJS.WriteStream) => { + if (!stream) return; + for await (const chunk of stream) { + out.write(chunk); + teeToLog(chunk); + } + }; + try { + await Promise.all([pump(proc.stdout as any, process.stdout), pump(proc.stderr as any, process.stderr)]); + exitCode = await proc.exited; + if (exitCode === null || exitCode === undefined) exitCode = 1; + } catch (e: any) { + warn(`stream error: ${e?.message ?? e}`); + try { + exitCode = await proc.exited; + } catch { + exitCode = 1; + } + } finally { + if (log) { + try { + closeSync(log.fd); + } catch {} + } + } + + record(paths, log?.path, label, commandString, cmdSha, exitCode, started); + return exitCode; +} + +function record( + paths: { dir: string; file: string } | undefined, + logPath: string | undefined, + label: string, + commandString: string, + cmdSha: string, + exitCode: number, + startedMs: number, +): void { + if (!paths) return; + try { + const { command, redacted } = safeCommandForRecord(commandString); + const rec: EvidenceRecord = { + ts: new Date().toISOString(), + label, + command, + cmd_sha256: cmdSha, + exit: exitCode, + duration_s: Math.round((Date.now() - startedMs) / 100) / 10, + }; + if (redacted) rec.redacted = true; + const commit = git(["rev-parse", "HEAD"]); + if (commit) { + rec.commit = commit; + rec.tree = git(["rev-parse", "HEAD^{tree}"]); + rec.dirty = (git(["status", "--porcelain", "-uno"]) ?? "") !== ""; + rec.wtree = currentWtree(); + } + if (logPath) rec.log_path = logPath; + appendJsonl(paths.file, rec, { mode: 0o600 }); + try { + chmodSync(paths.file, 0o600); + } catch {} + // Summary line on stderr so calling agents get the exit + log path even + // when the lane ran backgrounded. Never on stdout (stays transparent). + console.error(`gstack-evidence: recorded label=${label} exit=${exitCode} log=${logPath ?? "-"}`); + } catch (e: any) { + warn(`ledger append failed (${e?.message ?? e}) — the command result stands`); + } +} + +function cmdCheck(argv: string[]): number { + // Parse: repeated --label, each optionally followed (anywhere later) by its + // own --expect-cmd; pairing is positional — an --expect-cmd binds to the most + // recent --label before it. + const wanted: { label: string; expectCmd?: string }[] = []; + let all = false; + let maxAgeHours: number | undefined; + let allowPaths: string[] = []; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a === "--label") wanted.push({ label: argv[++i] ?? "" }); + else if (a === "--expect-cmd") { + if (wanted.length === 0) { + console.error("gstack-evidence: --expect-cmd requires a preceding --label"); + return 2; + } + wanted[wanted.length - 1].expectCmd = argv[++i] ?? ""; + } else if (a === "--all") all = true; + else if (a === "--max-age") maxAgeHours = Number(argv[++i]); + else if (a === "--allow-paths") allowPaths = (argv[++i] ?? "").split(",").map((s) => s.trim()).filter(Boolean); + } + if (!all && wanted.length === 0) { + console.error("usage: gstack-evidence check [--label [--expect-cmd ]]... | --all [--max-age ] [--allow-paths ]"); + return 2; + } + + let records: EvidenceRecord[] = []; + try { + records = readJsonl(ledgerPath().file); + } catch { + records = []; + } + + const labels = all + ? [...new Set(records.map((r) => r.label))].map((label) => ({ label, expectCmd: undefined as string | undefined })) + : wanted; + if (all && labels.length === 0) { + console.log("EVIDENCE: MISSING (ledger empty — no labels recorded)"); + return 1; + } + + const wtreeNow = currentWtree(); + let allFresh = true; + for (const { label, expectCmd } of labels) { + const latest = [...records].reverse().find((r) => r.label === label); + if (!latest) { + console.log(`EVIDENCE: MISSING label=${label}`); + allFresh = false; + continue; + } + const detail = `label=${label} exit=${latest.exit} ts=${latest.ts}${latest.log_path ? ` log=${latest.log_path}` : ""}`; + let verdict: "FRESH" | "STALE" = "FRESH"; + let reason = ""; + if (latest.exit !== 0) { + verdict = "STALE"; + reason = "recorded run failed"; + } else if (maxAgeHours !== undefined && Number.isFinite(maxAgeHours)) { + const ageMs = Date.now() - Date.parse(latest.ts); + if (!(ageMs >= 0 && ageMs <= maxAgeHours * 3600 * 1000)) { + verdict = "STALE"; + reason = `older than ${maxAgeHours}h`; + } + } + if (verdict === "FRESH" && expectCmd !== undefined && sha256(expectCmd) !== latest.cmd_sha256) { + verdict = "STALE"; + reason = "command changed (cmd_sha256 mismatch)"; + } + if (verdict === "FRESH") { + // Content binding: identical working-tree fingerprint, or a diff confined + // to the allow-list. Any git failure (gc'd tree, not a repo) → STALE — + // never an error into the calling flow. + if (!latest.wtree || !wtreeNow) { + verdict = "STALE"; + reason = !latest.wtree ? "record has no content fingerprint" : "current fingerprint unavailable"; + } else if (latest.wtree !== wtreeNow) { + const diff = git(["diff", "--name-only", latest.wtree, wtreeNow]); + if (diff === undefined) { + verdict = "STALE"; + reason = "content changed (fingerprint diff unavailable)"; + } else { + const changed = diff.split("\n").map((s) => s.trim()).filter(Boolean); + const outside = changed.filter((f) => !allowPaths.some((a) => f === a || f.startsWith(a.replace(/\/$/, "") + "/"))); + if (changed.length === 0 || outside.length === 0) { + reason = changed.length ? `diff confined to allow-paths (${changed.length} file(s))` : ""; + } else { + verdict = "STALE"; + reason = `content changed: ${outside.slice(0, 5).join(", ")}${outside.length > 5 ? ", ..." : ""}`; + } + } + } + } + console.log(`EVIDENCE: ${verdict} ${detail}${reason ? ` reason=${reason}` : ""}`); + if (verdict !== "FRESH") allFresh = false; + } + return allFresh ? 0 : 1; +} + +const [, , sub, ...rest] = process.argv; +try { + if (sub === "run") { + process.exit(await cmdRun(rest)); + } else if (sub === "check") { + process.exit(cmdCheck(rest)); + } else { + console.error("usage: gstack-evidence run|check ..."); + process.exit(2); + } +} catch (e: any) { + // Never let the wrapper's own failure look like a command failure in a way + // that breaks a skill flow: `run` propagates the child's code from inside + // cmdRun; reaching here means bookkeeping blew up outside it. + warn(`unexpected error: ${e?.message ?? e}`); + process.exit(sub === "check" ? 1 : 1); +} diff --git a/land-and-deploy/SKILL.md b/land-and-deploy/SKILL.md index 444c07261..f4ac5b4ee 100644 --- a/land-and-deploy/SKILL.md +++ b/land-and-deploy/SKILL.md @@ -1369,16 +1369,29 @@ and tell the user: "I found and fixed a few issues during the review. The fixes ### 3.5b: Test results -**Free tests — run them now:** +**Free tests — cite fresh evidence or run them now:** -Read CLAUDE.md to find the project's test command. If not specified, use `bun test`. -Run the test command and capture the exit code and output. +Check the evidence ledger first: ```bash -bun test 2>&1 | tail -10 +~/.claude/skills/gstack/bin/gstack-evidence check --label tests --max-age 24 ``` -If tests fail: **BLOCKER.** Cannot merge with failing tests. +If it prints FRESH (exit 0), a green run is on record for THIS exact +working-tree content (fingerprint-bound, so a rebase or an identical-content +commit doesn't invalidate it) — cite the evidence line (exit, ts, log path) +instead of re-running. + +Otherwise (STALE/MISSING, or you want a live run anyway): read CLAUDE.md to +find the project's test command (default `bun test`) and run it wrapped, so +the fresh result is recorded: + +```bash +~/.claude/skills/gstack/bin/gstack-evidence run --label tests -- 'bun test 2>&1' +``` + +If tests fail: **BLOCKER.** Cannot merge with failing tests. (A failed evidence +CHECK is never a blocker — it just means run live; a failed RUN is.) **E2E tests — check recent results:** diff --git a/land-and-deploy/SKILL.md.tmpl b/land-and-deploy/SKILL.md.tmpl index 6a3f479ab..819b44247 100644 --- a/land-and-deploy/SKILL.md.tmpl +++ b/land-and-deploy/SKILL.md.tmpl @@ -465,16 +465,29 @@ and tell the user: "I found and fixed a few issues during the review. The fixes ### 3.5b: Test results -**Free tests — run them now:** +**Free tests — cite fresh evidence or run them now:** -Read CLAUDE.md to find the project's test command. If not specified, use `bun test`. -Run the test command and capture the exit code and output. +Check the evidence ledger first: ```bash -bun test 2>&1 | tail -10 +~/.claude/skills/gstack/bin/gstack-evidence check --label tests --max-age 24 ``` -If tests fail: **BLOCKER.** Cannot merge with failing tests. +If it prints FRESH (exit 0), a green run is on record for THIS exact +working-tree content (fingerprint-bound, so a rebase or an identical-content +commit doesn't invalidate it) — cite the evidence line (exit, ts, log path) +instead of re-running. + +Otherwise (STALE/MISSING, or you want a live run anyway): read CLAUDE.md to +find the project's test command (default `bun test`) and run it wrapped, so +the fresh result is recorded: + +```bash +~/.claude/skills/gstack/bin/gstack-evidence run --label tests -- 'bun test 2>&1' +``` + +If tests fail: **BLOCKER.** Cannot merge with failing tests. (A failed evidence +CHECK is never a blocker — it just means run live; a failed RUN is.) **E2E tests — check recent results:** diff --git a/ship/SKILL.md b/ship/SKILL.md index 94d4c714f..208cbddc9 100644 --- a/ship/SKILL.md +++ b/ship/SKILL.md @@ -1280,9 +1280,24 @@ EOF **IRON LAW: NO COMPLETION CLAIMS WITHOUT FRESH VERIFICATION EVIDENCE.** +The evidence ledger is the mechanical arm of this law. Check it FIRST: + +```bash +~/.claude/skills/gstack/bin/gstack-evidence check --label tests --label vitest --max-age 24 --allow-paths CHANGELOG.md,VERSION,package.json +``` + +- **Every line FRESH (exit 0):** the recorded runs were green and the working-tree + content is identical to what was tested, modulo the allow-listed release files + (this mechanizes the "CHANGELOG edits don't count" rule — VERSION/CHANGELOG + commits between Step 5 and here don't invalidate the run). Cite the evidence + lines (label, exit, ts, log path) as the verification evidence and continue. +- **Any STALE/MISSING (exit non-zero):** run live, wrapped, so the fresh run is + recorded: `~/.claude/skills/gstack/bin/gstack-evidence run --label -- ''`. + The check is an advisory guardrail — a failed CHECK never blocks; a failed RUN does. + Before pushing, re-verify if code changed during Steps 4-6: -1. **Test verification:** If ANY code changed after Step 5's test run (fixes from review findings, CHANGELOG edits don't count), re-run the test suite. Paste fresh output. Stale output from Step 5 is NOT acceptable. +1. **Test verification:** If ANY code changed after Step 5's test run (fixes from review findings, CHANGELOG edits don't count), re-run the test suite. The evidence check above IS this rule, mechanized — trust FRESH, re-run on STALE. Paste fresh output when you re-run. Stale output from Step 5 with changed content is NOT acceptable. 2. **Build verification:** If the project has a build step, run it. Paste output. diff --git a/ship/SKILL.md.tmpl b/ship/SKILL.md.tmpl index a01296688..4b2f459e4 100644 --- a/ship/SKILL.md.tmpl +++ b/ship/SKILL.md.tmpl @@ -375,9 +375,24 @@ EOF **IRON LAW: NO COMPLETION CLAIMS WITHOUT FRESH VERIFICATION EVIDENCE.** +The evidence ledger is the mechanical arm of this law. Check it FIRST: + +```bash +~/.claude/skills/gstack/bin/gstack-evidence check --label tests --label vitest --max-age 24 --allow-paths CHANGELOG.md,VERSION,package.json +``` + +- **Every line FRESH (exit 0):** the recorded runs were green and the working-tree + content is identical to what was tested, modulo the allow-listed release files + (this mechanizes the "CHANGELOG edits don't count" rule — VERSION/CHANGELOG + commits between Step 5 and here don't invalidate the run). Cite the evidence + lines (label, exit, ts, log path) as the verification evidence and continue. +- **Any STALE/MISSING (exit non-zero):** run live, wrapped, so the fresh run is + recorded: `~/.claude/skills/gstack/bin/gstack-evidence run --label -- ''`. + The check is an advisory guardrail — a failed CHECK never blocks; a failed RUN does. + Before pushing, re-verify if code changed during Steps 4-6: -1. **Test verification:** If ANY code changed after Step 5's test run (fixes from review findings, CHANGELOG edits don't count), re-run the test suite. Paste fresh output. Stale output from Step 5 is NOT acceptable. +1. **Test verification:** If ANY code changed after Step 5's test run (fixes from review findings, CHANGELOG edits don't count), re-run the test suite. The evidence check above IS this rule, mechanized — trust FRESH, re-run on STALE. Paste fresh output when you re-run. Stale output from Step 5 with changed content is NOT acceptable. 2. **Build verification:** If the project has a build step, run it. Paste output. diff --git a/ship/sections/tests.md b/ship/sections/tests.md index 2b93deab1..2ae1f2352 100644 --- a/ship/sections/tests.md +++ b/ship/sections/tests.md @@ -195,15 +195,22 @@ Only commit if there are changes. Stage all bootstrap files (config, test direct `db:test:prepare` internally, which loads the schema into the correct lane database. Running bare test migrations without INSTANCE hits an orphan DB and corrupts structure.sql. -Run both test suites in parallel: +Run both test suites in parallel, each wrapped in the evidence ledger. The +wrapper is transparent (streams output live, exit code passes through) and +records `{command, exit, working-tree fingerprint, log path}` to +`~/.gstack/projects//-evidence.jsonl` — Step 16 cites this +record instead of re-running when the content hasn't changed: ```bash -bin/test-lane 2>&1 | tee /tmp/ship_tests.txt & -npm run test 2>&1 | tee /tmp/ship_vitest.txt & +~/.claude/skills/gstack/bin/gstack-evidence run --label tests -- 'bin/test-lane 2>&1' & +~/.claude/skills/gstack/bin/gstack-evidence run --label vitest -- 'npm run test 2>&1' & wait ``` -After both complete, read the output files and check pass/fail. +After both complete, check the `gstack-evidence: recorded label=... exit=... +log=...` summary lines — each carries the lane's exit code and a per-run log +file (no shared /tmp collisions between concurrent ships). Read the log files +for failure detail. **If any test fails:** Do NOT immediately stop. Apply the Test Failure Ownership Triage: diff --git a/ship/sections/tests.md.tmpl b/ship/sections/tests.md.tmpl index e6d53d495..a8ffe8318 100644 --- a/ship/sections/tests.md.tmpl +++ b/ship/sections/tests.md.tmpl @@ -10,15 +10,22 @@ `db:test:prepare` internally, which loads the schema into the correct lane database. Running bare test migrations without INSTANCE hits an orphan DB and corrupts structure.sql. -Run both test suites in parallel: +Run both test suites in parallel, each wrapped in the evidence ledger. The +wrapper is transparent (streams output live, exit code passes through) and +records `{command, exit, working-tree fingerprint, log path}` to +`~/.gstack/projects//-evidence.jsonl` — Step 16 cites this +record instead of re-running when the content hasn't changed: ```bash -bin/test-lane 2>&1 | tee /tmp/ship_tests.txt & -npm run test 2>&1 | tee /tmp/ship_vitest.txt & +~/.claude/skills/gstack/bin/gstack-evidence run --label tests -- 'bin/test-lane 2>&1' & +~/.claude/skills/gstack/bin/gstack-evidence run --label vitest -- 'npm run test 2>&1' & wait ``` -After both complete, read the output files and check pass/fail. +After both complete, check the `gstack-evidence: recorded label=... exit=... +log=...` summary lines — each carries the lane's exit code and a per-run log +file (no shared /tmp collisions between concurrent ships). Read the log files +for failure detail. **If any test fails:** Do NOT immediately stop. Apply the Test Failure Ownership Triage: diff --git a/test/evidence.test.ts b/test/evidence.test.ts new file mode 100644 index 000000000..11a8d3b75 --- /dev/null +++ b/test/evidence.test.ts @@ -0,0 +1,286 @@ +import { describe, test, expect, beforeEach, afterEach } from 'bun:test'; +import { execSync, spawnSync } from 'child_process'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const EVIDENCE = path.join(ROOT, 'bin', 'gstack-evidence'); + +let gstackHome: string; +let repoDir: string; + +function git(args: string) { + execSync(`git -c user.email=t@test -c user.name=t ${args}`, { cwd: repoDir, encoding: 'utf-8', timeout: 10000 }); +} + +function run(args: string[], opts: { cwd?: string } = {}): { status: number; stdout: string; stderr: string } { + const r = spawnSync(EVIDENCE, args, { + cwd: opts.cwd ?? repoDir, + env: { ...process.env, GSTACK_HOME: gstackHome }, + encoding: 'utf-8', + timeout: 60000, + maxBuffer: 16 * 1024 * 1024, // the truncation test streams 3MB through the wrapper + }); + return { status: r.status ?? 1, stdout: r.stdout ?? '', stderr: r.stderr ?? '' }; +} + +function ledgerFile(): string { + const found: string[] = []; + const walk = (d: string) => { + if (!fs.existsSync(d)) return; + for (const e of fs.readdirSync(d, { withFileTypes: true })) { + const p = path.join(d, e.name); + if (e.isDirectory()) walk(p); + else if (e.name.endsWith('-evidence.jsonl')) found.push(p); + } + }; + walk(path.join(gstackHome, 'projects')); + expect(found.length).toBeGreaterThan(0); + return found[0]; +} + +function records(): any[] { + return fs + .readFileSync(ledgerFile(), 'utf-8') + .trim() + .split('\n') + .map((l) => JSON.parse(l)); +} + +beforeEach(() => { + gstackHome = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-evidence-home-')); + repoDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-evidence-repo-')); + git('init -q -b main'); + fs.writeFileSync(path.join(repoDir, 'src.txt'), 'v1\n'); + fs.writeFileSync(path.join(repoDir, '.gitignore'), 'scratch.txt\n'); + git('add src.txt .gitignore'); + git('commit -q -m init'); +}); + +afterEach(() => { + fs.rmSync(gstackHome, { recursive: true, force: true }); + fs.rmSync(repoDir, { recursive: true, force: true }); +}); + +describe('gstack-evidence run', () => { + test('records a complete evidence record and propagates exit 0', () => { + const r = run(['run', '--label', 'tests', '--', 'echo ok']); + expect(r.status).toBe(0); + expect(r.stdout).toContain('ok'); + expect(r.stderr).toContain('recorded label=tests exit=0'); + const rec = records().pop(); + expect(rec.label).toBe('tests'); + expect(rec.command).toBe('echo ok'); + expect(rec.cmd_sha256).toMatch(/^[0-9a-f]{64}$/); + expect(rec.exit).toBe(0); + expect(typeof rec.duration_s).toBe('number'); + expect(rec.commit).toMatch(/^[0-9a-f]{40}$/); + expect(rec.tree).toMatch(/^[0-9a-f]{40}$/); + expect(rec.wtree).toMatch(/^[0-9a-f]{40}$/); + expect(typeof rec.dirty).toBe('boolean'); + expect(fs.existsSync(rec.log_path)).toBe(true); + expect(fs.readFileSync(rec.log_path, 'utf-8')).toContain('ok'); + }); + + test('propagates a failing exit code and records it', () => { + const r = run(['run', '--label', 'tests', '--', 'exit 3']); + expect(r.status).toBe(3); + expect(records().pop().exit).toBe(3); + }); + + test('spawn failure (ENOENT, argv-direct form) records and propagates 127', () => { + const r = run(['run', '--label', 'tests', '--', '/nonexistent-gstack-binary', 'arg']); + expect(r.status).toBe(127); + expect(records().pop().exit).toBe(127); + }); + + test('TRANSPARENCY: ledger failure never breaks the command (append-failure injection)', () => { + // Point GSTACK_HOME somewhere mkdir cannot succeed. + const r = spawnSync(EVIDENCE, ['run', '--label', 'tests', '--', 'echo still-ran'], { + cwd: repoDir, + env: { ...process.env, GSTACK_HOME: '/dev/null/nope' }, + encoding: 'utf-8', + timeout: 60000, + }); + expect(r.status).toBe(0); + expect(r.stdout).toContain('still-ran'); + expect(r.stderr).toContain('warning'); + }); + + test('ledger and log files are 0600', () => { + run(['run', '--label', 'tests', '--', 'echo ok']); + const rec = records().pop(); + expect(fs.statSync(ledgerFile()).mode & 0o777).toBe(0o600); + expect(fs.statSync(rec.log_path).mode & 0o777).toBe(0o600); + }); + + test('two rapid runs get distinct log files (collision-safe exclusive open)', () => { + run(['run', '--label', 'tests', '--', 'echo one']); + run(['run', '--label', 'tests', '--', 'echo two']); + const [a, b] = records().slice(-2); + expect(a.log_path).not.toBe(b.log_path); + }); + + test('log truncates at 2MB with a marker; exit code unaffected', () => { + const r = run(['run', '--label', 'big', '--', 'head -c 3000000 /dev/zero | tr "\\0" a']); + expect(r.status).toBe(0); + const rec = records().pop(); + const size = fs.statSync(rec.log_path).size; + expect(size).toBeLessThanOrEqual(2 * 1024 * 1024 + 200); + expect(fs.readFileSync(rec.log_path, 'utf-8')).toContain('log truncated at 2MB'); + }); + + test('logs older than 30 days are pruned opportunistically', () => { + run(['run', '--label', 'tests', '--', 'echo ok']); + const logsDir = path.dirname(records().pop().log_path); + const oldLog = path.join(logsDir, 'ancient.log'); + fs.writeFileSync(oldLog, 'old'); + const past = new Date(Date.now() - 40 * 24 * 3600 * 1000); + fs.utimesSync(oldLog, past, past); + run(['run', '--label', 'tests', '--', 'echo again']); + expect(fs.existsSync(oldLog)).toBe(false); + }); + + test('works as a backgrounded job (ship Step 5 lanes run with & wait)', () => { + execSync(`bash -c '"${EVIDENCE}" run --label bg -- "echo backgrounded" & wait'`, { + cwd: repoDir, + env: { ...process.env, GSTACK_HOME: gstackHome }, + encoding: 'utf-8', + timeout: 60000, + }); + const rec = records().pop(); + expect(rec.label).toBe('bg'); + expect(rec.exit).toBe(0); + }); + + test('a HIGH credential in the command is stored redacted', () => { + const r = run(['run', '--label', 'sec', '--', 'echo ghp_A8bC2dE4fG6hI8jK0lM2nO4pQ6rS8tU0vW2x deploy']); + expect(r.status).toBe(0); + const rec = records().pop(); + expect(rec.command).not.toContain('ghp_A8bC2dE4fG6hI8jK0lM2nO4pQ6rS8tU0vW2x'); + expect(rec.redacted).toBe(true); + // The hash still binds to the ORIGINAL exact string (freshness key). + expect(rec.cmd_sha256).toMatch(/^[0-9a-f]{64}$/); + }); +}); + +describe('gstack-evidence check', () => { + test('KEYSTONE: evidence recorded on a dirty tree stays FRESH after committing the exact tested content', () => { + // Dirty the tree (this is /ship Step 5: tests run on uncommitted code). + fs.writeFileSync(path.join(repoDir, 'src.txt'), 'v2-tested\n'); + expect(run(['run', '--label', 'tests', '--', 'echo green']).status).toBe(0); + expect(records().pop().dirty).toBe(true); + + // Step 15: commit the exact same content. HEAD tree changes; working-tree + // content does not. + git('commit -q -am ship'); + + const chk = run(['check', '--label', 'tests']); + expect(chk.status).toBe(0); + expect(chk.stdout).toContain('EVIDENCE: FRESH'); + }); + + test('a content change after the run grades STALE', () => { + expect(run(['run', '--label', 'tests', '--', 'echo green']).status).toBe(0); + fs.writeFileSync(path.join(repoDir, 'src.txt'), 'changed-after-tests\n'); + const chk = run(['check', '--label', 'tests']); + expect(chk.status).toBe(1); + expect(chk.stdout).toContain('EVIDENCE: STALE'); + }); + + test('an untracked NEW source file grades STALE; gitignored scratch stays FRESH', () => { + expect(run(['run', '--label', 'tests', '--', 'echo green']).status).toBe(0); + + fs.writeFileSync(path.join(repoDir, 'scratch.txt'), 'conductor noise\n'); + expect(run(['check', '--label', 'tests']).status).toBe(0); + + fs.writeFileSync(path.join(repoDir, 'brand-new.ts'), 'export {}\n'); + const chk = run(['check', '--label', 'tests']); + expect(chk.status).toBe(1); + expect(chk.stdout).toContain('STALE'); + }); + + test('allow-paths carve-out: a CHANGELOG-only change stays FRESH with --allow-paths', () => { + expect(run(['run', '--label', 'tests', '--', 'echo green']).status).toBe(0); + fs.writeFileSync(path.join(repoDir, 'CHANGELOG.md'), '## v1\n'); + git('add CHANGELOG.md'); + git('commit -q -m changelog'); + + const without = run(['check', '--label', 'tests']); + expect(without.status).toBe(1); + + const withAllow = run(['check', '--label', 'tests', '--allow-paths', 'CHANGELOG.md,VERSION,package.json']); + expect(withAllow.status).toBe(0); + expect(withAllow.stdout).toContain('FRESH'); + + // A source change is NOT rescued by the allow-list. + fs.writeFileSync(path.join(repoDir, 'src.txt'), 'v3\n'); + expect(run(['check', '--label', 'tests', '--allow-paths', 'CHANGELOG.md']).status).toBe(1); + }); + + test('--expect-cmd binds the label to the exact command string', () => { + expect(run(['run', '--label', 'tests', '--', 'echo green']).status).toBe(0); + expect(run(['check', '--label', 'tests', '--expect-cmd', 'echo green']).status).toBe(0); + const mismatch = run(['check', '--label', 'tests', '--expect-cmd', 'echo cheaper-command']); + expect(mismatch.status).toBe(1); + expect(mismatch.stdout).toContain('cmd_sha256 mismatch'); + }); + + test('a recorded FAILING run is never FRESH', () => { + run(['run', '--label', 'tests', '--', 'exit 1']); + const chk = run(['check', '--label', 'tests']); + expect(chk.status).toBe(1); + expect(chk.stdout).toContain('recorded run failed'); + }); + + test('--max-age expires old records', () => { + expect(run(['run', '--label', 'tests', '--', 'echo green']).status).toBe(0); + const file = ledgerFile(); + const rec = JSON.parse(fs.readFileSync(file, 'utf-8').trim()); + rec.ts = new Date(Date.now() - 48 * 3600 * 1000).toISOString(); + fs.writeFileSync(file, JSON.stringify(rec) + '\n'); + const chk = run(['check', '--label', 'tests', '--max-age', '24']); + expect(chk.status).toBe(1); + expect(chk.stdout).toContain('older than 24h'); + }); + + test('a gc-d / fabricated stored fingerprint degrades to STALE, never a crash', () => { + expect(run(['run', '--label', 'tests', '--', 'echo green']).status).toBe(0); + const file = ledgerFile(); + const rec = JSON.parse(fs.readFileSync(file, 'utf-8').trim()); + rec.wtree = 'deadbeefdeadbeefdeadbeefdeadbeefdeadbeef'; + fs.writeFileSync(file, JSON.stringify(rec) + '\n'); + const chk = run(['check', '--label', 'tests']); + expect(chk.status).toBe(1); + expect(chk.stdout).toContain('STALE'); + }); + + test('a green lane never masks a red sibling: every named label must be FRESH', () => { + expect(run(['run', '--label', 'tests', '--', 'echo green']).status).toBe(0); + run(['run', '--label', 'vitest', '--', 'exit 1']); + const chk = run(['check', '--label', 'tests', '--label', 'vitest']); + expect(chk.status).toBe(1); + expect(chk.stdout).toContain('EVIDENCE: FRESH label=tests'); + expect(chk.stdout).toContain('EVIDENCE: STALE label=vitest'); + }); + + test('MISSING for a label that never ran (explicit labels prove expected lanes)', () => { + expect(run(['run', '--label', 'tests', '--', 'echo green']).status).toBe(0); + const chk = run(['check', '--label', 'tests', '--label', 'never-ran']); + expect(chk.status).toBe(1); + expect(chk.stdout).toContain('MISSING label=never-ran'); + }); + + test('check never errors outside a git repo — degrades to STALE', () => { + expect(run(['run', '--label', 'tests', '--', 'echo green']).status).toBe(0); + const nonGit = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-evidence-nongit-')); + try { + const chk = run(['check', '--label', 'tests'], { cwd: nonGit }); + expect([0, 1]).toContain(chk.status); // different slug → MISSING; the point is: no crash + expect(chk.status).toBe(1); + } finally { + fs.rmSync(nonGit, { recursive: true, force: true }); + } + }); +});