mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-18 19:02:18 +02:00
feat(security): hash-chained egress receipt ledger (core)
Port lib/egress-receipt from the v2 fork as TypeScript: writeReceipt (sync, fail-closed via typed EGRESS_RECEIPT_FAILED), best-effort writeOutcome, readLedger/listReceipts/verifyLedger, GSTACK_HOME -> GSTACK_STATE_DIR -> ~/.gstack resolution, 0600 ledger under a 0700 security dir, and an mkdir spin lock (2.5s budget) with documented >10s-mtime stale-lock reclaim. Changes vs the fork: - lastRawLine tail-reads the final 4KB instead of loading the whole ledger, so appends stay O(1) as the file grows. - WARN-at-size: past 25MB writeReceipt emits one self-explanatory stderr warning per process (what the ledger is, how to inspect it, rotation TODO); verifyLedger gains a sizeWarning field. Rotation TODO carries the chain-genesis sketch (new generation's first record embeds the prior file's tail hash). bin/gstack-egress-receipt is a bun script bridging shell callers: write|outcome subcommands, exit 3 + EGRESS_RECEIPT_FAILED on stderr on failure; --no-payload records sha256:null for git-class ops. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> (cherry picked from commit 619726a3d77d987a2e50151a5727b3faaaf5fc6a)
This commit is contained in:
Executable
+84
@@ -0,0 +1,84 @@
|
|||||||
|
#!/usr/bin/env bun
|
||||||
|
// gstack-egress-receipt — bun script that BRIDGES shell callers (the bash
|
||||||
|
// egress sinks: gstack-telemetry-sync, gstack-update-check, gstack-brain-sync,
|
||||||
|
// and the sourced helpers in gstack-egress-lib.sh) into lib/egress-receipt.ts.
|
||||||
|
//
|
||||||
|
// Usage:
|
||||||
|
// gstack-egress-receipt write --sink S --host H --class C \
|
||||||
|
// (--payload-file F | --no-payload) [--consent "key=value"]
|
||||||
|
// → prints the receipt id on stdout, exit 0.
|
||||||
|
// → exit 3 + "EGRESS_RECEIPT_FAILED: ..." on stderr when the receipt
|
||||||
|
// cannot be written. Fail-closed callers MUST then refuse the send.
|
||||||
|
//
|
||||||
|
// gstack-egress-receipt outcome <receipt-id> <status>
|
||||||
|
// → best-effort response-status record; never blocks anything.
|
||||||
|
//
|
||||||
|
// Home: GSTACK_HOME, legacy GSTACK_STATE_DIR, else ~/.gstack.
|
||||||
|
// The payload file is hashed as-is: pass the SAME file to curl (`-d @file`)
|
||||||
|
// so the receipt hash matches the exact bytes sent (scan-at-sink precedent).
|
||||||
|
|
||||||
|
import fs from 'node:fs';
|
||||||
|
import { EGRESS_RECEIPT_FAILED, sha256Hex, writeOutcome, writeReceipt } from '../lib/egress-receipt';
|
||||||
|
|
||||||
|
function parseArgs(args: string[], valueFlags: string[], boolFlags: string[]) {
|
||||||
|
const values = new Map<string, string>();
|
||||||
|
const flags = new Set<string>();
|
||||||
|
for (let index = 0; index < args.length; index += 1) {
|
||||||
|
const arg = args[index];
|
||||||
|
if (valueFlags.includes(arg)) {
|
||||||
|
const value = args[++index];
|
||||||
|
if (value == null) usage(`${arg} requires a value`);
|
||||||
|
values.set(arg, value);
|
||||||
|
} else if (boolFlags.includes(arg)) {
|
||||||
|
flags.add(arg);
|
||||||
|
} else {
|
||||||
|
usage(`unknown option: ${arg}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return { values, flags };
|
||||||
|
}
|
||||||
|
|
||||||
|
function usage(message: string): never {
|
||||||
|
process.stderr.write(`gstack-egress-receipt: ${message}\n`);
|
||||||
|
process.exit(2);
|
||||||
|
}
|
||||||
|
|
||||||
|
const [command, ...rest] = process.argv.slice(2);
|
||||||
|
|
||||||
|
if (command === 'write') {
|
||||||
|
const { values, flags } = parseArgs(rest,
|
||||||
|
['--sink', '--host', '--class', '--payload-file', '--consent'], ['--no-payload']);
|
||||||
|
const payloadFile = values.get('--payload-file');
|
||||||
|
if (!payloadFile && !flags.has('--no-payload')) usage('write requires --payload-file or --no-payload');
|
||||||
|
try {
|
||||||
|
let bytes = 0;
|
||||||
|
let sha256: string | null = null; // --no-payload = git-class op, subprocess owns the bytes
|
||||||
|
if (payloadFile) {
|
||||||
|
const payload = fs.readFileSync(payloadFile);
|
||||||
|
bytes = payload.byteLength;
|
||||||
|
sha256 = sha256Hex(payload);
|
||||||
|
}
|
||||||
|
const { id } = writeReceipt({
|
||||||
|
sink: values.get('--sink') as string,
|
||||||
|
host: values.get('--host') as string,
|
||||||
|
payloadClass: values.get('--class') as string,
|
||||||
|
bytes,
|
||||||
|
sha256,
|
||||||
|
consent: values.get('--consent') ?? 'unspecified',
|
||||||
|
});
|
||||||
|
process.stdout.write(`${id}\n`);
|
||||||
|
} catch (error) {
|
||||||
|
process.stderr.write(`${EGRESS_RECEIPT_FAILED}: ${(error as Error)?.message ?? error}\n`);
|
||||||
|
process.exit(3);
|
||||||
|
}
|
||||||
|
} else if (command === 'outcome') {
|
||||||
|
const [receipt, status, ...extra] = rest;
|
||||||
|
if (!receipt || !status || extra.length) usage('Usage: gstack-egress-receipt outcome <receipt-id> <status>');
|
||||||
|
try {
|
||||||
|
writeOutcome({ receipt, status });
|
||||||
|
} catch {
|
||||||
|
// Best-effort: the pre-send receipt is the invariant, the outcome is bookkeeping.
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
usage('Usage: gstack-egress-receipt write|outcome ...');
|
||||||
|
}
|
||||||
@@ -0,0 +1,376 @@
|
|||||||
|
/**
|
||||||
|
* egress-receipt — hash-chained, content-free receipts for every
|
||||||
|
* gstack-initiated off-machine send (`~/.gstack/security/egress.jsonl`, 0600).
|
||||||
|
*
|
||||||
|
* THREAT MODEL: the egress ledger is forensic observability — it records
|
||||||
|
* ATTEMPTED egress so accidents are auditable; it is not an exfiltration
|
||||||
|
* control. Receipts are written before send, outcomes are best-effort, and
|
||||||
|
* fail-open sinks can send unrecorded with a warning.
|
||||||
|
*
|
||||||
|
* Semantics:
|
||||||
|
* - Receipt-before-send: the receipt line is appended BEFORE the network
|
||||||
|
* call. Fail-closed sinks MUST refuse the send with the typed code
|
||||||
|
* EGRESS_RECEIPT_FAILED when it cannot be written; fail-open sinks warn
|
||||||
|
* on stderr and proceed.
|
||||||
|
* - Content-free: never payload text, never credentials — a sha256 of the
|
||||||
|
* exact bytes sent plus a byte count only (semantic-reviews.jsonl
|
||||||
|
* precedent). Sinks where a subprocess/SDK owns the bytes record
|
||||||
|
* sha256: null.
|
||||||
|
* - Tamper-evident: each line carries `prev` = sha256 of the previous raw
|
||||||
|
* line ("" for line 1). `verifyLedger` recomputes the chain.
|
||||||
|
*
|
||||||
|
* Node builtins only, so bun TS binaries and the compiled browse binary can
|
||||||
|
* both import it (same constraint as browse/src/security.ts: no native
|
||||||
|
* modules).
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { createHash } from 'node:crypto';
|
||||||
|
import fs from 'node:fs';
|
||||||
|
import os from 'node:os';
|
||||||
|
import path from 'node:path';
|
||||||
|
|
||||||
|
export const EGRESS_RECEIPT_FAILED = 'EGRESS_RECEIPT_FAILED';
|
||||||
|
|
||||||
|
const SHA256_HEX = /^[0-9a-f]{64}$/;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* WARN-at-size threshold. Above this the ledger still appends (never blocks
|
||||||
|
* on size), but writeReceipt emits one stderr warning per process so the
|
||||||
|
* user learns the file exists and how to inspect it before it gets silly.
|
||||||
|
*/
|
||||||
|
export const LEDGER_WARN_BYTES = 25 * 1024 * 1024;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Tail window for reading the last raw line. Receipt lines are ~300 bytes;
|
||||||
|
* 4KB covers any legal line with an order of magnitude to spare.
|
||||||
|
*/
|
||||||
|
const TAIL_READ_BYTES = 4096;
|
||||||
|
|
||||||
|
// TODO(rotation): ledger rotation. Chain-genesis sketch: when the ledger
|
||||||
|
// exceeds the size threshold, rename it to egress.jsonl.1 and start a new
|
||||||
|
// generation whose FIRST record embeds `genesis: sha256(<tail raw line of
|
||||||
|
// the prior file>)`, so verifyLedger can walk generations end-to-end
|
||||||
|
// (verify each file's internal chain, then check each genesis hash against
|
||||||
|
// the previous generation's last line). Until then we warn at 25MB.
|
||||||
|
|
||||||
|
type Env = Record<string, string | undefined>;
|
||||||
|
|
||||||
|
export interface WriteReceiptOptions {
|
||||||
|
/** gstack home; resolved from env when omitted */
|
||||||
|
home?: string;
|
||||||
|
/** env for home resolution (tests) */
|
||||||
|
env?: Env;
|
||||||
|
/** which gstack component is sending */
|
||||||
|
sink: string;
|
||||||
|
/** destination host[:port] */
|
||||||
|
host: string;
|
||||||
|
/** content-free payload description */
|
||||||
|
payloadClass: string;
|
||||||
|
/** exact byte count sent (0 for bodyless requests) */
|
||||||
|
bytes?: number;
|
||||||
|
/** sha256 hex of the exact bytes sent; null when a subprocess/SDK owns the bytes */
|
||||||
|
sha256?: string | null;
|
||||||
|
/** the consent key+value that authorizes this send */
|
||||||
|
consent: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface WriteOutcomeOptions {
|
||||||
|
home?: string;
|
||||||
|
env?: Env;
|
||||||
|
/** receipt id returned by writeReceipt */
|
||||||
|
receipt: string;
|
||||||
|
status?: string | number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface LedgerLine {
|
||||||
|
lineNo: number;
|
||||||
|
raw: string;
|
||||||
|
record: Record<string, unknown> | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface VerifyResult {
|
||||||
|
ok: boolean;
|
||||||
|
count: number;
|
||||||
|
brokenLine: number | null;
|
||||||
|
reason: string | null;
|
||||||
|
/** present when the ledger file exceeds LEDGER_WARN_BYTES */
|
||||||
|
sizeWarning: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Same resolution order as the rest of gstack (shell sinks, selection code):
|
||||||
|
* GSTACK_HOME, legacy GSTACK_STATE_DIR, then $HOME/.gstack.
|
||||||
|
*/
|
||||||
|
export function resolveEgressHome(env: Env = process.env): string {
|
||||||
|
const configured = env.GSTACK_HOME || env.GSTACK_STATE_DIR;
|
||||||
|
if (configured) return path.resolve(configured);
|
||||||
|
return path.join(env.HOME || os.homedir(), '.gstack');
|
||||||
|
}
|
||||||
|
|
||||||
|
export function egressLedgerPath(home: string): string {
|
||||||
|
return path.join(home, 'security', 'egress.jsonl');
|
||||||
|
}
|
||||||
|
|
||||||
|
export function sha256Hex(data: string | Uint8Array): string {
|
||||||
|
return createHash('sha256').update(data).digest('hex');
|
||||||
|
}
|
||||||
|
|
||||||
|
function receiptError(message: string, cause?: unknown): Error & { code: string } {
|
||||||
|
const error = (cause === undefined ? new Error(message) : new Error(message, { cause })) as Error & { code: string };
|
||||||
|
error.code = EGRESS_RECEIPT_FAILED;
|
||||||
|
return error;
|
||||||
|
}
|
||||||
|
|
||||||
|
function requireString(value: unknown, name: string): string {
|
||||||
|
if (typeof value !== 'string' || !value) throw receiptError(`Egress receipt requires a non-empty ${name}`);
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* mkdir spin lock, ~2.5s budget. Egress events are rare (minutes apart); the
|
||||||
|
* lock only protects the read-last-line → append window.
|
||||||
|
*
|
||||||
|
* Stale-lock reclaim: a crashed writer strands the lock dir. Once the spin
|
||||||
|
* budget is exhausted, a lock dir whose mtime is >10s old is stale by
|
||||||
|
* definition (appends take milliseconds), so the waiter removes it and
|
||||||
|
* retries instead of failing. The rmdir/stat races with a concurrent
|
||||||
|
* reclaimer or the owner's own cleanup are harmless — losers just loop.
|
||||||
|
*/
|
||||||
|
function withLedgerLock<T>(ledger: string, callback: () => T): T {
|
||||||
|
const lock = `${ledger}.lock`;
|
||||||
|
const deadline = Date.now() + 2500;
|
||||||
|
for (;;) {
|
||||||
|
try {
|
||||||
|
fs.mkdirSync(lock);
|
||||||
|
break;
|
||||||
|
} catch (error) {
|
||||||
|
if ((error as NodeJS.ErrnoException)?.code !== 'EEXIST') throw error;
|
||||||
|
if (Date.now() > deadline) {
|
||||||
|
try {
|
||||||
|
const age = Date.now() - fs.statSync(lock).mtimeMs;
|
||||||
|
if (age > 10_000) { fs.rmdirSync(lock); continue; }
|
||||||
|
} catch { /* raced with the owner's cleanup — retry */ }
|
||||||
|
throw receiptError(`Egress ledger is locked: ${lock}`);
|
||||||
|
}
|
||||||
|
// Sync sleep (node + bun): the API is sync on purpose so shell, bun,
|
||||||
|
// and node callers all share one implementation.
|
||||||
|
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return callback();
|
||||||
|
} finally {
|
||||||
|
try { fs.rmdirSync(lock); } catch { /* best-effort unlock */ }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Last raw line via a tail read: open the file, read the final
|
||||||
|
* TAIL_READ_BYTES, take the last newline-terminated chunk. Never loads the
|
||||||
|
* whole ledger, so appends stay O(1) as the file grows.
|
||||||
|
*/
|
||||||
|
function lastRawLine(ledger: string): string | null {
|
||||||
|
let fd: number;
|
||||||
|
try {
|
||||||
|
fd = fs.openSync(ledger, 'r');
|
||||||
|
} catch (error) {
|
||||||
|
if ((error as NodeJS.ErrnoException)?.code === 'ENOENT') return null;
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const size = fs.fstatSync(fd).size;
|
||||||
|
if (size === 0) return null;
|
||||||
|
const length = Math.min(size, TAIL_READ_BYTES);
|
||||||
|
const buffer = Buffer.alloc(length);
|
||||||
|
fs.readSync(fd, buffer, 0, length, size - length);
|
||||||
|
const tail = buffer.toString('utf8');
|
||||||
|
const lines = tail.split('\n').filter((line) => line.length > 0);
|
||||||
|
return lines.length ? lines[lines.length - 1] : null;
|
||||||
|
} finally {
|
||||||
|
fs.closeSync(fd);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let warnedLedgerSize = false;
|
||||||
|
|
||||||
|
/** Test-only: re-arm the once-per-process size warning. */
|
||||||
|
export function resetLedgerSizeWarningForTests(): void {
|
||||||
|
warnedLedgerSize = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
function warnLedgerSizeOnce(ledger: string): void {
|
||||||
|
let size: number;
|
||||||
|
try {
|
||||||
|
size = fs.statSync(ledger).size;
|
||||||
|
} catch {
|
||||||
|
return; // no file yet — nothing to warn about
|
||||||
|
}
|
||||||
|
if (size <= LEDGER_WARN_BYTES || warnedLedgerSize) return;
|
||||||
|
warnedLedgerSize = true;
|
||||||
|
process.stderr.write(ledgerSizeWarning(ledger, size) + '\n');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Self-explanatory size warning: says what the ledger is (records what
|
||||||
|
* gstack ATTEMPTS to send off-machine), how to inspect it, and that
|
||||||
|
* trimming arrives with rotation.
|
||||||
|
*/
|
||||||
|
export function ledgerSizeWarning(ledger: string, size: number): string {
|
||||||
|
const mb = (size / (1024 * 1024)).toFixed(1);
|
||||||
|
return (
|
||||||
|
`gstack: egress ledger is large (${mb}MB): ${ledger}. ` +
|
||||||
|
`This file records what gstack ATTEMPTS to send off-machine (content-free receipts, for auditing). ` +
|
||||||
|
`Inspect it with 'gstack-egress list'. Trimming arrives with ledger rotation (TODO); until then it only grows.`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function appendChained(
|
||||||
|
homeOrNull: string | null,
|
||||||
|
record: Record<string, unknown>,
|
||||||
|
env?: Env,
|
||||||
|
): { id: string; path: string } {
|
||||||
|
const home = homeOrNull ?? resolveEgressHome(env);
|
||||||
|
const ledger = egressLedgerPath(home);
|
||||||
|
try {
|
||||||
|
fs.mkdirSync(path.dirname(ledger), { recursive: true, mode: 0o700 });
|
||||||
|
return withLedgerLock(ledger, () => {
|
||||||
|
const previous = lastRawLine(ledger);
|
||||||
|
const line = JSON.stringify({ ...record, prev: previous == null ? '' : sha256Hex(previous) });
|
||||||
|
const existed = fs.existsSync(ledger);
|
||||||
|
fs.appendFileSync(ledger, `${line}\n`, { mode: 0o600 });
|
||||||
|
if (!existed) fs.chmodSync(ledger, 0o600); // umask must not weaken the ledger
|
||||||
|
return { id: sha256Hex(line), path: ledger };
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
if ((error as NodeJS.ErrnoException)?.code === EGRESS_RECEIPT_FAILED) throw error;
|
||||||
|
throw receiptError(
|
||||||
|
`Egress receipt could not be written to ${ledger}: ${(error as Error)?.message ?? error}`,
|
||||||
|
error,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Append one content-free receipt BEFORE a network send.
|
||||||
|
*
|
||||||
|
* @returns id = sha256 of the written line (for writeOutcome)
|
||||||
|
* @throws Error with code EGRESS_RECEIPT_FAILED — fail-closed callers must
|
||||||
|
* refuse the send; fail-open callers warn and proceed
|
||||||
|
*/
|
||||||
|
export function writeReceipt(opts: WriteReceiptOptions): { id: string; path: string } {
|
||||||
|
const sink = requireString(opts.sink, 'sink');
|
||||||
|
const host = requireString(opts.host, 'host');
|
||||||
|
const payloadClass = requireString(opts.payloadClass, 'payloadClass');
|
||||||
|
const consent = requireString(opts.consent, 'consent');
|
||||||
|
const bytes = opts.bytes ?? 0;
|
||||||
|
if (!Number.isSafeInteger(bytes) || bytes < 0) throw receiptError('Egress receipt bytes must be a non-negative integer');
|
||||||
|
const sha256 = opts.sha256 ?? null;
|
||||||
|
if (sha256 !== null && !SHA256_HEX.test(String(sha256))) throw receiptError('Egress receipt sha256 must be 64 lowercase hex chars or null');
|
||||||
|
const home = opts.home ?? resolveEgressHome(opts.env);
|
||||||
|
warnLedgerSizeOnce(egressLedgerPath(home));
|
||||||
|
return appendChained(home, {
|
||||||
|
ts: new Date().toISOString(),
|
||||||
|
type: 'egress',
|
||||||
|
sink,
|
||||||
|
host,
|
||||||
|
payload_class: payloadClass,
|
||||||
|
bytes,
|
||||||
|
sha256,
|
||||||
|
consent,
|
||||||
|
}, opts.env);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Append the response status for an earlier receipt (best-effort companion
|
||||||
|
* record — the pre-send receipt is the invariant, the outcome is
|
||||||
|
* bookkeeping). Chained like every other line.
|
||||||
|
*/
|
||||||
|
export function writeOutcome(opts: WriteOutcomeOptions): { id: string; path: string } {
|
||||||
|
const receipt = requireString(opts.receipt, 'receipt id');
|
||||||
|
return appendChained(opts.home ?? null, {
|
||||||
|
ts: new Date().toISOString(),
|
||||||
|
type: 'outcome',
|
||||||
|
receipt,
|
||||||
|
status: String(opts.status ?? 'unknown'),
|
||||||
|
}, opts.env);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Raw parsed lines: [{lineNo, raw, record|null}]. Missing ledger → []. */
|
||||||
|
export function readLedger(home: string): LedgerLine[] {
|
||||||
|
const ledger = egressLedgerPath(home);
|
||||||
|
let content: string;
|
||||||
|
try {
|
||||||
|
content = fs.readFileSync(ledger, 'utf8');
|
||||||
|
} catch (error) {
|
||||||
|
if ((error as NodeJS.ErrnoException)?.code === 'ENOENT') return [];
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
return content.split('\n').filter((line) => line.length > 0).map((raw, index) => {
|
||||||
|
let record: Record<string, unknown> | null = null;
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(raw);
|
||||||
|
if (parsed && typeof parsed === 'object') record = parsed;
|
||||||
|
} catch { /* malformed line — verifyLedger reports it */ }
|
||||||
|
return { lineNo: index + 1, raw, record };
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface Receipt {
|
||||||
|
ts: string;
|
||||||
|
type: 'egress';
|
||||||
|
sink: string;
|
||||||
|
host: string;
|
||||||
|
payload_class: string;
|
||||||
|
bytes: number;
|
||||||
|
sha256: string | null;
|
||||||
|
consent: string;
|
||||||
|
prev: string;
|
||||||
|
id: string;
|
||||||
|
status: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Receipts with their joined outcome status (`status: null` = none recorded). */
|
||||||
|
export function listReceipts(home: string): Receipt[] {
|
||||||
|
const lines = readLedger(home);
|
||||||
|
const receipts: Receipt[] = [];
|
||||||
|
const byId = new Map<string, Receipt>();
|
||||||
|
for (const { raw, record } of lines) {
|
||||||
|
if (!record) continue;
|
||||||
|
if (record.type === 'egress') {
|
||||||
|
const entry = { ...(record as unknown as Omit<Receipt, 'id' | 'status'>), id: sha256Hex(raw), status: null };
|
||||||
|
receipts.push(entry);
|
||||||
|
byId.set(entry.id, entry);
|
||||||
|
} else if (record.type === 'outcome' && byId.has(record.receipt as string)) {
|
||||||
|
byId.get(record.receipt as string)!.status = String(record.status);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return receipts;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Recompute the hash chain. `brokenLine` is the 1-indexed first line whose
|
||||||
|
* `prev` no longer matches the sha256 of the previous raw line (or that
|
||||||
|
* fails to parse). `sizeWarning` is set when the ledger exceeds
|
||||||
|
* LEDGER_WARN_BYTES.
|
||||||
|
*/
|
||||||
|
export function verifyLedger(home: string): VerifyResult {
|
||||||
|
const ledger = egressLedgerPath(home);
|
||||||
|
let sizeWarning: string | null = null;
|
||||||
|
try {
|
||||||
|
const size = fs.statSync(ledger).size;
|
||||||
|
if (size > LEDGER_WARN_BYTES) sizeWarning = ledgerSizeWarning(ledger, size);
|
||||||
|
} catch { /* missing ledger — verify of an empty chain below */ }
|
||||||
|
const lines = readLedger(home);
|
||||||
|
let previousRaw: string | null = null;
|
||||||
|
for (const { lineNo, raw, record } of lines) {
|
||||||
|
if (!record || typeof record.prev !== 'string') {
|
||||||
|
return { ok: false, count: lines.length, brokenLine: lineNo, reason: 'unparseable or missing prev', sizeWarning };
|
||||||
|
}
|
||||||
|
const expected = previousRaw == null ? '' : sha256Hex(previousRaw);
|
||||||
|
if (record.prev !== expected) {
|
||||||
|
return { ok: false, count: lines.length, brokenLine: lineNo, reason: 'prev hash does not match previous line', sizeWarning };
|
||||||
|
}
|
||||||
|
previousRaw = raw;
|
||||||
|
}
|
||||||
|
return { ok: true, count: lines.length, brokenLine: null, reason: null, sizeWarning };
|
||||||
|
}
|
||||||
@@ -0,0 +1,258 @@
|
|||||||
|
/**
|
||||||
|
* Egress receipts — chain, fail-closed, verify, shell bridge. Free tier, no network.
|
||||||
|
*
|
||||||
|
* THREAT MODEL: the egress ledger is forensic observability — it records
|
||||||
|
* ATTEMPTED egress so accidents are auditable; it is not an exfiltration
|
||||||
|
* control.
|
||||||
|
*
|
||||||
|
* Pins the auditor contract:
|
||||||
|
* - receipt-before-send fail-closed (EGRESS_RECEIPT_FAILED, no ledger = no send)
|
||||||
|
* - content-free lines chained by prev = sha256(previous raw line)
|
||||||
|
* - tail-read correctness (last line found without loading the whole file)
|
||||||
|
* - abandoned-lock reclaim (stale lock dir >10s old is removed, not fatal)
|
||||||
|
* - WARN-at-size (one self-explanatory stderr warning per process >25MB)
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
|
||||||
|
import * as fs from 'fs';
|
||||||
|
import * as os from 'os';
|
||||||
|
import * as path from 'path';
|
||||||
|
import { spawnSync } from 'child_process';
|
||||||
|
import {
|
||||||
|
EGRESS_RECEIPT_FAILED,
|
||||||
|
LEDGER_WARN_BYTES,
|
||||||
|
egressLedgerPath,
|
||||||
|
ledgerSizeWarning,
|
||||||
|
listReceipts,
|
||||||
|
resetLedgerSizeWarningForTests,
|
||||||
|
sha256Hex,
|
||||||
|
verifyLedger,
|
||||||
|
writeOutcome,
|
||||||
|
writeReceipt,
|
||||||
|
} from '../lib/egress-receipt';
|
||||||
|
|
||||||
|
const ROOT = path.resolve(new URL(import.meta.url).pathname, '..', '..');
|
||||||
|
|
||||||
|
let home: string;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
home = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-egress-'));
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
try { fs.chmodSync(path.join(home, 'security'), 0o700); } catch {} // undo fail-closed fixtures
|
||||||
|
fs.rmSync(home, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('egress receipt library', () => {
|
||||||
|
test('receipts chain: prev = sha256 of the previous raw line, "" for line 1', () => {
|
||||||
|
writeReceipt({ home, sink: 'a', host: 'h1', payloadClass: 'c', bytes: 3, sha256: sha256Hex('abc'), consent: 'k=v' });
|
||||||
|
writeReceipt({ home, sink: 'b', host: 'h2', payloadClass: 'c', bytes: 0, sha256: null, consent: 'k=v' });
|
||||||
|
const lines = fs.readFileSync(egressLedgerPath(home), 'utf-8').trim().split('\n');
|
||||||
|
expect(lines.length).toBe(2);
|
||||||
|
const first = JSON.parse(lines[0]);
|
||||||
|
const second = JSON.parse(lines[1]);
|
||||||
|
expect(first.prev).toBe('');
|
||||||
|
expect(second.prev).toBe(sha256Hex(lines[0]));
|
||||||
|
expect(first.sha256).toBe(sha256Hex('abc'));
|
||||||
|
expect(second.sha256).toBeNull();
|
||||||
|
expect(verifyLedger(home)).toMatchObject({ ok: true, count: 2 });
|
||||||
|
});
|
||||||
|
|
||||||
|
test('ledger is 0600 and the security dir 0700', () => {
|
||||||
|
writeReceipt({ home, sink: 'a', host: 'h', payloadClass: 'c', consent: 'k=v' });
|
||||||
|
const ledger = egressLedgerPath(home);
|
||||||
|
expect(fs.statSync(ledger).mode & 0o777).toBe(0o600);
|
||||||
|
expect(fs.statSync(path.dirname(ledger)).mode & 0o777).toBe(0o700);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('fail-closed: unwritable security dir throws typed EGRESS_RECEIPT_FAILED', () => {
|
||||||
|
if (process.platform === 'win32' || process.getuid?.() === 0) return; // chmod is advisory there
|
||||||
|
writeReceipt({ home, sink: 'a', host: 'h', payloadClass: 'c', consent: 'k=v' });
|
||||||
|
fs.chmodSync(path.join(home, 'security'), 0o500);
|
||||||
|
try {
|
||||||
|
expect(() =>
|
||||||
|
writeReceipt({ home, sink: 'a', host: 'h', payloadClass: 'c', consent: 'k=v' }),
|
||||||
|
).toThrow();
|
||||||
|
try {
|
||||||
|
writeReceipt({ home, sink: 'a', host: 'h', payloadClass: 'c', consent: 'k=v' });
|
||||||
|
} catch (err: any) {
|
||||||
|
expect(err.code).toBe(EGRESS_RECEIPT_FAILED);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
fs.chmodSync(path.join(home, 'security'), 0o700);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('outcome records join back onto their receipt in listReceipts', () => {
|
||||||
|
const { id } = writeReceipt({ home, sink: 'a', host: 'h', payloadClass: 'c', consent: 'k=v' });
|
||||||
|
writeOutcome({ home, receipt: id, status: 204 });
|
||||||
|
const receipts = listReceipts(home);
|
||||||
|
expect(receipts.length).toBe(1);
|
||||||
|
expect(receipts[0].status).toBe('204');
|
||||||
|
expect(verifyLedger(home)).toMatchObject({ ok: true, count: 2 });
|
||||||
|
});
|
||||||
|
|
||||||
|
test('tampering with a middle line breaks verification at that line', () => {
|
||||||
|
for (let i = 0; i < 3; i += 1) {
|
||||||
|
writeReceipt({ home, sink: `s${i}`, host: 'h', payloadClass: 'c', consent: 'telemetry=community' });
|
||||||
|
}
|
||||||
|
const ledger = egressLedgerPath(home);
|
||||||
|
const lines = fs.readFileSync(ledger, 'utf-8').trim().split('\n');
|
||||||
|
lines[1] = lines[1].replace('community', 'communitX');
|
||||||
|
fs.writeFileSync(ledger, `${lines.join('\n')}\n`);
|
||||||
|
// Line 2's edited bytes no longer hash to line 3's recorded prev.
|
||||||
|
expect(verifyLedger(home)).toMatchObject({ ok: false, brokenLine: 3 });
|
||||||
|
});
|
||||||
|
|
||||||
|
test('validation rejects garbage before touching the ledger', () => {
|
||||||
|
expect(() => writeReceipt({ home, sink: '', host: 'h', payloadClass: 'c', consent: 'k' } as any)).toThrow();
|
||||||
|
expect(() => writeReceipt({ home, sink: 's', host: 'h', payloadClass: 'c', consent: 'k', bytes: -1 })).toThrow();
|
||||||
|
expect(() => writeReceipt({ home, sink: 's', host: 'h', payloadClass: 'c', consent: 'k', sha256: 'nothex' })).toThrow();
|
||||||
|
expect(fs.existsSync(egressLedgerPath(home))).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('abandoned lock: a stale lock dir (>10s-old mtime) is reclaimed by the next writer', () => {
|
||||||
|
// First write creates the security dir so the lock path's parent exists.
|
||||||
|
writeReceipt({ home, sink: 'a', host: 'h', payloadClass: 'c', consent: 'k=v' });
|
||||||
|
const lock = `${egressLedgerPath(home)}.lock`;
|
||||||
|
fs.mkdirSync(lock);
|
||||||
|
const old = new Date(Date.now() - 60_000);
|
||||||
|
fs.utimesSync(lock, old, old);
|
||||||
|
const started = Date.now();
|
||||||
|
const { id } = writeReceipt({ home, sink: 'b', host: 'h', payloadClass: 'c', consent: 'k=v' });
|
||||||
|
expect(id).toMatch(/^[0-9a-f]{64}$/);
|
||||||
|
// Spin budget is 2.5s; reclaim happens right after budget exhaustion.
|
||||||
|
expect(Date.now() - started).toBeLessThan(10_000);
|
||||||
|
expect(fs.existsSync(lock)).toBe(false);
|
||||||
|
expect(verifyLedger(home)).toMatchObject({ ok: true, count: 2 });
|
||||||
|
});
|
||||||
|
|
||||||
|
test('a fresh (recent-mtime) lock held past the budget fails closed instead of being stolen', () => {
|
||||||
|
writeReceipt({ home, sink: 'a', host: 'h', payloadClass: 'c', consent: 'k=v' });
|
||||||
|
const lock = `${egressLedgerPath(home)}.lock`;
|
||||||
|
fs.mkdirSync(lock);
|
||||||
|
// Keep the mtime fresh so the reclaim path never fires: refresh it in the
|
||||||
|
// background while the writer spins out its 2.5s budget.
|
||||||
|
const refresher = setInterval(() => {
|
||||||
|
const now = new Date();
|
||||||
|
try { fs.utimesSync(lock, now, now); } catch { /* test teardown race */ }
|
||||||
|
}, 1000);
|
||||||
|
try {
|
||||||
|
expect(() =>
|
||||||
|
writeReceipt({ home, sink: 'b', host: 'h', payloadClass: 'c', consent: 'k=v' }),
|
||||||
|
).toThrow(/locked/);
|
||||||
|
} finally {
|
||||||
|
clearInterval(refresher);
|
||||||
|
fs.rmdirSync(lock);
|
||||||
|
}
|
||||||
|
}, 15_000);
|
||||||
|
|
||||||
|
test('tail-read: last line is found correctly on a multi-record ledger larger than the tail window', () => {
|
||||||
|
// 30 records ≈ 9KB > the 4KB tail window, so the append path must find
|
||||||
|
// the true last line from a partial read.
|
||||||
|
for (let i = 0; i < 30; i += 1) {
|
||||||
|
writeReceipt({
|
||||||
|
home,
|
||||||
|
sink: `sink-${i}`,
|
||||||
|
host: 'h',
|
||||||
|
payloadClass: `class-${'x'.repeat(200)}-${i}`,
|
||||||
|
consent: 'k=v',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const ledger = egressLedgerPath(home);
|
||||||
|
expect(fs.statSync(ledger).size).toBeGreaterThan(4096);
|
||||||
|
const result = verifyLedger(home);
|
||||||
|
expect(result).toMatchObject({ ok: true, count: 30 });
|
||||||
|
});
|
||||||
|
|
||||||
|
test('chain verify stays intact across 100+ records', () => {
|
||||||
|
for (let i = 0; i < 120; i += 1) {
|
||||||
|
writeReceipt({ home, sink: `s${i}`, host: 'h', payloadClass: 'c', consent: 'k=v' });
|
||||||
|
if (i % 10 === 0) writeOutcome({ home, receipt: 'f'.repeat(64), status: 200 });
|
||||||
|
}
|
||||||
|
const result = verifyLedger(home);
|
||||||
|
expect(result.ok).toBe(true);
|
||||||
|
expect(result.count).toBe(132);
|
||||||
|
expect(listReceipts(home).length).toBe(120);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('WARN-at-size: >25MB ledger emits one self-explanatory stderr warning per process', () => {
|
||||||
|
const ledger = egressLedgerPath(home);
|
||||||
|
fs.mkdirSync(path.dirname(ledger), { recursive: true, mode: 0o700 });
|
||||||
|
// Grow the file past the threshold with valid-looking filler; the warning
|
||||||
|
// keys off file size only.
|
||||||
|
const filler = `${JSON.stringify({ type: 'egress', pad: 'x'.repeat(1024) })}\n`;
|
||||||
|
const chunk = filler.repeat(1024); // ~1MB
|
||||||
|
const writes = Math.ceil(LEDGER_WARN_BYTES / chunk.length) + 1;
|
||||||
|
for (let i = 0; i < writes; i += 1) fs.appendFileSync(ledger, chunk);
|
||||||
|
expect(fs.statSync(ledger).size).toBeGreaterThan(LEDGER_WARN_BYTES);
|
||||||
|
|
||||||
|
resetLedgerSizeWarningForTests();
|
||||||
|
const captured: string[] = [];
|
||||||
|
const originalWrite = process.stderr.write.bind(process.stderr);
|
||||||
|
(process.stderr as any).write = (chunk: string) => { captured.push(String(chunk)); return true; };
|
||||||
|
try {
|
||||||
|
writeReceipt({ home, sink: 'a', host: 'h', payloadClass: 'c', consent: 'k=v' });
|
||||||
|
writeReceipt({ home, sink: 'b', host: 'h', payloadClass: 'c', consent: 'k=v' });
|
||||||
|
} finally {
|
||||||
|
(process.stderr as any).write = originalWrite;
|
||||||
|
}
|
||||||
|
const warnings = captured.filter((c) => c.includes('egress ledger is large'));
|
||||||
|
expect(warnings.length).toBe(1); // once per process, not per write
|
||||||
|
// Self-explanatory shape: what the ledger is, how to inspect, what's coming.
|
||||||
|
expect(warnings[0]).toContain('ATTEMPTS to send off-machine');
|
||||||
|
expect(warnings[0]).toContain('gstack-egress list');
|
||||||
|
expect(warnings[0]).toContain('rotation');
|
||||||
|
expect(warnings[0]).toContain(ledger);
|
||||||
|
|
||||||
|
// verifyLedger surfaces the same warning as data.
|
||||||
|
const message = ledgerSizeWarning(ledger, fs.statSync(ledger).size);
|
||||||
|
expect(message).toContain('MB');
|
||||||
|
expect(verifyLedger(home).sizeWarning).toBe(message);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('gstack-egress-receipt shell bridge', () => {
|
||||||
|
const bin = path.join(ROOT, 'bin', 'gstack-egress-receipt');
|
||||||
|
|
||||||
|
test('write hashes the exact payload file, prints the receipt id; outcome joins', () => {
|
||||||
|
const payload = path.join(home, 'payload.json');
|
||||||
|
fs.writeFileSync(payload, '[{"v":1}]');
|
||||||
|
const write = spawnSync(bin, ['write', '--sink', 'telemetry-sync', '--host', '127.0.0.1:8399',
|
||||||
|
'--class', 'telemetry-events', '--payload-file', payload, '--consent', 'telemetry=community'],
|
||||||
|
{ encoding: 'utf-8', env: { ...process.env, GSTACK_HOME: home } });
|
||||||
|
expect(write.status).toBe(0);
|
||||||
|
const id = write.stdout.trim();
|
||||||
|
expect(id).toMatch(/^[0-9a-f]{64}$/);
|
||||||
|
const outcome = spawnSync(bin, ['outcome', id, '204'],
|
||||||
|
{ encoding: 'utf-8', env: { ...process.env, GSTACK_HOME: home } });
|
||||||
|
expect(outcome.status).toBe(0);
|
||||||
|
const receipts = listReceipts(home);
|
||||||
|
expect(receipts.length).toBe(1);
|
||||||
|
expect(receipts[0].bytes).toBe(9);
|
||||||
|
expect(receipts[0].sha256).toBe(sha256Hex('[{"v":1}]'));
|
||||||
|
expect(receipts[0].status).toBe('204');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('--no-payload records sha256:null (git-class: a subprocess owns the bytes)', () => {
|
||||||
|
const write = spawnSync(bin, ['write', '--sink', 'brain-sync', '--host', 'github.com',
|
||||||
|
'--class', 'git-push', '--no-payload', '--consent', 'artifacts_sync_mode=auto'],
|
||||||
|
{ encoding: 'utf-8', env: { ...process.env, GSTACK_HOME: home } });
|
||||||
|
expect(write.status).toBe(0);
|
||||||
|
const receipts = listReceipts(home);
|
||||||
|
expect(receipts.length).toBe(1);
|
||||||
|
expect(receipts[0].sha256).toBeNull();
|
||||||
|
expect(receipts[0].bytes).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('write exits 3 with EGRESS_RECEIPT_FAILED when the ledger is unwritable', () => {
|
||||||
|
if (process.platform === 'win32' || process.getuid?.() === 0) return;
|
||||||
|
fs.mkdirSync(path.join(home, 'security'), { recursive: true, mode: 0o500 });
|
||||||
|
const write = spawnSync(bin, ['write', '--sink', 's', '--host', 'h', '--class', 'c', '--no-payload'],
|
||||||
|
{ encoding: 'utf-8', env: { ...process.env, GSTACK_HOME: home } });
|
||||||
|
expect(write.status).toBe(3);
|
||||||
|
expect(write.stderr).toContain('EGRESS_RECEIPT_FAILED');
|
||||||
|
fs.chmodSync(path.join(home, 'security'), 0o700);
|
||||||
|
});
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user