mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-02 17:40:02 +02:00
garrytan
d93d61f7ba
Mechanical reformat only. Minified transpile output is byte-identical for 21 of 22 files; witness.ts differs only in three regex flag orders (/mi -> /im), which JavaScript canonicalizes. Source-text assertions over lib/cso now compare whitespace-insensitively with the same tokens.
474 lines
18 KiB
TypeScript
474 lines
18 KiB
TypeScript
import * as fs from 'node:fs';
|
|
import { join } from 'node:path';
|
|
import { tmpdir, userInfo } from 'node:os';
|
|
import { randomBytes } from 'node:crypto';
|
|
import { CsoError, sha256 } from './contracts';
|
|
import { discardAtomicNoReplaceTemp, recoverAtomicNoReplaceJson, secureDirectory } from './state';
|
|
import { atomicWriteSync } from '../fs-atomic';
|
|
|
|
export const GROUP_LIMITS = {
|
|
cpu: 2,
|
|
memoryMiB: 4096,
|
|
pids: 256,
|
|
writableMiB: 2048,
|
|
outputBytes: 1024 * 1024,
|
|
} as const;
|
|
export const ROLE_LIMITS = {
|
|
anchor: { cpu: 0.05, memoryMiB: 64, pids: 8, writableMiB: 16 },
|
|
app: { cpu: 0.85, memoryMiB: 2304, pids: 96, writableMiB: 1264 },
|
|
verifier: { cpu: 0.55, memoryMiB: 512, pids: 32, writableMiB: 256 },
|
|
tests: { cpu: 0.55, memoryMiB: 1280, pids: 64, writableMiB: 1024 },
|
|
postgres: { cpu: 0.25, memoryMiB: 1024, pids: 96, writableMiB: 512 },
|
|
browser: { cpu: 0.3, memoryMiB: 512, pids: 16, writableMiB: 256 },
|
|
} as const;
|
|
export type Role = keyof typeof ROLE_LIMITS;
|
|
export interface Lease {
|
|
endpoint: string;
|
|
slot: number;
|
|
path: string;
|
|
runId: string;
|
|
ownerPid: number;
|
|
expiresAt: number;
|
|
token: string;
|
|
supervised: boolean;
|
|
}
|
|
function alive(pid: number): boolean {
|
|
try {
|
|
process.kill(pid, 0);
|
|
return true;
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
function processIdentity(pid: number): string | undefined {
|
|
if (process.platform !== 'linux') return;
|
|
try {
|
|
const raw = fs.readFileSync(`/proc/${pid}/stat`, 'utf8'),
|
|
tail = raw
|
|
.slice(raw.lastIndexOf(')') + 2)
|
|
.trim()
|
|
.split(/\s+/);
|
|
return /^\d+$/.test(tail[19] ?? '') ? `linux:${tail[19]}` : undefined;
|
|
} catch {
|
|
return;
|
|
}
|
|
}
|
|
function sameDirectory(left: fs.Stats, right: fs.Stats): boolean {
|
|
return (
|
|
left.dev === right.dev && left.ino === right.ino && left.uid === right.uid && left.mode === right.mode
|
|
);
|
|
}
|
|
function sameFile(left: fs.Stats, right: fs.Stats): boolean {
|
|
return (
|
|
left.dev === right.dev &&
|
|
left.ino === right.ino &&
|
|
left.uid === right.uid &&
|
|
left.mode === right.mode &&
|
|
left.nlink === right.nlink
|
|
);
|
|
}
|
|
function privateDirectory(path: string, label: string): fs.Stats {
|
|
const stat = fs.lstatSync(path);
|
|
if (
|
|
!stat.isDirectory() ||
|
|
stat.isSymbolicLink() ||
|
|
(process.getuid && stat.uid !== process.getuid()) ||
|
|
(stat.mode & 0o077) !== 0
|
|
)
|
|
throw new CsoError('UNSAFE_PATH', `${label} is not a private owned directory`);
|
|
return stat;
|
|
}
|
|
function privateFile(path: string, label: string): fs.Stats {
|
|
const stat = fs.lstatSync(path);
|
|
if (
|
|
!stat.isFile() ||
|
|
stat.isSymbolicLink() ||
|
|
stat.nlink !== 1 ||
|
|
(process.getuid && stat.uid !== process.getuid()) ||
|
|
(stat.mode & 0o077) !== 0 ||
|
|
stat.size > 1024 * 1024
|
|
)
|
|
throw new CsoError('UNSAFE_PATH', `${label} is not a private regular file`);
|
|
return stat;
|
|
}
|
|
type Claim = { path: string; token: string; identity: fs.Stats; pid: number; processIdentity: string | null };
|
|
type ClaimOwner = { pid: number; processIdentity: string | null; token: string; createdAt: number };
|
|
function validateClaimOwner(value: unknown, expectedToken?: string, publisherPid?: number): ClaimOwner {
|
|
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
throw new CsoError('INCOMPATIBLE_INPUT', 'Reproduction recovery owner is invalid');
|
|
const owner = value as Record<string, unknown>;
|
|
if (
|
|
Object.keys(owner).sort().join(',') !== 'createdAt,pid,processIdentity,token' ||
|
|
!Number.isSafeInteger(owner.pid) ||
|
|
Number(owner.pid) <= 1 ||
|
|
typeof owner.token !== 'string' ||
|
|
!/^[a-f0-9]{32}$/.test(owner.token) ||
|
|
(expectedToken !== undefined && owner.token !== expectedToken) ||
|
|
!Number.isFinite(owner.createdAt) ||
|
|
Number(owner.createdAt) < 0 ||
|
|
!(
|
|
owner.processIdentity === null ||
|
|
(typeof owner.processIdentity === 'string' && /^linux:\d+$/.test(owner.processIdentity))
|
|
) ||
|
|
(publisherPid !== undefined && Number(owner.pid) !== publisherPid)
|
|
)
|
|
throw new CsoError('INCOMPATIBLE_INPUT', 'Reproduction recovery owner is invalid');
|
|
return {
|
|
pid: Number(owner.pid),
|
|
processIdentity: owner.processIdentity as string | null,
|
|
token: owner.token,
|
|
createdAt: Number(owner.createdAt),
|
|
};
|
|
}
|
|
function inspectClaim(path: string, expectedToken?: string): Claim {
|
|
const before = privateFile(path, 'Reproduction recovery claim');
|
|
if (before.size <= 0 || before.size > 4096)
|
|
throw new CsoError('UNSAFE_PATH', 'Reproduction recovery claim has an invalid size');
|
|
let owner: any;
|
|
try {
|
|
owner = JSON.parse(fs.readFileSync(path, 'utf8'));
|
|
} catch {
|
|
throw new CsoError('INCOMPATIBLE_INPUT', 'Reproduction recovery owner is invalid');
|
|
}
|
|
const after = privateFile(path, 'Reproduction recovery claim');
|
|
if (!sameFile(before, after))
|
|
throw new CsoError('INCOMPATIBLE_INPUT', 'Reproduction recovery owner is invalid');
|
|
owner = validateClaimOwner(owner, expectedToken);
|
|
return {
|
|
path,
|
|
token: owner.token,
|
|
identity: after,
|
|
pid: owner.pid,
|
|
processIdentity: owner.processIdentity,
|
|
};
|
|
}
|
|
function releaseClaim(claim: Claim): void {
|
|
const current = inspectClaim(claim.path, claim.token);
|
|
if (!sameFile(current.identity, claim.identity))
|
|
throw new CsoError('PERSISTENCE_FAILED', 'Reproduction recovery ownership changed');
|
|
const final = privateFile(claim.path, 'Reproduction recovery claim');
|
|
if (!sameFile(final, claim.identity))
|
|
throw new CsoError('PERSISTENCE_FAILED', 'Reproduction recovery ownership changed');
|
|
fs.unlinkSync(claim.path);
|
|
}
|
|
function acquireClaim(parent: string, expected: fs.Stats): Claim {
|
|
const path = join(parent, '.recovery'),
|
|
assertParent = () => {
|
|
const current = privateDirectory(parent, 'Reproduction lease slot');
|
|
if (!sameDirectory(expected, current))
|
|
throw new CsoError('INSUFFICIENT_CAPACITY', 'Reproduction lease changed during recovery');
|
|
};
|
|
const recoverPublications = () => {
|
|
const pattern = /^\.recovery\.tmp\.(\d{1,10})\.[a-f0-9]{8}$/;
|
|
for (const name of fs.readdirSync(parent)) {
|
|
const match = name.match(pattern);
|
|
if (!match) continue;
|
|
const publisherPid = Number(match[1]),
|
|
temporary = join(parent, name),
|
|
options = {
|
|
label: 'Reproduction recovery claim',
|
|
maxBytes: 4096,
|
|
validate: (value: unknown, pid: number) => {
|
|
validateClaimOwner(value, undefined, pid);
|
|
},
|
|
};
|
|
assertParent();
|
|
if (fs.existsSync(path)) recoverAtomicNoReplaceJson(path, options);
|
|
if (fs.existsSync(temporary)) discardAtomicNoReplaceTemp(temporary, publisherPid, options);
|
|
assertParent();
|
|
}
|
|
};
|
|
for (let attempt = 0; attempt < 64; attempt++) {
|
|
assertParent();
|
|
recoverPublications();
|
|
const token = randomBytes(16).toString('hex');
|
|
try {
|
|
atomicWriteSync(
|
|
path,
|
|
JSON.stringify({
|
|
pid: process.pid,
|
|
processIdentity: processIdentity(process.pid) ?? null,
|
|
token,
|
|
createdAt: Date.now(),
|
|
}) + '\n',
|
|
{ mode: 0o600, noReplace: true },
|
|
);
|
|
const claim = inspectClaim(path, token);
|
|
try {
|
|
assertParent();
|
|
} catch (error) {
|
|
try {
|
|
releaseClaim(claim);
|
|
} catch {}
|
|
throw error;
|
|
}
|
|
return claim;
|
|
} catch (error: any) {
|
|
if (error instanceof CsoError) throw error;
|
|
if (error?.code !== 'EEXIST')
|
|
throw new CsoError('PERSISTENCE_FAILED', 'Reproduction recovery claim could not be created');
|
|
}
|
|
assertParent();
|
|
const observed = inspectClaim(path),
|
|
isAlive = alive(observed.pid),
|
|
identity = isAlive ? processIdentity(observed.pid) : undefined;
|
|
if (
|
|
isAlive &&
|
|
!(
|
|
typeof observed.processIdentity === 'string' &&
|
|
identity !== undefined &&
|
|
identity !== observed.processIdentity
|
|
)
|
|
)
|
|
throw new CsoError('INSUFFICIENT_CAPACITY', 'Another helper is recovering the reproduction lease');
|
|
try {
|
|
releaseClaim(observed);
|
|
} catch (error) {
|
|
if (error instanceof CsoError && error.code === 'PERSISTENCE_FAILED') continue;
|
|
throw error;
|
|
}
|
|
}
|
|
throw new CsoError('INSUFFICIENT_CAPACITY', 'Reproduction recovery claim changed repeatedly');
|
|
}
|
|
/** One host-user pool shared by every workspace/state root on this machine. */
|
|
export function machinePoolRoot(): string {
|
|
const uid = process.getuid?.() ?? userInfo().uid;
|
|
return secureDirectory(join(fs.realpathSync(tmpdir()), `gstack-cso-pool-${uid}`));
|
|
}
|
|
function reclaimSlot(
|
|
path: string,
|
|
pool: string,
|
|
slot: number,
|
|
observed: fs.Stats,
|
|
expectedToken?: string,
|
|
): boolean {
|
|
let claim: Claim;
|
|
try {
|
|
claim = acquireClaim(path, observed);
|
|
} catch (error) {
|
|
if (error instanceof CsoError && error.code === 'INSUFFICIENT_CAPACITY') return false;
|
|
throw error;
|
|
}
|
|
try {
|
|
const current = privateDirectory(path, 'Reproduction lease slot');
|
|
if (!sameDirectory(observed, current)) {
|
|
releaseClaim(claim);
|
|
return false;
|
|
}
|
|
if (expectedToken) {
|
|
privateFile(join(path, 'lease.json'), 'Reproduction lease');
|
|
const lease = JSON.parse(fs.readFileSync(join(path, 'lease.json'), 'utf8'));
|
|
if (lease.token !== expectedToken) {
|
|
releaseClaim(claim);
|
|
return false;
|
|
}
|
|
}
|
|
const tomb = join(pool, `.slot-${slot}.stale-${process.pid}-${randomBytes(8).toString('hex')}`);
|
|
fs.renameSync(path, tomb);
|
|
const moved = privateDirectory(tomb, 'Reproduction lease tomb');
|
|
if (!sameDirectory(observed, moved))
|
|
throw new CsoError('SNAPSHOT_RACE', 'Reproduction lease changed while quarantined');
|
|
fs.mkdirSync(path, { mode: 0o700 });
|
|
releaseClaim({ ...claim, path: join(tomb, '.recovery') });
|
|
for (const name of fs.readdirSync(tomb)) {
|
|
if (
|
|
!['lease.json', 'lease.token'].includes(name) &&
|
|
!/^lease\.json\.tmp\.\d+\.[a-f0-9]{8}$/.test(name) &&
|
|
!/^\.recovery\.tmp\.\d+\.[a-f0-9]{8}$/.test(name)
|
|
)
|
|
throw new CsoError('UNSAFE_PATH', 'Stale reproduction lease contains an unexpected object');
|
|
privateFile(join(tomb, name), 'Stale reproduction lease file');
|
|
fs.unlinkSync(join(tomb, name));
|
|
}
|
|
fs.rmdirSync(tomb);
|
|
return true;
|
|
} catch (error) {
|
|
if (error instanceof CsoError) throw error;
|
|
return false;
|
|
}
|
|
}
|
|
function slotControl(pool: string, slot: number): { path: string; stat: fs.Stats } {
|
|
const path = join(pool, `.slot-${slot}.control`);
|
|
try {
|
|
fs.mkdirSync(path, { mode: 0o700 });
|
|
} catch (error: any) {
|
|
if (error?.code !== 'EEXIST')
|
|
throw new CsoError('PERSISTENCE_FAILED', 'Reproduction slot control directory could not be created');
|
|
}
|
|
const stat = privateDirectory(path, 'Reproduction slot control directory');
|
|
for (const name of fs.readdirSync(path))
|
|
if (name !== '.recovery' && !/^\.recovery\.tmp\.\d+\.[a-f0-9]{8}$/.test(name))
|
|
throw new CsoError('UNSAFE_PATH', 'Reproduction slot control directory contains an unexpected object');
|
|
return { path, stat };
|
|
}
|
|
function writeLease(lease: Lease): void {
|
|
const keys = Object.keys(lease).sort().join(','),
|
|
expected = 'endpoint,expiresAt,ownerPid,path,runId,slot,supervised,token';
|
|
if (
|
|
keys !== expected ||
|
|
!/^unix:\/\/[/.A-Za-z0-9_-]+$/.test(lease.endpoint) ||
|
|
![0, 1].includes(lease.slot) ||
|
|
!/^[A-Za-z0-9_.-]{1,100}$/.test(lease.runId) ||
|
|
lease.ownerPid !== process.pid ||
|
|
!Number.isSafeInteger(lease.expiresAt) ||
|
|
!/^[a-f0-9]{32}$/.test(lease.token) ||
|
|
typeof lease.supervised !== 'boolean'
|
|
)
|
|
throw new CsoError('PERSISTENCE_FAILED', 'Reproduction lease metadata is invalid');
|
|
const expectedPath = join(machinePoolRoot(), sha256(lease.endpoint).slice(0, 24), `slot-${lease.slot}`);
|
|
if (lease.path !== expectedPath)
|
|
throw new CsoError('PERSISTENCE_FAILED', 'Reproduction lease path is invalid');
|
|
// This exact helper-owned schema contains only control metadata. In
|
|
// particular, its random capability may resemble a wallet address and must
|
|
// remain byte-identical to lease.token; untrusted reports still use writeJson.
|
|
atomicWriteSync(join(lease.path, 'lease.json'), JSON.stringify(lease) + '\n', { mode: 0o600 });
|
|
}
|
|
export function admit(endpoint: string, runId: string, deadline: number): Lease {
|
|
if (!/^unix:\/\/[/.A-Za-z0-9_-]+$/.test(endpoint))
|
|
throw new CsoError(
|
|
'ISOLATION_FAILED',
|
|
'Only a pinned local Unix Docker endpoint is admitted on this host',
|
|
);
|
|
const pool = secureDirectory(join(machinePoolRoot(), sha256(endpoint).slice(0, 24)));
|
|
for (let slot = 0; slot < 2; slot++) {
|
|
const path = join(pool, `slot-${slot}`),
|
|
control = slotControl(pool, slot);
|
|
let mutation: Claim;
|
|
try {
|
|
mutation = acquireClaim(control.path, control.stat);
|
|
} catch (error) {
|
|
if (error instanceof CsoError && error.code === 'INSUFFICIENT_CAPACITY') continue;
|
|
throw error;
|
|
}
|
|
try {
|
|
try {
|
|
fs.mkdirSync(path, { mode: 0o700 });
|
|
} catch (error: any) {
|
|
if (error?.code !== 'EEXIST')
|
|
throw new CsoError('PERSISTENCE_FAILED', 'Reproduction lease slot could not be created');
|
|
try {
|
|
const observed = privateDirectory(path, 'Reproduction lease slot');
|
|
const old = JSON.parse(fs.readFileSync(join(path, 'lease.json'), 'utf8'));
|
|
// A supervised lease is removed only after its watchdog or owner has
|
|
// confirmed exact-resource cleanup. This preserves the two-group cap
|
|
// through supervisor death and daemon outages.
|
|
if (old.supervised === true || (typeof old.ownerPid === 'number' && alive(old.ownerPid))) continue;
|
|
// Unsupervised stale slots cannot have created containers: supervision
|
|
// is acknowledged before the anchor create call.
|
|
if (!reclaimSlot(path, pool, slot, observed, typeof old.token === 'string' ? old.token : undefined))
|
|
continue;
|
|
} catch (recoveryError) {
|
|
if (recoveryError instanceof CsoError) throw recoveryError;
|
|
// No live initializer can publish into this path while this stable
|
|
// slot-control claim is held. Recover a crashed partial publication
|
|
// only after the compatibility grace period.
|
|
let stat: fs.Stats;
|
|
try {
|
|
stat = privateDirectory(path, 'Reproduction lease slot');
|
|
} catch (statError) {
|
|
if (statError instanceof CsoError) throw statError;
|
|
continue;
|
|
}
|
|
if (Date.now() - stat.mtimeMs <= 5000) continue;
|
|
if (!reclaimSlot(path, pool, slot, stat)) continue;
|
|
}
|
|
}
|
|
// Both authenticated records become visible as one logical publication
|
|
// when the stable slot-control claim is released.
|
|
const lease: Lease = {
|
|
endpoint,
|
|
slot,
|
|
path,
|
|
runId,
|
|
ownerPid: process.pid,
|
|
expiresAt: deadline,
|
|
token: randomBytes(16).toString('hex'),
|
|
supervised: false,
|
|
};
|
|
writeLease(lease);
|
|
fs.writeFileSync(join(path, 'lease.token'), lease.token + '\n', { mode: 0o600, flag: 'wx' });
|
|
return lease;
|
|
} finally {
|
|
releaseClaim(mutation);
|
|
}
|
|
}
|
|
throw new CsoError(
|
|
'INSUFFICIENT_CAPACITY',
|
|
'Two reproduction groups are already admitted for this Docker endpoint',
|
|
);
|
|
}
|
|
export function markSupervised(lease: Lease): void {
|
|
const current = JSON.parse(fs.readFileSync(join(lease.path, 'lease.json'), 'utf8'));
|
|
if (current.token !== lease.token || current.ownerPid !== lease.ownerPid)
|
|
throw new CsoError('INSUFFICIENT_CAPACITY', 'Reproduction lease changed before watchdog supervision');
|
|
lease.supervised = true;
|
|
writeLease(lease);
|
|
}
|
|
export function release(lease: Lease): void {
|
|
let observed: fs.Stats;
|
|
try {
|
|
observed = privateDirectory(lease.path, 'Reproduction lease slot');
|
|
} catch (error: any) {
|
|
if (error?.code === 'ENOENT')
|
|
throw new CsoError('PERSISTENCE_FAILED', 'Exact reproduction lease was already missing');
|
|
throw error;
|
|
}
|
|
const claim = acquireClaim(lease.path, observed);
|
|
try {
|
|
const currentStat = privateDirectory(lease.path, 'Reproduction lease slot');
|
|
if (!sameDirectory(observed, currentStat))
|
|
throw new CsoError('PERSISTENCE_FAILED', 'Reproduction lease changed before exact release');
|
|
const names = fs
|
|
.readdirSync(lease.path)
|
|
.filter((name) => name !== '.recovery' && !/^\.recovery\.tmp\.\d+\.[a-f0-9]{8}$/.test(name))
|
|
.sort();
|
|
if (names.join('\0') !== 'lease.json\0lease.token')
|
|
throw new CsoError('PERSISTENCE_FAILED', 'Reproduction lease contents changed before exact release');
|
|
privateFile(join(lease.path, 'lease.json'), 'Reproduction lease');
|
|
privateFile(join(lease.path, 'lease.token'), 'Reproduction lease token');
|
|
const current = JSON.parse(fs.readFileSync(join(lease.path, 'lease.json'), 'utf8')),
|
|
token = fs.readFileSync(join(lease.path, 'lease.token'), 'utf8').trim();
|
|
if (
|
|
current.runId !== lease.runId ||
|
|
current.ownerPid !== lease.ownerPid ||
|
|
current.token !== lease.token ||
|
|
token !== lease.token
|
|
)
|
|
throw new CsoError('PERSISTENCE_FAILED', 'Reproduction lease ownership changed before exact release');
|
|
fs.unlinkSync(join(lease.path, 'lease.token'));
|
|
fs.unlinkSync(join(lease.path, 'lease.json'));
|
|
releaseClaim(claim);
|
|
fs.rmdirSync(lease.path);
|
|
if (fs.existsSync(lease.path))
|
|
throw new CsoError('PERSISTENCE_FAILED', 'Exact reproduction lease removal could not be proven');
|
|
} catch (error) {
|
|
try {
|
|
if (fs.existsSync(claim.path)) releaseClaim(claim);
|
|
} catch {}
|
|
if (error instanceof CsoError) throw error;
|
|
throw new CsoError('PERSISTENCE_FAILED', 'Exact reproduction lease removal failed');
|
|
}
|
|
}
|
|
export function total(roles: Role[]) {
|
|
const value = roles.reduce(
|
|
(a, r) => ({
|
|
cpu: a.cpu + ROLE_LIMITS[r].cpu,
|
|
memoryMiB: a.memoryMiB + ROLE_LIMITS[r].memoryMiB,
|
|
pids: a.pids + ROLE_LIMITS[r].pids,
|
|
writableMiB: a.writableMiB + ROLE_LIMITS[r].writableMiB,
|
|
}),
|
|
{ cpu: 0, memoryMiB: 0, pids: 0, writableMiB: 0 },
|
|
);
|
|
if (
|
|
value.cpu > GROUP_LIMITS.cpu ||
|
|
value.memoryMiB > GROUP_LIMITS.memoryMiB ||
|
|
value.pids > GROUP_LIMITS.pids ||
|
|
value.writableMiB > GROUP_LIMITS.writableMiB
|
|
)
|
|
throw new CsoError(
|
|
'INSUFFICIENT_CAPACITY',
|
|
'Requested sidecars exceed the aggregate reproduction-group limit',
|
|
);
|
|
return value;
|
|
}
|