mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-16 09:55:29 +02:00
v1.87.0.0 feat: add verified CSO audits and replayable repair bundles (#2852)
* feat(cso): add verified audits and replayable repair bundles * fix(cso): harden qualification and setup boundaries * fix(cso): assemble security canaries at runtime * fix(cso): bound release proof and maintenance work Co-Authored-By: OpenAI Codex <noreply@openai.com> * fix(cso): require complete evaluation reports Co-Authored-By: OpenAI Codex <noreply@openai.com> * fix(cso): replay expired snapshots from supplied source Co-Authored-By: OpenAI Codex <noreply@openai.com> * test(cso): synchronize DNS cancellation assertion Co-Authored-By: OpenAI Codex <noreply@openai.com> * chore(ship): exempt repository owner from liveness proof Co-Authored-By: OpenAI Codex <noreply@openai.com> * test(cso): make recheck retention overlap deterministic Co-Authored-By: OpenAI Codex <noreply@openai.com> * chore: bump version and changelog (v1.85.0.0) Co-Authored-By: OpenAI Codex <noreply@openai.com> * fix(cso): pass native release gates Co-Authored-By: OpenAI Codex <noreply@openai.com> * chore: move release to v1.86.0.0 Co-Authored-By: OpenAI Codex <noreply@openai.com> * fix(cso): resolve rechecks by finding Co-Authored-By: OpenAI Codex <noreply@openai.com> * chore: move release to v1.87.0.0 Co-Authored-By: OpenAI Codex <noreply@openai.com> * fix(cso): pass macOS and Windows release gates Normalize BSD wc output, compare Windows paths by filesystem identity, preserve portable snapshot race coverage, and narrow POSIX-only Windows fixtures. Co-Authored-By: OpenAI Codex <noreply@openai.com> * fix(cso): harden native verification gates * fix(cso): refine Windows native diagnostics * test(cso): isolate Windows Git startup failure * test(cso): stabilize Windows native diagnostics * fix(cso): support hardened Git on Windows * fix(cso): close final verification gaps * test(cso): bound cold Docker fixture setup * fix(cso): restore cross-platform free-suite gates --------- Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
9f81911136
commit
4a3c6a8a3c
@@ -0,0 +1,707 @@
|
||||
import * as fs from 'node:fs';
|
||||
import { createHash, randomBytes } from 'node:crypto';
|
||||
import { join, resolve, sep } from 'node:path';
|
||||
import { atomicWriteSync } from '../fs-atomic';
|
||||
import { CsoError } from './contracts';
|
||||
import { discardAtomicNoReplaceTemp, privateRoot, recoverAtomicNoReplaceJson, secureDirectory, withLock as withStateLock } from './state';
|
||||
|
||||
export const DEFAULT_PUBLIC_ARCHIVE_CACHE_BYTES = 10 * 1024 * 1024 * 1024;
|
||||
const METADATA_VERSION = 1;
|
||||
const METADATA_LIMIT = 4096;
|
||||
const COPY_BUFFER_BYTES = 64 * 1024;
|
||||
const MAX_CACHE_DIRECTORY_ENTRIES = 100_000;
|
||||
const CACHE_LOCK_PROTOCOL = 'immutable-cache-lease-set-v3';
|
||||
const SHA256 = /^[a-f0-9]{64}$/;
|
||||
const RELATIVE_STAGE_PATH = /^(?!\/)(?!.*(?:^|\/)\.\.?(?:\/|$))(?!.*\\)[^\0-\x1f\x7f]+$/;
|
||||
|
||||
export interface PublicArchiveCacheOptions {
|
||||
/** Defaults to the private CSO state namespace. */
|
||||
root?: string;
|
||||
/** Existing directory populated by the constrained acquisition step. */
|
||||
stagingRoot: string;
|
||||
/** Persistent archive-byte ceiling. Defaults to 10 GiB. */
|
||||
maxBytes?: number;
|
||||
/** Per-archive ceiling. Defaults to maxBytes. */
|
||||
maxEntryBytes?: number;
|
||||
/** Deterministic clock for tests. */
|
||||
now?: () => number;
|
||||
}
|
||||
|
||||
/** Optional cooperative bounds for synchronous cache work. */
|
||||
export interface CacheOperationControl {
|
||||
/** Absolute Unix timestamp in milliseconds. */
|
||||
deadline?: number;
|
||||
/** Checked between bounded filesystem operations. */
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export type CacheOperationInput = CacheOperationControl | number | undefined;
|
||||
type NormalizedCacheOperationControl = Readonly<{ deadline?: number; signal?: AbortSignal }>;
|
||||
|
||||
export interface PublicArchiveCacheEntry {
|
||||
sha256: string;
|
||||
path: string;
|
||||
bytes: number;
|
||||
createdAt: number;
|
||||
lastAccessedAt: number;
|
||||
}
|
||||
|
||||
export interface PublicArchiveCacheStats {
|
||||
entries: number;
|
||||
bytes: number;
|
||||
maxBytes: number;
|
||||
}
|
||||
|
||||
export interface MaterializedArchive {
|
||||
sha256: string;
|
||||
path: string;
|
||||
bytes: number;
|
||||
}
|
||||
|
||||
interface Metadata {
|
||||
version: 1;
|
||||
sha256: string;
|
||||
bytes: number;
|
||||
createdAt: number;
|
||||
lastAccessedAt: number;
|
||||
}
|
||||
|
||||
interface StableStat {
|
||||
dev: number;
|
||||
ino: number;
|
||||
size: number;
|
||||
mode: number;
|
||||
nlink: number;
|
||||
mtimeMs: number;
|
||||
ctimeMs: number;
|
||||
uid: number;
|
||||
}
|
||||
|
||||
function fail(code: ConstructorParameters<typeof CsoError>[0], message: string): never {
|
||||
throw new CsoError(code, message);
|
||||
}
|
||||
|
||||
function operationControl(input?: CacheOperationInput): NormalizedCacheOperationControl {
|
||||
const value = typeof input === 'number' ? { deadline: input } : input ?? {};
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) fail('INVALID_ARGUMENT', 'Cache operation control must be an object or absolute deadline');
|
||||
if (value.deadline !== undefined && (!Number.isSafeInteger(value.deadline) || value.deadline <= 0))
|
||||
fail('INVALID_ARGUMENT', 'Cache deadline must be an absolute millisecond timestamp');
|
||||
if (value.signal !== undefined && typeof value.signal.aborted !== 'boolean')
|
||||
fail('INVALID_ARGUMENT', 'Cache cancellation signal is invalid');
|
||||
const control = Object.freeze({ ...(value.deadline === undefined ? {} : { deadline: value.deadline }),
|
||||
...(value.signal === undefined ? {} : { signal: value.signal }) });
|
||||
checkOperation(control);
|
||||
return control;
|
||||
}
|
||||
|
||||
function checkOperation(control: NormalizedCacheOperationControl): void {
|
||||
if (control.signal?.aborted) fail('CANCELLED', 'Archive-cache operation was cancelled');
|
||||
if (control.deadline !== undefined && Date.now() >= control.deadline)
|
||||
fail('DEADLINE', 'Archive-cache operation reached its deadline');
|
||||
}
|
||||
|
||||
function boundedDirectoryNames(path: string, label: string, control: NormalizedCacheOperationControl): string[] {
|
||||
checkOperation(control);
|
||||
const directory = fs.opendirSync(path), names: string[] = [];
|
||||
try {
|
||||
for (;;) {
|
||||
checkOperation(control);
|
||||
const entry = directory.readSync();
|
||||
if (!entry) break;
|
||||
if (names.length >= MAX_CACHE_DIRECTORY_ENTRIES) fail('INSUFFICIENT_CAPACITY', `${label} exceeds the cache entry limit`);
|
||||
names.push(entry.name);
|
||||
}
|
||||
} finally { directory.closeSync(); }
|
||||
checkOperation(control);
|
||||
return names;
|
||||
}
|
||||
|
||||
function assertEmptyDirectory(path: string, control: NormalizedCacheOperationControl): void {
|
||||
checkOperation(control);
|
||||
const directory = fs.opendirSync(path);
|
||||
try { if (directory.readSync()) fail('UNSAFE_PATH', 'Archive materialization directory must be empty'); }
|
||||
finally { directory.closeSync(); }
|
||||
checkOperation(control);
|
||||
}
|
||||
|
||||
function boundedPositiveInteger(value: number, name: string): number {
|
||||
if (!Number.isSafeInteger(value) || value <= 0) fail('INVALID_ARGUMENT', `${name} must be a positive safe integer`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function expectedDigest(value: string): string {
|
||||
if (!SHA256.test(value)) fail('INVALID_ARGUMENT', 'Archive SHA-256 must be 64 lowercase hexadecimal characters');
|
||||
return value;
|
||||
}
|
||||
|
||||
function stagedRelativePath(value: string): string {
|
||||
if (typeof value !== 'string' || value.length > 4096 || !RELATIVE_STAGE_PATH.test(value))
|
||||
fail('UNSAFE_PATH', 'Staged archive path must be a contained relative path');
|
||||
const parts = value.split('/');
|
||||
if (parts.some(part => !part || part === '.' || part === '..')) fail('UNSAFE_PATH', 'Staged archive path must be a contained relative path');
|
||||
return value;
|
||||
}
|
||||
|
||||
function stableStat(stat: fs.Stats): StableStat {
|
||||
return {
|
||||
dev: stat.dev, ino: stat.ino, size: stat.size, mode: stat.mode, nlink: stat.nlink,
|
||||
mtimeMs: stat.mtimeMs, ctimeMs: stat.ctimeMs, uid: stat.uid,
|
||||
};
|
||||
}
|
||||
|
||||
function sameStat(left: StableStat, right: StableStat): boolean {
|
||||
return left.dev === right.dev && left.ino === right.ino && left.size === right.size &&
|
||||
left.mode === right.mode && left.nlink === right.nlink && left.mtimeMs === right.mtimeMs &&
|
||||
left.ctimeMs === right.ctimeMs && left.uid === right.uid;
|
||||
}
|
||||
|
||||
function sameRenamedInode(left: StableStat, right: StableStat): boolean {
|
||||
return left.dev === right.dev && left.ino === right.ino && left.size === right.size &&
|
||||
left.mode === right.mode && left.nlink === right.nlink && left.mtimeMs === right.mtimeMs && left.uid === right.uid;
|
||||
}
|
||||
|
||||
|
||||
function assertOwnedRegular(stat: fs.Stats, label: string, maxBytes: number, immutable = false): void {
|
||||
if (!stat.isFile() || stat.isSymbolicLink()) fail('UNSAFE_PATH', `${label} must be a regular file`);
|
||||
if (stat.nlink !== 1) fail('UNSAFE_PATH', `${label} must not be hard-linked`);
|
||||
if (process.getuid && stat.uid !== process.getuid()) fail('UNSAFE_PATH', `${label} must be owned by the current user`);
|
||||
if (stat.size > maxBytes) fail('INSUFFICIENT_CAPACITY', `${label} exceeds its byte limit`);
|
||||
if (immutable && (stat.mode & 0o222) !== 0) fail('INCOMPATIBLE_INPUT', `${label} is unexpectedly writable`);
|
||||
}
|
||||
|
||||
function assertExistingDirectory(path: string, label: string): string {
|
||||
const requested = resolve(path);
|
||||
let requestedStat: fs.Stats;
|
||||
try { requestedStat = fs.lstatSync(requested); }
|
||||
catch { fail('MISSING_INPUT', `${label} does not exist`); }
|
||||
if (requestedStat!.isSymbolicLink()) fail('UNSAFE_PATH', `${label} must not be a symlink`);
|
||||
let canonical: string;
|
||||
try { canonical = fs.realpathSync(requested); }
|
||||
catch { fail('MISSING_INPUT', `${label} does not exist`); }
|
||||
const stat = fs.lstatSync(canonical!);
|
||||
if (!stat.isDirectory() || stat.isSymbolicLink()) fail('UNSAFE_PATH', `${label} must be a directory`);
|
||||
if (process.getuid && stat.uid !== process.getuid()) fail('UNSAFE_PATH', `${label} must be owned by the current user`);
|
||||
if ((stat.mode & 0o022) !== 0) fail('UNSAFE_PATH', `${label} must not be writable by another user`);
|
||||
return canonical!;
|
||||
}
|
||||
|
||||
function assertContainedAncestors(root: string, relativePath: string, control: NormalizedCacheOperationControl): string {
|
||||
const parts = relativePath.split('/');
|
||||
let cursor = root;
|
||||
for (const part of parts.slice(0, -1)) {
|
||||
checkOperation(control);
|
||||
cursor = join(cursor, part);
|
||||
let stat: fs.Stats;
|
||||
try { stat = fs.lstatSync(cursor); }
|
||||
catch { fail('MISSING_INPUT', `Staged archive directory is missing: ${part}`); }
|
||||
if (!stat.isDirectory() || stat.isSymbolicLink()) fail('UNSAFE_PATH', 'Staged archive has a symlink or non-directory ancestor');
|
||||
if (process.getuid && stat.uid !== process.getuid()) fail('UNSAFE_PATH', 'Staged archive ancestor has an unexpected owner');
|
||||
if ((stat.mode & 0o022) !== 0) fail('UNSAFE_PATH', 'Staged archive ancestor is writable by another user');
|
||||
}
|
||||
const path = resolve(root, ...parts);
|
||||
if (path !== root && !path.startsWith(`${root}${sep}`)) fail('UNSAFE_PATH', 'Staged archive escaped its staging directory');
|
||||
return path;
|
||||
}
|
||||
|
||||
function openNoFollow(path: string, flags: number, mode?: number): number {
|
||||
const noFollow = (fs.constants as Record<string, number>).O_NOFOLLOW ?? 0;
|
||||
const closeOnExec = (fs.constants as Record<string, number>).O_CLOEXEC ?? 0;
|
||||
try { return fs.openSync(path, flags | noFollow | closeOnExec, mode); }
|
||||
catch { fail('UNSAFE_PATH', 'Archive file could not be opened without following links'); }
|
||||
}
|
||||
|
||||
function readMetadata(path: string, digest: string, control: NormalizedCacheOperationControl): Metadata {
|
||||
checkOperation(control);
|
||||
let stat: fs.Stats;
|
||||
try { stat = fs.lstatSync(path); }
|
||||
catch { fail('INCOMPATIBLE_INPUT', `Cache metadata is missing for ${digest}`); }
|
||||
assertOwnedRegular(stat!, 'Cache metadata', METADATA_LIMIT);
|
||||
if ((stat!.mode & 0o077) !== 0) fail('INCOMPATIBLE_INPUT', 'Cache metadata permissions are not private');
|
||||
let value: unknown;
|
||||
try { checkOperation(control); value = JSON.parse(fs.readFileSync(path, 'utf8')); checkOperation(control); }
|
||||
catch (error) {
|
||||
if (error instanceof CsoError) throw error;
|
||||
fail('INCOMPATIBLE_INPUT', `Cache metadata is invalid for ${digest}`);
|
||||
}
|
||||
const record = value as Partial<Metadata>;
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value) || Object.keys(value).sort().join(',') !== 'bytes,createdAt,lastAccessedAt,sha256,version' ||
|
||||
record.version !== METADATA_VERSION || record.sha256 !== digest || !Number.isSafeInteger(record.bytes) || Number(record.bytes) < 0 ||
|
||||
!Number.isSafeInteger(record.createdAt) || Number(record.createdAt) < 0 || !Number.isSafeInteger(record.lastAccessedAt) ||
|
||||
Number(record.lastAccessedAt) < Number(record.createdAt)) fail('INCOMPATIBLE_INPUT', `Cache metadata is invalid for ${digest}`);
|
||||
return record as Metadata;
|
||||
}
|
||||
|
||||
function writeMetadata(path: string, metadata: Metadata, noReplace = false): void {
|
||||
try { atomicWriteSync(path, `${JSON.stringify(metadata)}\n`, { mode: 0o600, noReplace }); }
|
||||
catch { fail('PERSISTENCE_FAILED', 'Cache metadata could not be written atomically'); }
|
||||
}
|
||||
|
||||
function removeRegular(path: string, label: string): void {
|
||||
const stat = fs.lstatSync(path);
|
||||
assertOwnedRegular(stat, label, Number.MAX_SAFE_INTEGER);
|
||||
try { fs.unlinkSync(path); }
|
||||
catch { fail('PERSISTENCE_FAILED', `${label} could not be removed`); }
|
||||
}
|
||||
|
||||
function existsNoFollow(path: string): boolean {
|
||||
try { fs.lstatSync(path); return true; }
|
||||
catch (error: any) {
|
||||
if (error?.code === 'ENOENT') return false;
|
||||
fail('INCOMPATIBLE_INPUT', 'Cache object could not be inspected safely');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A content-addressed cache for already-acquired public archives. It never
|
||||
* performs downloads, runs package managers, or executes archive content.
|
||||
*/
|
||||
export class PublicArchiveCache {
|
||||
readonly root: string;
|
||||
readonly stagingRoot: string;
|
||||
readonly maxBytes: number;
|
||||
readonly maxEntryBytes: number;
|
||||
private readonly entriesDir: string;
|
||||
private readonly metadataDir: string;
|
||||
private readonly incomingDir: string;
|
||||
private readonly recoveryDir: string;
|
||||
private readonly lockDir: string;
|
||||
private readonly clock: () => number;
|
||||
|
||||
constructor(options: PublicArchiveCacheOptions) {
|
||||
if (!options || typeof options !== 'object') fail('INVALID_ARGUMENT', 'Cache options are required');
|
||||
this.maxBytes = boundedPositiveInteger(options.maxBytes ?? DEFAULT_PUBLIC_ARCHIVE_CACHE_BYTES, 'maxBytes');
|
||||
this.maxEntryBytes = boundedPositiveInteger(options.maxEntryBytes ?? this.maxBytes, 'maxEntryBytes');
|
||||
if (this.maxEntryBytes > this.maxBytes) fail('INVALID_ARGUMENT', 'maxEntryBytes cannot exceed maxBytes');
|
||||
this.clock = options.now ?? Date.now;
|
||||
const base = options.root ? resolve(options.root) : join(privateRoot(), 'public-cache');
|
||||
this.root = secureDirectory(base);
|
||||
this.entriesDir = secureDirectory(join(this.root, 'entries'));
|
||||
this.metadataDir = secureDirectory(join(this.root, 'metadata'));
|
||||
this.incomingDir = secureDirectory(join(this.root, 'incoming'));
|
||||
this.recoveryDir = secureDirectory(join(this.root, 'recovery'));
|
||||
this.lockDir = join(this.root, '.lock');
|
||||
this.stagingRoot = assertExistingDirectory(options.stagingRoot, 'Archive staging directory');
|
||||
if (this.root === this.stagingRoot || this.root.startsWith(`${this.stagingRoot}${sep}`) || this.stagingRoot.startsWith(`${this.root}${sep}`))
|
||||
fail('UNSAFE_PATH', 'Archive staging and cache directories must be separate');
|
||||
}
|
||||
|
||||
/** Promote a verified staging file. The staging file is never deleted. */
|
||||
promote(stagedPath: string, sha256: string, operation?: CacheOperationInput): PublicArchiveCacheEntry {
|
||||
const control = operationControl(operation);
|
||||
const digest = expectedDigest(sha256), relativePath = stagedRelativePath(stagedPath);
|
||||
const source = assertContainedAncestors(this.stagingRoot, relativePath, control);
|
||||
return this.withLock(() => {
|
||||
checkOperation(control);
|
||||
this.cleanIncoming(control);
|
||||
this.recoverInterruptedOperations(control);
|
||||
let initial: fs.Stats;
|
||||
try { initial = fs.lstatSync(source); }
|
||||
catch { fail('MISSING_INPUT', 'Staged archive is missing'); }
|
||||
assertOwnedRegular(initial!, 'Staged archive', this.maxEntryBytes);
|
||||
if ((initial!.mode & 0o022) !== 0) fail('UNSAFE_PATH', 'Staged archive must not be writable by another user');
|
||||
|
||||
const target = this.entryPath(digest), metadataPath = this.metadataPath(digest);
|
||||
const targetExists = existsNoFollow(target), metadataExists = existsNoFollow(metadataPath);
|
||||
if (targetExists !== metadataExists) fail('INCOMPATIBLE_INPUT', `Cache entry is incomplete for ${digest}`);
|
||||
|
||||
if (targetExists) {
|
||||
const staged = this.hashFile(source, this.maxEntryBytes, false, stableStat(initial!), control);
|
||||
if (staged.digest !== digest) fail('INCOMPATIBLE_INPUT', 'Staged archive does not match its caller-provided SHA-256');
|
||||
const metadata = this.verifiedEntry(digest, control);
|
||||
return this.touch(metadata, control);
|
||||
}
|
||||
|
||||
// Authenticate the complete staged object before it is allowed to
|
||||
// displace any already-verified cache entry. Copying below hashes it a
|
||||
// second time so a staging race still fails closed.
|
||||
const authenticated = this.hashFile(source, this.maxEntryBytes, false, stableStat(initial!), control);
|
||||
if (authenticated.digest !== digest) fail('INCOMPATIBLE_INPUT', 'Staged archive does not match its caller-provided SHA-256');
|
||||
this.evictToFit(initial!.size, control);
|
||||
const incoming = join(this.incomingDir, `.incoming-${process.pid}-${randomBytes(12).toString('hex')}`);
|
||||
let promoted = false;
|
||||
try {
|
||||
const staged = this.copyAndHash(source, incoming, this.maxEntryBytes, stableStat(initial!), control);
|
||||
if (staged.digest !== digest) fail('INCOMPATIBLE_INPUT', 'Staged archive does not match its caller-provided SHA-256');
|
||||
if (staged.bytes !== initial!.size) fail('SNAPSHOT_RACE', 'Staged archive changed during promotion');
|
||||
checkOperation(control);
|
||||
fs.chmodSync(incoming, 0o400);
|
||||
// A hard-link followed by unlink is an atomic no-replace publication on
|
||||
// the cache filesystem. rename(2) would silently replace a raced target.
|
||||
try { fs.linkSync(incoming, target); fs.unlinkSync(incoming); }
|
||||
catch { fail('PERSISTENCE_FAILED', 'Verified archive could not be promoted atomically'); }
|
||||
promoted = true;
|
||||
const now = this.timestamp();
|
||||
const metadata: Metadata = { version: 1, sha256: digest, bytes: staged.bytes, createdAt: now, lastAccessedAt: now };
|
||||
try { checkOperation(control); writeMetadata(metadataPath, metadata, true); }
|
||||
catch (error) {
|
||||
try { this.discardObject(target, 'entry', digest); } catch {}
|
||||
throw error;
|
||||
}
|
||||
return this.entry(metadata);
|
||||
} finally {
|
||||
if (!promoted && existsNoFollow(incoming)) {
|
||||
const stat = fs.lstatSync(incoming);
|
||||
if (stat.isFile() && !stat.isSymbolicLink()) fs.unlinkSync(incoming);
|
||||
}
|
||||
}
|
||||
}, control);
|
||||
}
|
||||
|
||||
/** Return a cache hit only after hashing every byte and validating metadata. */
|
||||
get(sha256: string, operation?: CacheOperationInput): PublicArchiveCacheEntry | undefined {
|
||||
const control = operationControl(operation);
|
||||
const digest = expectedDigest(sha256);
|
||||
return this.withLock(() => {
|
||||
checkOperation(control);
|
||||
this.cleanIncoming(control);
|
||||
this.recoverInterruptedOperations(control);
|
||||
const targetExists = existsNoFollow(this.entryPath(digest)), metadataExists = existsNoFollow(this.metadataPath(digest));
|
||||
if (!targetExists && !metadataExists) return undefined;
|
||||
if (targetExists !== metadataExists) fail('INCOMPATIBLE_INPUT', `Cache entry is incomplete for ${digest}`);
|
||||
return this.touch(this.verifiedEntry(digest, control), control);
|
||||
}, control);
|
||||
}
|
||||
|
||||
/** Inspect capacity without treating entries as execution-ready cache hits. */
|
||||
stats(operation?: CacheOperationInput): PublicArchiveCacheStats {
|
||||
const control = operationControl(operation);
|
||||
return this.withLock(() => {
|
||||
checkOperation(control);
|
||||
this.cleanIncoming(control);
|
||||
this.recoverInterruptedOperations(control);
|
||||
const entries = this.inventory(control);
|
||||
let bytes = 0;
|
||||
for (const entry of entries) { checkOperation(control); bytes += entry.bytes; }
|
||||
return { entries: entries.length, bytes, maxBytes: this.maxBytes };
|
||||
}, control);
|
||||
}
|
||||
|
||||
/**
|
||||
* Copy a complete digest set into one run-owned directory while holding the
|
||||
* cache lock. Callers mount these immutable copies, never eviction-prone
|
||||
* cache paths. Every source and every copy is fully hashed in the same
|
||||
* critical section.
|
||||
*/
|
||||
materialize(digests: string[], destinationRoot: string, operation?: CacheOperationInput): MaterializedArchive[] {
|
||||
const control = operationControl(operation);
|
||||
if (!Array.isArray(digests) || !digests.length)
|
||||
fail('INVALID_ARGUMENT', 'Archive materialization requires at least one SHA-256 digest');
|
||||
if (digests.length > MAX_CACHE_DIRECTORY_ENTRIES)
|
||||
fail('INSUFFICIENT_CAPACITY', 'Archive materialization exceeds the cache entry limit');
|
||||
const selectedSet=new Set<string>();for(const digest of digests){checkOperation(control);if(typeof digest!=='string')fail('INVALID_ARGUMENT', 'Archive materialization requires SHA-256 digest strings');selectedSet.add(expectedDigest(digest));}
|
||||
checkOperation(control);const selected=[...selectedSet].sort();checkOperation(control);
|
||||
const destination = assertExistingDirectory(destinationRoot, 'Archive materialization directory');
|
||||
assertEmptyDirectory(destination, control);
|
||||
if (destination === this.root || destination.startsWith(`${this.root}${sep}`) || this.root.startsWith(`${destination}${sep}`) ||
|
||||
destination === this.stagingRoot || destination.startsWith(`${this.stagingRoot}${sep}`) || this.stagingRoot.startsWith(`${destination}${sep}`))
|
||||
fail('UNSAFE_PATH', 'Archive materialization directory must be separate from cache and staging roots');
|
||||
return this.withLock(() => {
|
||||
checkOperation(control);
|
||||
this.cleanIncoming(control);
|
||||
this.recoverInterruptedOperations(control);
|
||||
const created: string[] = [], result: MaterializedArchive[] = [];
|
||||
try {
|
||||
for (const digest of selected) {
|
||||
checkOperation(control);
|
||||
const metadata = this.verifiedEntry(digest, control), source = this.entryPath(digest), initial = fs.lstatSync(source);
|
||||
assertOwnedRegular(initial, 'Cached archive', this.maxEntryBytes, true);
|
||||
const target = join(destination, digest);
|
||||
let copied: { digest: string; bytes: number };
|
||||
try { copied = this.copyAndHash(source, target, this.maxEntryBytes, stableStat(initial), control); }
|
||||
catch (error) {
|
||||
if (existsNoFollow(target)) try { removeRegular(target, 'Incomplete run-owned archive copy'); } catch {}
|
||||
throw error;
|
||||
}
|
||||
created.push(target);
|
||||
if (copied.digest !== digest || copied.bytes !== metadata.bytes) fail('SNAPSHOT_RACE', 'Cached archive changed while its run-owned copy was materialized');
|
||||
fs.chmodSync(target, 0o400);
|
||||
const verified = this.hashFile(target, this.maxEntryBytes, true, undefined, control);
|
||||
if (verified.digest !== digest || verified.bytes !== metadata.bytes) fail('SNAPSHOT_RACE', 'Run-owned archive copy failed verification');
|
||||
result.push(Object.freeze({ sha256: digest, path: target, bytes: verified.bytes }));
|
||||
}
|
||||
return result;
|
||||
} catch (error) {
|
||||
for (const path of created.reverse()) { try { removeRegular(path, 'Incomplete run-owned archive copy'); } catch {} }
|
||||
throw error;
|
||||
}
|
||||
}, control);
|
||||
}
|
||||
|
||||
private timestamp(): number {
|
||||
const value = this.clock();
|
||||
if (!Number.isSafeInteger(value) || value < 0) fail('PERSISTENCE_FAILED', 'Cache clock returned an invalid timestamp');
|
||||
return value;
|
||||
}
|
||||
|
||||
private entryPath(digest: string): string { return join(this.entriesDir, digest); }
|
||||
private metadataPath(digest: string): string { return join(this.metadataDir, `${digest}.json`); }
|
||||
|
||||
private entry(metadata: Metadata): PublicArchiveCacheEntry {
|
||||
return Object.freeze({ sha256: metadata.sha256, path: this.entryPath(metadata.sha256), bytes: metadata.bytes,
|
||||
createdAt: metadata.createdAt, lastAccessedAt: metadata.lastAccessedAt });
|
||||
}
|
||||
|
||||
private touch(metadata: Metadata, control: NormalizedCacheOperationControl): PublicArchiveCacheEntry {
|
||||
checkOperation(control);
|
||||
const updated: Metadata = { ...metadata, lastAccessedAt: Math.max(metadata.lastAccessedAt, this.timestamp()) };
|
||||
checkOperation(control);
|
||||
writeMetadata(this.metadataPath(metadata.sha256), updated);
|
||||
return this.entry(updated);
|
||||
}
|
||||
|
||||
private verifiedEntry(digest: string, control: NormalizedCacheOperationControl): Metadata {
|
||||
checkOperation(control);
|
||||
const metadata = readMetadata(this.metadataPath(digest), digest, control);
|
||||
const result = this.hashFile(this.entryPath(digest), this.maxEntryBytes, true, undefined, control);
|
||||
if (result.digest !== digest || result.bytes !== metadata.bytes) fail('INCOMPATIBLE_INPUT', `Cached archive failed SHA-256 verification: ${digest}`);
|
||||
return metadata;
|
||||
}
|
||||
|
||||
private hashFile(path: string, maxBytes: number, immutable: boolean, expected: StableStat | undefined,
|
||||
control: NormalizedCacheOperationControl): { digest: string; bytes: number } {
|
||||
checkOperation(control);
|
||||
const fd = openNoFollow(path, fs.constants.O_RDONLY);
|
||||
try {
|
||||
const beforeStat = fs.fstatSync(fd);
|
||||
assertOwnedRegular(beforeStat, immutable ? 'Cached archive' : 'Staged archive', maxBytes, immutable);
|
||||
const before = stableStat(beforeStat), hash = createHash('sha256'), buffer = Buffer.allocUnsafe(COPY_BUFFER_BYTES);
|
||||
if (expected && !sameStat(expected, before)) fail('SNAPSHOT_RACE', 'Archive changed before it could be verified');
|
||||
let bytes = 0;
|
||||
for (;;) {
|
||||
checkOperation(control);
|
||||
const read = fs.readSync(fd, buffer, 0, buffer.length, null);
|
||||
if (!read) break;
|
||||
bytes += read;
|
||||
if (bytes > maxBytes) fail('INSUFFICIENT_CAPACITY', 'Archive exceeded its byte limit while being read');
|
||||
hash.update(buffer.subarray(0, read));
|
||||
checkOperation(control);
|
||||
}
|
||||
checkOperation(control);
|
||||
const after = stableStat(fs.fstatSync(fd));
|
||||
if (!sameStat(before, after) || bytes !== before.size) fail('SNAPSHOT_RACE', 'Archive changed while it was being verified');
|
||||
return { digest: hash.digest('hex'), bytes };
|
||||
} finally { fs.closeSync(fd); }
|
||||
}
|
||||
|
||||
private copyAndHash(source: string, destination: string, maxBytes: number, expected: StableStat,
|
||||
control: NormalizedCacheOperationControl): { digest: string; bytes: number } {
|
||||
checkOperation(control);
|
||||
const sourceFd = openNoFollow(source, fs.constants.O_RDONLY);
|
||||
let destinationFd: number | undefined;
|
||||
try {
|
||||
const beforeStat = fs.fstatSync(sourceFd);
|
||||
assertOwnedRegular(beforeStat, 'Staged archive', maxBytes);
|
||||
const before = stableStat(beforeStat), hash = createHash('sha256'), buffer = Buffer.allocUnsafe(COPY_BUFFER_BYTES);
|
||||
if (!sameStat(expected, before)) fail('SNAPSHOT_RACE', 'Staged archive changed before promotion');
|
||||
destinationFd = openNoFollow(destination, fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL, 0o600);
|
||||
let bytes = 0;
|
||||
for (;;) {
|
||||
checkOperation(control);
|
||||
const read = fs.readSync(sourceFd, buffer, 0, buffer.length, null);
|
||||
if (!read) break;
|
||||
bytes += read;
|
||||
if (bytes > expected.size) fail('SNAPSHOT_RACE', 'Staged archive grew during promotion');
|
||||
if (bytes > maxBytes) fail('INSUFFICIENT_CAPACITY', 'Archive exceeded its byte limit during promotion');
|
||||
hash.update(buffer.subarray(0, read));
|
||||
let offset = 0;
|
||||
while (offset < read) {
|
||||
checkOperation(control);
|
||||
const written = fs.writeSync(destinationFd, buffer, offset, read - offset);
|
||||
if (written <= 0) fail('PERSISTENCE_FAILED', 'Archive copy stopped before the current chunk was written');
|
||||
offset += written;
|
||||
checkOperation(control);
|
||||
}
|
||||
}
|
||||
checkOperation(control);
|
||||
fs.fsyncSync(destinationFd);
|
||||
checkOperation(control);
|
||||
const after = stableStat(fs.fstatSync(sourceFd));
|
||||
if (!sameStat(before, after) || bytes !== before.size) fail('SNAPSHOT_RACE', 'Staged archive changed during promotion');
|
||||
return { digest: hash.digest('hex'), bytes };
|
||||
} finally {
|
||||
if (destinationFd !== undefined) fs.closeSync(destinationFd);
|
||||
fs.closeSync(sourceFd);
|
||||
}
|
||||
}
|
||||
|
||||
private inventory(control: NormalizedCacheOperationControl): Metadata[] {
|
||||
checkOperation(control);
|
||||
const entryNames = boundedDirectoryNames(this.entriesDir, 'Cache entries directory', control),
|
||||
metadataNames = boundedDirectoryNames(this.metadataDir, 'Cache metadata directory', control);
|
||||
const entrySet=new Set<string>(),metadataSet=new Set<string>();
|
||||
for (const name of entryNames) {checkOperation(control);if (!SHA256.test(name)) fail('INCOMPATIBLE_INPUT', 'Cache entries directory contains an unexpected object');entrySet.add(name);}
|
||||
for (const name of metadataNames) {checkOperation(control);if (!/^[a-f0-9]{64}\.json$/.test(name)) fail('INCOMPATIBLE_INPUT', 'Cache metadata directory contains an unexpected object');metadataSet.add(name.slice(0,-5));}
|
||||
if (entrySet.size !== metadataSet.size) fail('INCOMPATIBLE_INPUT', 'Cache entries and metadata are inconsistent');
|
||||
for(const name of entrySet){checkOperation(control);if(!metadataSet.has(name))
|
||||
fail('INCOMPATIBLE_INPUT', 'Cache entries and metadata are inconsistent');
|
||||
}
|
||||
const inventory = entryNames.map(digest => {
|
||||
checkOperation(control);
|
||||
const stat = fs.lstatSync(this.entryPath(digest));
|
||||
assertOwnedRegular(stat, 'Cached archive', this.maxEntryBytes, true);
|
||||
const metadata = readMetadata(this.metadataPath(digest), digest, control);
|
||||
if (metadata.bytes !== stat.size) fail('INCOMPATIBLE_INPUT', `Cache size metadata is inconsistent for ${digest}`);
|
||||
return metadata;
|
||||
});
|
||||
checkOperation(control);
|
||||
return inventory;
|
||||
}
|
||||
|
||||
private evictToFit(incomingBytes: number, control: NormalizedCacheOperationControl): void {
|
||||
if (!Number.isSafeInteger(incomingBytes) || incomingBytes < 0 || incomingBytes > this.maxBytes)
|
||||
fail('INSUFFICIENT_CAPACITY', 'Archive cannot fit within the public-cache limit');
|
||||
checkOperation(control);
|
||||
const entries = this.inventory(control);checkOperation(control);entries.sort((a, b) => a.lastAccessedAt - b.lastAccessedAt || a.createdAt - b.createdAt || a.sha256.localeCompare(b.sha256));
|
||||
checkOperation(control);
|
||||
let total = 0;
|
||||
for (const item of entries) { checkOperation(control); total += item.bytes; }
|
||||
for (const item of entries) {
|
||||
checkOperation(control);
|
||||
if (total + incomingBytes <= this.maxBytes) break;
|
||||
const archive = this.moveToRecovery(this.entryPath(item.sha256), 'entry', item.sha256);
|
||||
const metadata = this.moveToRecovery(this.metadataPath(item.sha256), 'metadata', item.sha256);
|
||||
removeRegular(archive, 'Evicted cache archive');
|
||||
removeRegular(metadata, 'Evicted cache metadata');
|
||||
total -= item.bytes;
|
||||
}
|
||||
if (total + incomingBytes > this.maxBytes) fail('INSUFFICIENT_CAPACITY', 'Archive cache could not free enough verified capacity');
|
||||
}
|
||||
|
||||
private cleanIncoming(control: NormalizedCacheOperationControl): void {
|
||||
const incomingNames = boundedDirectoryNames(this.incomingDir, 'Cache incoming directory', control);
|
||||
let publishedByInode:Map<string,string[]>|undefined;
|
||||
for (const name of incomingNames) {
|
||||
checkOperation(control);
|
||||
if (!/^\.incoming-\d+-[a-f0-9]{24}$/.test(name)) fail('INCOMPATIBLE_INPUT', 'Cache incoming directory contains an unexpected object');
|
||||
const incoming = join(this.incomingDir, name), stat = fs.lstatSync(incoming);
|
||||
if (!stat.isFile() || stat.isSymbolicLink() || ![1, 2].includes(stat.nlink) || stat.size > this.maxEntryBytes ||
|
||||
(process.getuid && stat.uid !== process.getuid())) fail('UNSAFE_PATH', 'Incomplete cache archive is not a bounded regular file');
|
||||
if (stat.nlink === 2) {
|
||||
if(!publishedByInode){
|
||||
publishedByInode=new Map();
|
||||
for(const entry of boundedDirectoryNames(this.entriesDir, 'Cache entries directory', control)){
|
||||
checkOperation(control);
|
||||
if (!SHA256.test(entry)) fail('INCOMPATIBLE_INPUT', 'Cache entries directory contains an unexpected object');
|
||||
const candidate=fs.lstatSync(this.entryPath(entry)),key=`${candidate.dev}:${candidate.ino}`,matches=publishedByInode.get(key)??[];
|
||||
matches.push(entry);publishedByInode.set(key,matches);
|
||||
}
|
||||
}
|
||||
const matches=publishedByInode.get(`${stat.dev}:${stat.ino}`)??[];
|
||||
if (matches.length !== 1) fail('UNSAFE_PATH', 'Incoming archive hard link does not match one published cache entry');
|
||||
const target = fs.lstatSync(this.entryPath(matches[0]));
|
||||
if (!target.isFile() || target.isSymbolicLink() || target.nlink !== 2 || target.size > this.maxEntryBytes ||
|
||||
(process.getuid && target.uid !== process.getuid()) || (target.mode & 0o222) !== 0)
|
||||
fail('SNAPSHOT_RACE', 'Incoming archive link count or identity changed during recovery');
|
||||
}
|
||||
try { fs.unlinkSync(incoming); } catch { fail('PERSISTENCE_FAILED', 'Incomplete cache archive could not be removed'); }
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Recover only artifacts whose names and inode types prove they belong to an
|
||||
* interrupted cache transaction. Unknown objects remain a hard failure.
|
||||
*/
|
||||
private recoverInterruptedOperations(control: NormalizedCacheOperationControl): void {
|
||||
for (const name of boundedDirectoryNames(this.recoveryDir, 'Cache recovery directory', control)) {
|
||||
checkOperation(control);
|
||||
if (!/^\.recovery-(?:entry|metadata)-[a-f0-9]{64}-\d+-[a-f0-9]{24}$/.test(name))
|
||||
fail('INCOMPATIBLE_INPUT', 'Cache recovery directory contains an unexpected object');
|
||||
removeRegular(join(this.recoveryDir, name), 'Interrupted cache transaction');
|
||||
}
|
||||
|
||||
const entries = boundedDirectoryNames(this.entriesDir, 'Cache entries directory', control),
|
||||
metadataObjects = boundedDirectoryNames(this.metadataDir, 'Cache metadata directory', control);
|
||||
for (const name of metadataObjects) {
|
||||
checkOperation(control);
|
||||
if (/^[a-f0-9]{64}\.json\.tmp\.\d+\.[a-f0-9]{8}$/.test(name)) this.recoverMetadataTemp(name, control);
|
||||
else if (!/^[a-f0-9]{64}\.json$/.test(name)) fail('INCOMPATIBLE_INPUT', 'Cache metadata directory contains an unexpected object');
|
||||
}
|
||||
const metadata = boundedDirectoryNames(this.metadataDir, 'Cache metadata directory', control);
|
||||
for (const name of entries) if (!SHA256.test(name)) fail('INCOMPATIBLE_INPUT', 'Cache entries directory contains an unexpected object');
|
||||
for (const name of metadata) if (!/^[a-f0-9]{64}\.json$/.test(name)) fail('INCOMPATIBLE_INPUT', 'Cache metadata directory contains an unexpected object');
|
||||
const entrySet=new Set<string>(),metadataSet=new Set<string>(),digests=new Set<string>();
|
||||
for(const name of entries){checkOperation(control);entrySet.add(name);digests.add(name);}
|
||||
for(const name of metadata){checkOperation(control);const digest=name.slice(0,-5);metadataSet.add(digest);digests.add(digest);}
|
||||
for (const digest of digests) {
|
||||
checkOperation(control);
|
||||
if (entrySet.has(digest) === metadataSet.has(digest)) continue;
|
||||
if (entrySet.has(digest)) this.discardObject(this.entryPath(digest), 'entry', digest);
|
||||
else this.discardObject(this.metadataPath(digest), 'metadata', digest);
|
||||
}
|
||||
}
|
||||
|
||||
private recoveryPath(kind: 'entry' | 'metadata', digest: string): string {
|
||||
return join(this.recoveryDir, `.recovery-${kind}-${digest}-${process.pid}-${randomBytes(12).toString('hex')}`);
|
||||
}
|
||||
|
||||
private moveToRecovery(path: string, kind: 'entry' | 'metadata', digest: string): string {
|
||||
const before = fs.lstatSync(path);
|
||||
assertOwnedRegular(before, kind === 'entry' ? 'Cached archive' : 'Cache metadata', kind === 'entry' ? this.maxEntryBytes : METADATA_LIMIT, kind === 'entry');
|
||||
if (kind === 'metadata' && (before.mode & 0o077) !== 0) fail('UNSAFE_PATH', 'Cache metadata permissions are not private');
|
||||
const destination = this.recoveryPath(kind, digest);
|
||||
try { fs.renameSync(path, destination); }
|
||||
catch { fail('PERSISTENCE_FAILED', 'Interrupted cache object could not be quarantined atomically'); }
|
||||
const after = fs.lstatSync(destination);
|
||||
// rename(2) can update ctime; stable inode identity, content size, mode,
|
||||
// link count, mtime, and ownership prove the moved object is the one read.
|
||||
if (!sameRenamedInode(stableStat(before), stableStat(after))) fail('SNAPSHOT_RACE', 'Cache object changed while it was quarantined');
|
||||
return destination;
|
||||
}
|
||||
|
||||
private discardObject(path: string, kind: 'entry' | 'metadata', digest: string): void {
|
||||
const quarantined = this.moveToRecovery(path, kind, digest);
|
||||
removeRegular(quarantined, 'Interrupted cache transaction');
|
||||
}
|
||||
|
||||
private recoverMetadataTemp(name: string, control: NormalizedCacheOperationControl): void {
|
||||
checkOperation(control);
|
||||
const path = join(this.metadataDir, name), stat = fs.lstatSync(path);
|
||||
if (!stat.isFile() || stat.isSymbolicLink() || ![1, 2].includes(stat.nlink) || stat.size > METADATA_LIMIT ||
|
||||
(process.getuid && stat.uid !== process.getuid()) || (stat.mode & 0o077) !== 0)
|
||||
fail('UNSAFE_PATH', 'Interrupted cache metadata write is not a private regular file');
|
||||
if (stat.nlink === 2) {
|
||||
const target = this.metadataPath(name.slice(0, 64));
|
||||
let targetStat: fs.Stats;
|
||||
try { targetStat = fs.lstatSync(target); } catch { fail('INCOMPATIBLE_INPUT', 'Hard-linked metadata temp has no published target'); }
|
||||
if (!targetStat!.isFile() || targetStat!.isSymbolicLink() || targetStat!.dev !== stat.dev || targetStat!.ino !== stat.ino || targetStat!.nlink !== 2)
|
||||
fail('UNSAFE_PATH', 'Interrupted metadata hard link does not match its published target');
|
||||
}
|
||||
try { fs.unlinkSync(path); } catch { fail('PERSISTENCE_FAILED', 'Interrupted cache metadata write could not be removed'); }
|
||||
}
|
||||
|
||||
private withLock<T>(callback: () => T, control: NormalizedCacheOperationControl): T {
|
||||
checkOperation(control);
|
||||
const marker = `${JSON.stringify({ protocol: CACHE_LOCK_PROTOCOL })}\n`;
|
||||
const options={label:'Cache lock protocol',maxBytes:METADATA_LIMIT,validate:(value:unknown)=>{
|
||||
if(!value||typeof value!=='object'||Array.isArray(value)||Object.keys(value).join(',')!=='protocol'||(value as any).protocol!==CACHE_LOCK_PROTOCOL)
|
||||
fail('INCOMPATIBLE_INPUT','Archive-cache lock protocol is invalid');
|
||||
}};
|
||||
const tempPattern=/^\.lock\.tmp\.(\d{1,10})\.[a-f0-9]{8}$/;
|
||||
for(const name of boundedDirectoryNames(this.root, 'Cache root directory', control)){
|
||||
checkOperation(control);
|
||||
const match=name.match(tempPattern);if(!match)continue;
|
||||
const temporary=join(this.root,name),publisherPid=Number(match[1]);
|
||||
if(existsNoFollow(this.lockDir))recoverAtomicNoReplaceJson(this.lockDir,options);
|
||||
if(existsNoFollow(temporary))discardAtomicNoReplaceTemp(temporary,publisherPid,options);
|
||||
}
|
||||
try { atomicWriteSync(this.lockDir, marker, { mode: 0o600, noReplace: true }); }
|
||||
catch (error: any) {
|
||||
if (error?.code !== 'EEXIST') fail('PERSISTENCE_FAILED', 'Archive-cache lock protocol could not be initialized');
|
||||
recoverAtomicNoReplaceJson(this.lockDir,options);
|
||||
const stat = fs.lstatSync(this.lockDir);
|
||||
if (stat.isDirectory() && !stat.isSymbolicLink())
|
||||
fail('INSUFFICIENT_CAPACITY', 'A legacy archive-cache helper may still own or initialize this cache; its lock was left intact');
|
||||
assertOwnedRegular(stat, 'Cache lock protocol', METADATA_LIMIT);
|
||||
if ((stat.mode & 0o077) !== 0) fail('UNSAFE_PATH', 'Cache lock protocol permissions are not private');
|
||||
let protocol:unknown;try{protocol=JSON.parse(fs.readFileSync(this.lockDir,'utf8')).protocol;}catch{}
|
||||
if(protocol!==CACHE_LOCK_PROTOCOL)fail('INCOMPATIBLE_INPUT','Archive-cache lock protocol is invalid');
|
||||
}
|
||||
const result=withStateLock(this.root,()=>{checkOperation(control);return callback();});
|
||||
if(result&&typeof (result as any).then==='function')fail('PERSISTENCE_FAILED','Archive-cache operation unexpectedly became asynchronous');
|
||||
return result as T;
|
||||
}
|
||||
}
|
||||
|
||||
export function publicArchiveCacheRoot(): string {
|
||||
return join(privateRoot(), 'public-cache');
|
||||
}
|
||||
Reference in New Issue
Block a user