v1.87.0.0 feat: add verified CSO audits and replayable repair bundles (#2852)

* feat(cso): add verified audits and replayable repair bundles

* fix(cso): harden qualification and setup boundaries

* fix(cso): assemble security canaries at runtime

* fix(cso): bound release proof and maintenance work

Co-Authored-By: OpenAI Codex <noreply@openai.com>

* fix(cso): require complete evaluation reports

Co-Authored-By: OpenAI Codex <noreply@openai.com>

* fix(cso): replay expired snapshots from supplied source

Co-Authored-By: OpenAI Codex <noreply@openai.com>

* test(cso): synchronize DNS cancellation assertion

Co-Authored-By: OpenAI Codex <noreply@openai.com>

* chore(ship): exempt repository owner from liveness proof

Co-Authored-By: OpenAI Codex <noreply@openai.com>

* test(cso): make recheck retention overlap deterministic

Co-Authored-By: OpenAI Codex <noreply@openai.com>

* chore: bump version and changelog (v1.85.0.0)

Co-Authored-By: OpenAI Codex <noreply@openai.com>

* fix(cso): pass native release gates

Co-Authored-By: OpenAI Codex <noreply@openai.com>

* chore: move release to v1.86.0.0

Co-Authored-By: OpenAI Codex <noreply@openai.com>

* fix(cso): resolve rechecks by finding

Co-Authored-By: OpenAI Codex <noreply@openai.com>

* chore: move release to v1.87.0.0

Co-Authored-By: OpenAI Codex <noreply@openai.com>

* fix(cso): pass macOS and Windows release gates

Normalize BSD wc output, compare Windows paths by filesystem identity, preserve portable snapshot race coverage, and narrow POSIX-only Windows fixtures.

Co-Authored-By: OpenAI Codex <noreply@openai.com>

* fix(cso): harden native verification gates

* fix(cso): refine Windows native diagnostics

* test(cso): isolate Windows Git startup failure

* test(cso): stabilize Windows native diagnostics

* fix(cso): support hardened Git on Windows

* fix(cso): close final verification gaps

* test(cso): bound cold Docker fixture setup

* fix(cso): restore cross-platform free-suite gates

---------

Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Garry Tan
2026-09-14 15:14:58 -07:00
committed by GitHub
co-authored by OpenAI Codex
parent 9f81911136
commit 4a3c6a8a3c
160 changed files with 24697 additions and 2288 deletions
+707
View File
@@ -0,0 +1,707 @@
import * as fs from 'node:fs';
import { createHash, randomBytes } from 'node:crypto';
import { join, resolve, sep } from 'node:path';
import { atomicWriteSync } from '../fs-atomic';
import { CsoError } from './contracts';
import { discardAtomicNoReplaceTemp, privateRoot, recoverAtomicNoReplaceJson, secureDirectory, withLock as withStateLock } from './state';
export const DEFAULT_PUBLIC_ARCHIVE_CACHE_BYTES = 10 * 1024 * 1024 * 1024;
const METADATA_VERSION = 1;
const METADATA_LIMIT = 4096;
const COPY_BUFFER_BYTES = 64 * 1024;
const MAX_CACHE_DIRECTORY_ENTRIES = 100_000;
const CACHE_LOCK_PROTOCOL = 'immutable-cache-lease-set-v3';
const SHA256 = /^[a-f0-9]{64}$/;
const RELATIVE_STAGE_PATH = /^(?!\/)(?!.*(?:^|\/)\.\.?(?:\/|$))(?!.*\\)[^\0-\x1f\x7f]+$/;
export interface PublicArchiveCacheOptions {
/** Defaults to the private CSO state namespace. */
root?: string;
/** Existing directory populated by the constrained acquisition step. */
stagingRoot: string;
/** Persistent archive-byte ceiling. Defaults to 10 GiB. */
maxBytes?: number;
/** Per-archive ceiling. Defaults to maxBytes. */
maxEntryBytes?: number;
/** Deterministic clock for tests. */
now?: () => number;
}
/** Optional cooperative bounds for synchronous cache work. */
export interface CacheOperationControl {
/** Absolute Unix timestamp in milliseconds. */
deadline?: number;
/** Checked between bounded filesystem operations. */
signal?: AbortSignal;
}
export type CacheOperationInput = CacheOperationControl | number | undefined;
type NormalizedCacheOperationControl = Readonly<{ deadline?: number; signal?: AbortSignal }>;
export interface PublicArchiveCacheEntry {
sha256: string;
path: string;
bytes: number;
createdAt: number;
lastAccessedAt: number;
}
export interface PublicArchiveCacheStats {
entries: number;
bytes: number;
maxBytes: number;
}
export interface MaterializedArchive {
sha256: string;
path: string;
bytes: number;
}
interface Metadata {
version: 1;
sha256: string;
bytes: number;
createdAt: number;
lastAccessedAt: number;
}
interface StableStat {
dev: number;
ino: number;
size: number;
mode: number;
nlink: number;
mtimeMs: number;
ctimeMs: number;
uid: number;
}
function fail(code: ConstructorParameters<typeof CsoError>[0], message: string): never {
throw new CsoError(code, message);
}
function operationControl(input?: CacheOperationInput): NormalizedCacheOperationControl {
const value = typeof input === 'number' ? { deadline: input } : input ?? {};
if (!value || typeof value !== 'object' || Array.isArray(value)) fail('INVALID_ARGUMENT', 'Cache operation control must be an object or absolute deadline');
if (value.deadline !== undefined && (!Number.isSafeInteger(value.deadline) || value.deadline <= 0))
fail('INVALID_ARGUMENT', 'Cache deadline must be an absolute millisecond timestamp');
if (value.signal !== undefined && typeof value.signal.aborted !== 'boolean')
fail('INVALID_ARGUMENT', 'Cache cancellation signal is invalid');
const control = Object.freeze({ ...(value.deadline === undefined ? {} : { deadline: value.deadline }),
...(value.signal === undefined ? {} : { signal: value.signal }) });
checkOperation(control);
return control;
}
function checkOperation(control: NormalizedCacheOperationControl): void {
if (control.signal?.aborted) fail('CANCELLED', 'Archive-cache operation was cancelled');
if (control.deadline !== undefined && Date.now() >= control.deadline)
fail('DEADLINE', 'Archive-cache operation reached its deadline');
}
function boundedDirectoryNames(path: string, label: string, control: NormalizedCacheOperationControl): string[] {
checkOperation(control);
const directory = fs.opendirSync(path), names: string[] = [];
try {
for (;;) {
checkOperation(control);
const entry = directory.readSync();
if (!entry) break;
if (names.length >= MAX_CACHE_DIRECTORY_ENTRIES) fail('INSUFFICIENT_CAPACITY', `${label} exceeds the cache entry limit`);
names.push(entry.name);
}
} finally { directory.closeSync(); }
checkOperation(control);
return names;
}
function assertEmptyDirectory(path: string, control: NormalizedCacheOperationControl): void {
checkOperation(control);
const directory = fs.opendirSync(path);
try { if (directory.readSync()) fail('UNSAFE_PATH', 'Archive materialization directory must be empty'); }
finally { directory.closeSync(); }
checkOperation(control);
}
function boundedPositiveInteger(value: number, name: string): number {
if (!Number.isSafeInteger(value) || value <= 0) fail('INVALID_ARGUMENT', `${name} must be a positive safe integer`);
return value;
}
function expectedDigest(value: string): string {
if (!SHA256.test(value)) fail('INVALID_ARGUMENT', 'Archive SHA-256 must be 64 lowercase hexadecimal characters');
return value;
}
function stagedRelativePath(value: string): string {
if (typeof value !== 'string' || value.length > 4096 || !RELATIVE_STAGE_PATH.test(value))
fail('UNSAFE_PATH', 'Staged archive path must be a contained relative path');
const parts = value.split('/');
if (parts.some(part => !part || part === '.' || part === '..')) fail('UNSAFE_PATH', 'Staged archive path must be a contained relative path');
return value;
}
function stableStat(stat: fs.Stats): StableStat {
return {
dev: stat.dev, ino: stat.ino, size: stat.size, mode: stat.mode, nlink: stat.nlink,
mtimeMs: stat.mtimeMs, ctimeMs: stat.ctimeMs, uid: stat.uid,
};
}
function sameStat(left: StableStat, right: StableStat): boolean {
return left.dev === right.dev && left.ino === right.ino && left.size === right.size &&
left.mode === right.mode && left.nlink === right.nlink && left.mtimeMs === right.mtimeMs &&
left.ctimeMs === right.ctimeMs && left.uid === right.uid;
}
function sameRenamedInode(left: StableStat, right: StableStat): boolean {
return left.dev === right.dev && left.ino === right.ino && left.size === right.size &&
left.mode === right.mode && left.nlink === right.nlink && left.mtimeMs === right.mtimeMs && left.uid === right.uid;
}
function assertOwnedRegular(stat: fs.Stats, label: string, maxBytes: number, immutable = false): void {
if (!stat.isFile() || stat.isSymbolicLink()) fail('UNSAFE_PATH', `${label} must be a regular file`);
if (stat.nlink !== 1) fail('UNSAFE_PATH', `${label} must not be hard-linked`);
if (process.getuid && stat.uid !== process.getuid()) fail('UNSAFE_PATH', `${label} must be owned by the current user`);
if (stat.size > maxBytes) fail('INSUFFICIENT_CAPACITY', `${label} exceeds its byte limit`);
if (immutable && (stat.mode & 0o222) !== 0) fail('INCOMPATIBLE_INPUT', `${label} is unexpectedly writable`);
}
function assertExistingDirectory(path: string, label: string): string {
const requested = resolve(path);
let requestedStat: fs.Stats;
try { requestedStat = fs.lstatSync(requested); }
catch { fail('MISSING_INPUT', `${label} does not exist`); }
if (requestedStat!.isSymbolicLink()) fail('UNSAFE_PATH', `${label} must not be a symlink`);
let canonical: string;
try { canonical = fs.realpathSync(requested); }
catch { fail('MISSING_INPUT', `${label} does not exist`); }
const stat = fs.lstatSync(canonical!);
if (!stat.isDirectory() || stat.isSymbolicLink()) fail('UNSAFE_PATH', `${label} must be a directory`);
if (process.getuid && stat.uid !== process.getuid()) fail('UNSAFE_PATH', `${label} must be owned by the current user`);
if ((stat.mode & 0o022) !== 0) fail('UNSAFE_PATH', `${label} must not be writable by another user`);
return canonical!;
}
function assertContainedAncestors(root: string, relativePath: string, control: NormalizedCacheOperationControl): string {
const parts = relativePath.split('/');
let cursor = root;
for (const part of parts.slice(0, -1)) {
checkOperation(control);
cursor = join(cursor, part);
let stat: fs.Stats;
try { stat = fs.lstatSync(cursor); }
catch { fail('MISSING_INPUT', `Staged archive directory is missing: ${part}`); }
if (!stat.isDirectory() || stat.isSymbolicLink()) fail('UNSAFE_PATH', 'Staged archive has a symlink or non-directory ancestor');
if (process.getuid && stat.uid !== process.getuid()) fail('UNSAFE_PATH', 'Staged archive ancestor has an unexpected owner');
if ((stat.mode & 0o022) !== 0) fail('UNSAFE_PATH', 'Staged archive ancestor is writable by another user');
}
const path = resolve(root, ...parts);
if (path !== root && !path.startsWith(`${root}${sep}`)) fail('UNSAFE_PATH', 'Staged archive escaped its staging directory');
return path;
}
function openNoFollow(path: string, flags: number, mode?: number): number {
const noFollow = (fs.constants as Record<string, number>).O_NOFOLLOW ?? 0;
const closeOnExec = (fs.constants as Record<string, number>).O_CLOEXEC ?? 0;
try { return fs.openSync(path, flags | noFollow | closeOnExec, mode); }
catch { fail('UNSAFE_PATH', 'Archive file could not be opened without following links'); }
}
function readMetadata(path: string, digest: string, control: NormalizedCacheOperationControl): Metadata {
checkOperation(control);
let stat: fs.Stats;
try { stat = fs.lstatSync(path); }
catch { fail('INCOMPATIBLE_INPUT', `Cache metadata is missing for ${digest}`); }
assertOwnedRegular(stat!, 'Cache metadata', METADATA_LIMIT);
if ((stat!.mode & 0o077) !== 0) fail('INCOMPATIBLE_INPUT', 'Cache metadata permissions are not private');
let value: unknown;
try { checkOperation(control); value = JSON.parse(fs.readFileSync(path, 'utf8')); checkOperation(control); }
catch (error) {
if (error instanceof CsoError) throw error;
fail('INCOMPATIBLE_INPUT', `Cache metadata is invalid for ${digest}`);
}
const record = value as Partial<Metadata>;
if (!value || typeof value !== 'object' || Array.isArray(value) || Object.keys(value).sort().join(',') !== 'bytes,createdAt,lastAccessedAt,sha256,version' ||
record.version !== METADATA_VERSION || record.sha256 !== digest || !Number.isSafeInteger(record.bytes) || Number(record.bytes) < 0 ||
!Number.isSafeInteger(record.createdAt) || Number(record.createdAt) < 0 || !Number.isSafeInteger(record.lastAccessedAt) ||
Number(record.lastAccessedAt) < Number(record.createdAt)) fail('INCOMPATIBLE_INPUT', `Cache metadata is invalid for ${digest}`);
return record as Metadata;
}
function writeMetadata(path: string, metadata: Metadata, noReplace = false): void {
try { atomicWriteSync(path, `${JSON.stringify(metadata)}\n`, { mode: 0o600, noReplace }); }
catch { fail('PERSISTENCE_FAILED', 'Cache metadata could not be written atomically'); }
}
function removeRegular(path: string, label: string): void {
const stat = fs.lstatSync(path);
assertOwnedRegular(stat, label, Number.MAX_SAFE_INTEGER);
try { fs.unlinkSync(path); }
catch { fail('PERSISTENCE_FAILED', `${label} could not be removed`); }
}
function existsNoFollow(path: string): boolean {
try { fs.lstatSync(path); return true; }
catch (error: any) {
if (error?.code === 'ENOENT') return false;
fail('INCOMPATIBLE_INPUT', 'Cache object could not be inspected safely');
}
}
/**
* A content-addressed cache for already-acquired public archives. It never
* performs downloads, runs package managers, or executes archive content.
*/
export class PublicArchiveCache {
readonly root: string;
readonly stagingRoot: string;
readonly maxBytes: number;
readonly maxEntryBytes: number;
private readonly entriesDir: string;
private readonly metadataDir: string;
private readonly incomingDir: string;
private readonly recoveryDir: string;
private readonly lockDir: string;
private readonly clock: () => number;
constructor(options: PublicArchiveCacheOptions) {
if (!options || typeof options !== 'object') fail('INVALID_ARGUMENT', 'Cache options are required');
this.maxBytes = boundedPositiveInteger(options.maxBytes ?? DEFAULT_PUBLIC_ARCHIVE_CACHE_BYTES, 'maxBytes');
this.maxEntryBytes = boundedPositiveInteger(options.maxEntryBytes ?? this.maxBytes, 'maxEntryBytes');
if (this.maxEntryBytes > this.maxBytes) fail('INVALID_ARGUMENT', 'maxEntryBytes cannot exceed maxBytes');
this.clock = options.now ?? Date.now;
const base = options.root ? resolve(options.root) : join(privateRoot(), 'public-cache');
this.root = secureDirectory(base);
this.entriesDir = secureDirectory(join(this.root, 'entries'));
this.metadataDir = secureDirectory(join(this.root, 'metadata'));
this.incomingDir = secureDirectory(join(this.root, 'incoming'));
this.recoveryDir = secureDirectory(join(this.root, 'recovery'));
this.lockDir = join(this.root, '.lock');
this.stagingRoot = assertExistingDirectory(options.stagingRoot, 'Archive staging directory');
if (this.root === this.stagingRoot || this.root.startsWith(`${this.stagingRoot}${sep}`) || this.stagingRoot.startsWith(`${this.root}${sep}`))
fail('UNSAFE_PATH', 'Archive staging and cache directories must be separate');
}
/** Promote a verified staging file. The staging file is never deleted. */
promote(stagedPath: string, sha256: string, operation?: CacheOperationInput): PublicArchiveCacheEntry {
const control = operationControl(operation);
const digest = expectedDigest(sha256), relativePath = stagedRelativePath(stagedPath);
const source = assertContainedAncestors(this.stagingRoot, relativePath, control);
return this.withLock(() => {
checkOperation(control);
this.cleanIncoming(control);
this.recoverInterruptedOperations(control);
let initial: fs.Stats;
try { initial = fs.lstatSync(source); }
catch { fail('MISSING_INPUT', 'Staged archive is missing'); }
assertOwnedRegular(initial!, 'Staged archive', this.maxEntryBytes);
if ((initial!.mode & 0o022) !== 0) fail('UNSAFE_PATH', 'Staged archive must not be writable by another user');
const target = this.entryPath(digest), metadataPath = this.metadataPath(digest);
const targetExists = existsNoFollow(target), metadataExists = existsNoFollow(metadataPath);
if (targetExists !== metadataExists) fail('INCOMPATIBLE_INPUT', `Cache entry is incomplete for ${digest}`);
if (targetExists) {
const staged = this.hashFile(source, this.maxEntryBytes, false, stableStat(initial!), control);
if (staged.digest !== digest) fail('INCOMPATIBLE_INPUT', 'Staged archive does not match its caller-provided SHA-256');
const metadata = this.verifiedEntry(digest, control);
return this.touch(metadata, control);
}
// Authenticate the complete staged object before it is allowed to
// displace any already-verified cache entry. Copying below hashes it a
// second time so a staging race still fails closed.
const authenticated = this.hashFile(source, this.maxEntryBytes, false, stableStat(initial!), control);
if (authenticated.digest !== digest) fail('INCOMPATIBLE_INPUT', 'Staged archive does not match its caller-provided SHA-256');
this.evictToFit(initial!.size, control);
const incoming = join(this.incomingDir, `.incoming-${process.pid}-${randomBytes(12).toString('hex')}`);
let promoted = false;
try {
const staged = this.copyAndHash(source, incoming, this.maxEntryBytes, stableStat(initial!), control);
if (staged.digest !== digest) fail('INCOMPATIBLE_INPUT', 'Staged archive does not match its caller-provided SHA-256');
if (staged.bytes !== initial!.size) fail('SNAPSHOT_RACE', 'Staged archive changed during promotion');
checkOperation(control);
fs.chmodSync(incoming, 0o400);
// A hard-link followed by unlink is an atomic no-replace publication on
// the cache filesystem. rename(2) would silently replace a raced target.
try { fs.linkSync(incoming, target); fs.unlinkSync(incoming); }
catch { fail('PERSISTENCE_FAILED', 'Verified archive could not be promoted atomically'); }
promoted = true;
const now = this.timestamp();
const metadata: Metadata = { version: 1, sha256: digest, bytes: staged.bytes, createdAt: now, lastAccessedAt: now };
try { checkOperation(control); writeMetadata(metadataPath, metadata, true); }
catch (error) {
try { this.discardObject(target, 'entry', digest); } catch {}
throw error;
}
return this.entry(metadata);
} finally {
if (!promoted && existsNoFollow(incoming)) {
const stat = fs.lstatSync(incoming);
if (stat.isFile() && !stat.isSymbolicLink()) fs.unlinkSync(incoming);
}
}
}, control);
}
/** Return a cache hit only after hashing every byte and validating metadata. */
get(sha256: string, operation?: CacheOperationInput): PublicArchiveCacheEntry | undefined {
const control = operationControl(operation);
const digest = expectedDigest(sha256);
return this.withLock(() => {
checkOperation(control);
this.cleanIncoming(control);
this.recoverInterruptedOperations(control);
const targetExists = existsNoFollow(this.entryPath(digest)), metadataExists = existsNoFollow(this.metadataPath(digest));
if (!targetExists && !metadataExists) return undefined;
if (targetExists !== metadataExists) fail('INCOMPATIBLE_INPUT', `Cache entry is incomplete for ${digest}`);
return this.touch(this.verifiedEntry(digest, control), control);
}, control);
}
/** Inspect capacity without treating entries as execution-ready cache hits. */
stats(operation?: CacheOperationInput): PublicArchiveCacheStats {
const control = operationControl(operation);
return this.withLock(() => {
checkOperation(control);
this.cleanIncoming(control);
this.recoverInterruptedOperations(control);
const entries = this.inventory(control);
let bytes = 0;
for (const entry of entries) { checkOperation(control); bytes += entry.bytes; }
return { entries: entries.length, bytes, maxBytes: this.maxBytes };
}, control);
}
/**
* Copy a complete digest set into one run-owned directory while holding the
* cache lock. Callers mount these immutable copies, never eviction-prone
* cache paths. Every source and every copy is fully hashed in the same
* critical section.
*/
materialize(digests: string[], destinationRoot: string, operation?: CacheOperationInput): MaterializedArchive[] {
const control = operationControl(operation);
if (!Array.isArray(digests) || !digests.length)
fail('INVALID_ARGUMENT', 'Archive materialization requires at least one SHA-256 digest');
if (digests.length > MAX_CACHE_DIRECTORY_ENTRIES)
fail('INSUFFICIENT_CAPACITY', 'Archive materialization exceeds the cache entry limit');
const selectedSet=new Set<string>();for(const digest of digests){checkOperation(control);if(typeof digest!=='string')fail('INVALID_ARGUMENT', 'Archive materialization requires SHA-256 digest strings');selectedSet.add(expectedDigest(digest));}
checkOperation(control);const selected=[...selectedSet].sort();checkOperation(control);
const destination = assertExistingDirectory(destinationRoot, 'Archive materialization directory');
assertEmptyDirectory(destination, control);
if (destination === this.root || destination.startsWith(`${this.root}${sep}`) || this.root.startsWith(`${destination}${sep}`) ||
destination === this.stagingRoot || destination.startsWith(`${this.stagingRoot}${sep}`) || this.stagingRoot.startsWith(`${destination}${sep}`))
fail('UNSAFE_PATH', 'Archive materialization directory must be separate from cache and staging roots');
return this.withLock(() => {
checkOperation(control);
this.cleanIncoming(control);
this.recoverInterruptedOperations(control);
const created: string[] = [], result: MaterializedArchive[] = [];
try {
for (const digest of selected) {
checkOperation(control);
const metadata = this.verifiedEntry(digest, control), source = this.entryPath(digest), initial = fs.lstatSync(source);
assertOwnedRegular(initial, 'Cached archive', this.maxEntryBytes, true);
const target = join(destination, digest);
let copied: { digest: string; bytes: number };
try { copied = this.copyAndHash(source, target, this.maxEntryBytes, stableStat(initial), control); }
catch (error) {
if (existsNoFollow(target)) try { removeRegular(target, 'Incomplete run-owned archive copy'); } catch {}
throw error;
}
created.push(target);
if (copied.digest !== digest || copied.bytes !== metadata.bytes) fail('SNAPSHOT_RACE', 'Cached archive changed while its run-owned copy was materialized');
fs.chmodSync(target, 0o400);
const verified = this.hashFile(target, this.maxEntryBytes, true, undefined, control);
if (verified.digest !== digest || verified.bytes !== metadata.bytes) fail('SNAPSHOT_RACE', 'Run-owned archive copy failed verification');
result.push(Object.freeze({ sha256: digest, path: target, bytes: verified.bytes }));
}
return result;
} catch (error) {
for (const path of created.reverse()) { try { removeRegular(path, 'Incomplete run-owned archive copy'); } catch {} }
throw error;
}
}, control);
}
private timestamp(): number {
const value = this.clock();
if (!Number.isSafeInteger(value) || value < 0) fail('PERSISTENCE_FAILED', 'Cache clock returned an invalid timestamp');
return value;
}
private entryPath(digest: string): string { return join(this.entriesDir, digest); }
private metadataPath(digest: string): string { return join(this.metadataDir, `${digest}.json`); }
private entry(metadata: Metadata): PublicArchiveCacheEntry {
return Object.freeze({ sha256: metadata.sha256, path: this.entryPath(metadata.sha256), bytes: metadata.bytes,
createdAt: metadata.createdAt, lastAccessedAt: metadata.lastAccessedAt });
}
private touch(metadata: Metadata, control: NormalizedCacheOperationControl): PublicArchiveCacheEntry {
checkOperation(control);
const updated: Metadata = { ...metadata, lastAccessedAt: Math.max(metadata.lastAccessedAt, this.timestamp()) };
checkOperation(control);
writeMetadata(this.metadataPath(metadata.sha256), updated);
return this.entry(updated);
}
private verifiedEntry(digest: string, control: NormalizedCacheOperationControl): Metadata {
checkOperation(control);
const metadata = readMetadata(this.metadataPath(digest), digest, control);
const result = this.hashFile(this.entryPath(digest), this.maxEntryBytes, true, undefined, control);
if (result.digest !== digest || result.bytes !== metadata.bytes) fail('INCOMPATIBLE_INPUT', `Cached archive failed SHA-256 verification: ${digest}`);
return metadata;
}
private hashFile(path: string, maxBytes: number, immutable: boolean, expected: StableStat | undefined,
control: NormalizedCacheOperationControl): { digest: string; bytes: number } {
checkOperation(control);
const fd = openNoFollow(path, fs.constants.O_RDONLY);
try {
const beforeStat = fs.fstatSync(fd);
assertOwnedRegular(beforeStat, immutable ? 'Cached archive' : 'Staged archive', maxBytes, immutable);
const before = stableStat(beforeStat), hash = createHash('sha256'), buffer = Buffer.allocUnsafe(COPY_BUFFER_BYTES);
if (expected && !sameStat(expected, before)) fail('SNAPSHOT_RACE', 'Archive changed before it could be verified');
let bytes = 0;
for (;;) {
checkOperation(control);
const read = fs.readSync(fd, buffer, 0, buffer.length, null);
if (!read) break;
bytes += read;
if (bytes > maxBytes) fail('INSUFFICIENT_CAPACITY', 'Archive exceeded its byte limit while being read');
hash.update(buffer.subarray(0, read));
checkOperation(control);
}
checkOperation(control);
const after = stableStat(fs.fstatSync(fd));
if (!sameStat(before, after) || bytes !== before.size) fail('SNAPSHOT_RACE', 'Archive changed while it was being verified');
return { digest: hash.digest('hex'), bytes };
} finally { fs.closeSync(fd); }
}
private copyAndHash(source: string, destination: string, maxBytes: number, expected: StableStat,
control: NormalizedCacheOperationControl): { digest: string; bytes: number } {
checkOperation(control);
const sourceFd = openNoFollow(source, fs.constants.O_RDONLY);
let destinationFd: number | undefined;
try {
const beforeStat = fs.fstatSync(sourceFd);
assertOwnedRegular(beforeStat, 'Staged archive', maxBytes);
const before = stableStat(beforeStat), hash = createHash('sha256'), buffer = Buffer.allocUnsafe(COPY_BUFFER_BYTES);
if (!sameStat(expected, before)) fail('SNAPSHOT_RACE', 'Staged archive changed before promotion');
destinationFd = openNoFollow(destination, fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL, 0o600);
let bytes = 0;
for (;;) {
checkOperation(control);
const read = fs.readSync(sourceFd, buffer, 0, buffer.length, null);
if (!read) break;
bytes += read;
if (bytes > expected.size) fail('SNAPSHOT_RACE', 'Staged archive grew during promotion');
if (bytes > maxBytes) fail('INSUFFICIENT_CAPACITY', 'Archive exceeded its byte limit during promotion');
hash.update(buffer.subarray(0, read));
let offset = 0;
while (offset < read) {
checkOperation(control);
const written = fs.writeSync(destinationFd, buffer, offset, read - offset);
if (written <= 0) fail('PERSISTENCE_FAILED', 'Archive copy stopped before the current chunk was written');
offset += written;
checkOperation(control);
}
}
checkOperation(control);
fs.fsyncSync(destinationFd);
checkOperation(control);
const after = stableStat(fs.fstatSync(sourceFd));
if (!sameStat(before, after) || bytes !== before.size) fail('SNAPSHOT_RACE', 'Staged archive changed during promotion');
return { digest: hash.digest('hex'), bytes };
} finally {
if (destinationFd !== undefined) fs.closeSync(destinationFd);
fs.closeSync(sourceFd);
}
}
private inventory(control: NormalizedCacheOperationControl): Metadata[] {
checkOperation(control);
const entryNames = boundedDirectoryNames(this.entriesDir, 'Cache entries directory', control),
metadataNames = boundedDirectoryNames(this.metadataDir, 'Cache metadata directory', control);
const entrySet=new Set<string>(),metadataSet=new Set<string>();
for (const name of entryNames) {checkOperation(control);if (!SHA256.test(name)) fail('INCOMPATIBLE_INPUT', 'Cache entries directory contains an unexpected object');entrySet.add(name);}
for (const name of metadataNames) {checkOperation(control);if (!/^[a-f0-9]{64}\.json$/.test(name)) fail('INCOMPATIBLE_INPUT', 'Cache metadata directory contains an unexpected object');metadataSet.add(name.slice(0,-5));}
if (entrySet.size !== metadataSet.size) fail('INCOMPATIBLE_INPUT', 'Cache entries and metadata are inconsistent');
for(const name of entrySet){checkOperation(control);if(!metadataSet.has(name))
fail('INCOMPATIBLE_INPUT', 'Cache entries and metadata are inconsistent');
}
const inventory = entryNames.map(digest => {
checkOperation(control);
const stat = fs.lstatSync(this.entryPath(digest));
assertOwnedRegular(stat, 'Cached archive', this.maxEntryBytes, true);
const metadata = readMetadata(this.metadataPath(digest), digest, control);
if (metadata.bytes !== stat.size) fail('INCOMPATIBLE_INPUT', `Cache size metadata is inconsistent for ${digest}`);
return metadata;
});
checkOperation(control);
return inventory;
}
private evictToFit(incomingBytes: number, control: NormalizedCacheOperationControl): void {
if (!Number.isSafeInteger(incomingBytes) || incomingBytes < 0 || incomingBytes > this.maxBytes)
fail('INSUFFICIENT_CAPACITY', 'Archive cannot fit within the public-cache limit');
checkOperation(control);
const entries = this.inventory(control);checkOperation(control);entries.sort((a, b) => a.lastAccessedAt - b.lastAccessedAt || a.createdAt - b.createdAt || a.sha256.localeCompare(b.sha256));
checkOperation(control);
let total = 0;
for (const item of entries) { checkOperation(control); total += item.bytes; }
for (const item of entries) {
checkOperation(control);
if (total + incomingBytes <= this.maxBytes) break;
const archive = this.moveToRecovery(this.entryPath(item.sha256), 'entry', item.sha256);
const metadata = this.moveToRecovery(this.metadataPath(item.sha256), 'metadata', item.sha256);
removeRegular(archive, 'Evicted cache archive');
removeRegular(metadata, 'Evicted cache metadata');
total -= item.bytes;
}
if (total + incomingBytes > this.maxBytes) fail('INSUFFICIENT_CAPACITY', 'Archive cache could not free enough verified capacity');
}
private cleanIncoming(control: NormalizedCacheOperationControl): void {
const incomingNames = boundedDirectoryNames(this.incomingDir, 'Cache incoming directory', control);
let publishedByInode:Map<string,string[]>|undefined;
for (const name of incomingNames) {
checkOperation(control);
if (!/^\.incoming-\d+-[a-f0-9]{24}$/.test(name)) fail('INCOMPATIBLE_INPUT', 'Cache incoming directory contains an unexpected object');
const incoming = join(this.incomingDir, name), stat = fs.lstatSync(incoming);
if (!stat.isFile() || stat.isSymbolicLink() || ![1, 2].includes(stat.nlink) || stat.size > this.maxEntryBytes ||
(process.getuid && stat.uid !== process.getuid())) fail('UNSAFE_PATH', 'Incomplete cache archive is not a bounded regular file');
if (stat.nlink === 2) {
if(!publishedByInode){
publishedByInode=new Map();
for(const entry of boundedDirectoryNames(this.entriesDir, 'Cache entries directory', control)){
checkOperation(control);
if (!SHA256.test(entry)) fail('INCOMPATIBLE_INPUT', 'Cache entries directory contains an unexpected object');
const candidate=fs.lstatSync(this.entryPath(entry)),key=`${candidate.dev}:${candidate.ino}`,matches=publishedByInode.get(key)??[];
matches.push(entry);publishedByInode.set(key,matches);
}
}
const matches=publishedByInode.get(`${stat.dev}:${stat.ino}`)??[];
if (matches.length !== 1) fail('UNSAFE_PATH', 'Incoming archive hard link does not match one published cache entry');
const target = fs.lstatSync(this.entryPath(matches[0]));
if (!target.isFile() || target.isSymbolicLink() || target.nlink !== 2 || target.size > this.maxEntryBytes ||
(process.getuid && target.uid !== process.getuid()) || (target.mode & 0o222) !== 0)
fail('SNAPSHOT_RACE', 'Incoming archive link count or identity changed during recovery');
}
try { fs.unlinkSync(incoming); } catch { fail('PERSISTENCE_FAILED', 'Incomplete cache archive could not be removed'); }
}
}
/**
* Recover only artifacts whose names and inode types prove they belong to an
* interrupted cache transaction. Unknown objects remain a hard failure.
*/
private recoverInterruptedOperations(control: NormalizedCacheOperationControl): void {
for (const name of boundedDirectoryNames(this.recoveryDir, 'Cache recovery directory', control)) {
checkOperation(control);
if (!/^\.recovery-(?:entry|metadata)-[a-f0-9]{64}-\d+-[a-f0-9]{24}$/.test(name))
fail('INCOMPATIBLE_INPUT', 'Cache recovery directory contains an unexpected object');
removeRegular(join(this.recoveryDir, name), 'Interrupted cache transaction');
}
const entries = boundedDirectoryNames(this.entriesDir, 'Cache entries directory', control),
metadataObjects = boundedDirectoryNames(this.metadataDir, 'Cache metadata directory', control);
for (const name of metadataObjects) {
checkOperation(control);
if (/^[a-f0-9]{64}\.json\.tmp\.\d+\.[a-f0-9]{8}$/.test(name)) this.recoverMetadataTemp(name, control);
else if (!/^[a-f0-9]{64}\.json$/.test(name)) fail('INCOMPATIBLE_INPUT', 'Cache metadata directory contains an unexpected object');
}
const metadata = boundedDirectoryNames(this.metadataDir, 'Cache metadata directory', control);
for (const name of entries) if (!SHA256.test(name)) fail('INCOMPATIBLE_INPUT', 'Cache entries directory contains an unexpected object');
for (const name of metadata) if (!/^[a-f0-9]{64}\.json$/.test(name)) fail('INCOMPATIBLE_INPUT', 'Cache metadata directory contains an unexpected object');
const entrySet=new Set<string>(),metadataSet=new Set<string>(),digests=new Set<string>();
for(const name of entries){checkOperation(control);entrySet.add(name);digests.add(name);}
for(const name of metadata){checkOperation(control);const digest=name.slice(0,-5);metadataSet.add(digest);digests.add(digest);}
for (const digest of digests) {
checkOperation(control);
if (entrySet.has(digest) === metadataSet.has(digest)) continue;
if (entrySet.has(digest)) this.discardObject(this.entryPath(digest), 'entry', digest);
else this.discardObject(this.metadataPath(digest), 'metadata', digest);
}
}
private recoveryPath(kind: 'entry' | 'metadata', digest: string): string {
return join(this.recoveryDir, `.recovery-${kind}-${digest}-${process.pid}-${randomBytes(12).toString('hex')}`);
}
private moveToRecovery(path: string, kind: 'entry' | 'metadata', digest: string): string {
const before = fs.lstatSync(path);
assertOwnedRegular(before, kind === 'entry' ? 'Cached archive' : 'Cache metadata', kind === 'entry' ? this.maxEntryBytes : METADATA_LIMIT, kind === 'entry');
if (kind === 'metadata' && (before.mode & 0o077) !== 0) fail('UNSAFE_PATH', 'Cache metadata permissions are not private');
const destination = this.recoveryPath(kind, digest);
try { fs.renameSync(path, destination); }
catch { fail('PERSISTENCE_FAILED', 'Interrupted cache object could not be quarantined atomically'); }
const after = fs.lstatSync(destination);
// rename(2) can update ctime; stable inode identity, content size, mode,
// link count, mtime, and ownership prove the moved object is the one read.
if (!sameRenamedInode(stableStat(before), stableStat(after))) fail('SNAPSHOT_RACE', 'Cache object changed while it was quarantined');
return destination;
}
private discardObject(path: string, kind: 'entry' | 'metadata', digest: string): void {
const quarantined = this.moveToRecovery(path, kind, digest);
removeRegular(quarantined, 'Interrupted cache transaction');
}
private recoverMetadataTemp(name: string, control: NormalizedCacheOperationControl): void {
checkOperation(control);
const path = join(this.metadataDir, name), stat = fs.lstatSync(path);
if (!stat.isFile() || stat.isSymbolicLink() || ![1, 2].includes(stat.nlink) || stat.size > METADATA_LIMIT ||
(process.getuid && stat.uid !== process.getuid()) || (stat.mode & 0o077) !== 0)
fail('UNSAFE_PATH', 'Interrupted cache metadata write is not a private regular file');
if (stat.nlink === 2) {
const target = this.metadataPath(name.slice(0, 64));
let targetStat: fs.Stats;
try { targetStat = fs.lstatSync(target); } catch { fail('INCOMPATIBLE_INPUT', 'Hard-linked metadata temp has no published target'); }
if (!targetStat!.isFile() || targetStat!.isSymbolicLink() || targetStat!.dev !== stat.dev || targetStat!.ino !== stat.ino || targetStat!.nlink !== 2)
fail('UNSAFE_PATH', 'Interrupted metadata hard link does not match its published target');
}
try { fs.unlinkSync(path); } catch { fail('PERSISTENCE_FAILED', 'Interrupted cache metadata write could not be removed'); }
}
private withLock<T>(callback: () => T, control: NormalizedCacheOperationControl): T {
checkOperation(control);
const marker = `${JSON.stringify({ protocol: CACHE_LOCK_PROTOCOL })}\n`;
const options={label:'Cache lock protocol',maxBytes:METADATA_LIMIT,validate:(value:unknown)=>{
if(!value||typeof value!=='object'||Array.isArray(value)||Object.keys(value).join(',')!=='protocol'||(value as any).protocol!==CACHE_LOCK_PROTOCOL)
fail('INCOMPATIBLE_INPUT','Archive-cache lock protocol is invalid');
}};
const tempPattern=/^\.lock\.tmp\.(\d{1,10})\.[a-f0-9]{8}$/;
for(const name of boundedDirectoryNames(this.root, 'Cache root directory', control)){
checkOperation(control);
const match=name.match(tempPattern);if(!match)continue;
const temporary=join(this.root,name),publisherPid=Number(match[1]);
if(existsNoFollow(this.lockDir))recoverAtomicNoReplaceJson(this.lockDir,options);
if(existsNoFollow(temporary))discardAtomicNoReplaceTemp(temporary,publisherPid,options);
}
try { atomicWriteSync(this.lockDir, marker, { mode: 0o600, noReplace: true }); }
catch (error: any) {
if (error?.code !== 'EEXIST') fail('PERSISTENCE_FAILED', 'Archive-cache lock protocol could not be initialized');
recoverAtomicNoReplaceJson(this.lockDir,options);
const stat = fs.lstatSync(this.lockDir);
if (stat.isDirectory() && !stat.isSymbolicLink())
fail('INSUFFICIENT_CAPACITY', 'A legacy archive-cache helper may still own or initialize this cache; its lock was left intact');
assertOwnedRegular(stat, 'Cache lock protocol', METADATA_LIMIT);
if ((stat.mode & 0o077) !== 0) fail('UNSAFE_PATH', 'Cache lock protocol permissions are not private');
let protocol:unknown;try{protocol=JSON.parse(fs.readFileSync(this.lockDir,'utf8')).protocol;}catch{}
if(protocol!==CACHE_LOCK_PROTOCOL)fail('INCOMPATIBLE_INPUT','Archive-cache lock protocol is invalid');
}
const result=withStateLock(this.root,()=>{checkOperation(control);return callback();});
if(result&&typeof (result as any).then==='function')fail('PERSISTENCE_FAILED','Archive-cache operation unexpectedly became asynchronous');
return result as T;
}
}
export function publicArchiveCacheRoot(): string {
return join(privateRoot(), 'public-cache');
}