CalvinBackup/anonymous_github
1
0 Fork 0
mirror of https://github.com/tdurieux/anonymous_github.git synced 2026-11-14 08:17:26 +01:00
Code Issues Packages Projects Releases Wiki Activity

fix: coordinate cache disk reservations and tolerate reader renewal contention

Browse Source
This commit is contained in:
Thomas Durieux committed 2026-11-02 13:42:10 +00:00
1 parent 4b8635bd53
commit d851fdb381
6 files changed
+160 -17
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/core/blob-cache.ts
+58 -14
View File
@@ -1,6 +1,6 @@
import { DatabaseSync } from "node:sqlite";
import { createHash, randomUUID } from "crypto";
import { createReadStream, createWriteStream, mkdirSync, unlinkSync, rmdirSync, promises as fs } from "fs";
import { createReadStream, createWriteStream, mkdirSync, unlinkSync, rmdirSync, statfsSync, promises as fs } from "fs";
import { join } from "path";
import { Readable, Transform } from "stream";
import { pipeline } from "stream/promises";
@@ -20,7 +20,7 @@ function cacheError(error: unknown): never {
}
export interface BlobRecord { digest: string; file: string; size: number; stored: number; allocated: number; encoding: "gzip" | "identity"; accessed: number }
export interface BlobProof { gitSha: string; sha256: string; size: number }
export interface BlobOptions { root: string; budget: number; maxObject: number; gzip?: boolean; saving?: number; freeBytes?: number; lowWater?: number; name?: string }
export interface BlobOptions { root: string; budget: number; maxObject: number; gzip?: boolean; saving?: number; freeBytes?: number; lowWater?: number; name?: string; reservationRoot?: string }
export type BlobStream = Readable & { cachedBlob?: { store: BlobCache; record: BlobRecord } };
const validDigest = (s: string) => /^(git-[a-f0-9]{40}|sha256-[a-f0-9]{64})$/.test(s);
@@ -28,6 +28,7 @@ const validDigest = (s: string) => /^(git-[a-f0-9]{40}|sha256-[a-f0-9]{64})$/.te
export class BlobCache {
readonly db: DatabaseSync;
readonly name: string;
private disk?: DatabaseSync;
constructor(readonly options: BlobOptions) {
if (!(options.budget > 0 && options.maxObject > 0)) throw new Error("invalid_cache_budget");
this.name = options.name || "source";
@@ -80,8 +81,14 @@ export class BlobCache {
BEGIN UPDATE summary SET ${values.map(([column, value]) => `${column}=${column}-OLD.${value}+NEW.${value}`).join(",")} WHERE id=1; END;`);
}
});
const reservationRoot = options.reservationRoot || options.root;
mkdirSync(reservationRoot, { recursive: true, mode: 0o700 });
this.disk = new DatabaseSync(join(reservationRoot, "disk-reservations.sqlite"));
this.disk.exec(`PRAGMA busy_timeout=5000; PRAGMA journal_mode=WAL; PRAGMA synchronous=FULL;
CREATE TABLE IF NOT EXISTS reservations(token TEXT PRIMARY KEY,bytes INTEGER);
PRAGMA busy_timeout=50;`);
this.db.exec("PRAGMA synchronous=NORMAL; PRAGMA busy_timeout=50;");
} catch (error) { this.db.close(); throw error; }
} catch (error) { this.disk?.close(); this.db.close(); throw error; }
}
// Reader leases, history and decisions are disposable after a system failure.
// Reservations and publication records use FULL before creating/renaming payloads,
@@ -98,6 +105,20 @@ export class BlobCache {
const excess = Number(this.db.prepare(`SELECT ${table} n FROM summary WHERE id=1`).get()!.n) - limit;
if (excess > 0) this.db.prepare(`DELETE FROM ${table} WHERE rowid IN (SELECT rowid FROM ${table} ORDER BY ${order} LIMIT ?)`).run(excess);
}
private reserveDisk(token: string, bytes: number) {
const disk = this.disk!;
disk.exec("BEGIN IMMEDIATE");
try {
// Take a fresh filesystem sample while holding the shared admission lock.
const space = statfsSync(this.options.root);
const pending = Number(disk.prepare("SELECT COALESCE(SUM(bytes),0) bytes FROM reservations").get()!.bytes);
if (space.bavail * space.bsize < pending + bytes + (this.options.freeBytes || 0))
throw new CacheAdmissionError("disk_headroom");
disk.prepare("INSERT INTO reservations VALUES(?,?)").run(token, bytes);
disk.exec("COMMIT");
} catch (error) { disk.exec("ROLLBACK"); throw error; }
}
private releaseDisk(token: string) { this.disk!.prepare("DELETE FROM reservations WHERE token=?").run(token); }
private metric(event: string, bytes = 0, ms = 0) { observeCache(`${this.name}_${event}`, bytes, ms); }
private path(record: BlobRecord) {
if (!validDigest(record.digest) || !/^[a-f0-9-]+\.(gz|raw)$/.test(record.file)) throw new Error("invalid_blob_record");
@@ -116,7 +137,11 @@ export class BlobCache {
catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; }
}
this.db.prepare("DELETE FROM publications WHERE token=?").run(row.token);
for (const suffix of ["raw", "gz"]) { try { unlinkSync(join(this.options.root, "tmp", `${row.token}.${suffix}`)); } catch { /* absent */ } }
for (const suffix of ["raw", "gz"]) {
try { unlinkSync(join(this.options.root, "tmp", `${row.token}.${suffix}`)); }
catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; }
}
this.releaseDisk(String(row.token));
}
this.db.prepare("DELETE FROM fills WHERE expires<?").run(now);
this.db.prepare("DELETE FROM readers WHERE expires<?").run(now);
@@ -170,10 +195,11 @@ export class BlobCache {
private async openReader(digest: string, encoded = false): Promise<BlobStream | undefined> {
if (!validDigest(digest)) return;
const token = randomUUID();
let leaseUntil = Date.now() + 120000;
const record = this.transaction(() => {
const r = this.db.prepare("SELECT * FROM blobs WHERE digest=?").get(digest) as unknown as BlobRecord | undefined;
if (!r) return;
this.db.prepare("INSERT INTO readers VALUES(?,?,?)").run(token, digest, Date.now() + 120000);
this.db.prepare("INSERT INTO readers VALUES(?,?,?)").run(token, digest, leaseUntil);
if (Date.now() - r.accessed > 30000) this.db.prepare("UPDATE blobs SET accessed=? WHERE digest=?").run(Date.now(), digest);
return r;
});
@@ -194,10 +220,22 @@ export class BlobCache {
const output = (needsDecode ? createGunzip() : source) as BlobStream;
output.cachedBlob = { store: this, record };
let start = performance.now();
let renewAt = Date.now() + 30000;
const heartbeat = setInterval(() => {
try { if (!this.db.prepare("UPDATE readers SET expires=? WHERE token=?").run(Date.now() + 120000, token).changes) output.destroy(new Error("cache_reader_lease_lost")); }
catch { output.destroy(new Error("cache_reader_lease_lost")); }
}, 30000); heartbeat.unref();
const now = Date.now();
if (now < renewAt) return;
if (now >= leaseUntil) { output.destroy(new Error("cache_reader_lease_lost")); return; }
try {
const renewedUntil = now + 120000;
if (!this.db.prepare("UPDATE readers SET expires=? WHERE token=? AND expires>?").run(renewedUntil, token, now).changes) {
output.destroy(new Error("cache_reader_lease_lost")); return;
}
leaseUntil = renewedUntil; renewAt = now + 30000;
} catch (error) {
if (!isCacheContention(error)) output.destroy(new Error("cache_reader_lease_lost"));
// Retry on the next tick while the previously confirmed lease is valid.
}
}, 1000); heartbeat.unref();
let released = false;
const release = () => {
if (released) return; released = true; clearInterval(heartbeat); source.destroy();
@@ -242,8 +280,6 @@ export class BlobCache {
const token = randomUUID(), reservation = this.options.maxObject * 3 + 1024;
const deadline = Date.now() + 120000;
for (;;) {
const space = await fs.statfs(this.options.root);
if (space.bavail * space.bsize < reservation + (this.options.freeBytes || 0)) throw new CacheAdmissionError("disk_headroom");
const owned = this.transaction(() => {
this.expire();
if (this.db.prepare("SELECT 1 FROM fills WHERE key=?").get(key)) return false;
@@ -268,6 +304,11 @@ export class BlobCache {
}, 30000); heartbeat.unref();
const timeout = setTimeout(() => input?.destroy(new Error("cache_fill_timeout")), 120000); timeout.unref();
try {
// The pool reservation is already durable before acquiring physical headroom.
this.transaction(() => {
if (!this.db.prepare("SELECT 1 FROM fills WHERE token=? AND expires>?").get(token, Date.now())) throw new CacheAdmissionError("cache_fill_lease_lost");
this.reserveDisk(token, reservation);
});
// Recheck after ownership; another producer may have published between the initial lookup and lease acquisition.
hit = await this.lookup(key); if (hit) return hit;
input = await produce();
@@ -312,13 +353,16 @@ export class BlobCache {
const result = await this.open(digest); if (!result) throw new Error("published_blob_missing"); return result;
} finally {
clearTimeout(timeout); clearInterval(heartbeat); input?.destroy();
await Promise.all([raw, gz, ...(file ? [file] : [])].map(p => fs.unlink(p).catch(() => {})));
await Promise.all([raw, gz, ...(file ? [file] : [])].map(p => fs.unlink(p).catch(error => {
if (error.code !== "ENOENT") throw error;
})));
this.releaseFill(key, token);
}
}
private releaseFill(key: string, token: string) {
try {
this.transaction(() => {
this.releaseDisk(token);
this.db.prepare("DELETE FROM readers WHERE token=?").run(token);
this.db.prepare("DELETE FROM publications WHERE token=?").run(token);
this.db.prepare("DELETE FROM fills WHERE key=? AND token=?").run(key, token);
@@ -341,19 +385,19 @@ export class BlobCache {
});
} catch (error) { if (!isCacheContention(error)) throw error; /* Optional decision can be recomputed. */ }
}
close() { this.db.close(); }
close() { this.disk?.close(); this.db.close(); }
}
let source: BlobCache | undefined, results: BlobCache | undefined;
export function sharedBlobCache() {
if (!config.SHARED_CACHE_ENABLED || config.STORAGE !== "filesystem") return;
try { return source ||= new BlobCache({ root: join(config.SHARED_CACHE_PATH, "source"), budget: config.SHARED_CACHE_BYTES,
try { return source ||= new BlobCache({ reservationRoot: config.SHARED_CACHE_PATH, root: join(config.SHARED_CACHE_PATH, "source"), budget: config.SHARED_CACHE_BYTES,
maxObject: config.SHARED_CACHE_MAX_FILE_BYTES, gzip: config.SHARED_CACHE_GZIP, saving: config.SHARED_CACHE_GZIP_MIN_SAVING,
freeBytes: config.SHARED_CACHE_FREE_BYTES, lowWater: config.SHARED_CACHE_LOW_WATER });
} catch (error) { if (!isCacheContention(error)) throw error; }
}
export function resultBlobCache() {
if (!sharedBlobCache() || !config.SHARED_CACHE_DECISIONS) return;
try { return results ||= new BlobCache({ root: join(config.SHARED_CACHE_PATH, "results"), budget: config.SHARED_CACHE_RESULT_BYTES,
try { return results ||= new BlobCache({ reservationRoot: config.SHARED_CACHE_PATH, root: join(config.SHARED_CACHE_PATH, "results"), budget: config.SHARED_CACHE_RESULT_BYTES,
maxObject: Math.min(config.SHARED_CACHE_MAX_FILE_BYTES, Math.floor(config.SHARED_CACHE_RESULT_BYTES / 8)), gzip: config.SHARED_CACHE_GZIP,
freeBytes: config.SHARED_CACHE_FREE_BYTES, name: "result" });
} catch (error) { if (!isCacheContention(error)) throw error; }
src/core/source/GitHubStream.ts
+4 -2
View File
@@ -391,7 +391,7 @@ export default class GitHubStream extends GitHubBase {
await assertActive();
if (expected.size != null && expected.size > config.MAX_FILE_SIZE) throw new AnonymousError("file_too_big", { httpStatus: 413 });
let legacy = false, allowLegacy = true;
const download = async () => {
const download = async (checkMinimum = false) => {
const parentToken = await this.data.getToken();
const token = expected.upstream ? "" : parentToken;
const sourceMiss = cachedContentMiss(this.data, parentToken);
@@ -425,6 +425,8 @@ export default class GitHubStream extends GitHubBase {
if (bytes > config.MAX_FILE_SIZE) throw new AnonymousError("file_too_big", { httpStatus: 413 });
yield chunk;
}
if (checkMinimum && expected.size != null && expected.size > 0 && bytes < expected.size)
throw new AnonymousError("storage_write_size_mismatch", { httpStatus: 502 });
} catch (error) { throw await normalize(error as Error); }
finally { clearTimeout(deadline); input.destroy(); }
})());
@@ -464,7 +466,7 @@ export default class GitHubStream extends GitHubBase {
return content;
} catch (error) {
if (!(error instanceof CacheAdmissionError)) throw error;
observeCache("source_bypass"); await assertActive(); return download();
observeCache("source_bypass"); await assertActive(); return download(true);
}
}
const hit = await measureStage("cache_lookup", cached);