fix: fence shared transformations and cancel abandoned cache work

This commit is contained in:
Thomas Durieux committed 2026-11-02 13:58:25 +00:00
1 parent d851fdb381
commit 074a9af315
6 files changed
+118 -10

No files matched your search

+3 -1
View File
@@ -287,9 +287,11 @@ export class BlobCache {
if (this.usage() + reserved + reservation > this.options.budget) {
this.evictTo(Math.max(0, Math.min(this.options.budget * (this.options.lowWater ?? 0.9), this.options.budget - reserved - reservation)));
}
if (this.usage() + reserved + reservation > this.options.budget) throw new CacheAdmissionError("cache_budget_busy");
// Eviction unlinks files: commit its metadata even if admission still fails.
if (this.usage() + reserved + reservation > this.options.budget) return null;
this.db.prepare("INSERT INTO fills VALUES(?,?,?,?)").run(key, token, Date.now() + 120000, reservation); return true;
}, true);
if (owned === null) throw new CacheAdmissionError("cache_budget_busy");
if (owned) break;
this.metric("fill_wait");
if (Date.now() >= deadline) throw new CacheAdmissionError("fill_timeout");
+24 -5
View File
@@ -5,6 +5,7 @@ import { pipeline } from "stream/promises";
import { Response } from "express";
import { performance } from "perf_hooks";
import config from "../config";
import { assertTransformScopeActive } from "./transformed-cache";
import { BlobStream, CacheAdmissionError, resultBlobCache } from "./blob-cache";
import { AnonymizeTransformer, classifyByName } from "./anonymize-utils";
import { observeCache } from "./cache-observability";
@@ -18,13 +19,18 @@ export function decisionKey(digest: string, options: unknown) {
export async function sendSharedContent(content: Readable, res: Response,
options: ConstructorParameters<typeof AnonymizeTransformer>[0]): Promise<boolean> {
try { return await sendCached(content, res, options); }
catch (error) { if (error instanceof CacheAdmissionError) return false; throw error; }
catch (error) { if (error instanceof CacheAdmissionError) return false; content.destroy(); throw error; }
}
async function sendCached(content: Readable, res: Response,
options: ConstructorParameters<typeof AnonymizeTransformer>[0]): Promise<boolean> {
const source = (content as BlobStream).cachedBlob;
const results = resultBlobCache();
if (!source || !results) return false;
const assertActive = async () => {
if (res.destroyed) throw new Error("response_cancelled");
await assertTransformScopeActive(options);
if (res.destroyed) throw new Error("response_cancelled");
};
const encoding = res.req.acceptsEncodings("gzip", "identity");
res.vary("Accept-Encoding");
if (!encoding) { content.destroy(); res.status(406).end(); return true; }
@@ -33,6 +39,7 @@ async function sendCached(content: Readable, res: Response,
const send = async (store: typeof results, digest: string, isText: boolean, onHit?: () => void) => {
const stream = await store.open(digest, encoding === "gzip");
if (!stream) return false;
try { await assertActive(); } catch (error) { stream.destroy(); throw error; }
const record = stream.cachedBlob!.record;
onHit?.();
content.destroy();
@@ -63,30 +70,42 @@ async function sendCached(content: Readable, res: Response,
let processing: Readable | undefined;
let transformer: AnonymizeTransformer | undefined;
const started = performance.now();
const abort = () => { if (!res.writableFinished) processing?.destroy(new Error("response_cancelled")); };
const abort = () => {
if (!res.writableFinished) {
const error = new Error("response_cancelled");
transformer?.destroy(error);
processing?.destroy(error);
}
};
res.once("close", abort);
try {
const output = await results.getOrFill(`render-${key}`, async () => {
await assertActive();
processing = await source.store.open(source.record.digest);
if (!processing) throw new CacheAdmissionError("source_evicted");
if (res.destroyed) throw new Error("response_cancelled");
const transform = transformer = new AnonymizeTransformer({ ...options, decisionManaged: true });
const stream = new PassThrough();
void pipeline(processing, transform, stream).catch(error => stream.destroy(error));
return stream;
}, async proof => {
await assertActive();
const unchanged = source.record.digest === `git-${proof.gitSha}` || source.record.digest === `sha256-${proof.sha256}`;
observeCache(unchanged ? "rewrite_unchanged" : "rewrite_changed", source.record.size, performance.now() - started);
if (unchanged) { results.saveDecision(key, source.record.digest, false, !!transformer?.isText); throw new Unchanged(); }
return `sha256-${proof.sha256}`;
}, source.record.size);
const digest = output.cachedBlob!.record.digest;
results.saveDecision(key, digest, true, true);
// Keep the result pinned until its HTTP reader has been opened.
try { if (await send(results, digest, true)) return true; } finally { output.destroy(); }
try {
await assertActive();
results.saveDecision(key, digest, true, true);
if (await send(results, digest, true)) return true;
} finally { output.destroy(); }
return false;
} catch (error) {
if (error instanceof Unchanged) return send(source.store, source.record.digest, !!transformer?.isText);
if (error instanceof CacheAdmissionError) { observeCache("decision_bypass", source.record.size); return false; }
content.destroy(); throw error;
} finally { res.removeListener("close", abort); processing?.destroy(); }
} finally { res.removeListener("close", abort); transformer?.destroy(); processing?.destroy(); }
}
+9 -3
View File
@@ -26,6 +26,14 @@ async function scopeActive(scope?: CacheScope) {
&& scope.revision === current.contentCacheRevision;
}
/** Shared and legacy result caches must enforce the same repository lifecycle. */
export async function assertTransformScopeActive(options: unknown) {
const settings = options as { repoId?: string; cacheGeneration?: string; cacheRevision?: string } | undefined;
const scope = typeof settings?.repoId === "string"
? { repoId: settings.repoId, generation: settings.cacheGeneration, revision: settings.cacheRevision } : undefined;
if (!await scopeActive(scope)) throw new AnonymousError("repository_changed", { httpStatus: 409 });
}
async function removeEntry(metadata: string) {
await Promise.all([metadata, metadata.replace(/\.json$/, ".data")].map(path => fs.rm(path, { force: true })));
}
@@ -36,9 +44,7 @@ export async function transformedFile(digest: string, options: unknown, output:
const settings = options as { repoId?: string; cacheGeneration?: string; cacheRevision?: string } | undefined;
const scope: CacheScope | undefined = typeof settings?.repoId === "string"
? { repoId: settings.repoId, generation: settings.cacheGeneration, revision: settings.cacheRevision } : undefined;
const assertActive = async () => {
if (!await scopeActive(scope)) throw new AnonymousError("repository_changed", { httpStatus: 409 });
};
const assertActive = () => assertTransformScopeActive(options);
const produceActive = async () => {
await assertActive();
const changed = await produce();