mirror of
https://github.com/tdurieux/anonymous_github.git
synced 2026-11-14 00:07:00 +01:00
fix: fence shared transformations and cancel abandoned cache work
This commit is contained in:
1 parent
d851fdb381
commit
074a9af315
6 files changed
+118
-10
No files matched your search
@@ -287,9 +287,11 @@ export class BlobCache {
|
||||
if (this.usage() + reserved + reservation > this.options.budget) {
|
||||
this.evictTo(Math.max(0, Math.min(this.options.budget * (this.options.lowWater ?? 0.9), this.options.budget - reserved - reservation)));
|
||||
}
|
||||
if (this.usage() + reserved + reservation > this.options.budget) throw new CacheAdmissionError("cache_budget_busy");
|
||||
// Eviction unlinks files: commit its metadata even if admission still fails.
|
||||
if (this.usage() + reserved + reservation > this.options.budget) return null;
|
||||
this.db.prepare("INSERT INTO fills VALUES(?,?,?,?)").run(key, token, Date.now() + 120000, reservation); return true;
|
||||
}, true);
|
||||
if (owned === null) throw new CacheAdmissionError("cache_budget_busy");
|
||||
if (owned) break;
|
||||
this.metric("fill_wait");
|
||||
if (Date.now() >= deadline) throw new CacheAdmissionError("fill_timeout");
|
||||
|
||||
@@ -5,6 +5,7 @@ import { pipeline } from "stream/promises";
|
||||
import { Response } from "express";
|
||||
import { performance } from "perf_hooks";
|
||||
import config from "../config";
|
||||
import { assertTransformScopeActive } from "./transformed-cache";
|
||||
import { BlobStream, CacheAdmissionError, resultBlobCache } from "./blob-cache";
|
||||
import { AnonymizeTransformer, classifyByName } from "./anonymize-utils";
|
||||
import { observeCache } from "./cache-observability";
|
||||
@@ -18,13 +19,18 @@ export function decisionKey(digest: string, options: unknown) {
|
||||
export async function sendSharedContent(content: Readable, res: Response,
|
||||
options: ConstructorParameters<typeof AnonymizeTransformer>[0]): Promise<boolean> {
|
||||
try { return await sendCached(content, res, options); }
|
||||
catch (error) { if (error instanceof CacheAdmissionError) return false; throw error; }
|
||||
catch (error) { if (error instanceof CacheAdmissionError) return false; content.destroy(); throw error; }
|
||||
}
|
||||
async function sendCached(content: Readable, res: Response,
|
||||
options: ConstructorParameters<typeof AnonymizeTransformer>[0]): Promise<boolean> {
|
||||
const source = (content as BlobStream).cachedBlob;
|
||||
const results = resultBlobCache();
|
||||
if (!source || !results) return false;
|
||||
const assertActive = async () => {
|
||||
if (res.destroyed) throw new Error("response_cancelled");
|
||||
await assertTransformScopeActive(options);
|
||||
if (res.destroyed) throw new Error("response_cancelled");
|
||||
};
|
||||
const encoding = res.req.acceptsEncodings("gzip", "identity");
|
||||
res.vary("Accept-Encoding");
|
||||
if (!encoding) { content.destroy(); res.status(406).end(); return true; }
|
||||
@@ -33,6 +39,7 @@ async function sendCached(content: Readable, res: Response,
|
||||
const send = async (store: typeof results, digest: string, isText: boolean, onHit?: () => void) => {
|
||||
const stream = await store.open(digest, encoding === "gzip");
|
||||
if (!stream) return false;
|
||||
try { await assertActive(); } catch (error) { stream.destroy(); throw error; }
|
||||
const record = stream.cachedBlob!.record;
|
||||
onHit?.();
|
||||
content.destroy();
|
||||
@@ -63,30 +70,42 @@ async function sendCached(content: Readable, res: Response,
|
||||
let processing: Readable | undefined;
|
||||
let transformer: AnonymizeTransformer | undefined;
|
||||
const started = performance.now();
|
||||
const abort = () => { if (!res.writableFinished) processing?.destroy(new Error("response_cancelled")); };
|
||||
const abort = () => {
|
||||
if (!res.writableFinished) {
|
||||
const error = new Error("response_cancelled");
|
||||
transformer?.destroy(error);
|
||||
processing?.destroy(error);
|
||||
}
|
||||
};
|
||||
res.once("close", abort);
|
||||
try {
|
||||
const output = await results.getOrFill(`render-${key}`, async () => {
|
||||
await assertActive();
|
||||
processing = await source.store.open(source.record.digest);
|
||||
if (!processing) throw new CacheAdmissionError("source_evicted");
|
||||
if (res.destroyed) throw new Error("response_cancelled");
|
||||
const transform = transformer = new AnonymizeTransformer({ ...options, decisionManaged: true });
|
||||
const stream = new PassThrough();
|
||||
void pipeline(processing, transform, stream).catch(error => stream.destroy(error));
|
||||
return stream;
|
||||
}, async proof => {
|
||||
await assertActive();
|
||||
const unchanged = source.record.digest === `git-${proof.gitSha}` || source.record.digest === `sha256-${proof.sha256}`;
|
||||
observeCache(unchanged ? "rewrite_unchanged" : "rewrite_changed", source.record.size, performance.now() - started);
|
||||
if (unchanged) { results.saveDecision(key, source.record.digest, false, !!transformer?.isText); throw new Unchanged(); }
|
||||
return `sha256-${proof.sha256}`;
|
||||
}, source.record.size);
|
||||
const digest = output.cachedBlob!.record.digest;
|
||||
results.saveDecision(key, digest, true, true);
|
||||
// Keep the result pinned until its HTTP reader has been opened.
|
||||
try { if (await send(results, digest, true)) return true; } finally { output.destroy(); }
|
||||
try {
|
||||
await assertActive();
|
||||
results.saveDecision(key, digest, true, true);
|
||||
if (await send(results, digest, true)) return true;
|
||||
} finally { output.destroy(); }
|
||||
return false;
|
||||
} catch (error) {
|
||||
if (error instanceof Unchanged) return send(source.store, source.record.digest, !!transformer?.isText);
|
||||
if (error instanceof CacheAdmissionError) { observeCache("decision_bypass", source.record.size); return false; }
|
||||
content.destroy(); throw error;
|
||||
} finally { res.removeListener("close", abort); processing?.destroy(); }
|
||||
} finally { res.removeListener("close", abort); transformer?.destroy(); processing?.destroy(); }
|
||||
}
|
||||
@@ -26,6 +26,14 @@ async function scopeActive(scope?: CacheScope) {
|
||||
&& scope.revision === current.contentCacheRevision;
|
||||
}
|
||||
|
||||
/** Shared and legacy result caches must enforce the same repository lifecycle. */
|
||||
export async function assertTransformScopeActive(options: unknown) {
|
||||
const settings = options as { repoId?: string; cacheGeneration?: string; cacheRevision?: string } | undefined;
|
||||
const scope = typeof settings?.repoId === "string"
|
||||
? { repoId: settings.repoId, generation: settings.cacheGeneration, revision: settings.cacheRevision } : undefined;
|
||||
if (!await scopeActive(scope)) throw new AnonymousError("repository_changed", { httpStatus: 409 });
|
||||
}
|
||||
|
||||
async function removeEntry(metadata: string) {
|
||||
await Promise.all([metadata, metadata.replace(/\.json$/, ".data")].map(path => fs.rm(path, { force: true })));
|
||||
}
|
||||
@@ -36,9 +44,7 @@ export async function transformedFile(digest: string, options: unknown, output:
|
||||
const settings = options as { repoId?: string; cacheGeneration?: string; cacheRevision?: string } | undefined;
|
||||
const scope: CacheScope | undefined = typeof settings?.repoId === "string"
|
||||
? { repoId: settings.repoId, generation: settings.cacheGeneration, revision: settings.cacheRevision } : undefined;
|
||||
const assertActive = async () => {
|
||||
if (!await scopeActive(scope)) throw new AnonymousError("repository_changed", { httpStatus: 409 });
|
||||
};
|
||||
const assertActive = () => assertTransformScopeActive(options);
|
||||
const produceActive = async () => {
|
||||
await assertActive();
|
||||
const changed = await produce();
|
||||
|
||||
Reference in new issue
Block a user