perf: bound request work and resource usage (#866)

* perf: bound request work and resource usage

* fix: address performance review regressions

* fix: guard content expiration and dashboard state

* fix: tolerate cache outages and retire derived paths

* fix: revalidate expiration claims and status pagination

* fix: retry interrupted repository metadata cleanup

* fix: retire obsolete content cache storage

* fix: activate tree metadata and recheck expiration claims

* fix: fence cache publication across expiration

* fix: recover staged trees and enforce cache lifecycle cleanup

* fix: prune expired transforms during database outages

* fix: finish conference removals and reload expiration candidates
This commit is contained in:
Thomas Durieux authored and GitHub committed 2026-10-08 03:38:19 +01:00
1 parent 32722169a2
commit 6434337923
74 files changed
+4752 -553

No files matched your search

+14 -4
View File
@@ -8,7 +8,9 @@ import config from "../config";
import router from "./route";
import { handleError } from "../server/routes/route-utils";
import AnonymousError from "../core/AnonymousError";
import { createLogger } from "../core/logger";
import { createLogger, serializeError } from "../core/logger";
import { connect } from "../server/database";
import { startTemporaryStorageMaintenance } from "../core/temporary-storage";
const logger = createLogger("streamer");
@@ -43,7 +45,15 @@ app.all("/{*path}", (req, res) => {
req
);
});
app.listen(config.PORT, (error?: Error) => {
if (error) throw error;
logger.info("streamer started", { port: config.PORT });
async function start() {
await connect({ appName: "Anonymous GitHub Streamer", maxPoolSize: 10, minPoolSize: 0 });
await startTemporaryStorageMaintenance();
app.listen(config.PORT, (error?: Error) => {
if (error) throw error;
logger.info("streamer started", { port: config.PORT });
});
}
void start().catch(error => {
logger.error("streamer startup failed", serializeError(error));
process.exit(1);
});
+8 -2
View File
@@ -32,7 +32,7 @@ router.post(
repoName: repoFullName[1],
commit,
getToken: () => token,
anonymizerOptions,
anonymizerOptions: { ...anonymizerOptions, repoId },
contentOptions,
},
res
@@ -57,7 +57,9 @@ router.post("/", async (req: express.Request, res: express.Response) => {
const commit = req.body.commit;
const filePath: string = req.body.filePath;
const anonymizerOptions = req.body.anonymizerOptions;
const anonymizer = new AnonymizeTransformer(anonymizerOptions);
const anonymizer = new AnonymizeTransformer({ ...anonymizerOptions, repoId,
cacheGeneration: typeof req.body.cacheGeneration === "string" ? req.body.cacheGeneration : anonymizerOptions?.cacheGeneration,
});
// Defence in depth: the parent server validates filePath against
// FileModel before calling us, but the streamer joins this directly
@@ -79,6 +81,8 @@ router.post("/", async (req: express.Request, res: express.Response) => {
organization: repoFullName[0],
repoName: repoFullName[1],
commit: commit,
cacheGeneration: typeof req.body.cacheGeneration === "string" ? req.body.cacheGeneration : undefined,
cacheRevision: typeof req.body.cacheRevision === "string" ? req.body.cacheRevision : anonymizerOptions?.cacheRevision,
getToken: () => token,
});
try {
@@ -87,6 +91,7 @@ router.post("/", async (req: express.Request, res: express.Response) => {
repoId,
() => ({ sha: fileSha, size: fileSize })
);
if (res.destroyed || res.writableEnded) { content.destroy(); anonymizer.destroy(); return; }
const mime = lookup(filePath);
if (mime && !filePath.endsWith(".ts")) {
res.contentType(mime);
@@ -122,6 +127,7 @@ router.post("/", async (req: express.Request, res: express.Response) => {
.pipe(res)
.on("error", handleStreamError)
.on("close", () => {
anonymizer.destroy();
if (!content.closed && !content.destroyed) {
content.destroy();
}