mirror of
https://github.com/tdurieux/anonymous_github.git
synced 2026-11-16 01:06:48 +01:00
perf: bound request work and resource usage (#866)
* perf: bound request work and resource usage * fix: address performance review regressions * fix: guard content expiration and dashboard state * fix: tolerate cache outages and retire derived paths * fix: revalidate expiration claims and status pagination * fix: retry interrupted repository metadata cleanup * fix: retire obsolete content cache storage * fix: activate tree metadata and recheck expiration claims * fix: fence cache publication across expiration * fix: recover staged trees and enforce cache lifecycle cleanup * fix: prune expired transforms during database outages * fix: finish conference removals and reload expiration candidates
This commit is contained in:
1 parent
32722169a2
commit
6434337923
74 files changed
+4752
-553
No files matched your search
+14
-4
@@ -8,7 +8,9 @@ import config from "../config";
|
||||
import router from "./route";
|
||||
import { handleError } from "../server/routes/route-utils";
|
||||
import AnonymousError from "../core/AnonymousError";
|
||||
import { createLogger } from "../core/logger";
|
||||
import { createLogger, serializeError } from "../core/logger";
|
||||
import { connect } from "../server/database";
|
||||
import { startTemporaryStorageMaintenance } from "../core/temporary-storage";
|
||||
|
||||
const logger = createLogger("streamer");
|
||||
|
||||
@@ -43,7 +45,15 @@ app.all("/{*path}", (req, res) => {
|
||||
req
|
||||
);
|
||||
});
|
||||
app.listen(config.PORT, (error?: Error) => {
|
||||
if (error) throw error;
|
||||
logger.info("streamer started", { port: config.PORT });
|
||||
async function start() {
|
||||
await connect({ appName: "Anonymous GitHub Streamer", maxPoolSize: 10, minPoolSize: 0 });
|
||||
await startTemporaryStorageMaintenance();
|
||||
app.listen(config.PORT, (error?: Error) => {
|
||||
if (error) throw error;
|
||||
logger.info("streamer started", { port: config.PORT });
|
||||
});
|
||||
}
|
||||
void start().catch(error => {
|
||||
logger.error("streamer startup failed", serializeError(error));
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -32,7 +32,7 @@ router.post(
|
||||
repoName: repoFullName[1],
|
||||
commit,
|
||||
getToken: () => token,
|
||||
anonymizerOptions,
|
||||
anonymizerOptions: { ...anonymizerOptions, repoId },
|
||||
contentOptions,
|
||||
},
|
||||
res
|
||||
@@ -57,7 +57,9 @@ router.post("/", async (req: express.Request, res: express.Response) => {
|
||||
const commit = req.body.commit;
|
||||
const filePath: string = req.body.filePath;
|
||||
const anonymizerOptions = req.body.anonymizerOptions;
|
||||
const anonymizer = new AnonymizeTransformer(anonymizerOptions);
|
||||
const anonymizer = new AnonymizeTransformer({ ...anonymizerOptions, repoId,
|
||||
cacheGeneration: typeof req.body.cacheGeneration === "string" ? req.body.cacheGeneration : anonymizerOptions?.cacheGeneration,
|
||||
});
|
||||
|
||||
// Defence in depth: the parent server validates filePath against
|
||||
// FileModel before calling us, but the streamer joins this directly
|
||||
@@ -79,6 +81,8 @@ router.post("/", async (req: express.Request, res: express.Response) => {
|
||||
organization: repoFullName[0],
|
||||
repoName: repoFullName[1],
|
||||
commit: commit,
|
||||
cacheGeneration: typeof req.body.cacheGeneration === "string" ? req.body.cacheGeneration : undefined,
|
||||
cacheRevision: typeof req.body.cacheRevision === "string" ? req.body.cacheRevision : anonymizerOptions?.cacheRevision,
|
||||
getToken: () => token,
|
||||
});
|
||||
try {
|
||||
@@ -87,6 +91,7 @@ router.post("/", async (req: express.Request, res: express.Response) => {
|
||||
repoId,
|
||||
() => ({ sha: fileSha, size: fileSize })
|
||||
);
|
||||
if (res.destroyed || res.writableEnded) { content.destroy(); anonymizer.destroy(); return; }
|
||||
const mime = lookup(filePath);
|
||||
if (mime && !filePath.endsWith(".ts")) {
|
||||
res.contentType(mime);
|
||||
@@ -122,6 +127,7 @@ router.post("/", async (req: express.Request, res: express.Response) => {
|
||||
.pipe(res)
|
||||
.on("error", handleStreamError)
|
||||
.on("close", () => {
|
||||
anonymizer.destroy();
|
||||
if (!content.closed && !content.destroyed) {
|
||||
content.destroy();
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user