mirror of
https://github.com/tdurieux/anonymous_github.git
synced 2026-10-19 06:07:21 +02:00
perf: bound request work and resource usage (#866)
* perf: bound request work and resource usage * fix: address performance review regressions * fix: guard content expiration and dashboard state * fix: tolerate cache outages and retire derived paths * fix: revalidate expiration claims and status pagination * fix: retry interrupted repository metadata cleanup * fix: retire obsolete content cache storage * fix: activate tree metadata and recheck expiration claims * fix: fence cache publication across expiration * fix: recover staged trees and enforce cache lifecycle cleanup * fix: prune expired transforms during database outages * fix: finish conference removals and reload expiration candidates
This commit is contained in:
Thomas Durieux authored and GitHub committed
1 parent
32722169a2
commit
6434337923
74 files changed
+4752
-553
No files matched your search
@@ -5,6 +5,7 @@ const { ContentAnonimizer, AnonymizeTransformer } = require("../src/core/anonymi
|
||||
|
||||
describe("literal term presence checks", function () {
|
||||
it("serves a large JavaScript file with absent names within the anonymization deadline", async function () {
|
||||
this.timeout(10000); // Includes worker startup; the matcher retains its one-second deadline.
|
||||
const input = Buffer.from('function render(){return "application content";}\n'.repeat(65000));
|
||||
const transformer = new AnonymizeTransformer({
|
||||
filePath: "main.js",
|
||||
|
||||
@@ -9,11 +9,12 @@ describe("dashboard refresh feedback", () => {
|
||||
async function run({ statuses = [], postError, getError } = {}) {
|
||||
let toast;
|
||||
const item = { _type: "repo", _id: "restore-me", repoId: "restore-me", status: "removed" };
|
||||
const state = { items: [item], addToast: value => { toast = value; } };
|
||||
const state = { items: [item], addToast: value => { toast = value; },
|
||||
needsAttention: item => item.status === "error" || (item._statusKey === "progress" && item._stale) };
|
||||
const pending = [];
|
||||
let reloads = 0;
|
||||
vm.runInNewContext(actions, {
|
||||
state, reactive: value => value,
|
||||
state, legacyDashboard: true, reactive: value => value,
|
||||
statusKey: status => ["ready", "error", "removed", "expired"].includes(status) ? status : "progress",
|
||||
http: {
|
||||
post: async () => { if (postError) throw postError; return {}; },
|
||||
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
const process = require("node:process");
|
||||
const { Readable } = require("node:stream");
|
||||
const { setTimeout: delay } = require("node:timers/promises");
|
||||
require("ts-node/register/transpile-only");
|
||||
const GitHubStream = require("../../src/core/source/GitHubStream").default;
|
||||
const source = new GitHubStream({ repoId: "replicas", organization: "owner", repoName: "repo", commit: "commit",
|
||||
cacheGeneration: process.env.TEST_GENERATION || "generation1", getToken: () => "token" });
|
||||
source.downloadWithFallback = async () => {
|
||||
process.send({ producer: true });
|
||||
return Readable.from((async function* () {
|
||||
yield Buffer.from("complete "); await delay(200); yield Buffer.from("content");
|
||||
})());
|
||||
};
|
||||
process.once("message", async () => {
|
||||
try {
|
||||
const contents = await Promise.all(Array.from({ length: 5 }, async () => {
|
||||
const input = await source.getFileContentCache("file.txt", "replicas", () => ({ sha: "blob", size: 16 }));
|
||||
let text = ""; for await (const chunk of input) text += chunk; return text;
|
||||
}));
|
||||
process.send({ contents }); process.exit(0);
|
||||
} catch (error) { process.send({ error: error.message }); process.exit(1); }
|
||||
});
|
||||
process.send({ ready: true });
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
const { Readable } = require("node:stream");
|
||||
const path = require("node:path");
|
||||
const compiled = process.env.PERFORMANCE_STREAMER_COMPILED === "1";
|
||||
if (!compiled) require("ts-node/register/transpile-only");
|
||||
const base = path.join(__dirname, compiled ? "../../build" : "../../src");
|
||||
const GitHubStream = require(path.join(base, "core/source/GitHubStream")).default;
|
||||
const FileSystem = require(path.join(base, "core/storage/FileSystem")).default;
|
||||
const maintenance = require(path.join(base, "core/temporary-storage"));
|
||||
const startMaintenance = maintenance.startTemporaryStorageMaintenance;
|
||||
// Exercise the real startup/timer wiring without waiting a production minute.
|
||||
maintenance.startTemporaryStorageMaintenance = () => startMaintenance(25);
|
||||
GitHubStream.prototype.downloadWithFallback = async function () {
|
||||
process.send({ download: true });
|
||||
return Readable.from([Buffer.alloc(300000, "x")]);
|
||||
};
|
||||
const write = FileSystem.prototype.write;
|
||||
let release, releaseConnection;
|
||||
if (process.env.PERFORMANCE_HOLD_CONNECT === "1") {
|
||||
const database = require(path.join(base, "server/database")), connect = database.connect;
|
||||
database.connect = async (...args) => {
|
||||
const held = new Promise(resolve => { releaseConnection = resolve; });
|
||||
process.send({ connectionHeld: true }); await held;
|
||||
return connect(...args);
|
||||
};
|
||||
}
|
||||
FileSystem.prototype.write = async function (...args) {
|
||||
if (args[1].endsWith("/hold.txt")) {
|
||||
const held = new Promise(resolve => { release = resolve; });
|
||||
process.send({ held: true }); await held;
|
||||
}
|
||||
return write.apply(this, args);
|
||||
};
|
||||
if (process.env.CREDENTIAL_KEYS !== "invalid") process.on("message", message => {
|
||||
if (message === "release") release?.();
|
||||
if (message === "connect") releaseConnection?.();
|
||||
if (message === "inspect") {
|
||||
const options = require("mongoose").connection.getClient().options;
|
||||
process.send({ connected: require(path.join(base, "server/database")).isConnected,
|
||||
maxPoolSize: options.maxPoolSize, minPoolSize: options.minPoolSize });
|
||||
}
|
||||
});
|
||||
require(path.join(base, "streamer/index"));
|
||||
Vendored
+62
@@ -0,0 +1,62 @@
|
||||
const http = require("node:http");
|
||||
const { URL } = require("node:url");
|
||||
const objects = new Map();
|
||||
module.exports = async function s3Fixture() {
|
||||
objects.clear();
|
||||
const requests = { lists: 0, deletions: [] };
|
||||
const server = http.createServer((req, res) => {
|
||||
const url = new URL(req.url, "http://localhost");
|
||||
const key = decodeURIComponent(url.pathname).split("/").slice(2).join("/");
|
||||
if (req.method === "POST" && url.searchParams.has("delete")) {
|
||||
const chunks = [];
|
||||
req.on("data", chunk => chunks.push(chunk));
|
||||
req.on("end", () => {
|
||||
const body = Buffer.concat(chunks).toString();
|
||||
const keys = [...body.matchAll(/<Key>(.*?)<\/Key>/g)].map(match => match[1].replace(/</g, "<").replace(/>/g, ">").replace(/&/g, "&"));
|
||||
requests.deletions.push(keys);
|
||||
keys.forEach(name => objects.delete(name));
|
||||
res.setHeader("Content-Type", "application/xml");
|
||||
res.end('<DeleteResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"/>');
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (req.method === "PUT") {
|
||||
const chunks = [];
|
||||
req.on("data", chunk => chunks.push(chunk));
|
||||
req.on("end", () => {
|
||||
let body = Buffer.concat(chunks);
|
||||
if (req.headers["content-encoding"]?.includes("aws-chunked")) {
|
||||
const decoded = []; let offset = 0;
|
||||
while (offset < body.length) {
|
||||
const end = body.indexOf("\r\n", offset); if (end < 0) break;
|
||||
const size = parseInt(body.toString("ascii", offset, end).split(";")[0], 16);
|
||||
if (!size) break;
|
||||
decoded.push(body.subarray(end + 2, end + 2 + size)); offset = end + 2 + size + 2;
|
||||
}
|
||||
body = Buffer.concat(decoded);
|
||||
}
|
||||
if (key) objects.set(key, body);
|
||||
res.setHeader("ETag", '"fixture-etag"'); res.end();
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (url.searchParams.has("list-type")) {
|
||||
const prefix = url.searchParams.get("prefix") || "";
|
||||
requests.lists++;
|
||||
const escaped = value => value.replace(/&/g, "&").replace(/</g, "<");
|
||||
const remaining = [...objects].filter(([name]) => name.startsWith(prefix) && name > (url.searchParams.get("continuation-token") || "")).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0);
|
||||
const page = remaining.slice(0, Number(url.searchParams.get("max-keys") || 1000));
|
||||
const truncated = remaining.length > page.length;
|
||||
const contents = page.map(([name, body]) => `<Contents><Key>${escaped(name)}</Key><Size>${body.length}</Size></Contents>`).join("");
|
||||
res.setHeader("Content-Type", "application/xml");
|
||||
res.end(`<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><IsTruncated>${truncated}</IsTruncated>${truncated ? `<NextContinuationToken>${escaped(page.at(-1)[0])}</NextContinuationToken>` : ""}${contents}</ListBucketResult>`); return;
|
||||
}
|
||||
if (!objects.has(key)) { res.statusCode = 404; res.end("<Error><Code>NoSuchKey</Code></Error>"); return; }
|
||||
const body = objects.get(key);
|
||||
res.setHeader("Content-Length", body.length); res.setHeader("Content-Type", "text/plain");
|
||||
res.setHeader("Last-Modified", new Date(0).toUTCString());
|
||||
if (req.method === "HEAD") res.end(); else res.end(body);
|
||||
});
|
||||
await new Promise(resolve => server.listen(0, "127.0.0.1", resolve));
|
||||
return { endpoint: `http://127.0.0.1:${server.address().port}`, requests, seed: (key, data) => objects.set(key, Buffer.from(data)), close: () => new Promise(resolve => server.close(resolve)) };
|
||||
};
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,244 @@
|
||||
const { expect } = require("chai");
|
||||
const { Readable, PassThrough } = require("node:stream");
|
||||
const { setImmediate } = require("node:timers");
|
||||
const { setTimeout: delay } = require("node:timers/promises");
|
||||
require("ts-node/register/transpile-only");
|
||||
const { AsyncCache } = require("../src/core/async-cache");
|
||||
const { streamResponse } = require("../src/core/response-stream");
|
||||
const Repository = require("../src/core/Repository").default;
|
||||
const User = require("../src/core/User").default;
|
||||
const UserModel = require("../src/core/model/users/users.model").default;
|
||||
const RepoModel = require("../src/core/model/anonymizedRepositories/anonymizedRepositories.model").default;
|
||||
const GitHubStream = require("../src/core/source/GitHubStream").default;
|
||||
const storage = require("../src/core/storage").default;
|
||||
const { dashboardPipeline } = require("../src/server/routes/dashboard-summary");
|
||||
|
||||
describe("performance regressions", function () {
|
||||
this.timeout(10000);
|
||||
const restore = [];
|
||||
function stub(object, key, value) { const before = object[key]; object[key] = value; restore.push(() => { object[key] = before; }); }
|
||||
afterEach(() => { restore.splice(0).reverse().forEach(fn => fn()); });
|
||||
|
||||
it("keeps name/folder search semantics and stops at 500 matches", async () => {
|
||||
const repo = new Repository(new RepoModel({ owner: new UserModel()._id, repoId: "search", options: { terms: [] } }));
|
||||
const rows = Array.from({ length: 10000 }, (_, i) => ({ name: `file-${i}.ts`, path: `src/MATCH-${i}/nested`, size: i }));
|
||||
let examined = 0;
|
||||
for (const row of rows) { const name = row.name; Object.defineProperty(row, "name", { get() { examined++; return name; } }); }
|
||||
repo.anonymizedFiles = async () => rows;
|
||||
const result = await repo.searchFiles("match");
|
||||
expect(result).to.have.length(500);
|
||||
expect(result[499].path).to.equal("src/MATCH-499/nested");
|
||||
expect(examined).to.be.lessThan(2000);
|
||||
expect(await repo.searchFiles("src/MATCH")).to.deep.equal([]);
|
||||
});
|
||||
|
||||
it("coalesces cache misses, caches zero, bounds keys, and retries failures", async () => {
|
||||
let now = 0, calls = 0;
|
||||
const cache = new AsyncCache(10, 3, () => now);
|
||||
const values = await Promise.all(Array.from({ length: 10 }, () => cache.get("zero", async () => { calls++; await delay(10); return 0; })));
|
||||
expect(values).to.deep.equal(Array(10).fill(0));
|
||||
expect(calls).to.equal(1);
|
||||
for (let i = 0; i < 1000; i++) await cache.get(String(i), async () => i);
|
||||
expect(cache.size).to.equal(3);
|
||||
now = 11;
|
||||
expect(cache.size).to.equal(0);
|
||||
await cache.get("failure", async () => { throw Error("temporary"); }).catch(() => {});
|
||||
expect(await cache.get("failure", async () => 1)).to.equal(1);
|
||||
});
|
||||
|
||||
for (const terminal of ["close", "finish", "error"]) {
|
||||
it(`destroys the upstream and removes listeners on response ${terminal}`, async () => {
|
||||
const input = new PassThrough(), response = new PassThrough();
|
||||
const completed = streamResponse(input, response, () => {});
|
||||
response.emit(terminal, new Error("disconnected"));
|
||||
await completed.catch(() => {});
|
||||
expect(input.destroyed).to.equal(true);
|
||||
expect(response.listenerCount("close")).to.equal(0);
|
||||
expect(response.listenerCount("finish")).to.equal(0);
|
||||
expect(response.listenerCount("error")).to.equal(0);
|
||||
});
|
||||
}
|
||||
|
||||
it("performs one cache fill for ten callers and gives each its own stream", async () => {
|
||||
const source = new GitHubStream({ repoId: "coalesced", organization: "owner", repoName: "repo", commit: "abc", getToken: async () => "token" });
|
||||
let data, downloads = 0, writes = 0, heads = 0;
|
||||
stub(storage, "fileInfo", async () => { heads++; if (!data) throw Error("missing"); return { size: data.length }; });
|
||||
stub(storage, "read", async () => Readable.from([data]));
|
||||
stub(storage, "write", async (_repo, _path, input) => { writes++; const chunks = []; for await (const chunk of input) chunks.push(chunk); await delay(10); data = Buffer.concat(chunks); });
|
||||
source.downloadWithFallback = async () => { downloads++; return Readable.from([Buffer.from("complete content")]); };
|
||||
const streams = await Promise.all(Array.from({ length: 10 }, () => source.getFileContentCache("file.txt", "coalesced", () => ({ sha: "blob", size: 16 }))));
|
||||
expect(downloads).to.equal(1); expect(writes).to.equal(1);
|
||||
expect(new Set(streams).size).to.equal(10);
|
||||
for (const input of streams) { let text = ""; for await (const chunk of input) text += chunk; expect(text).to.equal("complete content"); }
|
||||
heads = 0;
|
||||
const warm = await source.getFileContentCache("file.txt", "coalesced", () => ({ sha: "blob", size: 16 }));
|
||||
warm.destroy(); expect(heads).to.equal(1); expect(downloads).to.equal(1);
|
||||
});
|
||||
|
||||
for (const phase of ["before download", "during publication"]) {
|
||||
it(`rejects retired cache producers ${phase} and removes late output`, async () => {
|
||||
const { contentGenerationPrefix, contentRetirementMarker } = require("../src/core/content-generation");
|
||||
const cacheGeneration = `retired-${phase}`, prefix = contentGenerationPrefix(cacheGeneration);
|
||||
const marker = contentRetirementMarker(prefix), objects = new Map();
|
||||
const source = new GitHubStream({ repoId: `retirement-${phase}`, organization: "owner", repoName: "repo", commit: "abc",
|
||||
cacheGeneration, getToken: async () => "token" });
|
||||
let downloads = 0, started, release;
|
||||
const began = new Promise(resolve => { started = resolve; }), held = new Promise(resolve => { release = resolve; });
|
||||
stub(storage, "fileInfo", async (_repo, path) => {
|
||||
if (!objects.has(path)) throw Object.assign(Error("missing"), { code: "ENOENT" });
|
||||
return { size: objects.get(path).length };
|
||||
});
|
||||
stub(storage, "write", async (_repo, path, input) => {
|
||||
started(); await held;
|
||||
let content = ""; for await (const chunk of input) content += chunk;
|
||||
objects.set(path, content);
|
||||
});
|
||||
stub(storage, "rm", async (_repo, path) => { objects.delete(path); });
|
||||
stub(storage, "read", async (_repo, path) => Readable.from([objects.get(path)]));
|
||||
source.downloadWithFallback = async () => { downloads++; return Readable.from(["obsolete"]); };
|
||||
if (phase === "before download") { objects.set(marker, "retired"); release(); }
|
||||
const pending = source.getFileContentCache("private/old.txt", source.data.repoId, () => ({ sha: "blob", size: 8 }));
|
||||
try {
|
||||
if (phase === "during publication") {
|
||||
await began; objects.set(marker, "retired"); release();
|
||||
}
|
||||
try { await pending; throw Error("expected retired generation rejection"); }
|
||||
catch (error) { expect(error.message).to.equal("repository_changed"); expect(error.httpStatus).to.equal(409); }
|
||||
expect(downloads).to.equal(phase === "before download" ? 0 : 1);
|
||||
expect([...objects.keys()]).to.deep.equal([marker]);
|
||||
} finally { release(); await pending.catch(() => {}); }
|
||||
});
|
||||
}
|
||||
|
||||
it("fetches a complete root tree in one recursive request", async () => {
|
||||
const source = new GitHubStream({ repoId: "tree", organization: "owner", repoName: "repo", commit: "head", getToken: () => "token" });
|
||||
const calls = [];
|
||||
source.getGHTree = async (_oct, _token, sha, _count, options) => {
|
||||
calls.push({ sha, recursive: options.recursive });
|
||||
return { truncated: false, tree: Array.from({ length: 20 }, (_, i) => ({ path: `folder${i}/file.ts`, type: "blob", sha: "abc" })) };
|
||||
};
|
||||
const files = await source.getTruncatedTree("head");
|
||||
expect(files).to.have.length(20); expect(calls).to.deep.equal([{ sha: "head", recursive: true }]);
|
||||
});
|
||||
|
||||
for (const [upstream, code, status] of [
|
||||
[Object.assign(new Error("forbidden"), { response: { statusCode: 403 } }), "file_not_accessible", 403],
|
||||
[Object.assign(new Error("missing"), { response: { statusCode: 404 } }), "file_not_found", 404],
|
||||
[Object.assign(new Error("large"), { status: 422 }), "file_too_big", 422],
|
||||
[Object.assign(new Error("missing"), { httpStatus: 404 }), "file_not_found", 404],
|
||||
[Object.assign(new Error("reset"), { code: "ECONNRESET" }), "upstream_error", 502],
|
||||
[new (require("../src/core/AnonymousError").default)("file_not_accessible", { httpStatus: 403 }), "file_not_accessible", 403],
|
||||
]) {
|
||||
it(`preserves ${code}/${status} for failed downloads (${upstream.message})`, async () => {
|
||||
const source = new GitHubStream({ repoId: "failed-download", organization: "owner", repoName: "repo", getToken: () => "token" });
|
||||
const input = new PassThrough();
|
||||
stub(storage, "fileInfo", async () => { throw Error("cache miss"); });
|
||||
let committed = false;
|
||||
stub(storage, "write", async (_repo, _path, stream) => { for await (const chunk of stream) void chunk; committed = true; });
|
||||
source.downloadWithFallback = async () => { setImmediate(() => input.destroy(upstream)); return input; };
|
||||
try { await source.getFileContentCache("file.txt", "failed-download", () => ({ sha: "blob", size: 1 })); throw Error("expected failure"); }
|
||||
catch (error) {
|
||||
expect(error.message).to.equal(code); expect(error.httpStatus).to.equal(status);
|
||||
if (upstream instanceof require("../src/core/AnonymousError").default) expect(error).to.equal(upstream);
|
||||
else expect(error.cause).to.equal(upstream);
|
||||
}
|
||||
expect(committed).to.equal(false); expect(input.destroyed).to.equal(true);
|
||||
});
|
||||
}
|
||||
|
||||
it("falls back to shallow trees when GitHub truncates recursive results", async () => {
|
||||
const source = new GitHubStream({ repoId: "tree", organization: "owner", repoName: "repo", commit: "head", getToken: () => "token" });
|
||||
const calls = [];
|
||||
source.getGHTree = async (_oct, _token, sha, _count, options) => {
|
||||
calls.push(`${sha}:${options.recursive}`);
|
||||
if (sha === "head" && options.recursive) return { truncated: true, tree: [{ path: "ignored.ts", type: "blob" }] };
|
||||
if (sha === "head") return { truncated: false, tree: [{ path: "src", type: "tree", sha: "src" }] };
|
||||
return { truncated: false, tree: [{ path: "file.ts", type: "blob", sha: "abc" }] };
|
||||
};
|
||||
const files = await source.getTruncatedTree("head");
|
||||
expect(files.map(file => `${file.path}/${file.name}`)).to.deep.equal(["/src", "src/file.ts"]);
|
||||
expect(calls).to.deep.equal(["head:true", "head:false", "src:true"]);
|
||||
});
|
||||
|
||||
it("does no physical expiration work while reading 100 expired projects", async () => {
|
||||
const user = new User(new UserModel({ username: "owner" }));
|
||||
stub(RepoModel, "find", () => ({ exec: async () => Array.from({ length: 100 }, (_, i) => new RepoModel({ repoId: `expired${i}`, owner: user.model._id, status: "ready", options: { expirationMode: "remove", expirationDate: new Date(0) } })) }));
|
||||
stub(Repository.prototype, "expire", () => { throw Error("inline cleanup"); });
|
||||
const projects = await user.getRepositories();
|
||||
expect(projects).to.have.length(100);
|
||||
expect(projects.every(project => project.status === "expiring")).to.equal(true);
|
||||
});
|
||||
|
||||
it("projects dashboard summaries before combining types, and validates cursors", () => {
|
||||
const user = new User(new UserModel({ username: "owner" }));
|
||||
const pipeline = dashboardPipeline(user, { limit: 2, type: "pr" });
|
||||
const projection = pipeline.find(stage => stage.$project).$project;
|
||||
expect(projection).not.to.have.property("pullRequest");
|
||||
expect(pipeline.filter(stage => stage.$unionWith)).to.have.length(2);
|
||||
expect(() => dashboardPipeline(user, { cursor: "bogus" })).to.throw("invalid_cursor");
|
||||
expect(() => dashboardPipeline(user, { sort: "$where" })).to.throw("invalid_sort");
|
||||
});
|
||||
});
|
||||
|
||||
describe("shared home statistics", function () {
|
||||
this.timeout(10000);
|
||||
const stats = require("../src/server/dailyStatsSnapshot");
|
||||
const PR = require("../src/core/model/anonymizedPullRequests/anonymizedPullRequests.model").default;
|
||||
const Daily = require("../src/core/model/dailyStats/dailyStats.model").default;
|
||||
let originals, calculations, fail;
|
||||
beforeEach(() => {
|
||||
stats.clearStatsCache(); calculations = 0; fail = false;
|
||||
originals = [RepoModel.estimatedDocumentCount, RepoModel.collection.aggregate, PR.estimatedDocumentCount, Daily.find];
|
||||
RepoModel.estimatedDocumentCount = async () => { calculations++; if (fail) throw Error("temporary stats error"); return 0; };
|
||||
RepoModel.collection.aggregate = () => ({ toArray: async () => [] });
|
||||
PR.estimatedDocumentCount = async () => 0;
|
||||
Daily.find = () => ({ sort: () => ({ lean: async () => [] }) });
|
||||
});
|
||||
afterEach(() => {
|
||||
[RepoModel.estimatedDocumentCount, RepoModel.collection.aggregate, PR.estimatedDocumentCount, Daily.find] = originals;
|
||||
stats.clearStatsCache();
|
||||
});
|
||||
it("coalesces concurrent current/history calculations and caches zero across day ranges", async () => {
|
||||
const values = await Promise.all([...Array.from({ length: 10 }, () => stats.getCurrentStats()),
|
||||
stats.getStatsHistory(30, new Date("2026-10-07")), stats.getStatsHistory(60, new Date("2026-10-07"))]);
|
||||
expect(calculations).to.equal(1); expect(values[0].nbRepositories).to.equal(0);
|
||||
expect(values[10][0].date.toISOString()).to.equal("2026-10-07T00:00:00.000Z");
|
||||
const next = await stats.getStatsHistory(30, new Date("2026-10-08"));
|
||||
expect(next[0].date.toISOString()).to.equal("2026-10-08T00:00:00.000Z");
|
||||
expect(calculations).to.equal(1);
|
||||
});
|
||||
it("retries a failed calculation without retaining a rejected promise", async () => {
|
||||
fail = true; await stats.getCurrentStats().catch(() => {}); fail = false;
|
||||
expect((await stats.getCurrentStats()).nbPageViews).to.equal(0);
|
||||
expect(calculations).to.equal(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe("admin queue snapshot reuse", function () {
|
||||
it("filters a bounded raw snapshot for thousands of search terms and expires it", async function () {
|
||||
this.timeout(10000);
|
||||
const queues = require("../src/queue");
|
||||
const metrics = require("../src/queue/queueMetrics");
|
||||
const router = require("../src/server/routes/admin").default;
|
||||
const handler = router.stack.find(layer => layer.route?.path === "/queues").route.stack.at(-1).handle;
|
||||
const originals = [queues.downloadQueue, queues.removeQueue, queues.cacheQueue, metrics.queryMetrics, Date.now];
|
||||
let calls = 0, now = Date.now();
|
||||
const queue = { getJobCounts: async () => ({}), getWorkers: async () => [], isPaused: async () => false,
|
||||
getJobs: async ([state]) => { calls++; return state === "active" ? [{ asJSON: () => ({ id: "known", name: "known" }) }] : []; } };
|
||||
queues.downloadQueue = queue; queues.removeQueue = queue; queues.cacheQueue = queue;
|
||||
metrics.queryMetrics = async () => [];
|
||||
Date.now = () => now;
|
||||
try {
|
||||
for (let i = 0; i < 1000; i++) await handler({ query: { search: `unique-${i}` } }, { json: data => expect(data.jobs).to.have.length(0) });
|
||||
const firstCalls = calls;
|
||||
expect(firstCalls).to.equal(5);
|
||||
await handler({ query: { search: "known" } }, { json: data => expect(data.jobs).to.have.length(1) });
|
||||
expect(calls).to.equal(firstCalls);
|
||||
now += 10001;
|
||||
await handler({ query: {} }, { json() {} });
|
||||
expect(calls).to.equal(firstCalls * 2);
|
||||
} finally {
|
||||
[queues.downloadQueue, queues.removeQueue, queues.cacheQueue, metrics.queryMetrics, Date.now] = originals;
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,136 @@
|
||||
const { expect } = require("chai");
|
||||
const process = require("node:process");
|
||||
const { fork } = require("node:child_process");
|
||||
const { promises: fs } = require("node:fs");
|
||||
const { tmpdir } = require("node:os");
|
||||
const path = require("node:path");
|
||||
const { Readable } = require("node:stream");
|
||||
const { S3, CreateBucketCommand } = require("@aws-sdk/client-s3");
|
||||
const { createClient } = require("redis");
|
||||
require("ts-node/register/transpile-only");
|
||||
const config = require("../src/config").default;
|
||||
const S3Storage = require("../src/core/storage/S3").default;
|
||||
const GitHubStream = require("../src/core/source/GitHubStream").default;
|
||||
const storage = require("../src/core/storage").default;
|
||||
const { coordinatedFill, closeCacheRedis, cacheRedis } = require("../src/core/cache-coordination");
|
||||
const { setTimeout: delay } = require("node:timers/promises");
|
||||
|
||||
(process.env.PERFORMANCE_REDIS_PORT ? describe : describe.skip)("disposable Redis cache coordination", function () {
|
||||
this.timeout(20000);
|
||||
let root, redis, previousRedis;
|
||||
before(async () => {
|
||||
await closeCacheRedis();
|
||||
root = await fs.mkdtemp(path.join(tmpdir(), "anonymous-cache-test-"));
|
||||
previousRedis = { REDIS_HOSTNAME: config.REDIS_HOSTNAME, REDIS_PORT: config.REDIS_PORT };
|
||||
config.REDIS_HOSTNAME = "127.0.0.1"; config.REDIS_PORT = Number(process.env.PERFORMANCE_REDIS_PORT);
|
||||
redis = createClient({ socket: { host: "127.0.0.1", port: config.REDIS_PORT } });
|
||||
await redis.connect();
|
||||
});
|
||||
after(async () => { await closeCacheRedis(); await redis.quit(); Object.assign(config, previousRedis); await fs.rm(root, { force: true, recursive: true }); });
|
||||
it("downloads and publishes once across two streamer processes and ten callers", async () => {
|
||||
let producers = 0;
|
||||
const children = Array.from({ length: 2 }, () => fork(path.join(__dirname, "fixtures/performance-cache-worker.js"), [], { stdio: ["ignore", "ignore", "ignore", "ipc"], env: { ...process.env,
|
||||
FOLDER: root, REDIS_HOSTNAME: "127.0.0.1", REDIS_PORT: String(config.REDIS_PORT), NODE_ENV: "test" } }));
|
||||
try {
|
||||
const results = children.map(child => new Promise((resolve, reject) => {
|
||||
child.on("message", message => {
|
||||
if (message.ready) child.send("start");
|
||||
if (message.producer) producers++;
|
||||
if (message.contents) resolve(message.contents);
|
||||
if (message.error) reject(Error(message.error));
|
||||
});
|
||||
child.on("error", reject); child.on("exit", code => { if (code) reject(Error(`worker exited ${code}`)); });
|
||||
}));
|
||||
const contents = (await Promise.all(results)).flat();
|
||||
expect(contents).to.deep.equal(Array(10).fill("complete content"));
|
||||
expect(producers).to.equal(1);
|
||||
} finally { children.forEach(child => child.kill()); }
|
||||
});
|
||||
it("recovers an expired crashed-producer lease without publishing partial output", async () => {
|
||||
const key = `crash-test-${Date.now()}`;
|
||||
await redis.set(`perf:lease:${key}`, "dead-process", { PX: 150 });
|
||||
let value, producers = 0;
|
||||
const result = await coordinatedFill(key, async () => value, async () => { producers++; value = "verified"; return value; });
|
||||
expect(result).to.equal("verified"); expect(producers).to.equal(1);
|
||||
expect(await redis.get(`perf:lease:${key}`)).to.equal(null);
|
||||
});
|
||||
for (const failure of ["restart", "timeout"]) {
|
||||
it(`returns verified output despite a Redis heartbeat ${failure}`, async () => {
|
||||
await closeCacheRedis();
|
||||
const client = await cacheRedis(), originalEval = client.eval;
|
||||
let notify, published;
|
||||
const heartbeat = new Promise(resolve => { notify = resolve; });
|
||||
client.eval = function (script, options) {
|
||||
if (script.includes("pexpire")) {
|
||||
notify();
|
||||
return failure === "timeout" ? new Promise(() => {}) : Promise.reject(Error("Redis restarted"));
|
||||
}
|
||||
return originalEval.call(this, script, options);
|
||||
};
|
||||
const key = `heartbeat-${failure}-${Date.now()}`;
|
||||
try {
|
||||
const result = await coordinatedFill(key, async () => published, async () => {
|
||||
await heartbeat; await delay(failure === "timeout" ? 1100 : 20);
|
||||
published = "verified output"; return published;
|
||||
});
|
||||
expect(result).to.equal("verified output");
|
||||
expect(await coordinatedFill(key, async () => published, async () => published)).to.equal("verified output");
|
||||
} finally { client.eval = originalEval; await closeCacheRedis(); }
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
describe("S3 HTTP protocol integration", function () {
|
||||
this.timeout(20000);
|
||||
let backend, previous, client, fixture;
|
||||
before(async () => {
|
||||
previous = { ...config };
|
||||
if (!process.env.PERFORMANCE_S3_ENDPOINT) fixture = await require("./fixtures/s3-server")();
|
||||
Object.assign(config, { S3_BUCKET: "performance-test", S3_CLIENT_ID: "perf-test-user", S3_CLIENT_SECRET: "perf-test-password",
|
||||
S3_REGION: "us-east-1", S3_ENDPOINT: process.env.PERFORMANCE_S3_ENDPOINT || fixture.endpoint });
|
||||
client = new S3({ endpoint: config.S3_ENDPOINT, region: config.S3_REGION, forcePathStyle: true,
|
||||
credentials: { accessKeyId: config.S3_CLIENT_ID, secretAccessKey: config.S3_CLIENT_SECRET } });
|
||||
await client.send(new CreateBucketCommand({ Bucket: config.S3_BUCKET }));
|
||||
backend = new S3Storage();
|
||||
});
|
||||
after(async () => { client?.destroy(); for (const value of backend?.clients.values() || []) value.destroy(); Object.assign(config, previous); await fixture?.close(); });
|
||||
it("reuses clients and serves a warm file with one HEAD and one GET", async () => {
|
||||
await backend.write("s3repo", "file.txt", Readable.from([Buffer.from("complete content")]), undefined, 16);
|
||||
expect(backend.client(3000)).to.equal(backend.client(3000));
|
||||
let heads = 0, gets = 0;
|
||||
const common = backend.client(3000), oldHead = common.headObject.bind(common), oldSend = common.send.bind(common);
|
||||
common.headObject = (...args) => { heads++; return oldHead(...args); };
|
||||
common.send = (...args) => { if (args[0].constructor.name === "GetObjectCommand") gets++; return oldSend(...args); };
|
||||
const originals = { fileInfo: storage.fileInfo, read: storage.read, write: storage.write };
|
||||
try {
|
||||
storage.fileInfo = backend.fileInfo.bind(backend); storage.read = backend.read.bind(backend); storage.write = backend.write.bind(backend);
|
||||
const source = new GitHubStream({ repoId: "s3repo", organization: "owner", repoName: "repo", commit: "commit", getToken: () => { throw Error("warm cache requested token"); } });
|
||||
const input = await source.getFileContentCache("file.txt", "s3repo", () => ({ sha: "blob", size: 16 }));
|
||||
let text = ""; for await (const chunk of input) text += chunk;
|
||||
expect(text).to.equal("complete content"); expect(heads).to.equal(1); expect(gets).to.equal(1);
|
||||
} finally { Object.assign(storage, originals); }
|
||||
});
|
||||
it("refuses a short upload before committing the object", async () => {
|
||||
let error;
|
||||
try { await backend.write("s3repo", "short.txt", Readable.from([Buffer.from("short")]), undefined, 100); } catch (caught) { error = caught; }
|
||||
expect(error).to.be.instanceOf(Error);
|
||||
expect(await backend.exists("s3repo", "short.txt")).to.equal("not_found");
|
||||
});
|
||||
it("removes legacy S3 keys in bounded batches while retaining managed caches and other repositories", async function () {
|
||||
if (!fixture) this.skip();
|
||||
for (let i = 0; i < 1005; i++) fixture.seed(`legacy-s3/original/private/file-${String(i).padStart(4, "0")}.txt`, "obsolete");
|
||||
const active = require("../src/core/content-generation").contentGenerationPrefix("active");
|
||||
fixture.seed(`legacy-s3/original/${active}/blob/keep.txt`, "active");
|
||||
fixture.seed("legacy-s3-other/original/private/keep.txt", "keep");
|
||||
const before = fixture.requests.deletions.length;
|
||||
await backend.removeLegacyContent("legacy-s3");
|
||||
const batches = fixture.requests.deletions.slice(before);
|
||||
expect(batches).to.have.length(2);
|
||||
expect(batches.flat()).to.have.length(1005);
|
||||
expect(batches.every(batch => batch.length <= 1000)).to.equal(true);
|
||||
expect(await backend.exists("legacy-s3", "private/file-0000.txt")).to.equal("not_found");
|
||||
expect(await backend.exists("legacy-s3", "private/file-1004.txt")).to.equal("not_found");
|
||||
expect(await backend.exists("legacy-s3", `${active}/blob/keep.txt`)).to.equal("file");
|
||||
expect(await backend.exists("legacy-s3-other", "private/keep.txt")).to.equal("file");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,127 @@
|
||||
const { expect } = require("chai");
|
||||
const { fork } = require("node:child_process");
|
||||
const { promises: fs } = require("node:fs");
|
||||
const { tmpdir } = require("node:os");
|
||||
const path = require("node:path");
|
||||
const net = require("node:net");
|
||||
const { randomBytes } = require("node:crypto");
|
||||
const { once } = require("node:events");
|
||||
const { setTimeout } = require("node:timers");
|
||||
const { fetch } = globalThis;
|
||||
const mongoose = require("mongoose");
|
||||
require("ts-node/register/transpile-only");
|
||||
const db = require("../src/server/database");
|
||||
const Repo = require("../src/core/model/anonymizedRepositories/anonymizedRepositories.model").default;
|
||||
const Repository = require("../src/core/Repository").default;
|
||||
const config = require("../src/config").default;
|
||||
const uri = process.env.PERFORMANCE_MONGO_URI;
|
||||
|
||||
(uri ? describe : describe.skip)("production streamer lifecycle and idle cleanup", function () {
|
||||
this.timeout(30000);
|
||||
let root, previousFolder;
|
||||
before(async () => {
|
||||
if (!/\/perf_test_[A-Za-z0-9_-]+(?:\?|$)/.test(uri)) throw Error("An isolated perf_test_ database is required");
|
||||
await mongoose.connect(uri); db.isConnected = true;
|
||||
root = await fs.mkdtemp(path.join(tmpdir(), "streamer-lifecycle-"));
|
||||
previousFolder = config.FOLDER; config.FOLDER = path.join(root, "repositories");
|
||||
});
|
||||
after(async () => {
|
||||
db.isConnected = false; await mongoose.connection.dropDatabase(); await mongoose.disconnect();
|
||||
config.FOLDER = previousFolder; await fs.rm(root, { force: true, recursive: true });
|
||||
});
|
||||
beforeEach(async () => { await Repo.deleteMany({}); });
|
||||
|
||||
const missing = async file => { try { await fs.access(file); return false; } catch (error) { if (error.code === "ENOENT") return true; throw error; } };
|
||||
async function waitFor(check) {
|
||||
const deadline = Date.now() + 5000;
|
||||
while (!await check()) { if (Date.now() >= deadline) throw Error("Streamer cleanup did not finish"); await new Promise(resolve => setTimeout(resolve, 20)); }
|
||||
}
|
||||
async function launch(invalidCredentials = false, holdConnect = false) {
|
||||
const socket = net.createServer(); socket.listen(0, "127.0.0.1"); await once(socket, "listening");
|
||||
const port = socket.address().port; await new Promise(resolve => socket.close(resolve));
|
||||
const directory = await fs.mkdtemp(path.join(root, "streamer-temp-"));
|
||||
const stale = path.join(directory, "anonymous-text-crashed"); await fs.mkdir(stale);
|
||||
await fs.writeFile(path.join(stale, "input"), "original private bytes"); await fs.utimes(stale, new Date(0), new Date(0));
|
||||
const child = fork(path.join(__dirname, "fixtures/performance-streamer.js"), [], {
|
||||
execArgv: ["--max-old-space-size=512"], stdio: ["ignore", "pipe", "pipe", "ipc"],
|
||||
env: { ...process.env, NODE_ENV: "test", PORT: String(port), MONGODB_URI: uri, GITHUB_APP_ENABLED: "false", STORAGE: "filesystem",
|
||||
PERFORMANCE_HOLD_CONNECT: holdConnect ? "1" : "0",
|
||||
REDIS_HOSTNAME: "127.0.0.1", REDIS_PORT: process.env.PERFORMANCE_REDIS_PORT || "6379", TMPDIR: directory, FOLDER: config.FOLDER,
|
||||
CREDENTIAL_KEYS: invalidCredentials ? "invalid" : JSON.stringify({ test: randomBytes(32).toString("base64") }), CREDENTIAL_ACTIVE_KEY_ID: "test" },
|
||||
});
|
||||
const messages = []; child.on("message", message => messages.push(message));
|
||||
const ready = new Promise((resolve, reject) => {
|
||||
let logs = "";
|
||||
child.stdout.on("data", chunk => { logs += chunk; if (logs.includes("streamer started")) resolve(); });
|
||||
child.stderr.on("data", () => {});
|
||||
child.once("error", reject); child.once("exit", code => reject(Error(`Streamer exited ${code} before serving`)));
|
||||
});
|
||||
ready.catch(() => {});
|
||||
return { child, ready, messages, directory, stale, url: `http://127.0.0.1:${port}` };
|
||||
}
|
||||
async function stop(runtime) {
|
||||
if (runtime.child.exitCode === null && runtime.child.signalCode === null) {
|
||||
const exited = once(runtime.child, "exit"); runtime.child.kill(); await exited;
|
||||
}
|
||||
await fs.rm(runtime.directory, { force: true, recursive: true });
|
||||
}
|
||||
async function createRow(repoId) {
|
||||
return Repo.create({ repoId, status: "ready", statusDate: new Date(), lastView: new Date(), anonymizeDate: new Date(), treeGeneration: "active",
|
||||
source: { type: "GitHubStream", repositoryName: "owner/repo", commit: "abc" }, options: { terms: ["NeverMatches"], expirationMode: "remove", expirationDate: new Date(0) } });
|
||||
}
|
||||
async function request(runtime, row, filePath, legacy = false) {
|
||||
const options = new Repository(row).generateAnonymizeTransformer(filePath).opt;
|
||||
return fetch(runtime.url + "/api", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ token: "test-token",
|
||||
repoId: row.repoId, repoFullName: "owner/repo", commit: "abc", ...(legacy ? {} : { cacheGeneration: options.cacheGeneration }), sha: "blob", size: 300000, filePath, anonymizerOptions: options }) });
|
||||
}
|
||||
|
||||
it("connects before serving and rejects a fill published after expiration in the actual streamer entrypoint", async () => {
|
||||
const runtime = await launch(false, true); let pending;
|
||||
try {
|
||||
await waitFor(() => runtime.messages.some(message => message.connectionHeld));
|
||||
await fetch(runtime.url + "/healthcheck").then(() => { throw Error("Streamer served before connecting"); }, error => expect(error).to.be.instanceOf(Error));
|
||||
runtime.child.send("connect");
|
||||
await runtime.ready;
|
||||
expect(await (await fetch(runtime.url + "/healthcheck")).json()).to.deep.equal({ status: "ok" });
|
||||
runtime.child.send("inspect"); await waitFor(() => runtime.messages.some(message => message.connected !== undefined));
|
||||
expect(runtime.messages.find(message => message.connected !== undefined)).to.deep.equal({ connected: true, maxPoolSize: 10, minPoolSize: 0 });
|
||||
expect(await missing(runtime.stale)).to.equal(true);
|
||||
const row = await createRow("streamer-late-fill");
|
||||
pending = request(runtime, row, "hold.txt"); pending.catch(() => {});
|
||||
await waitFor(() => runtime.messages.some(message => message.held));
|
||||
await new Repository(await Repo.findById(row._id)).expire();
|
||||
runtime.child.send("release");
|
||||
const response = await pending; expect(response.status).to.equal(409); expect((await response.json()).error).to.equal("repository_changed");
|
||||
const sourceRoot = path.join(config.FOLDER, row.repoId, "original");
|
||||
const entries = await fs.readdir(sourceRoot, { recursive: true });
|
||||
expect(entries.filter(name => name.endsWith("hold.txt"))).to.deep.equal([]);
|
||||
const rejected = await request(runtime, row, "rejected.txt"); expect(rejected.status).to.equal(409); await rejected.text();
|
||||
const legacy = await request(runtime, row, "legacy.txt", true); expect(legacy.status).to.equal(409); await legacy.text();
|
||||
expect(runtime.messages.filter(message => message.download)).to.have.length(1);
|
||||
} finally { runtime.child.send("release"); await pending?.catch(() => {}); await stop(runtime); }
|
||||
});
|
||||
|
||||
it("purges another process's transformed cache and crash spools while idle after lifecycle changes", async () => {
|
||||
const runtime = await launch();
|
||||
try {
|
||||
await runtime.ready;
|
||||
const row = await createRow("streamer-cache"); const response = await request(runtime, row, "file.txt");
|
||||
expect(response.status).to.equal(200); expect((await response.text()).length).to.equal(300000);
|
||||
const cacheRoot = path.join(runtime.directory, "anonymous-transformed-v1");
|
||||
expect((await fs.readdir(cacheRoot)).filter(name => name.endsWith(".data"))).to.have.length(1);
|
||||
const recent = path.join(runtime.directory, "anonymous-text-idle-crash");
|
||||
await fs.mkdir(recent); await fs.writeFile(path.join(recent, "input"), "private original"); await fs.utimes(recent, new Date(0), new Date(0));
|
||||
await new Repository(await Repo.findById(row._id)).expire();
|
||||
await waitFor(async () => (await fs.readdir(cacheRoot)).length === 0 && await missing(recent));
|
||||
expect(runtime.messages.filter(message => message.download)).to.have.length(1);
|
||||
} finally { await stop(runtime); }
|
||||
});
|
||||
|
||||
it("refuses to serve when required startup configuration is invalid", async () => {
|
||||
const runtime = await launch(true);
|
||||
try {
|
||||
const [code] = await once(runtime.child, "exit"); expect(code).to.equal(1);
|
||||
await fetch(runtime.url + "/healthcheck").then(() => { throw Error("Streamer served before connecting"); }, error => expect(error).to.be.instanceOf(Error));
|
||||
} finally { await stop(runtime); }
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,117 @@
|
||||
const { expect } = require("chai");
|
||||
const { Readable } = require("node:stream");
|
||||
const { promises: fs } = require("node:fs");
|
||||
const { tmpdir } = require("node:os");
|
||||
const { join } = require("node:path");
|
||||
const { randomUUID } = require("node:crypto");
|
||||
const { setTimeout } = require("node:timers");
|
||||
require("ts-node/register/transpile-only");
|
||||
const { ContentAnonimizer, AnonymizeTransformer } = require("../src/core/anonymize-utils");
|
||||
const pool = require("../src/core/anonymization-pool");
|
||||
const { removeStaleTextSpools, startTemporaryStorageMaintenance } = require("../src/core/temporary-storage");
|
||||
const { cacheCommand } = require("../src/core/cache-coordination");
|
||||
|
||||
describe("large anonymization resources", function () {
|
||||
this.timeout(20000);
|
||||
async function transform(text, options) {
|
||||
const bytes = Buffer.from(text), parts = [];
|
||||
// Split both a name and a multibyte character at varying chunk boundaries.
|
||||
for (let offset = 0; offset < bytes.length; offset += 4093) parts.push(bytes.subarray(offset, offset + 4093));
|
||||
const transformer = new AnonymizeTransformer(options), output = [];
|
||||
Readable.from(parts).pipe(transformer);
|
||||
for await (const chunk of transformer) output.push(chunk);
|
||||
return Buffer.concat(output).toString("utf8");
|
||||
}
|
||||
it("preserves whole-text Unicode matches and isolates cached settings", async () => {
|
||||
const text = "Alice Σ 研究 https://example.com/x\n".repeat(18000);
|
||||
const options = { filePath: `parity-${randomUUID()}.txt`, terms: ["Alice=>Hidden", "研究=>Other"], image: true, link: true };
|
||||
const original = pool.anonymizeOnWorker;
|
||||
let jobs = 0;
|
||||
pool.anonymizeOnWorker = (...args) => { jobs++; return original(...args); };
|
||||
try {
|
||||
const expected = new ContentAnonimizer(options).anonymize(text);
|
||||
expect(await transform(text, options)).to.equal(expected);
|
||||
expect(await transform(text, options)).to.equal(expected);
|
||||
expect(jobs).to.equal(1);
|
||||
const edited = { ...options, terms: ["Alice=>Changed", "研究=>Other"] };
|
||||
expect(await transform(text, edited)).to.equal(new ContentAnonimizer(edited).anonymize(text));
|
||||
expect(jobs).to.equal(2);
|
||||
} finally { pool.anonymizeOnWorker = original; }
|
||||
});
|
||||
it("removes a spooled input when its subscriber cancels", async () => {
|
||||
const transformer = new AnonymizeTransformer({ filePath: "cancel.txt", terms: ["Alice"] });
|
||||
await new Promise((resolve, reject) => transformer.write(Buffer.alloc(300000, "x"), error => error ? reject(error) : resolve()));
|
||||
const spool = transformer.spool;
|
||||
const closed = new Promise(resolve => transformer.once("close", resolve));
|
||||
transformer.destroy(); await closed;
|
||||
let exists = true;
|
||||
try { await fs.access(spool); } catch { exists = false; }
|
||||
expect(exists).to.equal(false);
|
||||
});
|
||||
it("recovers worker capacity after a failed input", async () => {
|
||||
const controller = new AbortController();
|
||||
let failure;
|
||||
try { await pool.anonymizeOnWorker({ input: join(tmpdir(), randomUUID()), output: join(tmpdir(), randomUUID()),
|
||||
options: { terms: ["Alice"] }, maxOutput: 1000000, context: { mask: "ANON", hostname: "localhost" } }, 1, controller.signal); }
|
||||
catch (error) { failure = error; }
|
||||
expect(failure).to.be.instanceOf(Error);
|
||||
const text = "Alice ".repeat(50000);
|
||||
const options = { filePath: `recovery-${randomUUID()}.txt`, terms: ["Alice=>Hidden"] };
|
||||
expect(await transform(text, options)).to.equal(new ContentAnonimizer(options).anonymize(text));
|
||||
});
|
||||
it("cleans crash leftovers while preserving recent spools and unrelated files", async () => {
|
||||
const root = await fs.mkdtemp(join(tmpdir(), "anonymous-spool-test-"));
|
||||
try {
|
||||
const old = join(root, "anonymous-text-old"), current = join(root, "anonymous-text-current"), other = join(root, "other");
|
||||
await Promise.all([old, current, other].map(path => fs.mkdir(path)));
|
||||
await fs.writeFile(join(old, "input"), "private");
|
||||
await fs.utimes(old, new Date(0), new Date(0));
|
||||
await removeStaleTextSpools(root);
|
||||
expect((await fs.readdir(root)).sort()).to.deep.equal(["anonymous-text-current", "other"]);
|
||||
} finally { await fs.rm(root, { force: true, recursive: true }); }
|
||||
});
|
||||
it("bounds an unresponsive Redis command and disconnects its socket", async () => {
|
||||
let disconnected = false, error;
|
||||
const client = { isOpen: true, disconnect: async () => { disconnected = true; } };
|
||||
try { await cacheCommand(new Promise(() => {}), client, 20); } catch (failure) { error = failure; }
|
||||
expect(error.message).to.equal("cache_command_timeout"); expect(disconnected).to.equal(true);
|
||||
});
|
||||
|
||||
it("cleans stale spools and transformed entries at startup and while idle", async () => {
|
||||
const cacheRoot = join(tmpdir(), "anonymous-transformed-v1");
|
||||
const stale = await fs.mkdtemp(join(tmpdir(), "anonymous-text-startup-"));
|
||||
const recent = await fs.mkdtemp(join(tmpdir(), "anonymous-text-idle-"));
|
||||
const unrelated = await fs.mkdtemp(join(tmpdir(), "unrelated-cleanup-"));
|
||||
const keys = [randomUUID(), randomUUID(), randomUUID()], paths = keys.flatMap(key => [join(cacheRoot, key + ".json"), join(cacheRoot, key + ".data")]);
|
||||
let stop;
|
||||
const missing = async path => { try { await fs.access(path); return false; } catch (error) { if (error.code === "ENOENT") return true; throw error; } };
|
||||
const seed = async (key, version, created) => {
|
||||
await fs.writeFile(join(cacheRoot, key + ".data"), "private");
|
||||
await fs.writeFile(join(cacheRoot, key + ".json"), JSON.stringify({ version, created, size: 7, changed: false }));
|
||||
};
|
||||
try {
|
||||
await fs.mkdir(cacheRoot, { recursive: true });
|
||||
await fs.writeFile(join(stale, "input"), "original bytes"); await fs.utimes(stale, new Date(0), new Date(0));
|
||||
await fs.writeFile(join(recent, "input"), "recent original bytes");
|
||||
await fs.writeFile(join(unrelated, "input"), "keep");
|
||||
await seed(keys[0], 1, Date.now()); await seed(keys[1], 2, 0); await seed(keys[2], 2, Date.now());
|
||||
stop = await startTemporaryStorageMaintenance(20);
|
||||
expect(await missing(stale)).to.equal(true);
|
||||
expect(await missing(join(cacheRoot, keys[0] + ".data"))).to.equal(true);
|
||||
expect(await missing(join(cacheRoot, keys[1] + ".data"))).to.equal(true);
|
||||
expect(await missing(join(cacheRoot, keys[2] + ".data"))).to.equal(false);
|
||||
expect(await missing(recent)).to.equal(false); expect(await missing(unrelated)).to.equal(false);
|
||||
await fs.utimes(recent, new Date(0), new Date(0));
|
||||
await seed(keys[2], 2, 0);
|
||||
const deadline = Date.now() + 2000;
|
||||
while ((!await missing(recent) || !await missing(join(cacheRoot, keys[2] + ".data"))) && Date.now() < deadline) {
|
||||
await new Promise(resolve => setTimeout(resolve, 10));
|
||||
}
|
||||
expect(await missing(recent)).to.equal(true); expect(await missing(join(cacheRoot, keys[2] + ".json"))).to.equal(true);
|
||||
expect(await fs.readFile(join(unrelated, "input"), "utf8")).to.equal("keep");
|
||||
} finally {
|
||||
stop?.();
|
||||
await Promise.all([...paths, stale, recent, unrelated].map(path => fs.rm(path, { force: true, recursive: true })));
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -282,7 +282,7 @@ describe("production regressions", function () {
|
||||
const Conference = require("../src/core/Conference").default;
|
||||
const model = { status: "ready", save: async () => {} };
|
||||
const conference = new Conference(model);
|
||||
conference.repositories = async () => [{ expire: async () => { throw new Error("storage failed"); } }];
|
||||
conference.repositories = async () => [{ markExpired: async () => { throw new Error("storage failed"); } }];
|
||||
try { await conference.expire(); throw new Error("expected rejection"); }
|
||||
catch (error) { expect(error.message).to.equal("storage failed"); }
|
||||
expect(model.status).to.equal("ready");
|
||||
|
||||
@@ -935,6 +935,164 @@ describe("Vue 3 UI", function () {
|
||||
expect(ui.errors).to.deep.equal([]);
|
||||
});
|
||||
|
||||
it("loads dashboard summaries once initially and reloads when search or status changes", async () => {
|
||||
ui = await browser("/dashboard", {
|
||||
"/api/user": { username: "tester" },
|
||||
"/api/user/dashboard": { items: [], total: 0, filtered: 0, attention: 0, cursor: null },
|
||||
});
|
||||
const requests = () => ui.requests.filter(request => request.url.pathname === "/api/user/dashboard");
|
||||
await delay(200);
|
||||
expect(requests()).to.have.length(1);
|
||||
await ui.input('input[type="search"]', "package");
|
||||
await delay(200);
|
||||
expect(requests()).to.have.length(2); expect(requests()[1].url.searchParams.get("q")).to.equal("package");
|
||||
ui.window.document.querySelector("#status-ready").click();
|
||||
await delay(200);
|
||||
expect(requests()).to.have.length(3); expect(requests()[2].url.searchParams.get("statuses")).not.to.include("ready");
|
||||
expect(ui.errors).to.deep.equal([]);
|
||||
});
|
||||
|
||||
it("shows repository commit badges and details from dashboard summaries", async () => {
|
||||
const commit = "abcdef1234567890";
|
||||
ui = await browser("/dashboard", {
|
||||
"/api/user": { username: "tester" },
|
||||
"/api/user/dashboard": { items: [{ _type: "repo", repoId: "package", status: "ready",
|
||||
source: { fullName: "owner/package", commit } }], total: 1, filtered: 1, attention: 0, cursor: null },
|
||||
});
|
||||
expect(ui.window.document.querySelector(".commit-hash").textContent).to.equal(commit.slice(0, 8));
|
||||
const state = ui.window.document.querySelector("#search")._field.binding.state;
|
||||
await state.showProjectDetails(state.items[0]);
|
||||
expect(ui.window.document.querySelector(".project-details").textContent).to.include(commit);
|
||||
expect(ui.errors).to.deep.equal([]);
|
||||
});
|
||||
|
||||
it("distinguishes empty server-filtered pages from an empty account", async () => {
|
||||
const item = { _type: "repo", repoId: "package", status: "ready", source: { fullName: "owner/package" } };
|
||||
ui = await browser("/dashboard", {
|
||||
"/api/user": { username: "tester" },
|
||||
"/api/user/dashboard": request => {
|
||||
const empty = !!request.url.searchParams.get("q") || !request.url.searchParams.get("statuses").split(",").includes("ready");
|
||||
return { items: empty ? [] : [item], total: 2, filtered: empty ? 0 : 1, attention: 0, cursor: null };
|
||||
},
|
||||
});
|
||||
const check = () => {
|
||||
const empty = ui.window.document.querySelector(".paper-table-empty");
|
||||
expect(empty.textContent).to.include("Nothing matches the current filters.").not.to.include("You have no anonymizations yet.");
|
||||
expect(empty.querySelector('a[href="/anonymize"]')).to.equal(null);
|
||||
expect(empty.querySelector("button").textContent).to.include("Clear filters");
|
||||
};
|
||||
await ui.input('input[type="search"]', "missing"); await delay(200); check();
|
||||
const state = ui.window.document.querySelector("#search")._field.binding.state;
|
||||
state.clearFilters(); await delay(200);
|
||||
ui.window.document.querySelector("#status-ready").click(); await delay(20); check();
|
||||
expect(ui.errors).to.deep.equal([]);
|
||||
});
|
||||
|
||||
it("offers a new anonymization when the server reports an empty account", async () => {
|
||||
ui = await browser("/dashboard", {
|
||||
"/api/user": { username: "tester" },
|
||||
"/api/user/dashboard": { items: [], total: 0, filtered: 0, attention: 0, cursor: null },
|
||||
});
|
||||
const empty = ui.window.document.querySelector(".paper-table-empty");
|
||||
expect(empty.textContent).to.include("You have no anonymizations yet.").not.to.include("Nothing matches");
|
||||
expect(empty.querySelector('a[href="/anonymize"]')).not.to.equal(null);
|
||||
expect(ui.errors).to.deep.equal([]);
|
||||
});
|
||||
|
||||
for (const status of ["error", "download"]) {
|
||||
it(`updates the global attention badge after polling a repository in ${status}`, async () => {
|
||||
const item = { _type: "repo", repoId: "package", status, anonymizeDate: "2000-01-01", source: { fullName: "owner/package" } };
|
||||
let ready = false;
|
||||
ui = await browser("/dashboard", {
|
||||
"/api/user": { username: "tester" },
|
||||
"/api/user/dashboard": () => ({ items: ready ? [] : [item], total: 20,
|
||||
filtered: ready ? 6 : 7, attention: ready ? 6 : 7, cursor: ready ? "new-next" : "next" }),
|
||||
"/api/repo/package/refresh": {}, "/api/repo/package": () => { ready = true; return { status: "ready" }; },
|
||||
});
|
||||
const state = ui.window.document.querySelector("#search")._field.binding.state;
|
||||
state.setProjectView("attention"); await delay(200);
|
||||
expect(state.filteredItems).to.have.length(1); expect(state.attentionCount()).to.equal(7);
|
||||
const requests = ui.requests.filter(request => request.url.pathname === "/api/user/dashboard").length;
|
||||
const original = state.items[0];
|
||||
state.refreshItem(original); await delay(30);
|
||||
expect(state.filteredItems).to.have.length(0);
|
||||
expect(state.attentionCount()).to.equal(6);
|
||||
expect(ui.window.document.querySelector(".attention-count").textContent).to.equal("6");
|
||||
state.refreshItem(original); await delay(30);
|
||||
expect(state.attentionCount()).to.equal(6);
|
||||
expect(state.dashboardTotals.filtered).to.equal(6);
|
||||
expect(state.dashboardCursor).to.equal("new-next");
|
||||
expect(ui.requests.filter(request => request.url.pathname === "/api/user/dashboard")).to.have.length(requests + 1);
|
||||
expect(ui.errors).to.deep.equal([]);
|
||||
});
|
||||
}
|
||||
|
||||
for (const filtered of [false, true]) {
|
||||
it(`reloads pagination after a polled status change${filtered ? " under a status filter" : " sorted by status"}`, async () => {
|
||||
const original = { _type: "repo", repoId: "package", status: "error", source: { fullName: "owner/package" } };
|
||||
const other = { _type: "repo", repoId: "z-other", status: "error", source: { fullName: "owner/z-other" } };
|
||||
let ready = false, release;
|
||||
ui = await browser("/dashboard", {
|
||||
"/api/user": { username: "tester" },
|
||||
"/api/user/dashboard": request => {
|
||||
const cursor = request.url.searchParams.get("cursor");
|
||||
if (cursor === "old-next") return new Promise(resolve => { release = () => resolve({ items: [other], total: 2, filtered: 2, attention: 2, cursor: null }); });
|
||||
if (cursor === "new-next") return { items: [{ ...original, status: "ready" }], total: 2, filtered: 2, attention: 1, cursor: null };
|
||||
return { items: [ready ? other : original], total: 2, filtered: ready && filtered ? 1 : 2,
|
||||
attention: ready ? 1 : 2, cursor: ready ? (filtered ? null : "new-next") : "old-next" };
|
||||
},
|
||||
"/api/repo/package/refresh": {}, "/api/repo/package": () => { ready = true; return { status: "ready" }; },
|
||||
});
|
||||
const state = ui.window.document.querySelector("#search")._field.binding.state;
|
||||
state.orderBy = "status";
|
||||
if (filtered) Object.keys(state.filters.status).forEach(key => { state.filters.status[key] = key === "error"; });
|
||||
await delay(200);
|
||||
await state.showProjectDetails(state.items[0]);
|
||||
const selected = state.items[0];
|
||||
const pending = state.loadMoreProjects(); await delay(10);
|
||||
expect(release).to.be.a("function");
|
||||
const count = ui.requests.filter(request => request.url.pathname === "/api/user/dashboard").length;
|
||||
state.refreshItem(selected); await delay(50);
|
||||
const reloads = ui.requests.filter(request => request.url.pathname === "/api/user/dashboard").slice(count);
|
||||
expect(reloads).to.have.length(1); expect(reloads[0].url.searchParams.has("cursor")).to.equal(false);
|
||||
expect(reloads[0].url.searchParams.get("sort")).to.equal("status");
|
||||
if (filtered) expect(reloads[0].url.searchParams.get("statuses")).to.equal("error");
|
||||
expect(state.dashboardTotals.filtered).to.equal(filtered ? 1 : 2);
|
||||
expect(state.dashboardCursor).to.equal(filtered ? null : "new-next");
|
||||
expect(state.selectedProject).to.equal(null);
|
||||
release(); await pending;
|
||||
expect(state.items.map(item => item._id)).to.deep.equal(["z-other"]);
|
||||
if (!filtered) {
|
||||
await state.loadMoreProjects();
|
||||
expect(state.items.map(item => item._id)).to.deep.equal(["z-other", "package"]);
|
||||
expect(state.items[1].status).to.equal("ready");
|
||||
}
|
||||
expect(ui.errors).to.deep.equal([]);
|
||||
});
|
||||
}
|
||||
|
||||
it("reloads later status transitions after a repository leaves the loaded page", async () => {
|
||||
const item = { _type: "repo", repoId: "package", status: "error", source: { fullName: "owner/package" } };
|
||||
let status = "error", polls = 0;
|
||||
ui = await browser("/dashboard", {
|
||||
"/api/user": { username: "tester" },
|
||||
"/api/user/dashboard": () => ({ items: status === "error" ? [{ ...item, status }] : [], total: 1,
|
||||
filtered: status === "error" ? 1 : 0, attention: status === "error" ? 1 : 0, cursor: null }),
|
||||
"/api/repo/package/refresh": {},
|
||||
"/api/repo/package": () => { status = ++polls === 1 ? "download" : "error"; return { status, statusMessage: "failed refresh" }; },
|
||||
});
|
||||
const state = ui.window.document.querySelector("#search")._field.binding.state;
|
||||
Object.keys(state.filters.status).forEach(key => { state.filters.status[key] = key === "error"; });
|
||||
await delay(20);
|
||||
state.refreshItem(state.items[0]); await delay(50);
|
||||
expect(state.items).to.have.length(0); expect(state.dashboardTotals.filtered).to.equal(0);
|
||||
await delay(2600);
|
||||
expect(polls).to.equal(2); expect(state.items.map(item => item._id)).to.deep.equal(["package"]);
|
||||
expect(state.items[0].status).to.equal("error");
|
||||
expect(state.dashboardTotals.filtered).to.equal(1); expect(state.attentionCount()).to.equal(1);
|
||||
expect(ui.errors).to.deep.equal([]);
|
||||
});
|
||||
|
||||
it("finds errors and stalled downloads without treating fresh queued projects as failures", async () => {
|
||||
ui = await browser("/dashboard", {
|
||||
"/api/user": { username: "tester" },
|
||||
@@ -1003,6 +1161,38 @@ describe("Vue 3 UI", function () {
|
||||
expect(ui.errors).to.deep.equal([]);
|
||||
});
|
||||
|
||||
for (const filtered of [false, true]) {
|
||||
it(`reloads summary pagination after renaming a project${filtered ? " in a search" : " sorted by label"}`, async () => {
|
||||
let name = "Old name", queryCount = 0;
|
||||
const item = () => ({ _type: "repo", repoId: "package", status: "ready", projectName: name, source: { fullName: "owner/package" } });
|
||||
ui = await browser("/dashboard", {
|
||||
"/api/user": { username: "tester" },
|
||||
"/api/user/dashboard": request => {
|
||||
queryCount++;
|
||||
const match = !request.url.searchParams.get("q") || name.includes(request.url.searchParams.get("q"));
|
||||
return { items: match ? [item()] : [], total: 2, filtered: match ? 2 : 0, attention: 0,
|
||||
cursor: match ? name + "-cursor" : null };
|
||||
},
|
||||
"/api/user/project-name": request => { name = request.payload.name; return { name }; },
|
||||
});
|
||||
const state = ui.window.document.querySelector("#search")._field.binding.state;
|
||||
state.orderBy = "_label"; if (filtered) state.search = "Old";
|
||||
await delay(200);
|
||||
await state.showProjectDetails(state.items[0]);
|
||||
state.projectName = "New name";
|
||||
const before = queryCount;
|
||||
await state.saveProjectName(state.items[0]); await delay(10);
|
||||
expect(queryCount).to.equal(before + 1);
|
||||
expect(state.dashboardCursor).to.equal(filtered ? null : "New name-cursor");
|
||||
expect(state.dashboardTotals.filtered).to.equal(filtered ? 0 : 2);
|
||||
expect(state.items).to.have.length(filtered ? 0 : 1);
|
||||
expect(state.selectedProject).to.equal(filtered ? null : state.items[0]);
|
||||
const last = ui.requests.filter(request => request.url.pathname === "/api/user/dashboard").at(-1);
|
||||
expect(last.url.searchParams.has("cursor")).to.equal(false);
|
||||
expect(ui.errors).to.deep.equal([]);
|
||||
});
|
||||
}
|
||||
|
||||
it("shows full project details as text and restores keyboard focus when closed", async () => {
|
||||
const source = "owner/" + "very-long-project-name-".repeat(5);
|
||||
ui = await browser("/dashboard", {
|
||||
|
||||
Reference in new issue
Block a user