mirror of
https://github.com/tdurieux/anonymous_github.git
synced 2026-10-18 21:57:20 +02:00
Merge commit 'refs/security-review/pr849' into HEAD
# Conflicts: # src/server/service/review-consent-http.ts
This commit is contained in:
Thomas Durieux committed
commit
eb4fbb6b15
165 files changed
+16267
-22725
No files matched your search
@@ -0,0 +1,32 @@
|
||||
const { expect } = require("chai");
|
||||
|
||||
describe("admin queue jobs", function () {
|
||||
it("returns job payloads, failure reasons and stacks as values, not JSON strings", async function () {
|
||||
const queues = require("../src/queue");
|
||||
const metrics = require("../src/queue/queueMetrics");
|
||||
const router = require("../src/server/routes/admin").default;
|
||||
const handler = router.stack.find(layer => layer.route?.path === "/queues").route.stack.at(-1).handle;
|
||||
const originals = [queues.downloadQueue, queues.removeQueue, queues.cacheQueue, metrics.queryMetrics];
|
||||
const failed = {
|
||||
data: { repoId: "paper-7F2A" }, failedReason: "branch_not_found", stacktrace: ["Error: branch_not_found\n at fetch"], returnvalue: null,
|
||||
// BullMQ's asJSON() stringifies these four fields.
|
||||
asJSON() { return { id: "paper-7F2A", name: "paper-7F2A", data: JSON.stringify(this.data), failedReason: JSON.stringify(this.failedReason), stacktrace: JSON.stringify(this.stacktrace), returnvalue: JSON.stringify(this.returnvalue) }; },
|
||||
};
|
||||
const queue = { getJobCounts: async () => ({}), getWorkers: async () => [], isPaused: async () => false,
|
||||
getJobs: async ([state]) => state === "failed" ? [failed] : [] };
|
||||
queues.downloadQueue = queue; queues.removeQueue = queue; queues.cacheQueue = queue;
|
||||
metrics.queryMetrics = async () => [];
|
||||
try {
|
||||
let body;
|
||||
// The remove queue keeps this snapshot apart from other tests' cache entries.
|
||||
await handler({ query: { queue: "remove" } }, { json: data => { body = data; } });
|
||||
const job = body.jobs.find(j => j.id === "paper-7F2A");
|
||||
expect(job.data).to.deep.equal({ repoId: "paper-7F2A" });
|
||||
expect(job.failedReason).to.equal("branch_not_found");
|
||||
expect(job.stacktrace).to.deep.equal(["Error: branch_not_found\n at fetch"]);
|
||||
expect(job._state).to.equal("failed");
|
||||
} finally {
|
||||
[queues.downloadQueue, queues.removeQueue, queues.cacheQueue, metrics.queryMetrics] = originals;
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -5,6 +5,7 @@ const { ContentAnonimizer, AnonymizeTransformer } = require("../src/core/anonymi
|
||||
|
||||
describe("literal term presence checks", function () {
|
||||
it("serves a large JavaScript file with absent names within the anonymization deadline", async function () {
|
||||
this.timeout(10000); // Includes worker startup; the matcher retains its one-second deadline.
|
||||
const input = Buffer.from('function render(){return "application content";}\n'.repeat(65000));
|
||||
const transformer = new AnonymizeTransformer({
|
||||
filePath: "main.js",
|
||||
@@ -38,4 +39,31 @@ describe("literal term presence checks", function () {
|
||||
expect(optimized.wasAnonymized).to.equal(reference.wasAnonymized);
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps required-prefix absence checks conservative for arbitrary regex syntax", function () {
|
||||
const cases = [
|
||||
{ terms: ["Researcher[0-9]+"], text: "absent name Researcher12 Résearcher42" },
|
||||
{ terms: ["foo?bar"], text: "foobar fobar" },
|
||||
{ terms: ["foo*bar"], text: "foobar fobar foobarbar" },
|
||||
{ terms: ["foo{0,2}bar"], text: "fobar foobar fooobar" },
|
||||
{ terms: ["foo+bar"], text: "foobar fooobar" },
|
||||
{ terms: ["a?", "[ab]*", "[ab]{0,2}"], text: "x a ab" },
|
||||
{ terms: ["foo[a]{0}bar", "[ab]?foo"], text: "foobar foo afoo" },
|
||||
{ terms: ["😀?bar"], text: "bar 😀bar" },
|
||||
{ terms: ["foo|bar", "foo(bar|baz)"], text: "foo bar foobar foobaz" },
|
||||
{ terms: ["foo\\|bar", "foo\\.bar"], text: "foo|bar foo.bar" },
|
||||
{ terms: ["^foo", "(?:foo)?bar", "[A-Z]+foo"], text: "foo bar XXfoo" },
|
||||
{ terms: ["k[0-9]+", "s[0-9]+"], text: "k12 K12 s34 ſ34 ék12" },
|
||||
{ terms: ["😀[0-9]+", "研究[0-9]+"], text: "😀12 研究34" },
|
||||
{ terms: ["Alice=>Researcher12", "Researcher[0-9]+=>Hidden"], text: "Alice" },
|
||||
{ terms: ["Researcher[0-9]+"], text: "https://host/Researcher12/path" },
|
||||
];
|
||||
for (const { terms, text } of cases) {
|
||||
const reference = new ContentAnonimizer({ terms });
|
||||
for (const term of reference.compiledTerms) delete term.requiredPrefix;
|
||||
const optimized = new ContentAnonimizer({ terms });
|
||||
expect(optimized.anonymize(text), JSON.stringify(terms)).to.equal(reference.anonymize(text));
|
||||
expect(optimized.wasAnonymized).to.equal(reference.wasAnonymized);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,28 @@
|
||||
const { expect } = require("chai");
|
||||
const { Readable } = require("node:stream");
|
||||
require("ts-node/register/transpile-only");
|
||||
const { AnonymizeTransformer, ContentAnonimizer } = require("../src/core/anonymize-utils");
|
||||
const terms = ["first.last", "user@example.org", "dept.example.edu", "lab.io", "project.org", "Author X. Name",
|
||||
...Array.from({ length: 41 }, (_, i) => `Researcher${i}`)];
|
||||
|
||||
describe("large files with many anonymization terms", function () {
|
||||
this.timeout(10000);
|
||||
it("handles a 50 MB numeric dataset without exhausting the one-second deadline", function () {
|
||||
const input = "123,456,78.91234,-45.123456\n".repeat(1800000);
|
||||
expect(new ContentAnonimizer({ terms }).anonymize(input)).to.equal(input);
|
||||
});
|
||||
it("redacts dotted terms and their URLs around a large payload", async function () {
|
||||
const payload = "123,456,78.91234,-45.123456\n".repeat(200000);
|
||||
const input = Buffer.from(`first.last\n${payload}\nhttps://host/user@example.org/path\nfirstXlast`);
|
||||
const stream = new AnonymizeTransformer({ filePath: "coords.csv", terms });
|
||||
Readable.from([input.subarray(0, 6), input.subarray(6)]).pipe(stream);
|
||||
const chunks = [];
|
||||
for await (const chunk of stream) chunks.push(chunk);
|
||||
expect(Buffer.concat(chunks).toString()).to.equal(`XXXX-1\n${payload}\nXXXX-2\nXXXX-1`);
|
||||
});
|
||||
it("handles many repeated matches while preserving the surrounding values", function () {
|
||||
const input = "Alice,123,45.678,-23.456\n".repeat(100000);
|
||||
expect(new ContentAnonimizer({ terms: ["Alice", ...terms] }).anonymize(input))
|
||||
.to.equal("XXXX-1,123,45.678,-23.456\n".repeat(100000));
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,104 @@
|
||||
const { expect } = require("chai");
|
||||
require("ts-node/register/transpile-only");
|
||||
const config = require("../src/config").default;
|
||||
const app = require("../src/core/github-app");
|
||||
const utils = require("../src/server/routes/route-utils");
|
||||
const Repositories = require("../src/core/model/anonymizedRepositories/anonymizedRepositories.model").default;
|
||||
const PullRequests = require("../src/core/model/anonymizedPullRequests/anonymizedPullRequests.model").default;
|
||||
const { githubAppRouter } = require("../src/server/routes/github-app");
|
||||
const migrate = githubAppRouter.stack.find(layer => layer.route?.path === "/connections/migrate").route.stack[0].handle;
|
||||
|
||||
for (const type of ["repository", "pull-request"]) {
|
||||
describe(`${type} App reinstall recovery`, function () {
|
||||
let restores, model, selected, writes, checks, owner;
|
||||
function stub(object, key, value) {
|
||||
const previous = object[key]; restores.push(() => { object[key] = previous; }); object[key] = value;
|
||||
}
|
||||
beforeEach(function () {
|
||||
restores = []; writes = []; checks = []; owner = "owner";
|
||||
model = { _id: "saved-resource", owner: "owner", status: "ready", statusDate: new Date(),
|
||||
repoId: "published-url", pullRequestId: "published-url",
|
||||
source: { repositoryName: "owner/private", repositoryFullName: "owner/private", commit: "saved-sha", pullRequestId: 7 },
|
||||
githubAccess: { kind: "github-app", repositoryId: 42, installationId: 100, revision: "old" },
|
||||
};
|
||||
selected = { token: "fresh-token", binding: { kind: "github-app", repositoryId: 42, installationId: 200, revision: "new" } };
|
||||
stub(config, "GITHUB_APP_ENABLED", true); stub(config, "GITHUB_APP_NEW_CONNECTIONS", true);
|
||||
stub(utils, "getUser", async () => ({ id: owner }));
|
||||
stub(utils, "handleError", error => { throw error; });
|
||||
stub(app, "selectRepositoryAccess", async (id, name, connection, repositoryId) => {
|
||||
expect(repositoryId).to.equal(42);
|
||||
expect([id, name, connection]).to.deep.equal(["owner", "owner/private", "github-app"]);
|
||||
return selected;
|
||||
});
|
||||
stub(app, "githubRequest", async (url, token) => { checks.push({ url, token }); return {}; });
|
||||
const Model = type === "repository" ? Repositories : PullRequests;
|
||||
stub(Model, "findOne", async filter => filter.owner === model.owner ? model : null);
|
||||
stub(Model, "updateOne", async (filter, update) => { writes.push({ filter, update }); return { modifiedCount: 1 }; });
|
||||
});
|
||||
afterEach(function () { while (restores.length) restores.pop()(); });
|
||||
async function run(preview = false) {
|
||||
let response;
|
||||
await migrate({ body: { type, id: "published-url", connection: "github-app", preview } }, { json: body => { response = body; } });
|
||||
return response;
|
||||
}
|
||||
async function fails(message) {
|
||||
let error; try { await run(); } catch (caught) { error = caught; }
|
||||
expect(error?.message).to.equal(message); expect(writes).to.have.length(0);
|
||||
}
|
||||
it("previews fresh installation access without changing the saved binding", async function () {
|
||||
expect(await run(true)).to.deep.equal({ eligible: true, connection: "github-app" });
|
||||
expect(writes).to.have.length(0); expect(model.githubAccess.installationId).to.equal(100);
|
||||
});
|
||||
it("replaces only the binding while preserving the URL, source and snapshot", async function () {
|
||||
expect(await run()).to.deep.equal({ connection: "github-app" });
|
||||
expect(writes).to.have.length(1);
|
||||
expect(writes[0].update).to.deep.equal({ $set: { githubAccess: selected.binding } });
|
||||
expect(writes[0].filter.githubAccess).to.deep.equal(model.githubAccess);
|
||||
expect(writes[0].filter.source).to.deep.equal(model.source);
|
||||
expect(checks).to.deep.equal([{ url: type === "repository" ? "/repos/owner/private/commits/saved-sha" : "/repos/owner/private/pulls/7", token: "fresh-token" }]);
|
||||
});
|
||||
it("saves a resolved rename atomically with the binding after preview", async function () {
|
||||
selected.fullName = "owner/renamed";
|
||||
await run(true);
|
||||
expect(writes).to.have.length(0);
|
||||
await run();
|
||||
expect(writes).to.have.length(1);
|
||||
expect(writes[0].update).to.deep.equal({ $set: {
|
||||
githubAccess: selected.binding,
|
||||
[type === "repository" ? "source.repositoryName" : "source.repositoryFullName"]: "owner/renamed",
|
||||
} });
|
||||
expect(writes[0].filter.source).to.deep.equal(model.source);
|
||||
expect(writes[0].filter.githubAccess).to.deep.equal(model.githubAccess);
|
||||
expect(checks.every(check => check.url.startsWith("/repos/owner/renamed/"))).to.equal(true);
|
||||
});
|
||||
it("rejects a replacement repository at the same name", async function () {
|
||||
selected.binding.repositoryId = 99; await fails("connection_changed"); expect(checks).to.have.length(0);
|
||||
});
|
||||
for (const status of ["preparing", "queue", "download", "removing", "expiring"]) {
|
||||
it("rejects reconnect during " + status, async function () {
|
||||
model.status = status;
|
||||
await fails("repository_busy");
|
||||
expect(checks).to.have.length(0);
|
||||
});
|
||||
}
|
||||
it("requires access to the existing snapshot before saving", async function () {
|
||||
stub(app, "githubRequest", async () => { throw new Error("github_app_access_required"); });
|
||||
await fails("github_app_access_required");
|
||||
});
|
||||
if (type === "pull-request") {
|
||||
it("allows stale downloads but guards their timestamp against a new refresh", async function () {
|
||||
model.status = "download";
|
||||
model.statusDate = new Date(Date.now() - 6 * 60 * 1000);
|
||||
await run();
|
||||
expect(writes[0].filter.statusDate).to.equal(model.statusDate);
|
||||
});
|
||||
}
|
||||
it("does not let another user repair the binding", async function () {
|
||||
owner = "other"; await fails("repo_not_found");
|
||||
});
|
||||
it("reports a concurrent change instead of overwriting it", async function () {
|
||||
stub(type === "repository" ? Repositories : PullRequests, "updateOne", async () => ({ modifiedCount: 0 }));
|
||||
await fails("connection_changed");
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -66,6 +66,23 @@ describe("asset build", function () {
|
||||
}
|
||||
});
|
||||
|
||||
it("ships the upstream PDF.js bytes without a second minification pass", function () {
|
||||
const upstream = fs.readFileSync(path.join(path.dirname(gulpfile), groups.pdfFiles[0]));
|
||||
fs.writeFileSync(path.join(directory, groups.pdfFiles[0]), upstream);
|
||||
const result = build();
|
||||
expect(result.status, result.stderr).to.equal(0);
|
||||
const bundled = fs.readFileSync(path.join(directory, "public/script/pdf.min.js"));
|
||||
expect(bundled.equals(upstream)).to.equal(true);
|
||||
});
|
||||
|
||||
it("rejects invalid PDF.js without publishing a manifest", function () {
|
||||
fs.writeFileSync(path.join(directory, groups.pdfFiles[0]), "function {");
|
||||
const result = build();
|
||||
expect(result.status).not.to.equal(0);
|
||||
expect(result.stderr).to.include("SyntaxError");
|
||||
expect(fs.existsSync(path.join(directory, "public/asset-manifest.json"))).to.equal(false);
|
||||
});
|
||||
|
||||
it("fails on missing input without publishing a manifest", function () {
|
||||
fs.unlinkSync(path.join(directory, groups.coreJsFiles[0]));
|
||||
const result = build();
|
||||
@@ -75,7 +92,7 @@ describe("asset build", function () {
|
||||
});
|
||||
|
||||
it("fails on invalid JavaScript without publishing a manifest", function () {
|
||||
fs.writeFileSync(path.join(directory, groups.pdfFiles[0]), "function {");
|
||||
fs.writeFileSync(path.join(directory, groups.markdownFiles[0]), "function {");
|
||||
const result = build();
|
||||
expect(result.status).not.to.equal(0);
|
||||
expect(result.stderr).to.include("uglify");
|
||||
|
||||
@@ -9,11 +9,13 @@ describe("dashboard refresh feedback", () => {
|
||||
async function run({ statuses = [], postError, getError } = {}) {
|
||||
let toast;
|
||||
const item = { _type: "repo", _id: "restore-me", repoId: "restore-me", status: "removed" };
|
||||
const state = { items: [item], addToast: value => { toast = value; } };
|
||||
const state = { items: [item], addToast: value => { toast = value; },
|
||||
needsAttention: item => item.status === "error" || (item._statusKey === "progress" && item._stale) };
|
||||
const pending = [];
|
||||
let reloads = 0;
|
||||
vm.runInNewContext(actions, {
|
||||
state, reactive: value => value,
|
||||
state, legacyDashboard: true, reactive: value => value,
|
||||
statusKey: status => ["ready", "error", "removed", "expired"].includes(status) ? status : "progress",
|
||||
http: {
|
||||
post: async () => { if (postError) throw postError; return {}; },
|
||||
get: async () => { if (getError) throw getError; return { data: statuses.shift() }; },
|
||||
|
||||
@@ -93,10 +93,11 @@ describe("dashboard UI", function () {
|
||||
expect(divider).to.be.greaterThan(-1);
|
||||
});
|
||||
it("labels hidden statuses as hidden, not as active filters", function () {
|
||||
expect(html).to.match(/Hiding \{\{\s*statusKeyLabels\[f\]\s*\}\}/);
|
||||
expect(html).to.match(/\{\{\s*statusKeyLabels\[f\]\s*\}\} hidden/);
|
||||
});
|
||||
it("shows an unlimited quota without a full bar", function () {
|
||||
expect(html).to.match(/quota-fill" v-if="!quota\[q\?\.key\]\.unlimited"/);
|
||||
const quota = fs.readFileSync(path.join(__dirname, "../public/partials/quotaUsage.htm"), "utf8");
|
||||
expect(quota).to.match(/quota-fill" v-if="!quota\[q\?\.key\]\.unlimited"/);
|
||||
expect(html).to.not.match(/bg-success|bg-warning|bg-danger/);
|
||||
});
|
||||
it("right-aligns the Views column and marks it sortable", function () {
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
/* global fetch */
|
||||
const { expect } = require("chai");
|
||||
const express = require("express");
|
||||
const session = require("express-session");
|
||||
const { createClient } = require("redis");
|
||||
const { RedisStore: SessionStore } = require("connect-redis");
|
||||
const { RedisStore: RateLimitStore } = require("rate-limit-redis");
|
||||
const { rateLimit } = require("express-rate-limit");
|
||||
const { execFile } = require("node:child_process");
|
||||
const { promisify } = require("node:util");
|
||||
const path = require("node:path");
|
||||
const port = Number(process.env.PERFORMANCE_REDIS_PORT);
|
||||
|
||||
(port ? describe : describe.skip)("upgraded Redis stores and session cleanup", function () {
|
||||
this.timeout(15000);
|
||||
let redis, server, base;
|
||||
const prefix = `dependency-test:${process.pid}:`;
|
||||
before(async function () {
|
||||
redis = createClient({ socket: { host: "127.0.0.1", port, reconnectStrategy: false } });
|
||||
redis.on("error", () => {});
|
||||
await redis.connect();
|
||||
const app = express();
|
||||
app.use(session({ secret: "dependency-test", resave: false, saveUninitialized: false,
|
||||
store: new SessionStore({ client: redis, prefix: `${prefix}session:` }) }));
|
||||
app.get("/session", (req, res) => {
|
||||
req.session.visits = (req.session.visits || 0) + 1;
|
||||
res.json({ visits: req.session.visits });
|
||||
});
|
||||
const limiter = () => rateLimit({ windowMs: 60000, limit: 1, keyGenerator: () => "client",
|
||||
store: new RateLimitStore({ prefix: `${prefix}limit:`, sendCommand: (...args) => redis.sendCommand(args) }) });
|
||||
app.get("/first", limiter(), (_req, res) => res.send("ok"));
|
||||
app.get("/second", limiter(), (_req, res) => res.send("ok"));
|
||||
await new Promise(resolve => { server = app.listen(0, "127.0.0.1", resolve); });
|
||||
base = `http://127.0.0.1:${server.address().port}`;
|
||||
});
|
||||
after(async function () {
|
||||
if (server) {
|
||||
server.closeAllConnections();
|
||||
await new Promise(resolve => server.close(resolve));
|
||||
}
|
||||
if (redis?.isOpen) {
|
||||
for await (const keys of redis.scanIterator({ MATCH: `${prefix}*` })) {
|
||||
if (keys.length) await redis.del(keys);
|
||||
}
|
||||
await redis.del([`anoGH_session:${prefix}legacy`, `anoGH_session:${prefix}current`]);
|
||||
await redis.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("persists sessions across HTTP requests", async function () {
|
||||
const first = await fetch(`${base}/session`);
|
||||
expect(await first.json()).to.deep.equal({ visits: 1 });
|
||||
const cookie = first.headers.get("set-cookie").split(";")[0];
|
||||
const second = await fetch(`${base}/session`, { headers: { Cookie: cookie } });
|
||||
expect(await second.json()).to.deep.equal({ visits: 2 });
|
||||
});
|
||||
|
||||
it("shares a request quota between separate Redis store instances", async function () {
|
||||
expect((await fetch(`${base}/first`)).status).to.equal(200);
|
||||
expect((await fetch(`${base}/second`)).status).to.equal(429);
|
||||
});
|
||||
|
||||
it("scans session batches, preserves current sessions and applies legacy cleanup", async function () {
|
||||
const legacy = `anoGH_session:${prefix}legacy`, current = `anoGH_session:${prefix}current`;
|
||||
await redis.set(legacy, JSON.stringify({ passport: { user: { id: "legacy" } } }));
|
||||
await redis.set(current, JSON.stringify({ passport: { user: "current" } }));
|
||||
const run = async args => {
|
||||
const result = await promisify(execFile)(process.execPath,
|
||||
["-r", "ts-node/register", "src/scripts/purge-legacy-sessions.ts", ...args], {
|
||||
cwd: path.resolve(__dirname, ".."), timeout: 10000,
|
||||
env: { ...process.env, NODE_ENV: "test", REDIS_HOSTNAME: "127.0.0.1", REDIS_PORT: String(port) },
|
||||
});
|
||||
return JSON.parse(result.stdout.trim().split("\n").at(-1));
|
||||
};
|
||||
expect(await run([])).to.deep.equal({ found: 1, removed: 0 });
|
||||
expect(await redis.exists(legacy)).to.equal(1);
|
||||
expect(await run(["--apply"])).to.deep.equal({ found: 1, removed: 1 });
|
||||
expect(await redis.exists(legacy)).to.equal(0);
|
||||
expect(await redis.get(current)).to.equal(JSON.stringify({ passport: { user: "current" } }));
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,86 @@
|
||||
/* global fetch */
|
||||
const { expect } = require("chai");
|
||||
const http = require("node:http");
|
||||
const net = require("node:net");
|
||||
const path = require("node:path");
|
||||
const { spawn } = require("node:child_process");
|
||||
const { once } = require("node:events");
|
||||
const { setTimeout, clearTimeout } = require("node:timers");
|
||||
|
||||
describe("development proxy", function () {
|
||||
this.timeout(15000);
|
||||
let upstream, child, base, target;
|
||||
before(async function () {
|
||||
upstream = http.createServer((req, res) => {
|
||||
if (req.url === "/api/headers") {
|
||||
res.setHeader("content-type", "application/json");
|
||||
return res.end(JSON.stringify({ origin: req.headers.origin, referer: req.headers.referer }));
|
||||
}
|
||||
res.setHeader("set-cookie", "test=value; Domain=localhost; Secure; SameSite=None; Path=/");
|
||||
if (req.url === "/api/redirect") {
|
||||
res.writeHead(302, { location: `${target}/api/next` });
|
||||
return res.end();
|
||||
}
|
||||
res.setHeader("content-type", "text/html");
|
||||
res.end(`<a href="${target}/api/next">next</a>`);
|
||||
});
|
||||
await new Promise(resolve => upstream.listen(0, "127.0.0.1", resolve));
|
||||
target = `http://127.0.0.1:${upstream.address().port}`;
|
||||
const socket = net.createServer();
|
||||
await new Promise(resolve => socket.listen(0, "127.0.0.1", resolve));
|
||||
const port = socket.address().port;
|
||||
await new Promise(resolve => socket.close(resolve));
|
||||
base = `http://127.0.0.1:${port}`;
|
||||
child = spawn(process.execPath, ["scripts/dev-proxy.js"], {
|
||||
cwd: path.resolve(__dirname, ".."), env: { ...process.env, UPSTREAM: target, PORT: String(port) },
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
await new Promise((resolve, reject) => {
|
||||
let output = "", errors = "";
|
||||
const timer = setTimeout(() => reject(new Error("Development proxy startup timed out")), 10000);
|
||||
child.stderr.on("data", chunk => { errors += chunk; });
|
||||
child.stdout.on("data", chunk => {
|
||||
output += chunk;
|
||||
if (output.includes("dev-proxy")) { clearTimeout(timer); resolve(); }
|
||||
});
|
||||
child.once("exit", code => { clearTimeout(timer); reject(new Error(`Proxy exited with ${code}: ${errors}`)); });
|
||||
});
|
||||
});
|
||||
after(async function () {
|
||||
if (child && child.exitCode === null && child.signalCode === null) {
|
||||
const exited = once(child, "exit");
|
||||
child.kill();
|
||||
await exited;
|
||||
}
|
||||
if (upstream) {
|
||||
upstream.closeAllConnections();
|
||||
await new Promise(resolve => upstream.close(resolve));
|
||||
}
|
||||
});
|
||||
|
||||
it("forwards the request hooks to the upstream", async function () {
|
||||
const response = await fetch(`${base}/api/headers`);
|
||||
expect(response.status).to.equal(200);
|
||||
expect(await response.json()).to.deep.equal({ origin: target, referer: `${target}/api/headers` });
|
||||
});
|
||||
|
||||
it("rewrites HTML, cookies and redirects for localhost", async function () {
|
||||
const response = await fetch(`${base}/api/html`);
|
||||
expect(await response.text()).to.equal('<a href="/api/next">next</a>');
|
||||
const cookie = response.headers.get("set-cookie");
|
||||
expect(cookie).to.include("SameSite=Lax");
|
||||
expect(cookie).not.to.match(/Secure|Domain=/i);
|
||||
const redirect = await fetch(`${base}/api/redirect`, { redirect: "manual" });
|
||||
expect(redirect.status).to.equal(302);
|
||||
expect(redirect.headers.get("location")).to.equal("/api/next");
|
||||
});
|
||||
|
||||
it("serves the local UI with resolved asset names", async function () {
|
||||
const response = await fetch(base);
|
||||
expect(response.status).to.equal(200);
|
||||
const body = await response.text();
|
||||
expect(body).to.include('id="app"');
|
||||
expect(body).not.to.include("__CORE_JS__");
|
||||
expect(body).not.to.include("__VENDOR_JS__");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,107 @@
|
||||
const { expect } = require("chai");
|
||||
const express = require("express");
|
||||
const http = require("node:http");
|
||||
const { Readable } = require("node:stream");
|
||||
require("ts-node/register/transpile-only");
|
||||
const utils = require("../src/server/routes/route-utils");
|
||||
const AnonymizedFile = require("../src/core/AnonymizedFile").default;
|
||||
const fileRouter = require("../src/server/routes/file").default;
|
||||
const webRouter = require("../src/server/routes/webview").default;
|
||||
const config = require("../src/config").default;
|
||||
|
||||
describe("download headers and website revalidation", function () {
|
||||
let server, restore, repo, transfers, blobSha;
|
||||
beforeEach(async function () {
|
||||
const originalGetRepo = utils.getRepo;
|
||||
const originals = Object.fromEntries(["originalPath", "sha", "size", "send", "getFileInfo", "anonymizedContent"].map(key => [key, AnonymizedFile.prototype[key]]));
|
||||
restore = () => { utils.getRepo = originalGetRepo; Object.assign(AnonymizedFile.prototype, originals); };
|
||||
transfers = 0; blobSha = "first";
|
||||
repo = { repoId: "fixture", options: { terms: ["author"], image: true, pdf: true, page: true, pageSource: { branch: "main", path: "/" } },
|
||||
model: { source: { commit: "commit", branch: "main" }, treeGeneration: "first" }, isReady: async () => true, countView: async () => {} };
|
||||
utils.getRepo = async () => repo;
|
||||
AnonymizedFile.prototype.originalPath = async function () { return this.anonymizedPath; };
|
||||
AnonymizedFile.prototype.sha = async () => blobSha;
|
||||
AnonymizedFile.prototype.size = async () => 10;
|
||||
AnonymizedFile.prototype.getFileInfo = async () => ({ name: "page.md", path: "", size: 10 });
|
||||
AnonymizedFile.prototype.send = async function (res) { transfers++; res.send("fixture content"); };
|
||||
AnonymizedFile.prototype.anonymizedContent = async () => { transfers++; return Readable.from(["# Anonymized page"]); };
|
||||
const app = express(); app.use("/api/repo", fileRouter); app.use("/w", webRouter);
|
||||
server = await new Promise(resolve => { const s = app.listen(0, "127.0.0.1", () => resolve(s)); });
|
||||
});
|
||||
afterEach(async function () { server.closeAllConnections(); await new Promise(resolve => server.close(resolve)); restore(); });
|
||||
function fetch(path, method = "GET", headers = {}) {
|
||||
return new Promise((resolve, reject) => {
|
||||
http.request({ host: "127.0.0.1", port: server.address().port, path, method, headers }, res => {
|
||||
let body = ""; res.on("data", chunk => { body += chunk; });
|
||||
res.on("end", () => resolve({ status: res.statusCode, headers: res.headers, body }));
|
||||
}).on("error", reject).end();
|
||||
});
|
||||
}
|
||||
it("serves file HEAD metadata without reading content", async function () {
|
||||
const response = await fetch("/api/repo/fixture/file/data.txt", "HEAD");
|
||||
expect(response.status).to.equal(200); expect(response.body).to.equal("");
|
||||
expect(response.headers["etag"]).to.match(/^"f-/); expect(response.headers["content-type"]).to.include("text/plain");
|
||||
expect(transfers).to.equal(0);
|
||||
});
|
||||
it("preserves content gates for HEAD", async function () {
|
||||
repo.options.pdf = false;
|
||||
expect((await fetch("/api/repo/fixture/file/private.pdf", "HEAD")).status).to.equal(403);
|
||||
expect(transfers).to.equal(0);
|
||||
});
|
||||
it("rejects oversized file HEAD without downloading", async function () {
|
||||
AnonymizedFile.prototype.size = async () => config.MAX_FILE_SIZE + 1;
|
||||
expect((await fetch("/api/repo/fixture/file/huge.txt", "HEAD")).status).to.equal(413); expect(transfers).to.equal(0);
|
||||
});
|
||||
it("accepts weak and multiple validators without transferring file content", async function () {
|
||||
const first = await fetch("/api/repo/fixture/file/data.txt", "HEAD");
|
||||
const response = await fetch("/api/repo/fixture/file/data.txt", "GET", { "If-None-Match": `"unrelated", W/${first.headers.etag}` });
|
||||
expect(response.status).to.equal(304); expect(transfers).to.equal(0);
|
||||
});
|
||||
it("revalidates a Markdown website before fetching or rendering it again", async function () {
|
||||
const first = await fetch("/w/fixture/page.md");
|
||||
expect(first.status).to.equal(200); expect(first.body).to.include("Anonymized page"); expect(transfers).to.equal(1);
|
||||
const second = await fetch("/w/fixture/page.md", "GET", { "If-None-Match": `W/${first.headers.etag}` });
|
||||
expect(second.status).to.equal(304); expect(second.body).to.equal(""); expect(transfers).to.equal(1);
|
||||
expect(second.headers["content-security-policy"]).to.include("sandbox");
|
||||
});
|
||||
it("renders Markdown again after a transient failure served the raw fallback", async function () {
|
||||
const marked = require("marked").marked, render = marked.parse;
|
||||
let first;
|
||||
try {
|
||||
marked.parse = () => { throw Error("transient rendering failure"); };
|
||||
first = await fetch("/w/fixture/page.md");
|
||||
expect(first.status).to.equal(200); expect(first.body).to.equal("fixture content");
|
||||
} finally { marked.parse = render; }
|
||||
const recovered = await fetch("/w/fixture/page.md", "GET", { "If-None-Match": first.headers.etag });
|
||||
expect(recovered.status).to.equal(200); expect(recovered.body).to.include("Anonymized page");
|
||||
expect(recovered.headers.etag).not.to.equal(first.headers.etag);
|
||||
const revalidated = await fetch("/w/fixture/page.md", "GET", { "If-None-Match": recovered.headers.etag });
|
||||
expect(revalidated.status).to.equal(304); expect(transfers).to.equal(3);
|
||||
});
|
||||
it("serves website HEAD without reading or rendering Markdown", async function () {
|
||||
const response = await fetch("/w/fixture/page.md", "HEAD");
|
||||
expect(response.status).to.equal(200); expect(response.headers["content-type"]).to.include("text/html"); expect(transfers).to.equal(0);
|
||||
});
|
||||
it("serves TypeScript website HEAD with the same text MIME policy as GET", async function () {
|
||||
const response = await fetch("/w/fixture/code.ts", "HEAD");
|
||||
expect(response.status).to.equal(200); expect(response.headers["content-type"]).to.include("text/plain"); expect(transfers).to.equal(0);
|
||||
});
|
||||
for (const change of ["terms", "commit", "tree", "cache", "blob", "source name"]) {
|
||||
it(`invalidates the website validator when ${change} changes`, async function () {
|
||||
const first = await fetch("/w/fixture/page.md");
|
||||
if (change === "terms") repo.options.terms = ["new author"];
|
||||
if (change === "commit") repo.model.source.commit = "new commit";
|
||||
if (change === "tree") repo.model.treeGeneration = "new tree";
|
||||
if (change === "cache") repo.model.contentCacheRevision = "new cache";
|
||||
if (change === "blob") blobSha = "new blob";
|
||||
if (change === "source name") repo.model.source.repositoryName = "new/source";
|
||||
const second = await fetch("/w/fixture/page.md", "GET", { "If-None-Match": first.headers.etag });
|
||||
expect(second.status).to.equal(200); expect(second.headers.etag).not.to.equal(first.headers.etag); expect(transfers).to.equal(2);
|
||||
});
|
||||
}
|
||||
it("checks website permissions before honoring an old validator", async function () {
|
||||
const first = await fetch("/w/fixture/page.md"); repo.options.page = false;
|
||||
expect((await fetch("/w/fixture/page.md", "GET", { "If-None-Match": first.headers.etag })).status).to.equal(400);
|
||||
expect(transfers).to.equal(1);
|
||||
});
|
||||
});
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
const process = require("node:process");
|
||||
const { Readable } = require("node:stream");
|
||||
const { setTimeout: delay } = require("node:timers/promises");
|
||||
require("ts-node/register/transpile-only");
|
||||
const GitHubStream = require("../../src/core/source/GitHubStream").default;
|
||||
const source = new GitHubStream({ repoId: "replicas", organization: "owner", repoName: "repo", commit: "commit",
|
||||
cacheGeneration: process.env.TEST_GENERATION || "generation1", getToken: () => "token" });
|
||||
source.downloadWithFallback = async () => {
|
||||
process.send({ producer: true });
|
||||
return Readable.from((async function* () {
|
||||
yield Buffer.from("complete "); await delay(200); yield Buffer.from("content");
|
||||
})());
|
||||
};
|
||||
process.once("message", async () => {
|
||||
try {
|
||||
const contents = await Promise.all(Array.from({ length: 5 }, async () => {
|
||||
const input = await source.getFileContentCache("file.txt", "replicas", () => ({ sha: "blob", size: 16 }));
|
||||
let text = ""; for await (const chunk of input) text += chunk; return text;
|
||||
}));
|
||||
process.send({ contents }); process.exit(0);
|
||||
} catch (error) { process.send({ error: error.message }); process.exit(1); }
|
||||
});
|
||||
process.send({ ready: true });
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
const { Readable } = require("node:stream");
|
||||
const path = require("node:path");
|
||||
const compiled = process.env.PERFORMANCE_STREAMER_COMPILED === "1";
|
||||
if (!compiled) require("ts-node/register/transpile-only");
|
||||
const base = path.join(__dirname, compiled ? "../../build" : "../../src");
|
||||
const GitHubStream = require(path.join(base, "core/source/GitHubStream")).default;
|
||||
const FileSystem = require(path.join(base, "core/storage/FileSystem")).default;
|
||||
const maintenance = require(path.join(base, "core/temporary-storage"));
|
||||
const startMaintenance = maintenance.startTemporaryStorageMaintenance;
|
||||
// Exercise the real startup/timer wiring without waiting a production minute.
|
||||
maintenance.startTemporaryStorageMaintenance = () => startMaintenance(25);
|
||||
GitHubStream.prototype.downloadWithFallback = async function () {
|
||||
process.send({ download: true });
|
||||
return Readable.from([Buffer.alloc(300000, "x")]);
|
||||
};
|
||||
const write = FileSystem.prototype.write;
|
||||
let release, releaseConnection;
|
||||
if (process.env.PERFORMANCE_HOLD_CONNECT === "1") {
|
||||
const database = require(path.join(base, "server/database")), connect = database.connect;
|
||||
database.connect = async (...args) => {
|
||||
const held = new Promise(resolve => { releaseConnection = resolve; });
|
||||
process.send({ connectionHeld: true }); await held;
|
||||
return connect(...args);
|
||||
};
|
||||
}
|
||||
FileSystem.prototype.write = async function (...args) {
|
||||
if (args[1].endsWith("/hold.txt")) {
|
||||
const held = new Promise(resolve => { release = resolve; });
|
||||
process.send({ held: true }); await held;
|
||||
}
|
||||
return write.apply(this, args);
|
||||
};
|
||||
if (process.env.CREDENTIAL_KEYS !== "invalid") process.on("message", message => {
|
||||
if (message === "release") release?.();
|
||||
if (message === "connect") releaseConnection?.();
|
||||
if (message === "inspect") {
|
||||
const options = require("mongoose").connection.getClient().options;
|
||||
process.send({ connected: require(path.join(base, "server/database")).isConnected,
|
||||
maxPoolSize: options.maxPoolSize, minPoolSize: options.minPoolSize });
|
||||
}
|
||||
});
|
||||
require(path.join(base, "streamer/index"));
|
||||
Vendored
+62
@@ -0,0 +1,62 @@
|
||||
const http = require("node:http");
|
||||
const { URL } = require("node:url");
|
||||
const objects = new Map();
|
||||
module.exports = async function s3Fixture() {
|
||||
objects.clear();
|
||||
const requests = { lists: 0, deletions: [] };
|
||||
const server = http.createServer((req, res) => {
|
||||
const url = new URL(req.url, "http://localhost");
|
||||
const key = decodeURIComponent(url.pathname).split("/").slice(2).join("/");
|
||||
if (req.method === "POST" && url.searchParams.has("delete")) {
|
||||
const chunks = [];
|
||||
req.on("data", chunk => chunks.push(chunk));
|
||||
req.on("end", () => {
|
||||
const body = Buffer.concat(chunks).toString();
|
||||
const keys = [...body.matchAll(/<Key>(.*?)<\/Key>/g)].map(match => match[1].replace(/</g, "<").replace(/>/g, ">").replace(/&/g, "&"));
|
||||
requests.deletions.push(keys);
|
||||
keys.forEach(name => objects.delete(name));
|
||||
res.setHeader("Content-Type", "application/xml");
|
||||
res.end('<DeleteResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"/>');
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (req.method === "PUT") {
|
||||
const chunks = [];
|
||||
req.on("data", chunk => chunks.push(chunk));
|
||||
req.on("end", () => {
|
||||
let body = Buffer.concat(chunks);
|
||||
if (req.headers["content-encoding"]?.includes("aws-chunked")) {
|
||||
const decoded = []; let offset = 0;
|
||||
while (offset < body.length) {
|
||||
const end = body.indexOf("\r\n", offset); if (end < 0) break;
|
||||
const size = parseInt(body.toString("ascii", offset, end).split(";")[0], 16);
|
||||
if (!size) break;
|
||||
decoded.push(body.subarray(end + 2, end + 2 + size)); offset = end + 2 + size + 2;
|
||||
}
|
||||
body = Buffer.concat(decoded);
|
||||
}
|
||||
if (key) objects.set(key, body);
|
||||
res.setHeader("ETag", '"fixture-etag"'); res.end();
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (url.searchParams.has("list-type")) {
|
||||
const prefix = url.searchParams.get("prefix") || "";
|
||||
requests.lists++;
|
||||
const escaped = value => value.replace(/&/g, "&").replace(/</g, "<");
|
||||
const remaining = [...objects].filter(([name]) => name.startsWith(prefix) && name > (url.searchParams.get("continuation-token") || "")).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0);
|
||||
const page = remaining.slice(0, Number(url.searchParams.get("max-keys") || 1000));
|
||||
const truncated = remaining.length > page.length;
|
||||
const contents = page.map(([name, body]) => `<Contents><Key>${escaped(name)}</Key><Size>${body.length}</Size></Contents>`).join("");
|
||||
res.setHeader("Content-Type", "application/xml");
|
||||
res.end(`<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><IsTruncated>${truncated}</IsTruncated>${truncated ? `<NextContinuationToken>${escaped(page.at(-1)[0])}</NextContinuationToken>` : ""}${contents}</ListBucketResult>`); return;
|
||||
}
|
||||
if (!objects.has(key)) { res.statusCode = 404; res.end("<Error><Code>NoSuchKey</Code></Error>"); return; }
|
||||
const body = objects.get(key);
|
||||
res.setHeader("Content-Length", body.length); res.setHeader("Content-Type", "text/plain");
|
||||
res.setHeader("Last-Modified", new Date(0).toUTCString());
|
||||
if (req.method === "HEAD") res.end(); else res.end(body);
|
||||
});
|
||||
await new Promise(resolve => server.listen(0, "127.0.0.1", resolve));
|
||||
return { endpoint: `http://127.0.0.1:${server.address().port}`, requests, seed: (key, data) => objects.set(key, Buffer.from(data)), close: () => new Promise(resolve => server.close(resolve)) };
|
||||
};
|
||||
@@ -9,6 +9,7 @@ function harness(date) {
|
||||
const defs = {}, routes = {}, timers = new Map();
|
||||
let timerId = 0;
|
||||
const context = { reactive: value => value, console, Map, Set, Date: date || Date,
|
||||
formDraft: state => ({ restore: () => false, clear() {}, discard() {}, checkpoint: () => ({ terms: state.terms, options: state.options }), accept() {} }),
|
||||
navigator: { platform: "Linux" }, document: { location: { pathname: "/r/repo" }, addEventListener() {}, querySelector() {} },
|
||||
window: {}, Prism: { highlightAll() {} }, encodeURIComponent,
|
||||
encodePathForUrl: p => p.split("/").map(encodeURIComponent).join("/"),
|
||||
@@ -71,6 +72,11 @@ describe("frontend production regressions", function () {
|
||||
expect(h.scope.options.expirationDate.toISOString()).to.equal("2026-11-01T00:00:00.000Z");
|
||||
expect(h.scope.conference_data).not.to.equal(null);
|
||||
if (id === "repoId") {
|
||||
h.requests.find(r => r.url === "/api/repo/owner/repo/").resolve({ data: { repo: "repo", defaultBranch: "main" } });
|
||||
h.requests.find(r => r.url === "/api/repo/owner/repo/readme").resolve({ data: "README" });
|
||||
await h.flush();
|
||||
h.requests.find(r => r.url === "/api/repo/owner/repo/branches").resolve({ data: [{ name: "main", commit: "abcdef", readme: "README" }] });
|
||||
await h.flush();
|
||||
h.scope.anonymizeRepo({ target: {} });
|
||||
expect(h.requests.at(-1).body.options.update).to.equal(false);
|
||||
expect(h.requests.at(-1).url).to.equal("/api/repo/saved");
|
||||
@@ -163,9 +169,9 @@ describe("frontend production regressions", function () {
|
||||
it("translates profile save failures", async function () {
|
||||
const h = harness(); expect(h.defs.profileController.toString()).to.include("translate");
|
||||
const timeout = Object.assign(() => 0, { cancel() {} });
|
||||
h.defs.profileController(h.scope, h.http, key => Promise.resolve(key), timeout, { load: () => Promise.resolve({}) });
|
||||
h.defs.profileController(h.scope, h.http, key => Promise.resolve(key === "ERRORS.not_connected" ? "Please sign in again." : key), timeout, { load: () => Promise.resolve({}) });
|
||||
h.scope.saveDefault(); h.requests.at(-1).reject({ data: { error: "not_connected" } }); await h.flush();
|
||||
expect(h.scope.error).to.equal("ERRORS.not_connected");
|
||||
expect(h.scope.error).to.equal("Please sign in again.");
|
||||
});
|
||||
describe("landing page features", function () {
|
||||
const home = fs.readFileSync(path.join(__dirname, "..", "public", "partials", "home.htm"), "utf8");
|
||||
@@ -220,3 +226,41 @@ describe("frontend production regressions", function () {
|
||||
expect(h.scope.options.endDate.getTime()).to.be.greaterThan(h.scope.options.startDate.getTime());
|
||||
});
|
||||
});
|
||||
|
||||
describe("redaction highlighting", () => {
|
||||
const { JSDOM } = require("jsdom");
|
||||
function helpers() {
|
||||
const { window } = new JSDOM("");
|
||||
const h = harness();
|
||||
Object.assign(h.context, { DOMParser: window.DOMParser, NodeFilter: window.NodeFilter });
|
||||
return h.defs;
|
||||
}
|
||||
it("matches the mask with and without a counter", () => {
|
||||
const { redactionPattern } = helpers();
|
||||
expect("by XXXX-1 and XXXX-12 at XXXX.".match(redactionPattern("XXXX"))).to.deep.equal(["XXXX-1", "XXXX-12", "XXXX"]);
|
||||
expect("a.b-3 axb-3".match(redactionPattern("a.b"))).to.deep.equal(["a.b-3"]);
|
||||
});
|
||||
it("marks redactions in text without touching attributes", () => {
|
||||
const { highlightRedactions } = helpers();
|
||||
const html = highlightRedactions('<p>By XXXX-1 <a href="https://x.test/XXXX-2">link</a></p>', "XXXX");
|
||||
expect(html).to.include('By <mark class="redaction" title="Redacted by Anonymous GitHub">XXXX-1</mark>');
|
||||
expect(html).to.include('href="https://x.test/XXXX-2"');
|
||||
expect(highlightRedactions("<p>nothing here</p>", "XXXX")).to.equal("<p>nothing here</p>");
|
||||
});
|
||||
it("leaves Mermaid sources and Prism code blocks for their renderers", () => {
|
||||
const { highlightRedactions } = helpers();
|
||||
const html = highlightRedactions('<div class="mermaid">graph TD; XXXX-1-->B</div><pre><code class="language-py">print("XXXX-2")</code></pre><code>XXXX-3</code>', "XXXX");
|
||||
expect(html).to.include('<div class="mermaid">graph TD; XXXX-1-->B</div>');
|
||||
expect(html).to.include('<code class="language-py">print("XXXX-2")</code>');
|
||||
expect(html).to.include('<code><mark class="redaction" title="Redacted by Anonymous GitHub">XXXX-3</mark></code>');
|
||||
});
|
||||
it("marks a highlighted code block when Prism hands it over, using the configured mask", () => {
|
||||
const { window } = new (require("jsdom").JSDOM)('<pre><code class="language-py"><span class="token string">"MASK-4"</span></code></pre>');
|
||||
const h = harness();
|
||||
Object.assign(h.context, { DOMParser: window.DOMParser, NodeFilter: window.NodeFilter });
|
||||
h.defs.setRedactionMask("MASK");
|
||||
const code = window.document.querySelector("code");
|
||||
h.defs.markRedactionsIn(code);
|
||||
expect(code.querySelector("mark.redaction").textContent).to.equal("MASK-4");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,59 @@
|
||||
const { expect } = require("chai");
|
||||
require("ts-node/register/transpile-only");
|
||||
const github = require("../src/core/GitHubUtils");
|
||||
const Gist = require("../src/core/Gist").default;
|
||||
const Model = require("../src/core/model/anonymizedGists/anonymizedGists.model").default;
|
||||
const { handleError } = require("../src/server/routes/route-utils");
|
||||
|
||||
describe("gist authentication errors", function () {
|
||||
let original, gist, client;
|
||||
beforeEach(function () {
|
||||
original = github.octokit;
|
||||
gist = new Gist(new Model({ source: { gistId: "example" }, gist: { files: [{ filename: "saved.txt", content: "saved" }] } }));
|
||||
gist.getAccess = async () => ({ token: "rejected-token", connection: "oauth" });
|
||||
client = { rest: { gists: { get: async () => ({ data: { files: {} } }) } }, paginate: async () => [], request: async () => ({ data: "raw" }) };
|
||||
github.octokit = token => { expect(token).to.equal("rejected-token"); return client; };
|
||||
});
|
||||
afterEach(function () { github.octokit = original; });
|
||||
for (const operation of ["metadata", "comments", "raw file"]) {
|
||||
it("requests OAuth reconnection when GitHub rejects " + operation, async function () {
|
||||
const failure = Object.assign(new Error("Bad credentials"), { status: 401 });
|
||||
const reject = async () => { throw failure; };
|
||||
if (operation === "metadata") client.rest.gists.get = reject;
|
||||
if (operation === "comments") client.paginate = reject;
|
||||
if (operation === "raw file") {
|
||||
client.rest.gists.get = async () => ({ data: { files: { file: { filename: "large.txt", truncated: true, raw_url: "https://gist.githubusercontent.com/example/raw" } } } });
|
||||
client.request = reject;
|
||||
}
|
||||
let error;
|
||||
try { await gist.download(); } catch (caught) { error = caught; }
|
||||
expect(error?.message).to.equal("github_oauth_required");
|
||||
expect(error.cause).to.equal(failure);
|
||||
let status, body;
|
||||
handleError(error, { headersSent: false, status: value => { status = value; return { json: value => { body = value; } }; } });
|
||||
expect(status).to.equal(403);
|
||||
expect(body).to.deep.equal({ error: "github_oauth_required" });
|
||||
expect(gist.model.gist.files[0].content).to.equal("saved");
|
||||
});
|
||||
}
|
||||
for (const status of [403, 404, 429, 500]) {
|
||||
it("preserves non-authentication GitHub errors: " + status, async function () {
|
||||
const failure = Object.assign(new Error("Other GitHub failure"), { status });
|
||||
client.rest.gists.get = async () => { throw failure; };
|
||||
let error;
|
||||
try { await gist.download(); } catch (caught) { error = caught; }
|
||||
expect(error).to.equal(failure);
|
||||
});
|
||||
}
|
||||
it("requests App reconnection when an App user token is rejected", async function () {
|
||||
gist.getAccess = async () => ({ token: "rejected-token", connection: "github-app" });
|
||||
client.rest.gists.get = async () => { throw Object.assign(new Error("Bad credentials"), { status: 401 }); };
|
||||
let error;
|
||||
try { await gist.download(); } catch (caught) { error = caught; }
|
||||
expect(error?.message).to.equal("github_app_reconnect_required");
|
||||
});
|
||||
it("still downloads a gist when the credential is accepted", async function () {
|
||||
await gist.download();
|
||||
expect(gist.model.gist.files).to.have.length(0);
|
||||
});
|
||||
});
|
||||
+163
-2
@@ -175,7 +175,7 @@ describeMongo("GitHub App credential and repository integration", function () {
|
||||
const result = String(url).endsWith("/token/scoped")
|
||||
? scopeHandler ? await scopeHandler(body, options) : { status: 403 }
|
||||
: await handler(String(url), options, body);
|
||||
return new globalThis.Response(JSON.stringify(result.body || result), { status: result.status || 200, headers: { "content-type": "application/json" } });
|
||||
return new globalThis.Response(JSON.stringify(result.body || result), { status: result.status || 200, headers: { "content-type": "application/json", ...result.headers } });
|
||||
};
|
||||
}
|
||||
it("keeps OAuth and App grants separate, encrypted and migration-verifiable", async () => {
|
||||
@@ -191,6 +191,68 @@ describeMongo("GitHub App credential and repository integration", function () {
|
||||
expect(projected.encryptedToken).to.equal(undefined);
|
||||
expect(await verifyCredentials(mongoose.connection.db, createTokenCipher(keys, "test"))).to.deep.equal({ checked: 2, legacy: 0 });
|
||||
});
|
||||
for (const legacy of [false, true]) {
|
||||
it(`reads gist files and comments with an App user grant (legacy: ${legacy})`, async () => {
|
||||
if (legacy) await setCredential(owner.id, "legacy-secret");
|
||||
await app.saveAppGrant(owner.id, data("old", -1));
|
||||
mock((url, options, body) => {
|
||||
if (url.includes("/login/oauth/access_token")) {
|
||||
expect(body.refresh_token).to.equal("ghr_refreshold");
|
||||
return data("new");
|
||||
}
|
||||
expect(new globalThis.Headers(options.headers).get("authorization")).to.equal("token ghu_accessnew");
|
||||
if (url.includes("/comments")) return [{ body: "comment", user: { login: "owner" } }];
|
||||
expect(url).to.include("/gists/example");
|
||||
return { description: "gist", files: { file: { filename: "file.txt", content: "hello" } } };
|
||||
});
|
||||
const Gist = require("../src/core/Gist").default;
|
||||
const Model = require("../src/core/model/anonymizedGists/anonymizedGists.model").default;
|
||||
const gist = new Gist(new Model({ owner: owner.id, source: { gistId: "example" } }));
|
||||
await gist.download();
|
||||
expect(gist.model.gist.files[0].content).to.equal("hello");
|
||||
expect(gist.model.gist.comments[0].body).to.equal("comment");
|
||||
expect(calls).to.have.length(3);
|
||||
});
|
||||
}
|
||||
for (const rejected of [false, true]) {
|
||||
it(`renews App gist access during comment pagination (401: ${rejected})`, async () => {
|
||||
await app.saveAppGrant(owner.id, data("old"));
|
||||
let pages = 0, refreshed = false;
|
||||
mock(async (url, options) => {
|
||||
if (url.includes("/login/oauth/access_token")) { refreshed = true; return data("new"); }
|
||||
const authorization = new globalThis.Headers(options.headers).get("authorization");
|
||||
if (url.includes("/comments")) {
|
||||
pages++;
|
||||
if (pages === 1) {
|
||||
expect(authorization).to.equal("token ghu_accessold");
|
||||
await Credentials.updateOne({ ownerId: owner.id, provider: app.APP_PROVIDER }, { $set: { expiresAt: new Date(0) } });
|
||||
if (rejected) return { status: 401, body: { message: "Bad credentials" } };
|
||||
return { body: [{ body: "first", user: { login: "owner" } }], headers: { link: '<https://api.github.com/gists/example/comments?page=2>; rel="next"' } };
|
||||
}
|
||||
expect(authorization).to.equal("token ghu_accessnew");
|
||||
return [{ body: "last", user: { login: "owner" } }];
|
||||
}
|
||||
return { files: {} };
|
||||
});
|
||||
const Gist = require("../src/core/Gist").default;
|
||||
const Model = require("../src/core/model/anonymizedGists/anonymizedGists.model").default;
|
||||
const gist = new Gist(new Model({ owner: owner.id, source: { gistId: "example" } }));
|
||||
await gist.download();
|
||||
expect(refreshed).to.equal(true);
|
||||
expect(pages).to.equal(2);
|
||||
expect(gist.model.gist.comments.map(c => c.body)).to.deep.equal(rejected ? ["last"] : ["first", "last"]);
|
||||
});
|
||||
}
|
||||
it("keeps legacy gist access but does not bypass revoked App grants", async () => {
|
||||
await setCredential(owner.id, "legacy-secret");
|
||||
const Gist = require("../src/core/Gist").default;
|
||||
const Model = require("../src/core/model/anonymizedGists/anonymizedGists.model").default;
|
||||
const gist = new Gist(new Model({ owner: owner.id, source: { gistId: "example" } }));
|
||||
expect(await gist.getToken()).to.equal("legacy-secret");
|
||||
await app.saveAppGrant(owner.id, data());
|
||||
await Credentials.updateOne({ ownerId: owner.id, provider: app.APP_PROVIDER }, { $set: { revoked: true } });
|
||||
await rejects(gist.download(), "github_app_reconnect_required");
|
||||
});
|
||||
it("serializes refresh and atomically rotates the token pair", async () => {
|
||||
await app.saveAppGrant(owner.id, data("old", -1));
|
||||
mock(async (_url, _options, body) => {
|
||||
@@ -335,7 +397,7 @@ describeMongo("GitHub App credential and repository integration", function () {
|
||||
it("does not require scoped-token minting for an uninstalled public repository", async () => {
|
||||
await app.saveAppGrant(owner.id, data());
|
||||
mock(url => url.includes("/user/installations") ? { installations: [] } :
|
||||
{ id: 7, private: false, visibility: "public" }, () => ({ status: 403 }));
|
||||
{ id: 7, full_name: "other/public", private: false, visibility: "public" }, () => ({ status: 403 }));
|
||||
const selected = await app.selectRepositoryAccess(owner.id, "other/public", "github-app");
|
||||
expect(selected.token).to.match(/^public-read:/);
|
||||
expect(calls.some(call => call.url.endsWith("/token/scoped"))).to.equal(false);
|
||||
@@ -415,6 +477,48 @@ describeMongo("GitHub App credential and repository integration", function () {
|
||||
headers: { "Content-Type": "application/json", "X-CSRF-Token": csrf }, body: body ? JSON.stringify(body) : undefined });
|
||||
return { status: response.status, location: response.headers.get("location"), data: await response.json().catch(() => null) };
|
||||
}
|
||||
it("allows OAuth revocation with an existing gist and keeps gist access working", async () => {
|
||||
await setCredential(owner.id, "legacy-secret");
|
||||
await app.saveAppGrant(owner.id, data());
|
||||
const Model = require("../src/core/model/anonymizedGists/anonymizedGists.model").default;
|
||||
const model = await Model.create({ owner: owner.id, gistId: "oauth-revoke-gist", source: { gistId: "example" } });
|
||||
mock((url, options, body) => {
|
||||
if (url.includes("/user/installations")) {
|
||||
expect(new globalThis.Headers(options.headers).get("authorization")).to.equal("Bearer ghu_access1");
|
||||
return { installations: [] };
|
||||
}
|
||||
if (url.includes("/grant")) {
|
||||
expect(options.method).to.equal("DELETE");
|
||||
expect(body.access_token).to.equal("legacy-secret");
|
||||
return {};
|
||||
}
|
||||
expect(new globalThis.Headers(options.headers).get("authorization")).to.equal("token ghu_access1");
|
||||
return url.includes("/comments") ? [] : { files: { file: { filename: "file.txt", content: "still accessible" } } };
|
||||
});
|
||||
const result = await request("/github/connections/disconnect-oauth", {});
|
||||
expect(result.status).to.equal(200);
|
||||
expect(await getCredentialToken(owner.id)).not.to.be.ok;
|
||||
const Gist = require("../src/core/Gist").default;
|
||||
const gist = new Gist(model);
|
||||
await gist.download();
|
||||
expect(gist.model.gist.files[0].content).to.equal("still accessible");
|
||||
});
|
||||
for (const status of [401, 503]) {
|
||||
it(`preserves OAuth when GitHub cannot verify the App grant (${status})`, async () => {
|
||||
await setCredential(owner.id, "legacy-secret");
|
||||
await app.saveAppGrant(owner.id, data());
|
||||
mock((url, options) => {
|
||||
expect(url).to.include("/user/installations");
|
||||
expect(options.method).not.to.equal("DELETE");
|
||||
return { status, body: { message: "unavailable" } };
|
||||
});
|
||||
const result = await request("/github/connections/disconnect-oauth", {});
|
||||
expect(result.status).to.equal(status === 401 ? 403 : 502);
|
||||
expect(result.data.error).to.equal(status === 401 ? "github_app_reconnect_required" : "github_unavailable");
|
||||
expect(await getCredentialToken(owner.id)).to.equal("legacy-secret");
|
||||
expect(calls).to.have.length(1);
|
||||
});
|
||||
}
|
||||
it("links App authorization to the existing OAuth account without replacing its grant", async () => {
|
||||
await setCredential(owner.id, "legacy-secret");
|
||||
owner.isAdmin = true; await owner.save();
|
||||
@@ -496,6 +600,63 @@ describeMongo("GitHub App credential and repository integration", function () {
|
||||
expect(await Credentials.countDocuments()).to.equal(0);
|
||||
expect(await Users.countDocuments()).to.equal(1);
|
||||
});
|
||||
it("reconnects an uninstalled public repository with a real user token", async () => {
|
||||
const Repos = require("../src/core/model/anonymizedRepositories/anonymizedRepositories.model").default;
|
||||
await app.saveAppGrant(owner.id, data());
|
||||
const CachedRepos = require("../src/core/model/repositories/repositories.model").default;
|
||||
const cached = await CachedRepos.create({ externalId: "gh_42", name: "other/old-public", url: "https://github.com/other/old-public" });
|
||||
const resource = await Repos.create({ repoId: "public-reconnect", owner: owner.id, status: "ready",
|
||||
source: { type: "GitHubStream", repositoryName: "other/old-public", repositoryId: cached.id, commit: "abc", branch: "main" },
|
||||
githubAccess: { kind: "github-app", repositoryId: 42, publicRead: true, revision: "old" } });
|
||||
mock((url, options) => {
|
||||
if (url.includes("/user/installations?")) return { installations: [] };
|
||||
if (url.endsWith("/repositories/42") || url.endsWith("/repos/other/public")) return {
|
||||
id: 42, full_name: "other/public", private: false, visibility: "public",
|
||||
};
|
||||
if (url.endsWith("/repos/other/public/commits/abc")) {
|
||||
expect(new globalThis.Headers(options.headers).get("authorization")).to.equal("Bearer ghu_access1");
|
||||
return { sha: "abc" };
|
||||
}
|
||||
throw new Error("Unexpected request " + url);
|
||||
});
|
||||
const body = { type: "repository", id: resource.repoId, connection: "github-app" };
|
||||
expect((await request("/github/connections/migrate", { ...body, preview: true })).status).to.equal(200);
|
||||
expect((await CachedRepos.findById(cached.id)).name).to.equal("other/old-public");
|
||||
expect((await request("/github/connections/migrate", body)).status).to.equal(200);
|
||||
expect((await Repos.findById(resource.id)).githubAccess.publicRead).to.equal(true);
|
||||
expect((await Repos.findById(resource.id)).source.repositoryName).to.equal("other/public");
|
||||
expect((await CachedRepos.findById(cached.id)).name).to.equal("other/public");
|
||||
});
|
||||
|
||||
it("recovers a renamed private pull request through its stable repository ID", async () => {
|
||||
const PRs = require("../src/core/model/anonymizedPullRequests/anonymizedPullRequests.model").default;
|
||||
await app.saveAppGrant(owner.id, data());
|
||||
const resource = await PRs.create({ pullRequestId: "renamed-pr", owner: owner.id, status: "ready",
|
||||
source: { repositoryFullName: "owner/old-name", pullRequestId: 7 }, options: { terms: ["owner"] },
|
||||
githubAccess: { kind: "github-app", repositoryId: 42, installationId: 3, revision: "old" } });
|
||||
mock(url => {
|
||||
if (url.includes("/user/installations?")) return { installations: [{ id: 4, app_id: 123, account: { id: 10, login: "owner", type: "User" } }] };
|
||||
if (url.includes("/user/installations/4/repositories")) return { repositories: [
|
||||
{ id: 99, full_name: "owner/old-name", private: true }, { id: 42, full_name: "owner/new-name", private: true },
|
||||
] };
|
||||
if (url.endsWith("/repositories/42")) return { id: 42 };
|
||||
if (url.endsWith("/app/installations/4")) return { app_id: 123, permissions: { contents: "read", metadata: "read" } };
|
||||
if (url.endsWith("/access_tokens")) return { token: "ghs_reinstalled", expires_at: new Date(Date.now() + 3600000).toISOString() };
|
||||
if (url.endsWith("/repos/owner/new-name/pulls/7")) return { number: 7 };
|
||||
throw new Error("Unexpected request " + url);
|
||||
});
|
||||
const body = { type: "pull-request", id: resource.pullRequestId, connection: "github-app" };
|
||||
expect((await request("/github/connections/migrate", { ...body, preview: true })).status).to.equal(200);
|
||||
expect((await PRs.findById(resource.id)).githubAccess.installationId).to.equal(3);
|
||||
expect((await request("/github/connections/migrate", body)).status).to.equal(200);
|
||||
const repaired = await PRs.findById(resource.id);
|
||||
expect(repaired.githubAccess.repositoryId).to.equal(42);
|
||||
expect(repaired.githubAccess.installationId).to.equal(4);
|
||||
expect(repaired.source.repositoryFullName).to.equal("owner/new-name");
|
||||
expect(repaired.source.pullRequestId).to.equal(7);
|
||||
expect(repaired.pullRequestId).to.equal("renamed-pr");
|
||||
});
|
||||
|
||||
it("migrates only the owner's resource after checking its configured commit", async () => {
|
||||
const Repos = require("../src/core/model/anonymizedRepositories/anonymizedRepositories.model").default;
|
||||
await app.saveAppGrant(owner.id, data());
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
const { expect } = require("chai");
|
||||
const { Readable, PassThrough } = require("stream");
|
||||
const { once } = require("events");
|
||||
const http = require("node:http");
|
||||
const express = require("express");
|
||||
const compression = require("compression");
|
||||
const { randomBytes } = require("node:crypto");
|
||||
const { gunzipSync } = require("node:zlib");
|
||||
const archiver = require("archiver");
|
||||
const { Parse } = require("unzip-stream");
|
||||
require("ts-node/register/transpile-only");
|
||||
@@ -81,17 +86,69 @@ describe("large notebook downloads", function () {
|
||||
GitHubDownload.prototype.getZipUrl = async () => ({ url: "https://example.invalid/source.zip" });
|
||||
got.stream = () => upstream;
|
||||
});
|
||||
afterEach(function () {
|
||||
afterEach(async function () {
|
||||
got.stream = originalStream;
|
||||
GitHubDownload.prototype.getZipUrl = originalUrl;
|
||||
config.MAX_FILE_SIZE = originalLimit;
|
||||
upstream?.destroy();
|
||||
await require("../src/core/cache-coordination").closeCacheRedis();
|
||||
});
|
||||
const options = {
|
||||
repoId: "fixture", organization: "owner", repoName: "repo", commit: "HEAD",
|
||||
getToken: () => "test", anonymizerOptions: { filePath: "", terms: ["Alice"] },
|
||||
};
|
||||
|
||||
for (const compressed of [false, true]) {
|
||||
it(`finishes ZIP responses through Express compression with gzip ${compressed}`, async function () {
|
||||
const text = "Alice Σ 研究\n".repeat(25000), binary = randomBytes(128 * 1024);
|
||||
const input = await zip({ "root/empty.txt": "", "root/large.txt": text, "root/image.png": binary, "root/after.txt": "Alice" });
|
||||
upstream = Readable.from((async function* () {
|
||||
for (let offset = 0; offset < input.length; offset += 1021) {
|
||||
yield input.subarray(offset, offset + 1021);
|
||||
}
|
||||
})());
|
||||
const app = express();
|
||||
app.use(compression({ threshold: 0, highWaterMark: 1024, filter: () => compressed }));
|
||||
const drainEmitters = new Set();
|
||||
let drainSubscriptions = 0;
|
||||
app.get("/archive", async (_req, res) => {
|
||||
const on = res.on;
|
||||
res.on = function (event, listener) {
|
||||
const emitter = on.call(this, event, listener);
|
||||
if (event === "drain") { drainSubscriptions++; drainEmitters.add(emitter); }
|
||||
return emitter;
|
||||
};
|
||||
res.type("application/zip");
|
||||
await streamAnonymizedZip(options, res);
|
||||
});
|
||||
const server = await new Promise(resolve => {
|
||||
const listening = http.createServer({ highWaterMark: 1024 }, app).listen(0, "127.0.0.1", () => resolve(listening));
|
||||
});
|
||||
let client;
|
||||
try {
|
||||
const response = await new Promise((resolve, reject) => {
|
||||
client = http.get(`http://127.0.0.1:${server.address().port}/archive`, { headers: { "Accept-Encoding": "gzip" } }, resolve);
|
||||
client.setTimeout(8000, () => client.destroy(new Error("ZIP response stalled")));
|
||||
client.on("error", reject);
|
||||
});
|
||||
const bytes = await collect(response);
|
||||
expect(response.statusCode).to.equal(200);
|
||||
expect(response.headers["content-encoding"]).to.equal(compressed ? "gzip" : undefined);
|
||||
const entries = await unzip(compressed ? gunzipSync(bytes) : bytes);
|
||||
expect(Object.keys(entries)).to.have.members(["empty.txt", "large.txt", "image.png", "after.txt"]);
|
||||
expect(entries["empty.txt"]).to.have.length(0);
|
||||
expect(entries["large.txt"].toString()).to.equal(new ContentAnonimizer(options.anonymizerOptions).anonymize(text));
|
||||
expect(entries["image.png"]).to.deep.equal(binary);
|
||||
expect(entries["after.txt"].toString()).to.equal("XXXX-1");
|
||||
expect(drainSubscriptions).to.be.greaterThan(0);
|
||||
for (const emitter of drainEmitters) expect(emitter.listenerCount("drain")).to.equal(0);
|
||||
} finally {
|
||||
client?.destroy(); server.closeAllConnections();
|
||||
await new Promise(resolve => server.close(resolve));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
it("completes an archive containing a large notebook and subsequent entries", async function () {
|
||||
upstream = Readable.from([await zip({ "root/example.ipynb": notebook(), "root/after.txt": "Alice" })]);
|
||||
const response = new PassThrough();
|
||||
@@ -103,6 +160,19 @@ describe("large notebook downloads", function () {
|
||||
expect(JSON.parse(entries["example.ipynb"].toString()).cells[0].source[0]).to.equal("# XXXX-1's experiment");
|
||||
});
|
||||
|
||||
it("completes large safe regex entries and continues to the next ZIP entry", async function () {
|
||||
const payload = "123,456,78.91234,-45.123456\n".repeat(280000);
|
||||
upstream = Readable.from([await zip({ "root/large.csv": `Alice\n${payload}Alice`, "root/after.txt": "Alice" })]);
|
||||
const response = new PassThrough(), output = collect(response);
|
||||
await streamAnonymizedZip({ ...options, anonymizerOptions: {
|
||||
terms: ["Alice", ...Array.from({ length: 8 }, (_, i) => `Researcher${i}[0-9]+`)],
|
||||
} }, response);
|
||||
const entries = await unzip(await output);
|
||||
expect(entries["large.csv"].toString()).to.equal(`XXXX-1\n${payload}XXXX-1`);
|
||||
expect(entries["after.txt"].toString()).to.equal("XXXX-1");
|
||||
expect(response.writableFinished).to.equal(true);
|
||||
});
|
||||
|
||||
it("aborts the response when an entry cannot be anonymized", async function () {
|
||||
upstream = Readable.from([await zip({ "root/large.txt": "Alice".repeat(1000) })]);
|
||||
config.MAX_FILE_SIZE = 100;
|
||||
|
||||
@@ -34,7 +34,7 @@ const sanitizeOptions = {
|
||||
* sanitizeHtml(marked(content, opts), sanitizeOptions)
|
||||
*/
|
||||
function renderAndSanitize(markdown) {
|
||||
const raw = marked(markdown, { headerIds: false, mangle: false });
|
||||
const raw = marked(markdown, { async: false });
|
||||
return sanitizeHtml(raw, sanitizeOptions);
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
const { expect } = require("chai");
|
||||
const express = require("express");
|
||||
const http = require("node:http");
|
||||
require("ts-node/register/transpile-only");
|
||||
const utils = require("../src/server/routes/route-utils");
|
||||
const admin = require("../src/server/routes/admin").default;
|
||||
|
||||
describe("performance report access", function () {
|
||||
let server, original;
|
||||
before(async function () {
|
||||
original = utils.getUser;
|
||||
utils.getUser = async req => ({ isAdmin: req.get("x-fixture-user") === "admin", model: { id: "fixture" } });
|
||||
const app = express();
|
||||
app.use((req, _res, next) => { req.isAuthenticated = () => !!req.get("x-fixture-user"); next(); });
|
||||
app.use("/api/admin", admin);
|
||||
server = await new Promise(resolve => { const s = app.listen(0, "127.0.0.1", () => resolve(s)); });
|
||||
});
|
||||
after(async function () { server.closeAllConnections(); await new Promise(resolve => server.close(resolve)); utils.getUser = original; });
|
||||
async function request(user) {
|
||||
return new Promise((resolve, reject) => {
|
||||
http.get({ host: "127.0.0.1", port: server.address().port, path: "/api/admin/performance", headers: user ? { "x-fixture-user": user } : {} }, res => {
|
||||
let body = ""; res.on("data", chunk => { body += chunk; }); res.on("end", () => resolve({ status: res.statusCode, body, headers: res.headers }));
|
||||
}).on("error", reject);
|
||||
});
|
||||
}
|
||||
it("rejects anonymous requests", async function () { expect((await request()).status).to.equal(401); });
|
||||
it("rejects non-admin users", async function () { expect((await request("regular")).status).to.equal(401); });
|
||||
it("allows admins and disables HTTP caching", async function () {
|
||||
const result = await request("admin"); expect(result.status).to.equal(200);
|
||||
expect(JSON.parse(result.body).available).to.equal(false); expect(result.headers["cache-control"]).to.equal("no-store");
|
||||
});
|
||||
});
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,95 @@
|
||||
const { expect } = require("chai");
|
||||
const { createClient } = require("redis");
|
||||
const net = require("node:net");
|
||||
const { setTimeout } = require("node:timers");
|
||||
require("ts-node/register/transpile-only");
|
||||
const config = require("../src/config").default;
|
||||
const { requestMetrics } = require("../src/core/request-monitoring");
|
||||
const { startPerformanceMonitoring, flushPerformanceMetrics, performanceReport } = require("../src/core/performance-monitoring");
|
||||
const port = Number(process.env.PERFORMANCE_REDIS_PORT);
|
||||
|
||||
(port ? describe : describe.skip)("request monitoring persistence", function () {
|
||||
this.timeout(15000);
|
||||
let stop, redis, previous;
|
||||
before(async function () {
|
||||
previous = { host: config.REDIS_HOSTNAME, port: config.REDIS_PORT };
|
||||
config.REDIS_HOSTNAME = "127.0.0.1"; config.REDIS_PORT = port;
|
||||
redis = createClient({ socket: { host: "127.0.0.1", port, reconnectStrategy: false } }); await redis.connect();
|
||||
});
|
||||
beforeEach(async function () {
|
||||
const keys = [];
|
||||
for await (const batch of redis.scanIterator({ MATCH: "performance:v1:*" })) keys.push(...batch);
|
||||
if (keys.length) await redis.del(keys);
|
||||
requestMetrics.drain();
|
||||
});
|
||||
afterEach(async function () { if (stop) { await stop(); stop = null; } requestMetrics.drain(); });
|
||||
after(async function () { await redis.close(); config.REDIS_HOSTNAME = previous.host; config.REDIS_PORT = previous.port; });
|
||||
async function start(service) {
|
||||
stop = startPerformanceMonitoring(service, () => ({ running: 0, waiting: 0 }));
|
||||
const deadline = Date.now() + 4000;
|
||||
while (!(await performanceReport(15)).available) {
|
||||
if (Date.now() > deadline) throw Error("Monitoring did not connect");
|
||||
await new Promise(resolve => setTimeout(resolve, 20));
|
||||
}
|
||||
}
|
||||
it("batches latency, failures, interruptions and resource history with bounded retention", async function () {
|
||||
await start("api");
|
||||
requestMetrics.observe("request", "/api/repo/:repoId/zip", "GET", 8000, { bytes: 12345 });
|
||||
requestMetrics.observe("request", "/api/repo/:repoId/zip", "GET", 1500, { aborted: true, error: true });
|
||||
requestMetrics.observe("upstream", "all", "all", 90);
|
||||
await flushPerformanceMetrics();
|
||||
const result = await performanceReport(15);
|
||||
expect(result.available).to.equal(true); expect(result.routes).to.have.length(1);
|
||||
expect(result.routes[0]).to.include({ count: 2, aborted: 1, errors: 1, bytes: 12345, avgMs: 4750, p95UpperMs: 10000 });
|
||||
expect(result.stages[0]).to.include({ metric: "upstream", count: 1 });
|
||||
expect(result.instances).to.have.length(1); expect(result.instances[0].memory.rss).to.be.greaterThan(0);
|
||||
expect(result.runtimeSeries).to.have.length(1); expect(result.series[0].requests).to.equal(2);
|
||||
const minuteKey = `performance:v1:minute:api:${Math.floor(Date.now() / 60000) * 60000}`;
|
||||
expect(await redis.ttl(minuteKey)).to.be.within(7100, 7200);
|
||||
expect(await redis.ttl(`performance:v1:instance:${result.instances[0].instance}`)).to.be.within(80, 90);
|
||||
await flushPerformanceMetrics(); expect((await performanceReport(15)).routes[0].count).to.equal(2);
|
||||
});
|
||||
it("retains counters across process restarts and combines all streamers", async function () {
|
||||
await start("api"); requestMetrics.observe("request", "/api/repo/:repoId/file/:path", "GET", 20);
|
||||
await flushPerformanceMetrics(); await stop(); stop = null;
|
||||
await start("streamer"); requestMetrics.observe("request", "/api", "POST", 10); await flushPerformanceMetrics();
|
||||
const result = await performanceReport(60);
|
||||
expect(result.routes.map(row => row.service).sort()).to.deep.equal(["api", "streamer"]);
|
||||
expect(result.runtimeSeries.map(row => row.service).sort()).to.deep.equal(["api", "streamer"]);
|
||||
expect(result.windowMinutes).to.equal(60);
|
||||
});
|
||||
it("bounds requested history and excludes expired process heartbeats", async function () {
|
||||
await start("api"); await flushPerformanceMetrics();
|
||||
await redis.zAdd("performance:v1:instances", { score: Date.now() - 100000, value: "retired" });
|
||||
await redis.set("performance:v1:instance:retired", '{"instance":"retired"}');
|
||||
const result = await performanceReport(1000000);
|
||||
expect(result.windowMinutes).to.equal(15); expect(result.instances).to.have.length(1);
|
||||
expect(result.instances[0].instance).not.to.equal("retired");
|
||||
});
|
||||
it("returns unavailable promptly and stays bounded when the Redis endpoint never replies", async function () {
|
||||
const connections = [];
|
||||
const blackhole = net.createServer(socket => { connections.push(socket); socket.on("error", () => {}); });
|
||||
await new Promise(resolve => blackhole.listen(0, "127.0.0.1", resolve));
|
||||
config.REDIS_PORT = blackhole.address().port;
|
||||
try {
|
||||
stop = startPerformanceMonitoring("api");
|
||||
for (let i = 0; i < 10000; i++) requestMetrics.observe("request", "other", "GET", 10);
|
||||
const started = Date.now(); await flushPerformanceMetrics();
|
||||
expect((await performanceReport(60)).available).to.equal(false);
|
||||
expect(Date.now() - started).to.be.lessThan(500); expect(requestMetrics.size).to.equal(0);
|
||||
await stop(); stop = null;
|
||||
} finally {
|
||||
connections.forEach(socket => socket.destroy()); await new Promise(resolve => blackhole.close(resolve)); config.REDIS_PORT = port;
|
||||
}
|
||||
});
|
||||
it("cuts off an in-flight batch when a connected Redis server stops responding", async function () {
|
||||
await start("api");
|
||||
requestMetrics.observe("request", "other", "GET", 15);
|
||||
await redis.sendCommand(["CLIENT", "PAUSE", "2500", "ALL"]);
|
||||
const started = Date.now();
|
||||
await flushPerformanceMetrics();
|
||||
expect(Date.now() - started).to.be.within(1800, 2800);
|
||||
expect(requestMetrics.size).to.equal(0);
|
||||
expect((await performanceReport(15)).available).to.equal(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,244 @@
|
||||
const { expect } = require("chai");
|
||||
const { Readable, PassThrough } = require("node:stream");
|
||||
const { setImmediate } = require("node:timers");
|
||||
const { setTimeout: delay } = require("node:timers/promises");
|
||||
require("ts-node/register/transpile-only");
|
||||
const { AsyncCache } = require("../src/core/async-cache");
|
||||
const { streamResponse } = require("../src/core/response-stream");
|
||||
const Repository = require("../src/core/Repository").default;
|
||||
const User = require("../src/core/User").default;
|
||||
const UserModel = require("../src/core/model/users/users.model").default;
|
||||
const RepoModel = require("../src/core/model/anonymizedRepositories/anonymizedRepositories.model").default;
|
||||
const GitHubStream = require("../src/core/source/GitHubStream").default;
|
||||
const storage = require("../src/core/storage").default;
|
||||
const { dashboardPipeline } = require("../src/server/routes/dashboard-summary");
|
||||
|
||||
describe("performance regressions", function () {
|
||||
this.timeout(10000);
|
||||
const restore = [];
|
||||
function stub(object, key, value) { const before = object[key]; object[key] = value; restore.push(() => { object[key] = before; }); }
|
||||
afterEach(() => { restore.splice(0).reverse().forEach(fn => fn()); });
|
||||
|
||||
it("keeps name/folder search semantics and stops at 500 matches", async () => {
|
||||
const repo = new Repository(new RepoModel({ owner: new UserModel()._id, repoId: "search", options: { terms: [] } }));
|
||||
const rows = Array.from({ length: 10000 }, (_, i) => ({ name: `file-${i}.ts`, path: `src/MATCH-${i}/nested`, size: i }));
|
||||
let examined = 0;
|
||||
for (const row of rows) { const name = row.name; Object.defineProperty(row, "name", { get() { examined++; return name; } }); }
|
||||
repo.anonymizedFiles = async () => rows;
|
||||
const result = await repo.searchFiles("match");
|
||||
expect(result).to.have.length(500);
|
||||
expect(result[499].path).to.equal("src/MATCH-499/nested");
|
||||
expect(examined).to.be.lessThan(2000);
|
||||
expect(await repo.searchFiles("src/MATCH")).to.deep.equal([]);
|
||||
});
|
||||
|
||||
it("coalesces cache misses, caches zero, bounds keys, and retries failures", async () => {
|
||||
let now = 0, calls = 0;
|
||||
const cache = new AsyncCache(10, 3, () => now);
|
||||
const values = await Promise.all(Array.from({ length: 10 }, () => cache.get("zero", async () => { calls++; await delay(10); return 0; })));
|
||||
expect(values).to.deep.equal(Array(10).fill(0));
|
||||
expect(calls).to.equal(1);
|
||||
for (let i = 0; i < 1000; i++) await cache.get(String(i), async () => i);
|
||||
expect(cache.size).to.equal(3);
|
||||
now = 11;
|
||||
expect(cache.size).to.equal(0);
|
||||
await cache.get("failure", async () => { throw Error("temporary"); }).catch(() => {});
|
||||
expect(await cache.get("failure", async () => 1)).to.equal(1);
|
||||
});
|
||||
|
||||
for (const terminal of ["close", "finish", "error"]) {
|
||||
it(`destroys the upstream and removes listeners on response ${terminal}`, async () => {
|
||||
const input = new PassThrough(), response = new PassThrough();
|
||||
const completed = streamResponse(input, response, () => {});
|
||||
response.emit(terminal, new Error("disconnected"));
|
||||
await completed.catch(() => {});
|
||||
expect(input.destroyed).to.equal(true);
|
||||
expect(response.listenerCount("close")).to.equal(0);
|
||||
expect(response.listenerCount("finish")).to.equal(0);
|
||||
expect(response.listenerCount("error")).to.equal(0);
|
||||
});
|
||||
}
|
||||
|
||||
it("performs one cache fill for ten callers and gives each its own stream", async () => {
|
||||
const source = new GitHubStream({ repoId: "coalesced", organization: "owner", repoName: "repo", commit: "abc", getToken: async () => "token" });
|
||||
let data, downloads = 0, writes = 0, heads = 0;
|
||||
stub(storage, "fileInfo", async () => { heads++; if (!data) throw Error("missing"); return { size: data.length }; });
|
||||
stub(storage, "read", async () => Readable.from([data]));
|
||||
stub(storage, "write", async (_repo, _path, input) => { writes++; const chunks = []; for await (const chunk of input) chunks.push(chunk); await delay(10); data = Buffer.concat(chunks); });
|
||||
source.downloadWithFallback = async () => { downloads++; return Readable.from([Buffer.from("complete content")]); };
|
||||
const streams = await Promise.all(Array.from({ length: 10 }, () => source.getFileContentCache("file.txt", "coalesced", () => ({ sha: "blob", size: 16 }))));
|
||||
expect(downloads).to.equal(1); expect(writes).to.equal(1);
|
||||
expect(new Set(streams).size).to.equal(10);
|
||||
for (const input of streams) { let text = ""; for await (const chunk of input) text += chunk; expect(text).to.equal("complete content"); }
|
||||
heads = 0;
|
||||
const warm = await source.getFileContentCache("file.txt", "coalesced", () => ({ sha: "blob", size: 16 }));
|
||||
warm.destroy(); expect(heads).to.equal(1); expect(downloads).to.equal(1);
|
||||
});
|
||||
|
||||
for (const phase of ["before download", "during publication"]) {
|
||||
it(`rejects retired cache producers ${phase} and removes late output`, async () => {
|
||||
const { contentGenerationPrefix, contentRetirementMarker } = require("../src/core/content-generation");
|
||||
const cacheGeneration = `retired-${phase}`, prefix = contentGenerationPrefix(cacheGeneration);
|
||||
const marker = contentRetirementMarker(prefix), objects = new Map();
|
||||
const source = new GitHubStream({ repoId: `retirement-${phase}`, organization: "owner", repoName: "repo", commit: "abc",
|
||||
cacheGeneration, getToken: async () => "token" });
|
||||
let downloads = 0, started, release;
|
||||
const began = new Promise(resolve => { started = resolve; }), held = new Promise(resolve => { release = resolve; });
|
||||
stub(storage, "fileInfo", async (_repo, path) => {
|
||||
if (!objects.has(path)) throw Object.assign(Error("missing"), { code: "ENOENT" });
|
||||
return { size: objects.get(path).length };
|
||||
});
|
||||
stub(storage, "write", async (_repo, path, input) => {
|
||||
started(); await held;
|
||||
let content = ""; for await (const chunk of input) content += chunk;
|
||||
objects.set(path, content);
|
||||
});
|
||||
stub(storage, "rm", async (_repo, path) => { objects.delete(path); });
|
||||
stub(storage, "read", async (_repo, path) => Readable.from([objects.get(path)]));
|
||||
source.downloadWithFallback = async () => { downloads++; return Readable.from(["obsolete"]); };
|
||||
if (phase === "before download") { objects.set(marker, "retired"); release(); }
|
||||
const pending = source.getFileContentCache("private/old.txt", source.data.repoId, () => ({ sha: "blob", size: 8 }));
|
||||
try {
|
||||
if (phase === "during publication") {
|
||||
await began; objects.set(marker, "retired"); release();
|
||||
}
|
||||
try { await pending; throw Error("expected retired generation rejection"); }
|
||||
catch (error) { expect(error.message).to.equal("repository_changed"); expect(error.httpStatus).to.equal(409); }
|
||||
expect(downloads).to.equal(phase === "before download" ? 0 : 1);
|
||||
expect([...objects.keys()]).to.deep.equal([marker]);
|
||||
} finally { release(); await pending.catch(() => {}); }
|
||||
});
|
||||
}
|
||||
|
||||
it("fetches a complete root tree in one recursive request", async () => {
|
||||
const source = new GitHubStream({ repoId: "tree", organization: "owner", repoName: "repo", commit: "head", getToken: () => "token" });
|
||||
const calls = [];
|
||||
source.getGHTree = async (_oct, _token, sha, _count, options) => {
|
||||
calls.push({ sha, recursive: options.recursive });
|
||||
return { truncated: false, tree: Array.from({ length: 20 }, (_, i) => ({ path: `folder${i}/file.ts`, type: "blob", sha: "abc" })) };
|
||||
};
|
||||
const files = await source.getTruncatedTree("head");
|
||||
expect(files).to.have.length(20); expect(calls).to.deep.equal([{ sha: "head", recursive: true }]);
|
||||
});
|
||||
|
||||
for (const [upstream, code, status] of [
|
||||
[Object.assign(new Error("forbidden"), { response: { statusCode: 403 } }), "file_not_accessible", 403],
|
||||
[Object.assign(new Error("missing"), { response: { statusCode: 404 } }), "file_not_found", 404],
|
||||
[Object.assign(new Error("large"), { status: 422 }), "file_too_big", 422],
|
||||
[Object.assign(new Error("missing"), { httpStatus: 404 }), "file_not_found", 404],
|
||||
[Object.assign(new Error("reset"), { code: "ECONNRESET" }), "upstream_error", 502],
|
||||
[new (require("../src/core/AnonymousError").default)("file_not_accessible", { httpStatus: 403 }), "file_not_accessible", 403],
|
||||
]) {
|
||||
it(`preserves ${code}/${status} for failed downloads (${upstream.message})`, async () => {
|
||||
const source = new GitHubStream({ repoId: "failed-download", organization: "owner", repoName: "repo", getToken: () => "token" });
|
||||
const input = new PassThrough();
|
||||
stub(storage, "fileInfo", async () => { throw Error("cache miss"); });
|
||||
let committed = false;
|
||||
stub(storage, "write", async (_repo, _path, stream) => { for await (const chunk of stream) void chunk; committed = true; });
|
||||
source.downloadWithFallback = async () => { setImmediate(() => input.destroy(upstream)); return input; };
|
||||
try { await source.getFileContentCache("file.txt", "failed-download", () => ({ sha: "blob", size: 1 })); throw Error("expected failure"); }
|
||||
catch (error) {
|
||||
expect(error.message).to.equal(code); expect(error.httpStatus).to.equal(status);
|
||||
if (upstream instanceof require("../src/core/AnonymousError").default) expect(error).to.equal(upstream);
|
||||
else expect(error.cause).to.equal(upstream);
|
||||
}
|
||||
expect(committed).to.equal(false); expect(input.destroyed).to.equal(true);
|
||||
});
|
||||
}
|
||||
|
||||
it("falls back to shallow trees when GitHub truncates recursive results", async () => {
|
||||
const source = new GitHubStream({ repoId: "tree", organization: "owner", repoName: "repo", commit: "head", getToken: () => "token" });
|
||||
const calls = [];
|
||||
source.getGHTree = async (_oct, _token, sha, _count, options) => {
|
||||
calls.push(`${sha}:${options.recursive}`);
|
||||
if (sha === "head" && options.recursive) return { truncated: true, tree: [{ path: "ignored.ts", type: "blob" }] };
|
||||
if (sha === "head") return { truncated: false, tree: [{ path: "src", type: "tree", sha: "src" }] };
|
||||
return { truncated: false, tree: [{ path: "file.ts", type: "blob", sha: "abc" }] };
|
||||
};
|
||||
const files = await source.getTruncatedTree("head");
|
||||
expect(files.map(file => `${file.path}/${file.name}`)).to.deep.equal(["/src", "src/file.ts"]);
|
||||
expect(calls).to.deep.equal(["head:true", "head:false", "src:true"]);
|
||||
});
|
||||
|
||||
it("does no physical expiration work while reading 100 expired projects", async () => {
|
||||
const user = new User(new UserModel({ username: "owner" }));
|
||||
stub(RepoModel, "find", () => ({ exec: async () => Array.from({ length: 100 }, (_, i) => new RepoModel({ repoId: `expired${i}`, owner: user.model._id, status: "ready", options: { expirationMode: "remove", expirationDate: new Date(0) } })) }));
|
||||
stub(Repository.prototype, "expire", () => { throw Error("inline cleanup"); });
|
||||
const projects = await user.getRepositories();
|
||||
expect(projects).to.have.length(100);
|
||||
expect(projects.every(project => project.status === "expiring")).to.equal(true);
|
||||
});
|
||||
|
||||
it("projects dashboard summaries before combining types, and validates cursors", () => {
|
||||
const user = new User(new UserModel({ username: "owner" }));
|
||||
const pipeline = dashboardPipeline(user, { limit: 2, type: "pr" });
|
||||
const projection = pipeline.find(stage => stage.$project).$project;
|
||||
expect(projection).not.to.have.property("pullRequest");
|
||||
expect(pipeline.filter(stage => stage.$unionWith)).to.have.length(2);
|
||||
expect(() => dashboardPipeline(user, { cursor: "bogus" })).to.throw("invalid_cursor");
|
||||
expect(() => dashboardPipeline(user, { sort: "$where" })).to.throw("invalid_sort");
|
||||
});
|
||||
});
|
||||
|
||||
describe("shared home statistics", function () {
|
||||
this.timeout(10000);
|
||||
const stats = require("../src/server/dailyStatsSnapshot");
|
||||
const PR = require("../src/core/model/anonymizedPullRequests/anonymizedPullRequests.model").default;
|
||||
const Daily = require("../src/core/model/dailyStats/dailyStats.model").default;
|
||||
let originals, calculations, fail;
|
||||
beforeEach(() => {
|
||||
stats.clearStatsCache(); calculations = 0; fail = false;
|
||||
originals = [RepoModel.estimatedDocumentCount, RepoModel.collection.aggregate, PR.estimatedDocumentCount, Daily.find];
|
||||
RepoModel.estimatedDocumentCount = async () => { calculations++; if (fail) throw Error("temporary stats error"); return 0; };
|
||||
RepoModel.collection.aggregate = () => ({ toArray: async () => [] });
|
||||
PR.estimatedDocumentCount = async () => 0;
|
||||
Daily.find = () => ({ sort: () => ({ lean: async () => [] }) });
|
||||
});
|
||||
afterEach(() => {
|
||||
[RepoModel.estimatedDocumentCount, RepoModel.collection.aggregate, PR.estimatedDocumentCount, Daily.find] = originals;
|
||||
stats.clearStatsCache();
|
||||
});
|
||||
it("coalesces concurrent current/history calculations and caches zero across day ranges", async () => {
|
||||
const values = await Promise.all([...Array.from({ length: 10 }, () => stats.getCurrentStats()),
|
||||
stats.getStatsHistory(30, new Date("2026-10-07")), stats.getStatsHistory(60, new Date("2026-10-07"))]);
|
||||
expect(calculations).to.equal(1); expect(values[0].nbRepositories).to.equal(0);
|
||||
expect(values[10][0].date.toISOString()).to.equal("2026-10-07T00:00:00.000Z");
|
||||
const next = await stats.getStatsHistory(30, new Date("2026-10-08"));
|
||||
expect(next[0].date.toISOString()).to.equal("2026-10-08T00:00:00.000Z");
|
||||
expect(calculations).to.equal(1);
|
||||
});
|
||||
it("retries a failed calculation without retaining a rejected promise", async () => {
|
||||
fail = true; await stats.getCurrentStats().catch(() => {}); fail = false;
|
||||
expect((await stats.getCurrentStats()).nbPageViews).to.equal(0);
|
||||
expect(calculations).to.equal(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe("admin queue snapshot reuse", function () {
|
||||
it("filters a bounded raw snapshot for thousands of search terms and expires it", async function () {
|
||||
this.timeout(10000);
|
||||
const queues = require("../src/queue");
|
||||
const metrics = require("../src/queue/queueMetrics");
|
||||
const router = require("../src/server/routes/admin").default;
|
||||
const handler = router.stack.find(layer => layer.route?.path === "/queues").route.stack.at(-1).handle;
|
||||
const originals = [queues.downloadQueue, queues.removeQueue, queues.cacheQueue, metrics.queryMetrics, Date.now];
|
||||
let calls = 0, now = Date.now();
|
||||
const queue = { getJobCounts: async () => ({}), getWorkers: async () => [], isPaused: async () => false,
|
||||
getJobs: async ([state]) => { calls++; return state === "active" ? [{ asJSON: () => ({ id: "known", name: "known" }) }] : []; } };
|
||||
queues.downloadQueue = queue; queues.removeQueue = queue; queues.cacheQueue = queue;
|
||||
metrics.queryMetrics = async () => [];
|
||||
Date.now = () => now;
|
||||
try {
|
||||
for (let i = 0; i < 1000; i++) await handler({ query: { search: `unique-${i}` } }, { json: data => expect(data.jobs).to.have.length(0) });
|
||||
const firstCalls = calls;
|
||||
expect(firstCalls).to.equal(5);
|
||||
await handler({ query: { search: "known" } }, { json: data => expect(data.jobs).to.have.length(1) });
|
||||
expect(calls).to.equal(firstCalls);
|
||||
now += 10001;
|
||||
await handler({ query: {} }, { json() {} });
|
||||
expect(calls).to.equal(firstCalls * 2);
|
||||
} finally {
|
||||
[queues.downloadQueue, queues.removeQueue, queues.cacheQueue, metrics.queryMetrics, Date.now] = originals;
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,136 @@
|
||||
const { expect } = require("chai");
|
||||
const process = require("node:process");
|
||||
const { fork } = require("node:child_process");
|
||||
const { promises: fs } = require("node:fs");
|
||||
const { tmpdir } = require("node:os");
|
||||
const path = require("node:path");
|
||||
const { Readable } = require("node:stream");
|
||||
const { S3, CreateBucketCommand } = require("@aws-sdk/client-s3");
|
||||
const { createClient } = require("redis");
|
||||
require("ts-node/register/transpile-only");
|
||||
const config = require("../src/config").default;
|
||||
const S3Storage = require("../src/core/storage/S3").default;
|
||||
const GitHubStream = require("../src/core/source/GitHubStream").default;
|
||||
const storage = require("../src/core/storage").default;
|
||||
const { coordinatedFill, closeCacheRedis, cacheRedis } = require("../src/core/cache-coordination");
|
||||
const { setTimeout: delay } = require("node:timers/promises");
|
||||
|
||||
(process.env.PERFORMANCE_REDIS_PORT ? describe : describe.skip)("disposable Redis cache coordination", function () {
|
||||
this.timeout(20000);
|
||||
let root, redis, previousRedis;
|
||||
before(async () => {
|
||||
await closeCacheRedis();
|
||||
root = await fs.mkdtemp(path.join(tmpdir(), "anonymous-cache-test-"));
|
||||
previousRedis = { REDIS_HOSTNAME: config.REDIS_HOSTNAME, REDIS_PORT: config.REDIS_PORT };
|
||||
config.REDIS_HOSTNAME = "127.0.0.1"; config.REDIS_PORT = Number(process.env.PERFORMANCE_REDIS_PORT);
|
||||
redis = createClient({ socket: { host: "127.0.0.1", port: config.REDIS_PORT } });
|
||||
await redis.connect();
|
||||
});
|
||||
after(async () => { await closeCacheRedis(); await redis.close(); Object.assign(config, previousRedis); await fs.rm(root, { force: true, recursive: true }); });
|
||||
it("downloads and publishes once across two streamer processes and ten callers", async () => {
|
||||
let producers = 0;
|
||||
const children = Array.from({ length: 2 }, () => fork(path.join(__dirname, "fixtures/performance-cache-worker.js"), [], { stdio: ["ignore", "ignore", "ignore", "ipc"], env: { ...process.env,
|
||||
FOLDER: root, REDIS_HOSTNAME: "127.0.0.1", REDIS_PORT: String(config.REDIS_PORT), NODE_ENV: "test" } }));
|
||||
try {
|
||||
const results = children.map(child => new Promise((resolve, reject) => {
|
||||
child.on("message", message => {
|
||||
if (message.ready) child.send("start");
|
||||
if (message.producer) producers++;
|
||||
if (message.contents) resolve(message.contents);
|
||||
if (message.error) reject(Error(message.error));
|
||||
});
|
||||
child.on("error", reject); child.on("exit", code => { if (code) reject(Error(`worker exited ${code}`)); });
|
||||
}));
|
||||
const contents = (await Promise.all(results)).flat();
|
||||
expect(contents).to.deep.equal(Array(10).fill("complete content"));
|
||||
expect(producers).to.equal(1);
|
||||
} finally { children.forEach(child => child.kill()); }
|
||||
});
|
||||
it("recovers an expired crashed-producer lease without publishing partial output", async () => {
|
||||
const key = `crash-test-${Date.now()}`;
|
||||
await redis.set(`perf:lease:${key}`, "dead-process", { PX: 150 });
|
||||
let value, producers = 0;
|
||||
const result = await coordinatedFill(key, async () => value, async () => { producers++; value = "verified"; return value; });
|
||||
expect(result).to.equal("verified"); expect(producers).to.equal(1);
|
||||
expect(await redis.get(`perf:lease:${key}`)).to.equal(null);
|
||||
});
|
||||
for (const failure of ["restart", "timeout"]) {
|
||||
it(`returns verified output despite a Redis heartbeat ${failure}`, async () => {
|
||||
await closeCacheRedis();
|
||||
const client = await cacheRedis(), originalEval = client.eval;
|
||||
let notify, published;
|
||||
const heartbeat = new Promise(resolve => { notify = resolve; });
|
||||
client.eval = function (script, options) {
|
||||
if (script.includes("pexpire")) {
|
||||
notify();
|
||||
return failure === "timeout" ? new Promise(() => {}) : Promise.reject(Error("Redis restarted"));
|
||||
}
|
||||
return originalEval.call(this, script, options);
|
||||
};
|
||||
const key = `heartbeat-${failure}-${Date.now()}`;
|
||||
try {
|
||||
const result = await coordinatedFill(key, async () => published, async () => {
|
||||
await heartbeat; await delay(failure === "timeout" ? 1100 : 20);
|
||||
published = "verified output"; return published;
|
||||
});
|
||||
expect(result).to.equal("verified output");
|
||||
expect(await coordinatedFill(key, async () => published, async () => published)).to.equal("verified output");
|
||||
} finally { client.eval = originalEval; await closeCacheRedis(); }
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
describe("S3 HTTP protocol integration", function () {
|
||||
this.timeout(20000);
|
||||
let backend, previous, client, fixture;
|
||||
before(async () => {
|
||||
previous = { ...config };
|
||||
if (!process.env.PERFORMANCE_S3_ENDPOINT) fixture = await require("./fixtures/s3-server")();
|
||||
Object.assign(config, { S3_BUCKET: "performance-test", S3_CLIENT_ID: "perf-test-user", S3_CLIENT_SECRET: "perf-test-password",
|
||||
S3_REGION: "us-east-1", S3_ENDPOINT: process.env.PERFORMANCE_S3_ENDPOINT || fixture.endpoint });
|
||||
client = new S3({ endpoint: config.S3_ENDPOINT, region: config.S3_REGION, forcePathStyle: true,
|
||||
credentials: { accessKeyId: config.S3_CLIENT_ID, secretAccessKey: config.S3_CLIENT_SECRET } });
|
||||
await client.send(new CreateBucketCommand({ Bucket: config.S3_BUCKET }));
|
||||
backend = new S3Storage();
|
||||
});
|
||||
after(async () => { client?.destroy(); for (const value of backend?.clients.values() || []) value.destroy(); Object.assign(config, previous); await fixture?.close(); });
|
||||
it("reuses clients and serves a warm file with one HEAD and one GET", async () => {
|
||||
await backend.write("s3repo", "file.txt", Readable.from([Buffer.from("complete content")]), undefined, 16);
|
||||
expect(backend.client(3000)).to.equal(backend.client(3000));
|
||||
let heads = 0, gets = 0;
|
||||
const common = backend.client(3000), oldHead = common.headObject.bind(common), oldSend = common.send.bind(common);
|
||||
common.headObject = (...args) => { heads++; return oldHead(...args); };
|
||||
common.send = (...args) => { if (args[0].constructor.name === "GetObjectCommand") gets++; return oldSend(...args); };
|
||||
const originals = { fileInfo: storage.fileInfo, read: storage.read, write: storage.write };
|
||||
try {
|
||||
storage.fileInfo = backend.fileInfo.bind(backend); storage.read = backend.read.bind(backend); storage.write = backend.write.bind(backend);
|
||||
const source = new GitHubStream({ repoId: "s3repo", organization: "owner", repoName: "repo", commit: "commit", getToken: () => { throw Error("warm cache requested token"); } });
|
||||
const input = await source.getFileContentCache("file.txt", "s3repo", () => ({ sha: "blob", size: 16 }));
|
||||
let text = ""; for await (const chunk of input) text += chunk;
|
||||
expect(text).to.equal("complete content"); expect(heads).to.equal(1); expect(gets).to.equal(1);
|
||||
} finally { Object.assign(storage, originals); }
|
||||
});
|
||||
it("refuses a short upload before committing the object", async () => {
|
||||
let error;
|
||||
try { await backend.write("s3repo", "short.txt", Readable.from([Buffer.from("short")]), undefined, 100); } catch (caught) { error = caught; }
|
||||
expect(error).to.be.instanceOf(Error);
|
||||
expect(await backend.exists("s3repo", "short.txt")).to.equal("not_found");
|
||||
});
|
||||
it("removes legacy S3 keys in bounded batches while retaining managed caches and other repositories", async function () {
|
||||
if (!fixture) this.skip();
|
||||
for (let i = 0; i < 1005; i++) fixture.seed(`legacy-s3/original/private/file-${String(i).padStart(4, "0")}.txt`, "obsolete");
|
||||
const active = require("../src/core/content-generation").contentGenerationPrefix("active");
|
||||
fixture.seed(`legacy-s3/original/${active}/blob/keep.txt`, "active");
|
||||
fixture.seed("legacy-s3-other/original/private/keep.txt", "keep");
|
||||
const before = fixture.requests.deletions.length;
|
||||
await backend.removeLegacyContent("legacy-s3");
|
||||
const batches = fixture.requests.deletions.slice(before);
|
||||
expect(batches).to.have.length(2);
|
||||
expect(batches.flat()).to.have.length(1005);
|
||||
expect(batches.every(batch => batch.length <= 1000)).to.equal(true);
|
||||
expect(await backend.exists("legacy-s3", "private/file-0000.txt")).to.equal("not_found");
|
||||
expect(await backend.exists("legacy-s3", "private/file-1004.txt")).to.equal("not_found");
|
||||
expect(await backend.exists("legacy-s3", `${active}/blob/keep.txt`)).to.equal("file");
|
||||
expect(await backend.exists("legacy-s3-other", "private/keep.txt")).to.equal("file");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,129 @@
|
||||
const { expect } = require("chai");
|
||||
const { fork } = require("node:child_process");
|
||||
const { promises: fs } = require("node:fs");
|
||||
const { tmpdir } = require("node:os");
|
||||
const path = require("node:path");
|
||||
const net = require("node:net");
|
||||
const { randomBytes } = require("node:crypto");
|
||||
const { once } = require("node:events");
|
||||
const { setTimeout } = require("node:timers");
|
||||
const { fetch } = globalThis;
|
||||
const mongoose = require("mongoose");
|
||||
require("ts-node/register/transpile-only");
|
||||
const db = require("../src/server/database");
|
||||
const Repo = require("../src/core/model/anonymizedRepositories/anonymizedRepositories.model").default;
|
||||
const Repository = require("../src/core/Repository").default;
|
||||
const config = require("../src/config").default;
|
||||
const uri = process.env.PERFORMANCE_MONGO_URI;
|
||||
|
||||
(uri ? describe : describe.skip)("production streamer lifecycle and idle cleanup", function () {
|
||||
this.timeout(30000);
|
||||
let root, previousFolder;
|
||||
before(async () => {
|
||||
if (!/\/perf_test_[A-Za-z0-9_-]+(?:\?|$)/.test(uri)) throw Error("An isolated perf_test_ database is required");
|
||||
await mongoose.connect(uri); db.isConnected = true;
|
||||
root = await fs.mkdtemp(path.join(tmpdir(), "streamer-lifecycle-"));
|
||||
previousFolder = config.FOLDER; config.FOLDER = path.join(root, "repositories");
|
||||
});
|
||||
after(async () => {
|
||||
db.isConnected = false; await mongoose.connection.dropDatabase(); await mongoose.disconnect();
|
||||
config.FOLDER = previousFolder; await fs.rm(root, { force: true, recursive: true });
|
||||
});
|
||||
beforeEach(async () => { await Repo.deleteMany({}); });
|
||||
|
||||
const missing = async file => { try { await fs.access(file); return false; } catch (error) { if (error.code === "ENOENT") return true; throw error; } };
|
||||
async function waitFor(check, timeoutMs = 5000) {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (!await check()) { if (Date.now() >= deadline) throw Error("Streamer cleanup did not finish"); await new Promise(resolve => setTimeout(resolve, 20)); }
|
||||
}
|
||||
async function launch(invalidCredentials = false, holdConnect = false) {
|
||||
const socket = net.createServer(); socket.listen(0, "127.0.0.1"); await once(socket, "listening");
|
||||
const port = socket.address().port; await new Promise(resolve => socket.close(resolve));
|
||||
const directory = await fs.mkdtemp(path.join(root, "streamer-temp-"));
|
||||
const stale = path.join(directory, "anonymous-text-crashed"); await fs.mkdir(stale);
|
||||
await fs.writeFile(path.join(stale, "input"), "original private bytes"); await fs.utimes(stale, new Date(0), new Date(0));
|
||||
const child = fork(path.join(__dirname, "fixtures/performance-streamer.js"), [], {
|
||||
execArgv: ["--max-old-space-size=512"], stdio: ["ignore", "pipe", "pipe", "ipc"],
|
||||
env: { ...process.env, NODE_ENV: "test", PORT: String(port), MONGODB_URI: uri, GITHUB_APP_ENABLED: "false", STORAGE: "filesystem",
|
||||
PERFORMANCE_HOLD_CONNECT: holdConnect ? "1" : "0",
|
||||
REDIS_HOSTNAME: "127.0.0.1", REDIS_PORT: process.env.PERFORMANCE_REDIS_PORT || "6379", TMPDIR: directory, FOLDER: config.FOLDER,
|
||||
CREDENTIAL_KEYS: invalidCredentials ? "invalid" : JSON.stringify({ test: randomBytes(32).toString("base64") }), CREDENTIAL_ACTIVE_KEY_ID: "test" },
|
||||
});
|
||||
const messages = []; child.on("message", message => messages.push(message));
|
||||
const ready = new Promise((resolve, reject) => {
|
||||
let logs = "";
|
||||
child.stdout.on("data", chunk => { logs += chunk; if (logs.includes("streamer started")) resolve(); });
|
||||
child.stderr.on("data", () => {});
|
||||
child.once("error", reject); child.once("exit", code => reject(Error(`Streamer exited ${code} before serving`)));
|
||||
});
|
||||
ready.catch(() => {});
|
||||
return { child, ready, messages, directory, stale, url: `http://127.0.0.1:${port}` };
|
||||
}
|
||||
async function stop(runtime) {
|
||||
if (runtime.child.exitCode === null && runtime.child.signalCode === null) {
|
||||
const exited = once(runtime.child, "exit"); runtime.child.kill(); await exited;
|
||||
}
|
||||
await fs.rm(runtime.directory, { force: true, recursive: true });
|
||||
}
|
||||
async function createRow(repoId) {
|
||||
return Repo.create({ repoId, status: "ready", statusDate: new Date(), lastView: new Date(), anonymizeDate: new Date(), treeGeneration: "active",
|
||||
source: { type: "GitHubStream", repositoryName: "owner/repo", commit: "abc" }, options: { terms: ["NeverMatches"], expirationMode: "remove", expirationDate: new Date(0) } });
|
||||
}
|
||||
async function request(runtime, row, filePath, legacy = false) {
|
||||
const options = new Repository(row).generateAnonymizeTransformer(filePath).opt;
|
||||
return fetch(runtime.url + "/api", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ token: "test-token",
|
||||
repoId: row.repoId, repoFullName: "owner/repo", commit: "abc", ...(legacy ? {} : { cacheGeneration: options.cacheGeneration }), sha: "blob", size: 300000, filePath, anonymizerOptions: options }) });
|
||||
}
|
||||
|
||||
it("connects before serving and rejects a fill published after expiration in the actual streamer entrypoint", async () => {
|
||||
const runtime = await launch(false, true); let pending;
|
||||
try {
|
||||
// Loading TypeScript and the Redis client in a fresh process can take
|
||||
// longer than the cleanup deadline used after the streamer starts.
|
||||
await waitFor(() => runtime.messages.some(message => message.connectionHeld), 10000);
|
||||
await fetch(runtime.url + "/healthcheck").then(() => { throw Error("Streamer served before connecting"); }, error => expect(error).to.be.instanceOf(Error));
|
||||
runtime.child.send("connect");
|
||||
await runtime.ready;
|
||||
expect(await (await fetch(runtime.url + "/healthcheck")).json()).to.deep.equal({ status: "ok" });
|
||||
runtime.child.send("inspect"); await waitFor(() => runtime.messages.some(message => message.connected !== undefined));
|
||||
expect(runtime.messages.find(message => message.connected !== undefined)).to.deep.equal({ connected: true, maxPoolSize: 10, minPoolSize: 0 });
|
||||
expect(await missing(runtime.stale)).to.equal(true);
|
||||
const row = await createRow("streamer-late-fill");
|
||||
pending = request(runtime, row, "hold.txt"); pending.catch(() => {});
|
||||
await waitFor(() => runtime.messages.some(message => message.held));
|
||||
await new Repository(await Repo.findById(row._id)).expire();
|
||||
runtime.child.send("release");
|
||||
const response = await pending; expect(response.status).to.equal(409); expect((await response.json()).error).to.equal("repository_changed");
|
||||
const sourceRoot = path.join(config.FOLDER, row.repoId, "original");
|
||||
const entries = await fs.readdir(sourceRoot, { recursive: true });
|
||||
expect(entries.filter(name => name.endsWith("hold.txt"))).to.deep.equal([]);
|
||||
const rejected = await request(runtime, row, "rejected.txt"); expect(rejected.status).to.equal(409); await rejected.text();
|
||||
const legacy = await request(runtime, row, "legacy.txt", true); expect(legacy.status).to.equal(409); await legacy.text();
|
||||
expect(runtime.messages.filter(message => message.download)).to.have.length(1);
|
||||
} finally { runtime.child.send("release"); await pending?.catch(() => {}); await stop(runtime); }
|
||||
});
|
||||
|
||||
it("purges another process's transformed cache and crash spools while idle after lifecycle changes", async () => {
|
||||
const runtime = await launch();
|
||||
try {
|
||||
await runtime.ready;
|
||||
const row = await createRow("streamer-cache"); const response = await request(runtime, row, "file.txt");
|
||||
expect(response.status).to.equal(200); expect((await response.text()).length).to.equal(300000);
|
||||
const cacheRoot = path.join(runtime.directory, "anonymous-transformed-v1");
|
||||
expect((await fs.readdir(cacheRoot)).filter(name => name.endsWith(".data"))).to.have.length(1);
|
||||
const recent = path.join(runtime.directory, "anonymous-text-idle-crash");
|
||||
await fs.mkdir(recent); await fs.writeFile(path.join(recent, "input"), "private original"); await fs.utimes(recent, new Date(0), new Date(0));
|
||||
await new Repository(await Repo.findById(row._id)).expire();
|
||||
await waitFor(async () => (await fs.readdir(cacheRoot)).length === 0 && await missing(recent));
|
||||
expect(runtime.messages.filter(message => message.download)).to.have.length(1);
|
||||
} finally { await stop(runtime); }
|
||||
});
|
||||
|
||||
it("refuses to serve when required startup configuration is invalid", async () => {
|
||||
const runtime = await launch(true);
|
||||
try {
|
||||
const [code] = await once(runtime.child, "exit"); expect(code).to.equal(1);
|
||||
await fetch(runtime.url + "/healthcheck").then(() => { throw Error("Streamer served before connecting"); }, error => expect(error).to.be.instanceOf(Error));
|
||||
} finally { await stop(runtime); }
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,155 @@
|
||||
const { expect } = require("chai");
|
||||
const { Readable } = require("node:stream");
|
||||
const { promises: fs } = require("node:fs");
|
||||
const { tmpdir } = require("node:os");
|
||||
const { join } = require("node:path");
|
||||
const { randomUUID } = require("node:crypto");
|
||||
const { setTimeout } = require("node:timers");
|
||||
require("ts-node/register/transpile-only");
|
||||
const { ContentAnonimizer, AnonymizeTransformer } = require("../src/core/anonymize-utils");
|
||||
const pool = require("../src/core/anonymization-pool");
|
||||
const { removeStaleTextSpools, startTemporaryStorageMaintenance } = require("../src/core/temporary-storage");
|
||||
const { cacheCommand } = require("../src/core/cache-coordination");
|
||||
|
||||
describe("large anonymization resources", function () {
|
||||
this.timeout(20000);
|
||||
async function transform(text, options) {
|
||||
const bytes = Buffer.from(text), parts = [];
|
||||
// Split both a name and a multibyte character at varying chunk boundaries.
|
||||
for (let offset = 0; offset < bytes.length; offset += 4093) parts.push(bytes.subarray(offset, offset + 4093));
|
||||
const transformer = new AnonymizeTransformer(options), output = [];
|
||||
Readable.from(parts).pipe(transformer);
|
||||
for await (const chunk of transformer) output.push(chunk);
|
||||
return Buffer.concat(output).toString("utf8");
|
||||
}
|
||||
it("preserves whole-text Unicode matches and isolates cached settings", async () => {
|
||||
const text = "Alice Σ 研究 https://example.com/x\n".repeat(18000);
|
||||
const options = { filePath: `parity-${randomUUID()}.txt`, terms: ["Alice=>Hidden", "研究=>Other"], image: true, link: true };
|
||||
const original = pool.anonymizeOnWorker;
|
||||
let jobs = 0;
|
||||
pool.anonymizeOnWorker = (...args) => { jobs++; return original(...args); };
|
||||
try {
|
||||
const expected = new ContentAnonimizer(options).anonymize(text);
|
||||
expect(await transform(text, options)).to.equal(expected);
|
||||
expect(await transform(text, options)).to.equal(expected);
|
||||
expect(jobs).to.equal(1);
|
||||
const edited = { ...options, terms: ["Alice=>Changed", "研究=>Other"] };
|
||||
expect(await transform(text, edited)).to.equal(new ContentAnonimizer(edited).anonymize(text));
|
||||
expect(jobs).to.equal(2);
|
||||
} finally { pool.anonymizeOnWorker = original; }
|
||||
});
|
||||
it("removes a spooled input when its subscriber cancels", async () => {
|
||||
const transformer = new AnonymizeTransformer({ filePath: "cancel.txt", terms: ["Alice"] });
|
||||
await new Promise((resolve, reject) => transformer.write(Buffer.alloc(300000, "x"), error => error ? reject(error) : resolve()));
|
||||
const spool = transformer.spool;
|
||||
const closed = new Promise(resolve => transformer.once("close", resolve));
|
||||
transformer.destroy(); await closed;
|
||||
let exists = true;
|
||||
try { await fs.access(spool); } catch { exists = false; }
|
||||
expect(exists).to.equal(false);
|
||||
});
|
||||
it("recovers worker capacity after a failed input", async () => {
|
||||
const controller = new AbortController();
|
||||
let failure;
|
||||
try { await pool.anonymizeOnWorker({ input: join(tmpdir(), randomUUID()), output: join(tmpdir(), randomUUID()),
|
||||
options: { terms: ["Alice"] }, maxOutput: 1000000, context: { mask: "ANON", hostname: "localhost" } }, 1, controller.signal); }
|
||||
catch (error) { failure = error; }
|
||||
expect(failure).to.be.instanceOf(Error);
|
||||
const text = "Alice ".repeat(50000);
|
||||
const options = { filePath: `recovery-${randomUUID()}.txt`, terms: ["Alice=>Hidden"] };
|
||||
expect(await transform(text, options)).to.equal(new ContentAnonimizer(options).anonymize(text));
|
||||
});
|
||||
it("finishes large files containing many safe regex rules", async () => {
|
||||
const payload = "123,456,78.91234,-45.123456\n".repeat(280000);
|
||||
const terms = ["Alice", ...Array.from({ length: 8 }, (_, i) => `Researcher${i}[0-9]+`)];
|
||||
const output = await transform(`Alice\n${payload}Alice`, {
|
||||
filePath: `safe-regex-${randomUUID()}.csv`, terms,
|
||||
});
|
||||
expect(output).to.equal(`XXXX-1\n${payload}XXXX-1`);
|
||||
});
|
||||
|
||||
it("uses the worker budget for dense Unicode replacements", async () => {
|
||||
const input = "Álice,123,45.678,-23.456\n".repeat(800000);
|
||||
expect(await transform(input, { filePath: `dense-unicode-${randomUUID()}.csv`, terms: ["Alice"] }))
|
||||
.to.equal("XXXX-1,123,45.678,-23.456\n".repeat(800000));
|
||||
});
|
||||
|
||||
it("retains the short deadline for backtracking regexes on a worker and recovers", async () => {
|
||||
const root = await fs.mkdtemp(join(tmpdir(), "worker-regex-deadline-"));
|
||||
const input = join(root, "input"), output = join(root, "output");
|
||||
const controller = new AbortController();
|
||||
try {
|
||||
const text = "a".repeat(250);
|
||||
await fs.writeFile(input, text);
|
||||
let failure;
|
||||
const started = Date.now();
|
||||
try {
|
||||
await pool.anonymizeOnWorker({ input, output, options: { terms: ["a+a+a+a+a+a+b(?=x)"] },
|
||||
maxOutput: 1000000, context: { mask: "XXXX", hostname: "localhost" } }, text.length, controller.signal);
|
||||
} catch (error) { failure = error; }
|
||||
expect(failure).to.be.instanceOf(Error);
|
||||
expect(failure.message).to.match(/timed out after 1000ms/);
|
||||
expect(Date.now() - started).to.be.lessThan(8000);
|
||||
let exists = true;
|
||||
try { await fs.access(output); } catch (error) { if (error.code === "ENOENT") exists = false; else throw error; }
|
||||
expect(exists).to.equal(false);
|
||||
expect(await transform("Alice ".repeat(50000), { filePath: `regex-recovery-${randomUUID()}.txt`, terms: ["Alice"] }))
|
||||
.to.equal("XXXX-1 ".repeat(50000));
|
||||
} finally { controller.abort(); await fs.rm(root, { force: true, recursive: true }); }
|
||||
});
|
||||
it("cleans crash leftovers while preserving recent spools and unrelated files", async () => {
|
||||
const root = await fs.mkdtemp(join(tmpdir(), "anonymous-spool-test-"));
|
||||
try {
|
||||
const old = join(root, "anonymous-text-old"), current = join(root, "anonymous-text-current"), other = join(root, "other");
|
||||
await Promise.all([old, current, other].map(path => fs.mkdir(path)));
|
||||
await fs.writeFile(join(old, "input"), "private");
|
||||
await fs.utimes(old, new Date(0), new Date(0));
|
||||
await removeStaleTextSpools(root);
|
||||
expect((await fs.readdir(root)).sort()).to.deep.equal(["anonymous-text-current", "other"]);
|
||||
} finally { await fs.rm(root, { force: true, recursive: true }); }
|
||||
});
|
||||
it("bounds an unresponsive Redis command and disconnects its socket", async () => {
|
||||
let disconnected = false, error;
|
||||
const client = { isOpen: true, destroy: () => { disconnected = true; } };
|
||||
try { await cacheCommand(new Promise(() => {}), client, 20); } catch (failure) { error = failure; }
|
||||
expect(error.message).to.equal("cache_command_timeout"); expect(disconnected).to.equal(true);
|
||||
});
|
||||
|
||||
it("cleans stale spools and transformed entries at startup and while idle", async () => {
|
||||
const cacheRoot = join(tmpdir(), "anonymous-transformed-v1");
|
||||
const stale = await fs.mkdtemp(join(tmpdir(), "anonymous-text-startup-"));
|
||||
const recent = await fs.mkdtemp(join(tmpdir(), "anonymous-text-idle-"));
|
||||
const unrelated = await fs.mkdtemp(join(tmpdir(), "unrelated-cleanup-"));
|
||||
const keys = [randomUUID(), randomUUID(), randomUUID()], paths = keys.flatMap(key => [join(cacheRoot, key + ".json"), join(cacheRoot, key + ".data")]);
|
||||
let stop;
|
||||
const missing = async path => { try { await fs.access(path); return false; } catch (error) { if (error.code === "ENOENT") return true; throw error; } };
|
||||
const seed = async (key, version, created) => {
|
||||
await fs.writeFile(join(cacheRoot, key + ".data"), "private");
|
||||
await fs.writeFile(join(cacheRoot, key + ".json"), JSON.stringify({ version, created, size: 7, changed: false }));
|
||||
};
|
||||
try {
|
||||
await fs.mkdir(cacheRoot, { recursive: true });
|
||||
await fs.writeFile(join(stale, "input"), "original bytes"); await fs.utimes(stale, new Date(0), new Date(0));
|
||||
await fs.writeFile(join(recent, "input"), "recent original bytes");
|
||||
await fs.writeFile(join(unrelated, "input"), "keep");
|
||||
await seed(keys[0], 1, Date.now()); await seed(keys[1], 2, 0); await seed(keys[2], 2, Date.now());
|
||||
stop = await startTemporaryStorageMaintenance(20);
|
||||
expect(await missing(stale)).to.equal(true);
|
||||
expect(await missing(join(cacheRoot, keys[0] + ".data"))).to.equal(true);
|
||||
expect(await missing(join(cacheRoot, keys[1] + ".data"))).to.equal(true);
|
||||
expect(await missing(join(cacheRoot, keys[2] + ".data"))).to.equal(false);
|
||||
expect(await missing(recent)).to.equal(false); expect(await missing(unrelated)).to.equal(false);
|
||||
await fs.utimes(recent, new Date(0), new Date(0));
|
||||
await seed(keys[2], 2, 0);
|
||||
const deadline = Date.now() + 2000;
|
||||
while ((!await missing(recent) || !await missing(join(cacheRoot, keys[2] + ".data"))) && Date.now() < deadline) {
|
||||
await new Promise(resolve => setTimeout(resolve, 10));
|
||||
}
|
||||
expect(await missing(recent)).to.equal(true); expect(await missing(join(cacheRoot, keys[2] + ".json"))).to.equal(true);
|
||||
expect(await fs.readFile(join(unrelated, "input"), "utf8")).to.equal("keep");
|
||||
} finally {
|
||||
stop?.();
|
||||
await Promise.all([...paths, stale, recent, unrelated].map(path => fs.rm(path, { force: true, recursive: true })));
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -166,7 +166,7 @@ describe("production regressions", function () {
|
||||
it("fetches complete truncated gist content", async function () {
|
||||
const Gist = require("../src/core/Gist").default;
|
||||
const model = new (require("../src/core/model/anonymizedGists/anonymizedGists.model").default)({ source: { gistId: "123" } });
|
||||
const gist = new Gist(model); gist.getToken = async () => "private-token";
|
||||
const gist = new Gist(model); gist.getAccess = async () => ({ token: "private-token", connection: "github-app" });
|
||||
stub(gh, "octokit", token => {
|
||||
expect(token).to.equal("private-token");
|
||||
return { rest: { gists: { get: async () => ({ data: { files: { file: { filename: "file", content: "prefix", truncated: true, raw_url: "https://gist.githubusercontent.com/raw", size: 20 } } } }) } },
|
||||
@@ -217,6 +217,7 @@ describe("production regressions", function () {
|
||||
|
||||
function response() {
|
||||
return { headers: {}, header(key, value) { this.headers[key] = value; return this; },
|
||||
set(key, value) { return this.header(key, value); },
|
||||
contentType() { return this; }, status(value) { this.statusCode = value; return this; }, end() {}, send() {} };
|
||||
}
|
||||
function fileRoute(originalName) {
|
||||
@@ -282,7 +283,7 @@ describe("production regressions", function () {
|
||||
const Conference = require("../src/core/Conference").default;
|
||||
const model = { status: "ready", save: async () => {} };
|
||||
const conference = new Conference(model);
|
||||
conference.repositories = async () => [{ expire: async () => { throw new Error("storage failed"); } }];
|
||||
conference.repositories = async () => [{ markExpired: async () => { throw new Error("storage failed"); } }];
|
||||
try { await conference.expire(); throw new Error("expected rejection"); }
|
||||
catch (error) { expect(error.message).to.equal("storage failed"); }
|
||||
expect(model.status).to.equal("ready");
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
const { expect } = require("chai");
|
||||
require("ts-node/register/transpile-only");
|
||||
const User = require("../src/core/User").default;
|
||||
const UserModel = require("../src/core/model/users/users.model").default;
|
||||
const { projectNameKey, saveProjectName } = require("../src/server/routes/project-names");
|
||||
|
||||
describe("private project names", () => {
|
||||
let updateOne, writes, user;
|
||||
beforeEach(() => {
|
||||
updateOne = UserModel.updateOne;
|
||||
writes = [];
|
||||
UserModel.updateOne = async (query, update) => { writes.push({ query, update }); };
|
||||
user = new User(new UserModel({ username: "owner" }));
|
||||
user.getRepositories = async () => [{ repoId: "artifact.v2" }];
|
||||
user.getPullRequests = async () => [{ pullRequestId: "artifact.v2" }];
|
||||
user.getGists = async () => [{ gistId: "artifact.v2" }];
|
||||
});
|
||||
afterEach(() => { UserModel.updateOne = updateOne; });
|
||||
|
||||
for (const type of ["repo", "pr", "gist"]) {
|
||||
it(`saves a ${type} name only on the current user's record`, async () => {
|
||||
const name = await saveProjectName(user, { type, id: "artifact.v2", name: " Reproduction package " });
|
||||
expect(name).to.equal("Reproduction package");
|
||||
expect(writes).to.have.length(1);
|
||||
expect(String(writes[0].query._id)).to.equal(String(user.model._id));
|
||||
expect(writes[0].update).to.deep.equal({ $set: { ["projectNames." + projectNameKey(type, "artifact.v2")]: name } });
|
||||
expect(user.model.projectNames.get(projectNameKey(type, "artifact.v2"))).to.equal(name);
|
||||
});
|
||||
}
|
||||
it("keeps names for different artifact types distinct", () => {
|
||||
expect(projectNameKey("repo", "same.id")).not.to.equal(projectNameKey("pr", "same.id"));
|
||||
expect(projectNameKey("repo", "same.id")).to.match(/^[a-f0-9]{64}$/);
|
||||
});
|
||||
it("removes a saved name when the field is cleared", async () => {
|
||||
await saveProjectName(user, { type: "repo", id: "artifact.v2", name: "Package" });
|
||||
await saveProjectName(user, { type: "repo", id: "artifact.v2", name: " " });
|
||||
expect(writes[1].update).to.deep.equal({ $unset: { ["projectNames." + projectNameKey("repo", "artifact.v2")]: "" } });
|
||||
expect(user.model.projectNames.has(projectNameKey("repo", "artifact.v2"))).to.equal(false);
|
||||
});
|
||||
it("rejects naming a project outside the user's accessible records", async () => {
|
||||
let error;
|
||||
try { await saveProjectName(user, { type: "repo", id: "someone-elses-project", name: "Mine" }); }
|
||||
catch (caught) { error = caught; }
|
||||
expect(error.httpStatus).to.equal(403);
|
||||
expect(writes).to.have.length(0);
|
||||
});
|
||||
it("rejects invalid names and object-shaped identifiers before writing", async () => {
|
||||
for (const input of [null, { type: ["repo"], id: "artifact.v2", name: "Bad" },
|
||||
{ type: "repo", id: { $ne: null }, name: "Bad" }, { type: "repo", id: "artifact.v2", name: "x".repeat(101) },
|
||||
{ type: "repo", id: "artifact.v2", name: "Two\nlines" },
|
||||
...[0, 31, 127].map(code => ({ type: "repo", id: "artifact.v2", name: "Name" + String.fromCharCode(code) }))]) {
|
||||
let error;
|
||||
try { await saveProjectName(user, input); } catch (caught) { error = caught; }
|
||||
expect(error.httpStatus).to.equal(400);
|
||||
}
|
||||
expect(writes).to.have.length(0);
|
||||
});
|
||||
it("does not change the in-memory name when persistence fails", async () => {
|
||||
UserModel.updateOne = async () => { throw Error("Database unavailable"); };
|
||||
let error;
|
||||
try { await saveProjectName(user, { type: "repo", id: "artifact.v2", name: "Package" }); }
|
||||
catch (caught) { error = caught; }
|
||||
expect(error.message).to.equal("Database unavailable");
|
||||
expect(user.model.projectNames).to.equal(undefined);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,47 @@
|
||||
/* global fetch */
|
||||
const { expect } = require("chai");
|
||||
const express = require("express");
|
||||
const { rateLimit } = require("express-rate-limit");
|
||||
require("ts-node/register/transpile-only");
|
||||
const { requestRateLimitKey } = require("../src/server/rate-limit-key");
|
||||
|
||||
describe("rate limiter client addresses", function () {
|
||||
const key = (ip, headers = {}, remoteAddress = "127.0.0.1") =>
|
||||
requestRateLimitKey({ ip, headers, socket: { remoteAddress } });
|
||||
|
||||
it("normalizes IPv4 ports and mapped IPv6 without truncating native IPv6", function () {
|
||||
expect(key("192.0.2.1:1234")).to.equal("192.0.2.1");
|
||||
expect(key("::ffff:192.0.2.1")).to.equal("192.0.2.1");
|
||||
expect(key("2001:db8:abcd:1200::1234")).to.equal("2001:db8:abcd:1200::/56");
|
||||
expect(key("2001:db8:abcd:1300::1234")).not.to.equal(key("2001:db8:abcd:1200::1234"));
|
||||
});
|
||||
|
||||
it("accepts the visitor header only for a Cloudflare edge address", function () {
|
||||
expect(key("192.0.2.1", { "cf-connecting-ip": "198.51.100.1" })).to.equal("192.0.2.1");
|
||||
expect(key("104.16.0.1", { "cf-connecting-ip": " 198.51.100.1 " })).to.equal("198.51.100.1");
|
||||
expect(key("104.16.0.1", { "cf-connecting-ip": " " })).to.equal("104.16.0.1");
|
||||
expect(key(undefined, {}, "::ffff:192.0.2.1")).to.equal("192.0.2.1");
|
||||
});
|
||||
|
||||
it("shares limits across IPv6 addresses within a subnet and ignores forged Cloudflare headers", async function () {
|
||||
const app = express();
|
||||
app.set("trust proxy", "loopback");
|
||||
app.use(rateLimit({ windowMs: 60000, limit: 1, keyGenerator: requestRateLimitKey }));
|
||||
app.get("/", (_req, res) => res.send("ok"));
|
||||
let server;
|
||||
await new Promise(resolve => { server = app.listen(0, "127.0.0.1", resolve); });
|
||||
const request = (ip, visitor) => fetch(`http://127.0.0.1:${server.address().port}/`, {
|
||||
headers: { "x-forwarded-for": ip, "cf-connecting-ip": visitor },
|
||||
});
|
||||
try {
|
||||
expect((await request("2001:db8:abcd:1200::1", "198.51.100.1")).status).to.equal(200);
|
||||
expect((await request("2001:db8:abcd:12ff::2", "198.51.100.2")).status).to.equal(429);
|
||||
expect((await request("2001:db8:abcd:1300::1", "198.51.100.1")).status).to.equal(200);
|
||||
expect((await request("192.0.2.1", "198.51.100.1")).status).to.equal(200);
|
||||
expect((await request("::ffff:192.0.2.1", "198.51.100.2")).status).to.equal(429);
|
||||
} finally {
|
||||
server.closeAllConnections();
|
||||
await new Promise(resolve => server.close(resolve));
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -92,5 +92,15 @@ describe("repository activity dates", () => {
|
||||
expect(body.publishedAt).to.equal(undefined);
|
||||
expect(body.settingsSavedAt).to.equal(undefined);
|
||||
expect(body.lastUpdateDate).to.deep.equal(repo.model.anonymizeDate);
|
||||
expect(body.expirationDate).to.equal(null);
|
||||
});
|
||||
it("tells reviewers when an expiring repository stops being available", async () => {
|
||||
const repo = repository(); auth(repo);
|
||||
repo.model.options.expirationMode = "remove";
|
||||
repo.model.options.expirationDate = new Date("2099-01-01");
|
||||
stub(Files, "exists", async () => null);
|
||||
let body;
|
||||
await handler("repository-public", "/:repoId/options", "get")({}, { header: () => {}, json: value => { body = value; } });
|
||||
expect(body.expirationDate).to.deep.equal(new Date("2099-01-01"));
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,159 @@
|
||||
const { expect } = require("chai");
|
||||
require("ts-node/register/transpile-only");
|
||||
const Repository = require("../src/core/Repository").default;
|
||||
const Model = require("../src/core/model/anonymizedRepositories/anonymizedRepositories.model").default;
|
||||
const utils = require("../src/server/routes/route-utils");
|
||||
const github = require("../src/core/source/GitHubRepository");
|
||||
const app = require("../src/core/github-app");
|
||||
const tokens = require("../src/core/GitHubUtils");
|
||||
const db = require("../src/server/database");
|
||||
const queue = require("../src/queue");
|
||||
const router = require("../src/server/routes/repository-private").default;
|
||||
const handler = (path, method = "get") => router.stack.find(layer =>
|
||||
layer.route?.path === path && layer.route.methods[method]).route.stack[0].handle;
|
||||
|
||||
describe("reconnecting a recreated repository", () => {
|
||||
const restores = [];
|
||||
function stub(object, key, value) {
|
||||
const original = object[key]; restores.push(() => { object[key] = original; }); object[key] = value;
|
||||
}
|
||||
let repo, user, fresh, selected, writes, removals, jobs, commitChecks;
|
||||
beforeEach(() => {
|
||||
repo = new Repository(new Model({ repoId: "recreated", status: "ready",
|
||||
owner: "507f1f77bcf86cd799439011",
|
||||
githubAccess: { kind: "github-app", publicRead: true, repositoryId: 1380842178, revision: "old" },
|
||||
source: { repositoryName: "owner/repo", repositoryId: "old-record", branch: "main", commit: "deadbeef" },
|
||||
options: { terms: ["author"], expirationMode: "never" },
|
||||
}));
|
||||
user = { id: repo.owner.id, model: { id: repo.owner.id } };
|
||||
selected = { token: "fresh-token", binding: { kind: "github-app", publicRead: true, repositoryId: 1396237353, revision: "fresh" } };
|
||||
writes = []; removals = []; jobs = []; commitChecks = [];
|
||||
fresh = { id: "gh_1396237353", model: { id: "new-record" }, fullName: "owner/repo",
|
||||
toJSON: () => ({ externalId: "gh_1396237353" }),
|
||||
branches: async () => [{ name: "main", commit: "abcdef" }],
|
||||
readme: async () => "replacement readme",
|
||||
getCommitInfo: async sha => { commitChecks.push(sha); return { commit: { committer: { date: "2026-09-29T00:00:00Z" } } }; },
|
||||
};
|
||||
stub(utils, "getRepo", async () => repo);
|
||||
stub(utils, "getUser", async () => user);
|
||||
stub(utils, "handleError", error => { throw error; });
|
||||
stub(db, "getRepository", async () => repo);
|
||||
stub(app, "selectRepositoryAccess", async (owner, name, kind) => {
|
||||
expect([owner, name, kind]).to.deep.equal([repo.owner.id, "owner/repo", "github-app"]);
|
||||
return selected;
|
||||
});
|
||||
stub(app, "boundAppToken", async () => { throw new Error("old binding is inaccessible"); });
|
||||
stub(tokens, "getToken", async () => { throw new Error("old binding is inaccessible"); });
|
||||
stub(github, "getRepositoryFromGitHub", async options => {
|
||||
expect(options.accessToken).to.equal("fresh-token");
|
||||
expect(options.force).to.equal(true);
|
||||
expect(options.repositoryID).to.equal(undefined);
|
||||
return fresh;
|
||||
});
|
||||
stub(Model, "updateOne", (filter, update) => ({ exec: async () => {
|
||||
writes.push({ filter, update }); return { matchedCount: 1 };
|
||||
} }));
|
||||
repo.remove = async options => { removals.push(options); };
|
||||
repo.updateStatus = async status => { repo.model.status = status; };
|
||||
stub(queue, "downloadQueue", { add: async (...args) => { jobs.push(args); } });
|
||||
});
|
||||
afterEach(() => { while (restores.length) restores.pop()(); });
|
||||
function preview(query = {}) {
|
||||
return { params: { owner: "owner", repo: "repo" }, query: {
|
||||
anonymizedRepoId: repo.repoId, repositoryID: "old-record", reconnect: "1", ...query,
|
||||
} };
|
||||
}
|
||||
function form(extra = {}) {
|
||||
return { body: { repoId: repo.repoId, fullName: "owner/repo", reconnectRepositoryId: "gh_1396237353",
|
||||
source: { branch: "main", commit: "abcdef" }, terms: ["author"], options: { expirationMode: "never" }, ...extra } };
|
||||
}
|
||||
async function failure(work, message) {
|
||||
let error;
|
||||
try { await work(); } catch (caught) { error = caught; }
|
||||
expect(error?.message).to.equal(message);
|
||||
expect(writes).to.have.length(0);
|
||||
expect(removals).to.have.length(0);
|
||||
expect(jobs).to.have.length(0);
|
||||
}
|
||||
|
||||
for (const path of ["/:owner/:repo/", "/:owner/:repo/branches", "/:owner/:repo/readme"]) {
|
||||
it(`previews ${path} with fresh access without changing the anonymization`, async () => {
|
||||
let result;
|
||||
await handler(path)(preview(), { json: value => { result = value; }, send: value => { result = value; } });
|
||||
expect(result).not.to.equal(undefined);
|
||||
expect(repo.model.githubAccess.repositoryId).to.equal(1380842178);
|
||||
expect(writes).to.have.length(0);
|
||||
});
|
||||
it(`keeps the bound identity for ordinary ${path} requests`, async () => {
|
||||
await failure(() => handler(path)(preview({ reconnect: undefined }), {}), "old binding is inaccessible");
|
||||
});
|
||||
it(`denies a non-owner reconnect preview at ${path}`, async () => {
|
||||
user = { id: "admin", isAdmin: true };
|
||||
await failure(() => handler(path)(preview(), {}), "not_owner");
|
||||
});
|
||||
}
|
||||
|
||||
it("rebinds the source and downloads the replacement even when the old commit no longer exists", async () => {
|
||||
await handler("/:repoId/", "post")(form(), { json: () => {} });
|
||||
expect(commitChecks).to.deep.equal(["abcdef"]);
|
||||
expect(writes).to.have.length(1);
|
||||
const saved = writes[0];
|
||||
expect(saved.filter["githubAccess.revision"]).to.equal("old");
|
||||
expect(saved.update.$set.githubAccess.repositoryId).to.equal(1396237353);
|
||||
expect(saved.update.$set.githubAccess.revision).not.to.equal("old");
|
||||
expect(saved.update.$set.source.repositoryId).to.equal("new-record");
|
||||
expect(saved.update.$set.source.commit).to.equal("abcdef");
|
||||
expect(saved.update.$set.source.commitDate).to.deep.equal(new Date("2026-09-29T00:00:00Z"));
|
||||
expect(removals).to.deep.equal([{ accessRevision: "old" }]);
|
||||
expect(jobs[0][1]).to.deep.equal({ repoId: "recreated" });
|
||||
});
|
||||
it("rebuilds when only the repository identity changed", async () => {
|
||||
await handler("/:repoId/", "post")(form({ source: { branch: "main", commit: "deadbeef" } }), { json: () => {} });
|
||||
expect(jobs).to.have.length(1);
|
||||
});
|
||||
it("does not reconnect implicitly on an ordinary edit", async () => {
|
||||
await failure(() => handler("/:repoId/", "post")(form({ reconnectRepositoryId: undefined }), {}), "old binding is inaccessible");
|
||||
});
|
||||
it("rejects a non-owner save", async () => {
|
||||
user = { id: "admin", isAdmin: true };
|
||||
await failure(() => handler("/:repoId/", "post")(form(), {}), "not_owner");
|
||||
});
|
||||
it("rejects a coauthor's reconnect preview and save", async () => {
|
||||
repo.model.coauthors = [{ username: "coauthor", githubId: "42" }];
|
||||
user = { id: "other", username: "coauthor", model: { id: "other", externalIDs: { github: "42" } } };
|
||||
await failure(() => handler("/:owner/:repo/branches")(preview(), {}), "not_owner");
|
||||
await failure(() => handler("/:repoId/", "post")(form(), {}), "not_owner");
|
||||
});
|
||||
it("does not remove files if the connection changed during validation", async () => {
|
||||
repo.remove = Repository.prototype.remove.bind(repo);
|
||||
repo.resetSate = async () => { throw new Error("must not remove files"); };
|
||||
stub(Model, "updateOne", async () => ({ matchedCount: 0 }));
|
||||
await failure(() => handler("/:repoId/", "post")(form(), {}), "connection_changed");
|
||||
});
|
||||
it("rejects replacement after the owner previewed the repository", async () => {
|
||||
fresh.id = "gh_999999";
|
||||
await failure(() => handler("/:repoId/", "post")(form(), {}), "connection_changed");
|
||||
});
|
||||
it("rejects a binding that differs from the resolved repository", async () => {
|
||||
selected.binding.repositoryId = 999999;
|
||||
await failure(() => handler("/:repoId/", "post")(form(), {}), "connection_changed");
|
||||
});
|
||||
it("preserves the snapshot when new access is denied", async () => {
|
||||
stub(app, "selectRepositoryAccess", async () => { throw new Error("github_app_access_required"); });
|
||||
await failure(() => handler("/:repoId/", "post")(form(), {}), "github_app_access_required");
|
||||
});
|
||||
it("rejects a missing replacement branch", async () => {
|
||||
fresh.branches = async () => [];
|
||||
await failure(() => handler("/:repoId/", "post")(form(), {}), "branch_not_specified");
|
||||
});
|
||||
it("preserves the snapshot when the selected commit is inaccessible", async () => {
|
||||
fresh.getCommitInfo = async () => { throw new Error("commit_not_found"); };
|
||||
await failure(() => handler("/:repoId/", "post")(form(), {}), "commit_not_found");
|
||||
});
|
||||
for (const status of ["preparing", "queue", "download", "removing", "expiring"]) {
|
||||
it(`does not reconnect during ${status}`, async () => {
|
||||
repo.model.status = status;
|
||||
await failure(() => handler("/:repoId/", "post")(form(), {}), "invalid_status");
|
||||
});
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,176 @@
|
||||
const { expect } = require("chai");
|
||||
const express = require("express");
|
||||
const compression = require("compression");
|
||||
const http = require("node:http");
|
||||
const net = require("node:net");
|
||||
const { setTimeout } = require("node:timers");
|
||||
const { gunzipSync } = require("node:zlib");
|
||||
const { Readable } = require("node:stream");
|
||||
const { setTimeout: delay } = require("node:timers/promises");
|
||||
require("ts-node/register/transpile-only");
|
||||
const { RequestMetrics, monitorRequests, requestRoute, requestHeaders, startStage, requestMetrics, activeRequestCount } = require("../src/core/request-monitoring");
|
||||
const { percentileUpperBound, socketSnapshot, performanceReport } = require("../src/core/performance-monitoring");
|
||||
const GitHubStream = require("../src/core/source/GitHubStream").default;
|
||||
const storage = require("../src/core/storage").default;
|
||||
|
||||
function fetch(server, path, options = {}) {
|
||||
return new Promise((resolve, reject) => {
|
||||
http.request({ host: "127.0.0.1", port: server.address().port, path, ...options }, res => {
|
||||
const chunks = []; res.on("data", chunk => chunks.push(chunk)); res.on("error", reject);
|
||||
res.on("end", () => resolve({ status: res.statusCode, headers: res.headers, body: Buffer.concat(chunks) }));
|
||||
}).on("error", reject).end();
|
||||
});
|
||||
}
|
||||
|
||||
describe("bounded request monitoring", function () {
|
||||
let server, metrics, logs, app;
|
||||
beforeEach(function () {
|
||||
metrics = new RequestMetrics(); logs = []; requestMetrics.drain();
|
||||
app = express(); app.use(monitorRequests("api", metrics, { info: (...args) => logs.push(args), warn: (...args) => logs.push(args) }));
|
||||
app.use(compression({ filter: () => true }));
|
||||
});
|
||||
afterEach(async function () {
|
||||
if (server) { server.closeAllConnections(); await new Promise(resolve => server.close(resolve)); server = null; }
|
||||
requestMetrics.drain();
|
||||
});
|
||||
async function listen() { server = await new Promise(resolve => { const s = app.listen(0, "127.0.0.1", () => resolve(s)); }); }
|
||||
|
||||
it("keeps labels bounded and removes source identities and query values", function () {
|
||||
expect(requestRoute("/api/repo/private-name/file/secret.txt?token=abc", "api")).to.equal("/api/repo/:repoId/file/:path");
|
||||
expect(requestRoute("/api/repo/secret-owner/secret-repo/readme?branch=secret", "api")).to.equal("/api/repo/:owner/:repo/readme");
|
||||
expect(requestRoute("/api/repo/id/files/counts", "api")).to.equal("/api/repo/:repoId/files/counts");
|
||||
const labels = new Set(Array.from({ length: 10000 }, (_, i) => requestRoute(`/unknown-${i}/private-${i}`, "api")));
|
||||
expect([...labels]).to.deep.equal(["other"]);
|
||||
expect(requestRoute("/healthcheck", "streamer")).to.equal("/healthcheck");
|
||||
const bounded = new RequestMetrics(3);
|
||||
for (let i = 0; i < 1000; i++) bounded.observe("request", String(i), "GET", 10);
|
||||
expect(bounded.size).to.equal(3); expect(bounded.dropped).to.equal(997);
|
||||
});
|
||||
|
||||
it("records histogram upper bounds and releases the batch after flushing", function () {
|
||||
[1, 20, 51, 150, 501, 50000, 70000].forEach(ms => metrics.observe("request", "other", "GET", ms));
|
||||
const row = metrics.drain()[0].data;
|
||||
expect(row.count).to.equal(7); expect(row.sumMs).to.equal(120723);
|
||||
expect(percentileUpperBound(row.buckets, .5)).to.equal(250);
|
||||
expect(percentileUpperBound(row.buckets, .95)).to.equal(null);
|
||||
expect(metrics.size).to.equal(0); expect(metrics.drain()).to.deep.equal([]);
|
||||
});
|
||||
|
||||
for (const gzip of [false, true]) {
|
||||
it(`preserves backpressure and ${gzip ? "gzip" : "plain"} bytes`, async function () {
|
||||
this.timeout(10000);
|
||||
const body = Buffer.alloc(2 * 1024 * 1024, "content-");
|
||||
let drains = 0, callbacks = 0;
|
||||
app.get("/api/repo/:repoId/file/*path", async (_req, res) => {
|
||||
res.type("text/plain");
|
||||
for (let i = 0; i < body.length; i += 16384) {
|
||||
if (!res.write(body.subarray(i, i + 16384), () => callbacks++)) {
|
||||
drains++;
|
||||
await new Promise(resolve => {
|
||||
const drained = () => { emitter.removeListener("drain", drained); resolve(); };
|
||||
const emitter = res.on("drain", drained);
|
||||
});
|
||||
}
|
||||
}
|
||||
res.end();
|
||||
});
|
||||
await listen();
|
||||
const result = await fetch(server, "/api/repo/private/file/private.txt?token=secret", { headers: { "Accept-Encoding": gzip ? "gzip" : "identity" } });
|
||||
expect(gzip ? gunzipSync(result.body) : result.body).to.deep.equal(body);
|
||||
expect(drains).to.be.greaterThan(0);
|
||||
// Compression omits write callbacks; the plain writable must retain them.
|
||||
expect(callbacks).to.equal(gzip ? 0 : body.length / 16384);
|
||||
const row = metrics.drain().find(row => row.data.metric === "request").data;
|
||||
expect(row.bytes).to.equal(result.body.length); expect(row.aborted).to.equal(0); expect(row.count).to.equal(1);
|
||||
expect(JSON.stringify(logs)).not.to.include("private.txt"); expect(JSON.stringify(logs)).not.to.include("secret");
|
||||
expect(logs[0][1].requestId).to.equal(result.headers["x-request-id"]);
|
||||
expect(activeRequestCount()).to.equal(0);
|
||||
});
|
||||
}
|
||||
|
||||
it("counts HEAD as a zero-byte response even when a handler writes a body", async function () {
|
||||
app.get("/", (_req, res) => res.send("body omitted by Node")); await listen();
|
||||
const result = await fetch(server, "/", { method: "HEAD" });
|
||||
expect(result.body.length).to.equal(0);
|
||||
expect(metrics.drain().find(row => row.data.metric === "request").data.bytes).to.equal(0);
|
||||
});
|
||||
|
||||
for (const headersSent of [false, true]) {
|
||||
it(`records a disconnect ${headersSent ? "after" : "before"} headers exactly once`, async function () {
|
||||
let arrived; const ready = new Promise(resolve => { arrived = resolve; });
|
||||
let finished; const terminal = new Promise(resolve => { finished = resolve; });
|
||||
app.get("/", (_req, res) => { if (headersSent) { res.type("text/plain"); res.write("partial"); } arrived(); res.on("close", finished); });
|
||||
await listen();
|
||||
const client = http.get({ host: "127.0.0.1", port: server.address().port, path: "/" }); client.on("error", () => {});
|
||||
await ready; client.destroy(); await terminal;
|
||||
expect(logs).to.have.length(1); expect(logs[0][0]).to.equal("request interrupted");
|
||||
expect(logs[0][1].outcome).to.equal("interrupted");
|
||||
const row = metrics.drain().find(row => row.data.metric === "request").data;
|
||||
expect(row.count).to.equal(1); expect(row.aborted).to.equal(1); expect(activeRequestCount()).to.equal(0);
|
||||
if (!headersSent) expect(logs[0][1].firstByteMs).to.equal(null);
|
||||
});
|
||||
}
|
||||
|
||||
it("keeps overlapping request IDs and asynchronous stages isolated", async function () {
|
||||
let doneA, doneB;
|
||||
app.get("/a", (_req, res) => { doneA = startStage("upstream"); setTimeout(() => { doneA(); doneA(); res.json(requestHeaders()); }, 30); });
|
||||
app.get("/b", (_req, res) => { doneB = startStage("repository"); setTimeout(() => { doneB(); res.json(requestHeaders()); }, 10); });
|
||||
await listen(); const results = await Promise.all([fetch(server, "/a"), fetch(server, "/b")]);
|
||||
for (const result of results) expect(JSON.parse(result.body)["x-request-id"]).to.equal(result.headers["x-request-id"]);
|
||||
expect(results[0].headers["x-request-id"]).not.to.equal(results[1].headers["x-request-id"]);
|
||||
expect(logs.filter(row => row[1].stages.upstream)).to.have.length(1);
|
||||
expect(logs.filter(row => row[1].stages.repository)).to.have.length(1);
|
||||
expect(requestMetrics.drain().reduce((count, row) => count + row.data.count, 0)).to.equal(2);
|
||||
expect(requestHeaders()).to.deep.equal({});
|
||||
});
|
||||
|
||||
it("covers delays and errors before route handling", async function () {
|
||||
app.use((_req, _res, next) => setTimeout(next, 35)); app.use(express.json());
|
||||
app.get("/", (_req, res) => res.status(503).json({ error: "unavailable" })); await listen();
|
||||
const response = await fetch(server, "/"); expect(response.status).to.equal(503);
|
||||
const row = metrics.drain().find(row => row.data.metric === "request").data;
|
||||
expect(row.errors).to.equal(1); expect(row.sumMs).to.be.at.least(30);
|
||||
});
|
||||
|
||||
for (const failure of [false, true]) {
|
||||
it(`includes GitHub response-header waits in upstream timing${failure ? " on failure" : ""}`, async function () {
|
||||
const originals = { fileInfo: storage.fileInfo, write: storage.write, read: storage.read };
|
||||
const source = new GitHubStream({ repoId: `headers-${failure}`, organization: "owner", repoName: "repo", commit: "abc", getToken: async () => "token" });
|
||||
const upstreamError = Error("headers failed");
|
||||
source.downloadWithFallback = async () => {
|
||||
await delay(120);
|
||||
if (failure) throw upstreamError;
|
||||
return Readable.from([Buffer.from("content")]);
|
||||
};
|
||||
storage.fileInfo = async () => { throw Error("cache miss"); };
|
||||
storage.write = async (_repo, _path, input) => { for await (const chunk of input) void chunk; };
|
||||
storage.read = async () => Readable.from([Buffer.from("content")]);
|
||||
app.get("/", async (_req, res) => {
|
||||
try {
|
||||
const input = await source.getFileContentCache("file.txt", source.data.repoId, () => ({ sha: "blob", size: 7 }));
|
||||
input.destroy(); res.send("ok");
|
||||
} catch (error) { res.status(502).send(error === upstreamError ? "original error" : "unexpected error"); }
|
||||
});
|
||||
try {
|
||||
await listen(); const response = await fetch(server, "/");
|
||||
expect(response.status).to.equal(failure ? 502 : 200);
|
||||
if (failure) expect(response.body.toString()).to.equal("original error");
|
||||
expect(logs[0][1].stages.upstream).to.be.at.least(100);
|
||||
const row = requestMetrics.drain().find(entry => entry.data.metric === "upstream").data;
|
||||
expect(row.count).to.equal(1); expect(row.errors).to.equal(Number(failure));
|
||||
} finally { Object.assign(storage, originals); }
|
||||
});
|
||||
}
|
||||
|
||||
it("counts PID-owned network sockets without blocking the event loop", async function () {
|
||||
const listener = net.createServer(socket => socket.end()); await new Promise(resolve => listener.listen(0, "127.0.0.1", resolve));
|
||||
try {
|
||||
const snapshot = await socketSnapshot();
|
||||
if (process.platform === "linux") { expect(snapshot.listening).to.be.at.least(1); expect(snapshot.truncated).to.equal(false); }
|
||||
} finally { await new Promise(resolve => listener.close(resolve)); }
|
||||
});
|
||||
|
||||
it("reports missing monitoring as unavailable", async function () {
|
||||
expect((await performanceReport(60000)).available).to.equal(false);
|
||||
});
|
||||
});
|
||||
@@ -3,6 +3,7 @@ const { expect } = require("chai");
|
||||
const express = require("express");
|
||||
const session = require("express-session");
|
||||
const http = require("http");
|
||||
const { setImmediate } = require("timers");
|
||||
const { randomBytes } = require("crypto");
|
||||
const {
|
||||
createReviewConsentRouter,
|
||||
@@ -267,6 +268,55 @@ describe("review consent browser HTTP transport", function () {
|
||||
).to.equal(403);
|
||||
expect(calls).to.have.length(0);
|
||||
});
|
||||
for (const operation of ["preview", "confirm", "handoff"]) {
|
||||
for (const change of ["logout", "account replacement"]) {
|
||||
it(`rejects ${operation} before backend work after ${change} during upload`, async () => {
|
||||
const original = store.get.bind(store);
|
||||
let reads = 0, ready;
|
||||
const authenticated = new Promise((resolve) => { ready = resolve; });
|
||||
store.get = (id, callback) => original(id, (error, value) => {
|
||||
callback(error, value);
|
||||
// The first read restores the session; the second authenticates
|
||||
// the request before the JSON parser waits for the remaining body.
|
||||
if (++reads === 2) setImmediate(ready);
|
||||
});
|
||||
const body = JSON.stringify(operation === "preview" ? previewInput() : operation === "handoff" ? { ticket: "synthetic-signed-ticket" } : confirmInput());
|
||||
let upload;
|
||||
const response = new Promise((resolve, reject) => {
|
||||
upload = http.request({
|
||||
host: "127.0.0.1", port: server.address().port,
|
||||
path: "/api/review-consent/" + operation, method: "POST",
|
||||
headers: {
|
||||
"X-Forwarded-Proto": "https", Cookie: cookie, Origin: origin,
|
||||
"Content-Type": "application/json", "Content-Length": Buffer.byteLength(body),
|
||||
"X-Review-CSRF": csrf,
|
||||
},
|
||||
}, (res) => {
|
||||
res.resume();
|
||||
res.on("end", () => resolve(res.statusCode));
|
||||
});
|
||||
upload.on("error", reject);
|
||||
upload.write(body.slice(0, 1));
|
||||
});
|
||||
try {
|
||||
await authenticated;
|
||||
if (change === "logout") {
|
||||
await new Promise((resolve) => store.destroy(sid, resolve));
|
||||
} else {
|
||||
const saved = await new Promise((resolve) => original(sid, (_error, value) => resolve(value)));
|
||||
saved.passport.user = "b".repeat(24);
|
||||
await new Promise((resolve) => store.set(sid, saved, resolve));
|
||||
}
|
||||
upload.end(body.slice(1));
|
||||
expect(await response).to.equal(401);
|
||||
expect(calls).to.have.length(0);
|
||||
} finally {
|
||||
upload.destroy();
|
||||
store.get = original;
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
it("reloads session authority before returning a pending policy preview", async () => {
|
||||
let arrived, release;
|
||||
const ready = new Promise((resolve) => {
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
const { expect } = require("chai");
|
||||
const fs = require("node:fs");
|
||||
const os = require("node:os");
|
||||
const path = require("node:path");
|
||||
const { PassThrough } = require("node:stream");
|
||||
require("ts-node/register/transpile-only");
|
||||
const config = require("../src/config").default;
|
||||
require("../src/core/storage");
|
||||
const FileSystem = require("../src/core/storage/FileSystem").default;
|
||||
|
||||
describe("cache write error origins", function () {
|
||||
let directory, previousFolder, messages, previousWarn, previousError;
|
||||
beforeEach(function () {
|
||||
directory = fs.mkdtempSync(path.join(os.tmpdir(), "cache-errors-"));
|
||||
previousFolder = config.FOLDER; config.FOLDER = directory;
|
||||
messages = [];
|
||||
previousWarn = console.warn; previousError = console.error;
|
||||
console.warn = (...args) => messages.push(["warn", args.join(" ")]);
|
||||
console.error = (...args) => messages.push(["error", args.join(" ")]);
|
||||
});
|
||||
afterEach(function () {
|
||||
console.warn = previousWarn; console.error = previousError;
|
||||
config.FOLDER = previousFolder;
|
||||
fs.rmSync(directory, { recursive: true, force: true });
|
||||
});
|
||||
it("warns on a source failure and preserves existing cached bytes", async function () {
|
||||
const storage = new FileSystem();
|
||||
await storage.write("repo", "file.txt", "previous");
|
||||
const input = new PassThrough();
|
||||
const failure = Object.assign(new Error("upstream missing"), { httpStatus: 404 });
|
||||
input.once("newListener", event => {
|
||||
if (event === "error") globalThis.queueMicrotask(() => input.destroy(failure));
|
||||
});
|
||||
let caught;
|
||||
try { await storage.write("repo", "file.txt", input); } catch (error) { caught = error; }
|
||||
expect(caught).to.equal(failure);
|
||||
expect(messages.some(([level, text]) => level === "warn" && text.includes("source stream failed"))).to.equal(true);
|
||||
expect(messages.some(([level, text]) => level === "error" && text.includes("write failed"))).to.equal(false);
|
||||
expect(fs.readFileSync(path.join(directory, "repo/original/file.txt"), "utf8")).to.equal("previous");
|
||||
expect(fs.readdirSync(path.join(directory, "repo/original"))).to.deep.equal(["file.txt"]);
|
||||
});
|
||||
it("keeps filesystem failures at error level", async function () {
|
||||
const storage = new FileSystem();
|
||||
await storage.mk("repo", "file.txt");
|
||||
let caught;
|
||||
try { await storage.write("repo", "file.txt", "new content"); } catch (error) { caught = error; }
|
||||
expect(caught).to.be.instanceOf(Error);
|
||||
expect(messages.some(([level, text]) => level === "error" && text.includes("write failed"))).to.equal(true);
|
||||
expect(messages.some(([level]) => level === "warn")).to.equal(false);
|
||||
expect(fs.readdirSync(path.join(directory, "repo/original"))).to.deep.equal(["file.txt"]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,75 @@
|
||||
const { expect } = require("chai");
|
||||
require("ts-node/register/transpile-only");
|
||||
const { TermPresenceIndex } = require("../src/core/term-presence");
|
||||
const { compileTerms, ContentAnonimizer } = require("../src/core/anonymize-utils");
|
||||
|
||||
function reference(terms, text) {
|
||||
const anonymizer = new ContentAnonimizer({ terms });
|
||||
anonymizer.termPresence = null;
|
||||
for (const term of anonymizer.compiledTerms) delete term.literalPrefilter;
|
||||
return anonymizer.anonymize(text);
|
||||
}
|
||||
|
||||
describe("term candidate index", function () {
|
||||
it("retains shared prefixes, nested matches and Unicode-folded alternatives", function () {
|
||||
const patterns = [/Alice/iu, /AliceSmith/iu, /Smith/iu, /[sśŝşšș]/iu, /ſ/iu, /K/iu, /k/iu, /研究/iu];
|
||||
const index = new TermPresenceIndex(patterns);
|
||||
for (const text of ["AliceSmith", "ALICESMITH K ſ 研究", "nothing", "ś", "k", "aAliceSmith"]) {
|
||||
for (let after = -1; after < patterns.length; after++) {
|
||||
const candidates = index.candidates(text, after);
|
||||
patterns.forEach((pattern, i) => {
|
||||
if (i > after && pattern.test(text)) expect(candidates.has(i), `${text}: ${i}`).to.equal(true);
|
||||
if (i <= after) expect(candidates.has(i)).to.equal(false);
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps dot's CR and Unicode line-separator matches", function () {
|
||||
for (const text of ["first\rlast", "first\u2028last", "first\u2029last", "first\nlast", "first😀last"]) {
|
||||
expect(new ContentAnonimizer({ terms: ["first.last"] }).anonymize(text)).to.equal(reference(["first.last"], text));
|
||||
}
|
||||
});
|
||||
|
||||
it("retains ordered cascades, URL removal, and matches created across replacement boundaries", function () {
|
||||
const cases = [
|
||||
[["Alice=>Bob", "Bob=>Hidden"], "Alice Bob https://host/Alice"],
|
||||
[["Alice=>", "@Bob=>Hidden"], "@AliceBob"],
|
||||
[["a-a", "Alice"], "xa-a-a Alice https://host/Alice/file"],
|
||||
[["first.last", "firstXlast=>other"], "firstXlast first.last"],
|
||||
[["Alice=>Bob", "Bob"], "https://host/Alice"],
|
||||
[["k", "s"], "K ſ ék k_ xK Kx"],
|
||||
[["Alice=>Bob", "B.b=>Done"], "Alice"],
|
||||
[["k=>Kk"], "https://host/k"],
|
||||
[["Alice=>$&", "Bob=>$$"], "Alice Bob"],
|
||||
[["\u0301"], "abc"],
|
||||
[["a.*b", "Alice"], "axxb Alice"],
|
||||
];
|
||||
for (const [terms, text] of cases) {
|
||||
expect(new ContentAnonimizer({ terms }).anonymize(text), JSON.stringify([terms, text])).to.equal(reference(terms, text));
|
||||
}
|
||||
});
|
||||
|
||||
it("matches the RE2 path for fixed-width terms and surrounding characters", function () {
|
||||
const terms = ["a-a", "a.a", "k", "s", "k.s", "Alice", "@Alice", "Σ", "研究", "😀", "..", "a.", "a..b"];
|
||||
const edges = ["", "x", "_", "é", "K", "ſ", "😀", "𐐀", "\r", "\n", "\u2028"];
|
||||
for (const term of terms) {
|
||||
const input = edges.flatMap(left => edges.map(right => `${left}${term}${right} ${left}${term.replaceAll(".", "\r")}${right}`)).join(" ");
|
||||
expect(new ContentAnonimizer({ terms: [term] }).anonymize(input), term).to.equal(reference([term], input));
|
||||
}
|
||||
});
|
||||
|
||||
it("does not reuse stale rules after the caller edits its term list", function () {
|
||||
const terms = ["Alice=>first"];
|
||||
expect(new ContentAnonimizer({ terms }).anonymize("Alice")).to.equal("first");
|
||||
terms[0] = "Alice=>second";
|
||||
expect(new ContentAnonimizer({ terms }).anonymize("Alice")).to.equal("second");
|
||||
expect(new ContentAnonimizer({ terms: ["Alice=>first"] }).anonymize("Alice")).to.equal("first");
|
||||
});
|
||||
|
||||
it("does not index quantified, capturing, or escaped user patterns", function () {
|
||||
for (const term of ["a.*b", "a+b", "(Alice)", "a\\.b", "a(?=b)"]) {
|
||||
expect(compileTerms([term]).every(c => !c.literalPrefilter), term).to.equal(true);
|
||||
}
|
||||
});
|
||||
});
|
||||
+913
-18
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,117 @@
|
||||
const { expect } = require("chai");
|
||||
const express = require("express");
|
||||
const compression = require("compression");
|
||||
const { gunzipSync } = require("node:zlib");
|
||||
const http = require("node:http");
|
||||
const { PassThrough } = require("node:stream");
|
||||
const got = require("got");
|
||||
require("ts-node/register/transpile-only");
|
||||
const config = require("../src/config").default;
|
||||
const utils = require("../src/server/routes/route-utils");
|
||||
const router = require("../src/server/routes/repository-public").default;
|
||||
|
||||
describe("ZIP proxy failures", function () {
|
||||
let restore, upstream, server;
|
||||
beforeEach(async function () {
|
||||
const previous = { download: config.ENABLE_DOWNLOAD, endpoint: config.STREAMER_ENTRYPOINT,
|
||||
getRepo: utils.getRepo, getUser: utils.getUser, stream: got.stream };
|
||||
restore = () => {
|
||||
config.ENABLE_DOWNLOAD = previous.download;
|
||||
config.STREAMER_ENTRYPOINT = previous.endpoint;
|
||||
utils.getRepo = previous.getRepo; utils.getUser = previous.getUser; got.stream = previous.stream;
|
||||
};
|
||||
config.ENABLE_DOWNLOAD = true;
|
||||
config.STREAMER_ENTRYPOINT = "http://streamer.test/";
|
||||
utils.getUser = async () => { throw new Error("anonymous"); };
|
||||
utils.getRepo = async () => ({ repoId: "test", options: {},
|
||||
model: { source: { repositoryName: "owner/repo", commit: "abc" } },
|
||||
countView: async () => {}, getToken: async () => "",
|
||||
generateAnonymizeTransformer: () => ({ opt: {} }),
|
||||
});
|
||||
upstream = new PassThrough();
|
||||
got.stream = () => upstream;
|
||||
const app = express(); app.use(compression({ filter: () => true })); app.use(router);
|
||||
server = await new Promise(resolve => {
|
||||
const listening = app.listen(0, "127.0.0.1", () => resolve(listening));
|
||||
});
|
||||
});
|
||||
afterEach(async function () {
|
||||
upstream.destroy(); server.closeAllConnections();
|
||||
await new Promise(resolve => server.close(resolve)); restore();
|
||||
});
|
||||
function request(onResponse, headers = {}) {
|
||||
return http.get(`http://127.0.0.1:${server.address().port}/test/zip`, { headers }, onResponse);
|
||||
}
|
||||
it("answers HEAD without counting a view, obtaining a token or starting a ZIP stream", async function () {
|
||||
utils.getRepo = async () => ({ repoId: "test", countView: () => { throw Error("view counted"); },
|
||||
getToken: () => { throw Error("token requested"); }, generateAnonymizeTransformer: () => { throw Error("transform requested"); } });
|
||||
got.stream = () => { throw Error("archive requested"); };
|
||||
const result = await new Promise((resolve, reject) => {
|
||||
http.request(`http://127.0.0.1:${server.address().port}/test/zip`, { method: "HEAD" }, res => {
|
||||
const chunks = []; res.on("data", chunk => chunks.push(chunk));
|
||||
res.on("end", () => resolve({ status: res.statusCode, headers: res.headers, body: Buffer.concat(chunks) }));
|
||||
}).on("error", reject).end();
|
||||
});
|
||||
expect(result.status).to.equal(200);
|
||||
expect(result.headers["content-disposition"]).to.include("test.zip");
|
||||
expect(result.body.length).to.equal(0);
|
||||
});
|
||||
it("still denies HEAD when downloads are disabled", async function () {
|
||||
config.ENABLE_DOWNLOAD = false;
|
||||
const status = await new Promise((resolve, reject) => {
|
||||
http.request(`http://127.0.0.1:${server.address().port}/test/zip`, { method: "HEAD" }, res => {
|
||||
res.resume(); res.on("end", () => resolve(res.statusCode));
|
||||
}).on("error", reject).end();
|
||||
});
|
||||
expect(status).to.equal(403);
|
||||
});
|
||||
it("aborts a partial ZIP when the upstream fails after headers", async function () {
|
||||
const result = new Promise((resolve, reject) => {
|
||||
const client = request(response => {
|
||||
expect(response.statusCode).to.equal(200);
|
||||
response.once("data", () => upstream.destroy(new Error("upstream disconnected")));
|
||||
response.on("aborted", resolve);
|
||||
response.on("end", () => reject(new Error("truncated ZIP reported as complete")));
|
||||
response.on("error", () => {});
|
||||
});
|
||||
client.on("error", reject);
|
||||
});
|
||||
upstream.write(Buffer.from("PK partial archive"));
|
||||
await result;
|
||||
});
|
||||
it("finishes compressed JSON errors before any ZIP bytes are sent", async function () {
|
||||
const error = "unavailable".repeat(300);
|
||||
const result = new Promise((resolve, reject) => {
|
||||
request(response => {
|
||||
const chunks = [];
|
||||
response.on("data", chunk => chunks.push(chunk));
|
||||
response.on("error", reject);
|
||||
response.on("end", () => resolve({
|
||||
status: response.statusCode, encoding: response.headers["content-encoding"],
|
||||
body: JSON.parse(gunzipSync(Buffer.concat(chunks)).toString()),
|
||||
}));
|
||||
}, { "Accept-Encoding": "gzip" }).on("error", reject);
|
||||
});
|
||||
upstream.once("newListener", event => {
|
||||
if (event === "error") globalThis.queueMicrotask(() => upstream.destroy(Object.assign(new Error("unavailable"), {
|
||||
response: { statusCode: 503, body: JSON.stringify({ error }) },
|
||||
})));
|
||||
});
|
||||
expect(await result).to.deep.equal({ status: 502, encoding: "gzip", body: { error } });
|
||||
});
|
||||
it("keeps JSON errors before any ZIP bytes are sent", async function () {
|
||||
const result = new Promise((resolve, reject) => {
|
||||
request(response => {
|
||||
let body = ""; response.on("data", chunk => { body += chunk; });
|
||||
response.on("end", () => resolve({ status: response.statusCode, body: JSON.parse(body) }));
|
||||
}).on("error", reject);
|
||||
});
|
||||
// Fail only after the route has attached its upstream listener.
|
||||
upstream.once("newListener", event => {
|
||||
if (event === "error") globalThis.queueMicrotask(() => upstream.destroy(Object.assign(new Error("unavailable"), {
|
||||
response: { statusCode: 503, body: '{"error":"zip_not_available"}' },
|
||||
})));
|
||||
});
|
||||
expect(await result).to.deep.equal({ status: 502, body: { error: "zip_not_available" } });
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user