mirror of
https://github.com/tdurieux/anonymous_github.git
synced 2026-11-14 00:07:00 +01:00
164 lines
9.2 KiB
JavaScript
164 lines
9.2 KiB
JavaScript
const { expect } = require("chai");
|
|
require("ts-node/register/transpile-only");
|
|
const mongoose = require("mongoose");
|
|
const db = require("../src/server/database");
|
|
const Repository = require("../src/core/Repository").default;
|
|
const Repo = require("../src/core/model/anonymizedRepositories/anonymizedRepositories.model").default;
|
|
const File = require("../src/core/model/files/files.model").default;
|
|
const Path = require("../src/core/model/anonymized-path").default;
|
|
const { folderCounts } = require("../src/core/folder-counts");
|
|
const { FILE_LISTING_LIMIT } = require("../src/core/file-tree");
|
|
const uri = process.env.PERFORMANCE_MONGO_URI;
|
|
|
|
describe("folder count request validation", () => {
|
|
for (const value of [[], {}, 1, "../private", "a/../b", "/src", "src/", "x".repeat(4097)]) {
|
|
it(`rejects invalid scope ${JSON.stringify(value).slice(0, 30)}`, async () => {
|
|
const error = await folderCounts({}, value).then(() => null, error => error);
|
|
expect(error.message).to.equal("invalid_path"); expect(error.httpStatus).to.equal(400);
|
|
});
|
|
}
|
|
for (const error of [Object.assign(Error("operation exceeded time limit"), { code: 50 }),
|
|
Object.assign(Error("unrelated bad value"), { code: 2 })]) {
|
|
it(`does not retry ${error.message}`, async () => {
|
|
const original = File.aggregate; let calls = 0;
|
|
File.aggregate = () => {
|
|
calls++;
|
|
return { option() { return this; }, exec: async () => { throw error; } };
|
|
};
|
|
try {
|
|
const caught = await folderCounts({ repoId: "fixture", model: { treeGeneration: "active" } }, "")
|
|
.then(() => null, error => error);
|
|
expect(caught).to.equal(error); expect(calls).to.equal(1);
|
|
} finally { File.aggregate = original; }
|
|
});
|
|
}
|
|
|
|
});
|
|
|
|
(uri ? describe : describe.skip)("scoped folder counts with MongoDB", function () {
|
|
this.timeout(60000);
|
|
let repo;
|
|
before(async () => {
|
|
if (!/\/perf_test_[A-Za-z0-9_-]+(?:\?|$)/.test(uri)) throw Error("An isolated perf_test_ database is required");
|
|
await mongoose.connect(uri); db.isConnected = true;
|
|
await Promise.all([Repo, File, Path].map(model => model.createIndexes()));
|
|
});
|
|
after(async () => { db.isConnected = false; await mongoose.connection.dropDatabase(); await mongoose.disconnect(); });
|
|
beforeEach(async () => {
|
|
await Promise.all([Repo, File, Path].map(model => model.deleteMany({})));
|
|
repo = new Repository(await Repo.create({ repoId: "counts", owner: new mongoose.Types.ObjectId(), status: "ready",
|
|
statusDate: new Date(), anonymizeDate: new Date(), treeGeneration: "active", source: { type: "GitHubStream", repositoryName: "fixture/repo" }, options: { terms: [] } }));
|
|
});
|
|
const insert = rows => File.insertMany(rows.map(row => ({ repoId: "counts", treeGeneration: "active", ...row })));
|
|
|
|
it("returns only the current directory and immediate children across a large deep tree", async () => {
|
|
await insert([{ name: "src", path: "" }, { name: "root.txt", path: "", size: 1 }]);
|
|
for (let offset = 0; offset < 12000; offset += 500) await insert(Array.from({ length: 500 }, (_, i) => ({
|
|
name: "file.txt", path: `src/deep/folder${offset + i}`, size: 1,
|
|
})));
|
|
await insert([{ name: "ignored.txt", path: "staged", size: 1, treeGeneration: "staged" },
|
|
{ name: "ignored.txt", path: "old", size: 1, treeGeneration: "old" }]);
|
|
expect(await folderCounts(repo, "")).to.deep.equal({ "": 12001, src: 12000 });
|
|
expect(await folderCounts(repo, "src")).to.deep.equal({ src: 12000, "src/deep": 12000 });
|
|
});
|
|
|
|
it("covers root counts without fetching documents and counts every file in a directory", async () => {
|
|
await insert(Array.from({ length: 2000 }, (_, i) => ({ name: `file${i}`, path: `src/folder${i % 20}`, size: i % 2 })));
|
|
await insert([{ name: "excluded-directory", path: "" },
|
|
{ name: "old", path: "old", size: 1, treeGeneration: "retired" }]);
|
|
let pipeline, query;
|
|
const aggregate = File.aggregate;
|
|
File.aggregate = function (stages, ...args) {
|
|
pipeline = stages; query = aggregate.call(this, stages, ...args); return query;
|
|
};
|
|
try { expect(await folderCounts(repo, "")).to.deep.equal({ "": 2000, src: 2000 }); }
|
|
finally { File.aggregate = aggregate; }
|
|
expect(query.options.hint).to.equal("file_counts_covering");
|
|
const explained = await File.aggregate(pipeline).option(query.options).explain("executionStats");
|
|
const cursor = explained.stages?.find(stage => stage.$cursor)?.$cursor || explained;
|
|
expect(cursor.executionStats.totalDocsExamined).to.equal(0);
|
|
expect(cursor.executionStats.totalKeysExamined).to.be.at.least(2000);
|
|
});
|
|
|
|
it("falls back before the covering index exists and uses it after creation", async () => {
|
|
await insert([{ name: "one", path: "src", size: 1 }]);
|
|
await File.collection.dropIndex("file_counts_covering");
|
|
try {
|
|
expect(await folderCounts(repo, "")).to.deep.equal({ "": 1, src: 1 });
|
|
} finally {
|
|
await File.collection.createIndex({ repoId: 1, treeGeneration: 1, path: 1, size: 1 }, { name: "file_counts_covering" });
|
|
}
|
|
expect(await folderCounts(repo, "")).to.deep.equal({ "": 1, src: 1 });
|
|
});
|
|
|
|
it("preserves counts for legacy repositories without a generation", async () => {
|
|
repo.model.treeGeneration = undefined; await repo.model.save();
|
|
await File.collection.insertMany([
|
|
{ repoId: "counts", name: "zero", path: "", size: 0 },
|
|
{ repoId: "counts", name: "one", path: "src", size: 1 },
|
|
{ repoId: "counts", name: "folder", path: "" },
|
|
{ repoId: "counts", name: "staged", path: "src", size: 1, treeGeneration: "future" },
|
|
]);
|
|
expect(await folderCounts(repo, "")).to.deep.equal({ "": 2, src: 1 });
|
|
});
|
|
|
|
it("escapes regex punctuation and handles Unicode scopes and empty directories", async () => {
|
|
await insert([{ path: "", name: "猫.[x]" }, { path: "猫.[x]", name: "empty" },
|
|
{ path: "猫.[x]/child", name: "a", size: 0 }, { path: "猫.[x]", name: "b", size: 2 },
|
|
{ path: "猫Xx/not-this-folder", name: "no", size: 3 }]);
|
|
expect(await folderCounts(repo, "猫.[x]")).to.deep.equal({ "猫.[x]": 2, "猫.[x]/child": 1 });
|
|
expect(await folderCounts(repo, "猫.[x]/empty")).to.deep.equal({ "猫.[x]/empty": 0 });
|
|
});
|
|
|
|
it("anonymizes child keys, sums collisions and resolves the same directory as browsing", async () => {
|
|
repo.model.options.terms = ["Alice|Bob"];
|
|
await repo.model.save();
|
|
await insert([{ name: "Alice", path: "" }, { name: "Bob", path: "" },
|
|
{ name: "one", path: "Alice/sub", size: 1 }, { name: "two", path: "Bob/sub", size: 1 }]);
|
|
const root = await folderCounts(repo, "");
|
|
expect(root).to.deep.equal({ "": 2, "XXXX-1": 2 });
|
|
const scoped = await folderCounts(repo, "XXXX-1");
|
|
expect(scoped).to.deep.equal({ "XXXX-1": 1, "XXXX-1/sub": 1 });
|
|
expect(JSON.stringify(scoped)).not.to.match(/Alice|Bob/);
|
|
});
|
|
|
|
for (const name of ["$data", "$$ROOT", "$$CURRENT"]) it(`treats directory ${name} as a literal in aggregation expressions`, async () => {
|
|
await insert([{ name, path: "" }, { name: "direct.txt", path: name, size: 1 },
|
|
{ name: "$nested", path: name }, { name: "child.txt", path: `${name}/$nested`, size: 1 }]);
|
|
expect(await folderCounts(repo, name)).to.deep.equal({ [name]: 2, [`${name}/$nested`]: 1 });
|
|
expect(await folderCounts(repo, `${name}/$nested`)).to.deep.equal({ [`${name}/$nested`]: 1 });
|
|
});
|
|
|
|
it("uses the active generation on each request without reusing obsolete counts", async () => {
|
|
await insert([{ path: "old", name: "a", size: 1 }]);
|
|
expect(await folderCounts(repo, "")).to.deep.equal({ "": 1, old: 1 });
|
|
await insert([{ path: "new", name: "b", size: 1, treeGeneration: "replacement" },
|
|
{ path: "new", name: "c", size: 1, treeGeneration: "replacement" }]);
|
|
repo.model.treeGeneration = "replacement"; await repo.model.save();
|
|
expect(await folderCounts(new Repository(await Repo.findById(repo.model._id)), "")).to.deep.equal({ "": 2, new: 2 });
|
|
});
|
|
|
|
it("keeps a hard bound for over-wide responses", async () => {
|
|
for (let offset = 0; offset <= FILE_LISTING_LIMIT; offset += 500) await insert(Array.from({
|
|
length: Math.min(500, FILE_LISTING_LIMIT + 1 - offset),
|
|
}, (_, i) => ({ name: "a", path: `folder${offset + i}`, size: 1 })));
|
|
const error = await folderCounts(repo, "").then(() => null, error => error);
|
|
expect(error.message).to.equal("file_listing_too_large"); expect(error.httpStatus).to.equal(413);
|
|
});
|
|
|
|
it("runs the scoped route only after authorization and rejects object scopes", async () => {
|
|
const utils = require("../src/server/routes/route-utils");
|
|
const original = { getRepo: utils.getRepo, handleError: utils.handleError };
|
|
const router = require("../src/server/routes/repository-public").default;
|
|
const handler = router.stack.find(layer => layer.route?.path === "/:repoId/files/counts").route.stack.at(-1).handle;
|
|
const res = { header() {}, json() { throw Error("must not return counts"); } };
|
|
try {
|
|
utils.getRepo = async () => null;
|
|
await handler({ query: { path: {} } }, res);
|
|
utils.getRepo = async () => repo; utils.handleError = error => { throw error; };
|
|
const error = await handler({ query: { path: {} } }, res).then(() => null, error => error);
|
|
expect(error.httpStatus).to.equal(400);
|
|
} finally { Object.assign(utils, original); }
|
|
});
|
|
});
|