CalvinBackup/anonymous_github
1
0 Fork 0
mirror of https://github.com/tdurieux/anonymous_github.git synced 2026-10-19 14:17:20 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge commit 'refs/security-review/pr844' into HEAD

Browse Source
This commit is contained in:
Thomas Durieux committed 2026-10-08 16:25:00 +00:00
2 parents 5485f57e02 37a87a0fba
commit 243396515e
163 files changed
+16198 -22725
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/server/dailyStatsSnapshot.ts
+39
View File
@@ -109,3 +109,42 @@ export async function ensureTodaySnapshot(): Promise<void> {
logger.error("ensureTodaySnapshot failed", serializeError(error));
}
}
// Shared by /stat and /stat/history; zero counts are valid cache entries.
import { AsyncCache } from "../core/async-cache";
import { cacheRedis, cacheCommand, coordinatedFill } from "../core/cache-coordination";
const currentStatsCache = new AsyncCache<HomeStats>(60 * 60_000, 1);
const historyCache = new AsyncCache<HomeStatsHistoryRow[]>(60 * 60_000, 16);
export function clearStatsCache() { currentStatsCache.clear(); historyCache.clear(); }
export function getCurrentStats(): Promise<HomeStats> {
return currentStatsCache.get("current", async () => {
const redis = await cacheRedis();
const key = "perf:home-stats:v1";
const read = async () => {
if (!redis?.isReady) return undefined;
const value = await cacheCommand(redis.get(key), redis).catch(() => null);
if (!value) return undefined;
try { return JSON.parse(value) as HomeStats; } catch { return undefined; }
};
return coordinatedFill(key, read, async () => {
const stats = await computeStats();
if (redis?.isReady) await cacheCommand(redis.set(key, JSON.stringify(stats), { PX: 60 * 60_000 }), redis).catch(() => {});
return stats;
});
});
}
export function getStatsHistory(days: number, now = new Date()) {
const count = Math.min(365, Math.max(1, Math.floor(days) || 30));
const today = utcMidnight(now);
return historyCache.get(`${today.toISOString()}:${count}`, async () => {
const since = new Date(today);
since.setUTCDate(since.getUTCDate() - count + 1);
const [docs, current] = await Promise.all([
DailyStatsModel.find({ date: { $gte: since } }).sort({ date: 1 }).lean(),
getCurrentStats(),
]);
return mergeCurrentStatsIntoHistory(docs.map(d => ({ date: d.date,
nbRepositories: d.nbRepositories, nbUsers: d.nbUsers,
nbPageViews: d.nbPageViews, nbPullRequests: d.nbPullRequests })), current, now);
});
}
src/server/database.ts
+2 -1
View File
@@ -22,12 +22,13 @@ export const database = mongoose.connection;
export let isConnected = false;
export async function connect() {
export async function connect(overrides: Pick<ConnectOptions, "appName" | "maxPoolSize" | "minPoolSize"> = {}) {
credentialCipher(); // Refuse to serve persisted credentials without a valid keyring.
mongoose.set("strictQuery", false);
const options: ConnectOptions = {
appName: "Anonymous GitHub Server",
compressors: "zstd",
...overrides,
};
if (!config.MONGODB_URI) options.authSource = "admin";
await mongoose.connect(getMongoUrl(), options);
src/server/index.ts
+18 -96
View File
@@ -12,6 +12,7 @@ import * as express from "express";
import * as compression from "compression";
import * as passport from "passport";
import { connect } from "./database";
import { startTemporaryStorageMaintenance } from "../core/temporary-storage";
import { initSession, router as connectionRouter } from "./routes/connection";
import { bearerTokenAuth } from "./routes/token-auth";
import router from "./routes";
@@ -27,16 +28,18 @@ import {
recoverStuckRemoving,
} from "../queue";
import {
computeStats,
getCurrentStats,
getStatsHistory,
ensureTodaySnapshot,
HomeStatsHistoryRow,
mergeCurrentStatsIntoHistory,
} from "./dailyStatsSnapshot";
import DailyStatsModel from "../core/model/dailyStats/dailyStats.model";
import { getUser } from "./routes/route-utils";
import config from "../config";
import { resolveTrustProxy, isCloudflareIP } from "./trustProxy";
import { resolveTrustProxy } from "./trustProxy";
import { requestRateLimitKey } from "./rate-limit-key";
import { createLogger, serializeError } from "../core/logger";
import { monitorRequests } from "../core/request-monitoring";
import { startPerformanceMonitoring } from "../core/performance-monitoring";
import { getAnonymizationPoolStats } from "../core/anonymization-pool";
import { createReviewCapabilities } from "./service/review-capabilities";
@@ -102,6 +105,8 @@ function indexResponse(req: express.Request, res: express.Response) {
export default async function start() {
const app = express();
app.use(monitorRequests("api"));
startPerformanceMonitoring("api", getAnonymizationPoolStats);
app.set("query parser", "extended");
app.use("/service", createReviewCapabilities(process.env.REVIEW_SERVICE_KEYS));
app.use("/github/app/webhook", githubAppWebhook);
@@ -155,37 +160,6 @@ export default async function start() {
await redisClient.connect();
function keyGenerator(
request: express.Request,
_response: express.Response
): string {
// Use request.ip, which Express resolves from X-Forwarded-For honouring
// the configured "trust proxy" setting. Do NOT key off the
// cf-connecting-ip header unconditionally: when the server isn't actually
// behind Cloudflare a client can set that header to an arbitrary value
// per request and trivially bypass the rate limiter (CWE-290).
let ip = request.ip;
if (!ip && request.socket.remoteAddress) {
logger.warn("request.ip is missing");
ip = request.socket.remoteAddress;
}
// If resolution stopped at a Cloudflare edge address (X-Forwarded-For no
// longer contains the visitor, e.g. after a Cloudflare-side change), fall
// back to cf-connecting-ip. This is safe: request.ip can only be a
// Cloudflare address when every hop up to it is trusted, i.e. the request
// genuinely traversed Cloudflare. A direct client forging the header is
// keyed by its own untrusted address instead, so the CWE-290 bypass above
// does not apply.
if (ip && isCloudflareIP(ip)) {
const cfConnectingIP = request.headers["cf-connecting-ip"];
if (typeof cfConnectingIP === "string" && cfConnectingIP) {
ip = cfConnectingIP.trim();
}
}
// remove port number from IPv4 addresses
return (ip || "").replace(/:\d+[^:]*$/, "");
}
const rate = rateLimit({
store: new RedisStore({
sendCommand: (...args: string[]) => redisClient.sendCommand(args),
@@ -200,7 +174,7 @@ export default async function start() {
}
return false;
},
max: async (request: express.Request, _response: express.Response) => {
limit: async (request: express.Request, _response: express.Response) => {
try {
const user = await getUser(request);
if (user) return config.RATE_LIMIT;
@@ -210,8 +184,8 @@ export default async function start() {
// if not logged in, limit to half the rate
return config.RATE_LIMIT / 2;
},
keyGenerator,
standardHeaders: true,
keyGenerator: requestRateLimitKey,
standardHeaders: "draft-6",
legacyHeaders: false,
message: (_request: express.Request, _response: express.Response) => {
return `You can only make ${config.RATE_LIMIT} requests every 15min. Please try again later.`;
@@ -222,28 +196,14 @@ export default async function start() {
delayAfter: 50,
delayMs: () => 150,
maxDelayMs: 5000,
keyGenerator,
keyGenerator: requestRateLimitKey,
});
const webViewSpeedLimiter = slowDown({
windowMs: 15 * 60 * 1000, // 15 minutes
delayAfter: 200,
delayMs: () => 150,
maxDelayMs: 5000,
keyGenerator,
});
app.use(function (req, res, next) {
const start = Date.now();
res.on("finish", function () {
const time = Date.now() - start;
logger.info("request", {
method: req.method,
status: res.statusCode,
url: join(req.baseUrl || "", req.url || ""),
ms: time,
});
});
next();
keyGenerator: requestRateLimitKey,
});
app.use("/github", rate, speedLimiter, githubAppRouter);
@@ -281,53 +241,14 @@ export default async function start() {
res.sendStatus(404);
});
let stat: Record<string, unknown> = {};
let history: HomeStatsHistoryRow[] | null = null;
let historyKey: number | null = null;
setInterval(() => {
stat = {};
history = null;
historyKey = null;
}, 1000 * 60 * 60);
apiRouter.get("/healthcheck", async (_, res) => {
res.json({ status: "ok" });
});
apiRouter.get("/stat", async (_, res) => {
if (stat.nbRepositories) {
res.json(stat);
return;
}
stat = { ...(await computeStats()) };
res.json(stat);
res.json(await getCurrentStats());
});
apiRouter.get("/stat/history", async (req, res) => {
const days = Math.min(
Math.max(parseInt(req.query.days as string) || 30, 1),
365
);
if (history && historyKey === days) {
res.json(history);
return;
}
const since = new Date();
since.setUTCDate(since.getUTCDate() - days + 1);
since.setUTCHours(0, 0, 0, 0);
const docs = await DailyStatsModel.find({ date: { $gte: since } })
.sort({ date: 1 })
.lean();
const rows = docs.map((d) => ({
date: d.date,
nbRepositories: d.nbRepositories,
nbUsers: d.nbUsers,
nbPageViews: d.nbPageViews,
nbPullRequests: d.nbPullRequests,
}));
history = mergeCurrentStatsIntoHistory(rows, await computeStats());
historyKey = days;
res.json(history);
res.json(await getStatsHistory(parseInt(String(req.query.days)) || 30));
});
// web view
@@ -382,6 +303,7 @@ export default async function start() {
dailyStatsSnapshot();
await connect();
await startTemporaryStorageMaintenance();
app.listen(config.PORT);
logger.info("server started", { port: config.PORT });
ensureTodaySnapshot().catch((err) =>
src/server/rate-limit-key.ts
+16
View File
@@ -0,0 +1,16 @@
import { Request } from "express";
import { ipKeyGenerator } from "express-rate-limit";
import { isCloudflareIP } from "./trustProxy";
export function requestRateLimitKey(request: Request): string {
// Express resolves request.ip using the configured trusted proxies.
let ip = request.ip || request.socket.remoteAddress || "";
// Only accept Cloudflare's visitor header when resolution reached its edge.
if (isCloudflareIP(ip)) {
const visitor = request.headers["cf-connecting-ip"];
if (typeof visitor === "string" && visitor.trim()) ip = visitor.trim();
}
// Some proxies append a port to IPv4 addresses. Do not truncate IPv6.
ip = ip.replace(/^(\d{1,3}(?:\.\d{1,3}){3}):\d+$/, "$1");
return ipKeyGenerator(ip);
}
src/server/routes/admin.ts
+45 -28
View File
@@ -1,5 +1,7 @@
import { AsyncCache } from "../../core/async-cache";
import * as os from "os";
import { execSync } from "child_process";
import { execFile } from "child_process";
import { promisify } from "util";
import { Queue, JobType } from "bullmq";
import * as express from "express";
import AnonymousError from "../../core/AnonymousError";
@@ -29,8 +31,11 @@ import {
} from "../../core/logger";
import { createClient, RedisClientType } from "redis";
import config from "../../config";
import { performanceReport } from "../../core/performance-monitoring";
const logger = createLogger("admin");
const execFileAsync = promisify(execFile);
const diskUsageCache = new AsyncCache<string>(30000, 1);
let errorLogClient: RedisClientType | null = null;
@@ -77,6 +82,12 @@ router.use(
router.use("/tokens", adminTokensRouter);
router.get("/performance", async (req, res) => {
res.set("Cache-Control", "no-store");
try { res.json(await performanceReport(Number(req.query.minutes))); }
catch { res.json({ available: false, instances: [], routes: [], stages: [] }); }
});
function dashboardCache(
_req: express.Request,
res: express.Response,
@@ -342,18 +353,11 @@ router.get("/queues/metrics", dashboardCache, async (req, res) => {
}
});
const queuesCache = new Map<string, { data: unknown; ts: number }>();
const QUEUES_CACHE_TTL = 10_000;
const queuesCache = new AsyncCache<{ queues: unknown; selectedQueue: string; jobs: Record<string, unknown>[] }>(10_000, 3);
router.get("/queues", dashboardCache, async (req, res) => {
const search = req.query.search ? String(req.query.search).toLowerCase() : "";
const queueName = req.query.queue ? String(req.query.queue) : "";
const cacheKey = `${queueName}|${search}`;
const cached = queuesCache.get(cacheKey);
if (cached && Date.now() - cached.ts < QUEUES_CACHE_TTL) {
return res.json(cached.data);
}
const allQueues: { key: string; label: string; queue: Queue }[] = [
{ key: "download", label: "Download", queue: downloadQueue },
{ key: "remove", label: "Remove", queue: removeQueue },
@@ -365,6 +369,7 @@ router.get("/queues", dashboardCache, async (req, res) => {
: allQueues[0];
const targetQueue = target ? target.queue : downloadQueue;
const data = await queuesCache.get(target?.key || "download", async () => {
const [statsResults, ...jobsByState] = await Promise.all([
Promise.all(
allQueues.map(async (q) => ({
@@ -376,7 +381,16 @@ router.get("/queues", dashboardCache, async (req, res) => {
...QUEUE_STATES.map(async (state) => {
const jobs = await targetQueue.getJobs([state], 0, 199);
return jobs.map((j) => {
const json: Record<string, unknown> = { ...j.asJSON(), _state: state };
// asJSON() serializes these fields to strings; the dashboard needs
// the objects (payload repoId, stack lines, failure reason).
const json: Record<string, unknown> = {
...j.asJSON(),
data: j.data,
failedReason: j.failedReason,
stacktrace: j.stacktrace,
returnvalue: j.returnvalue,
_state: state,
};
if (state === "delayed" && j.delay > 0) {
json.delayUntil = j.timestamp + j.delay;
}
@@ -385,6 +399,19 @@ router.get("/queues", dashboardCache, async (req, res) => {
}),
]);
const allJobs = (jobsByState as Record<string, unknown>[][]).flat();
const stateOrder: Record<string, number> = {
active: 0, waiting: 1, delayed: 2, failed: 3, completed: 4,
};
allJobs.sort((a, b) => (stateOrder[a._state as string] ?? 9) - (stateOrder[b._state as string] ?? 9));
return {
queues: statsResults,
selectedQueue: target?.key || "download",
jobs: allJobs,
};
});
const matches = (job: { id?: string | undefined; name?: string }) => {
if (!search) return true;
return (
@@ -392,20 +419,7 @@ router.get("/queues", dashboardCache, async (req, res) => {
(job.name || "").toLowerCase().includes(search)
);
};
const allJobs = (jobsByState as Record<string, unknown>[][]).flat().filter(matches);
const stateOrder: Record<string, number> = {
active: 0, waiting: 1, delayed: 2, failed: 3, completed: 4,
};
allJobs.sort((a, b) => (stateOrder[a._state as string] ?? 9) - (stateOrder[b._state as string] ?? 9));
const data = {
queues: statsResults,
selectedQueue: target?.key || "download",
jobs: allJobs,
};
queuesCache.set(cacheKey, { data, ts: Date.now() });
res.json(data);
res.json({ ...data, jobs: data.jobs.filter(matches) });
});
// Errors captured by the logger sink. Server-paginated to avoid pulling
@@ -595,15 +609,15 @@ router.delete("/errors", async (req, res) => {
// SCAN the hourly counter keys and del them along with the list and
// dropped counter so the admin page comes back to a clean slate.
const hourlyKeys: string[] = [];
let cursor = 0;
let cursor = "0";
do {
const reply = await client.scan(cursor, {
MATCH: `${ERROR_LOG_HOURLY_PREFIX}*`,
COUNT: 100,
});
cursor = Number(reply.cursor);
cursor = reply.cursor;
for (const k of reply.keys) hourlyKeys.push(k);
} while (cursor !== 0);
} while (cursor !== "0");
const pipe = client.multi();
pipe.del(ERROR_LOG_KEY);
pipe.del(ERROR_LOG_DROPPED_KEY);
@@ -632,7 +646,10 @@ router.get("/overview", async (req, res) => {
// Disk usage via df (root partition)
let diskTotal = 0, diskUsed = 0, diskFree = 0, diskPercent = 0, diskMount = "/";
try {
const dfOut = execSync("df -k / 2>/dev/null", { timeout: 3000 }).toString();
const dfOut = await diskUsageCache.get("root", async () => {
const result = await execFileAsync("df", ["-k", "/"], { timeout: 3000, maxBuffer: 16384 });
return result.stdout;
});
const lines = dfOut.trim().split("\n");
if (lines.length >= 2) {
const cols = lines[1].split(/\s+/);
src/server/routes/connection.ts
+1 -2
View File
@@ -1,7 +1,7 @@
import { createClient } from "redis";
import * as passport from "passport";
import * as session from "express-session";
import RedisStore from "connect-redis";
import { RedisStore } from "connect-redis";
import * as OAuth2Strategy from "passport-oauth2";
import { Profile, Strategy } from "passport-github2";
import * as express from "express";
@@ -166,7 +166,6 @@ passport.deserializeUser(async (id: string, done) => {
export function initSession() {
const redisClient = createClient({
legacyMode: false,
socket: {
port: config.REDIS_PORT,
host: config.REDIS_HOSTNAME,
src/server/routes/dashboard-names.ts
+48
View File
@@ -0,0 +1,48 @@
import { isConnected } from "../database";
import { AsyncCache } from "../../core/async-cache";
import DashboardName from "../../core/model/dashboard-name";
import Repo from "../../core/model/anonymizedRepositories/anonymizedRepositories.model";
import PR from "../../core/model/anonymizedPullRequests/anonymizedPullRequests.model";
import Gist from "../../core/model/anonymizedGists/anonymizedGists.model";
import User from "../../core/User";
import { projectNameKey } from "./project-names";
const backfills = new AsyncCache<void>(5 * 60_000, 64);
export function invalidateDashboardNames() { backfills.clear(); }
/** Backfill legacy private names using ID-only cursors, never artifact bodies. */
export function ensureDashboardNames(user: User) {
if (!isConnected) return Promise.resolve();
return backfills.get(user.id, async () => {
// The user map is authoritative, including after an interrupted dual write.
const existing = DashboardName.find({ owner: user.model._id }).lean().cursor();
try {
let batch: Parameters<typeof DashboardName.bulkWrite>[0] = [];
for await (const row of existing) {
const name = user.model.projectNames?.get(projectNameKey(row.type, row.artifactId));
batch.push(name ? { updateOne: { filter: { _id: row._id }, update: { $set: { name } } } }
: { deleteOne: { filter: { _id: row._id } } });
if (batch.length === 250) { await DashboardName.bulkWrite(batch); batch = []; }
}
if (batch.length) await DashboardName.bulkWrite(batch);
} finally { await existing.close(); }
if (!user.model.projectNames?.size) return;
const sources = [
{ type: "repo", cursor: Repo.find(user.repositoryMembership()).select("repoId").lean().cursor(), field: "repoId" },
{ type: "pr", cursor: PR.find({ owner: user.id }).select("pullRequestId").lean().cursor(), field: "pullRequestId" },
{ type: "gist", cursor: Gist.find({ owner: user.id }).select("gistId").lean().cursor(), field: "gistId" },
];
for (const { type, cursor, field } of sources) {
try {
let batch: Parameters<typeof DashboardName.bulkWrite>[0] = [];
for await (const row of cursor) {
const id = String((row as unknown as Record<string, unknown>)[field] || "");
const name = user.model.projectNames!.get(projectNameKey(type, id));
if (!name) continue;
batch.push({ updateOne: { filter: { owner: user.model._id, type, artifactId: id },
update: { $set: { name } }, upsert: true } });
if (batch.length === 250) { await DashboardName.bulkWrite(batch); batch = []; }
}
if (batch.length) await DashboardName.bulkWrite(batch);
} finally { await cursor.close(); }
}
});
}
src/server/routes/dashboard-summary.ts
+102
View File
@@ -0,0 +1,102 @@
import DashboardName from "../../core/model/dashboard-name";
import { ensureDashboardNames } from "./dashboard-names";
import { PipelineStage, Types } from "mongoose";
import User from "../../core/User";
import Repo from "../../core/model/anonymizedRepositories/anonymizedRepositories.model";
import PullRequest from "../../core/model/anonymizedPullRequests/anonymizedPullRequests.model";
import Gist from "../../core/model/anonymizedGists/anonymizedGists.model";
import AnonymousError from "../../core/AnonymousError";
const fields: Record<string, string> = { _name: "_anonymousId", _label: "_label", status: "status",
anonymizeDate: "anonymizeDate", lastView: "lastView", pageView: "pageView", "options.expirationDate": "options.expirationDate" };
export function dashboardPipeline(user: User, query: Record<string, unknown>, now = new Date()): PipelineStage[] {
const sort = String(query.sort || "-anonymizeDate");
const field = sort.replace(/^-/, "");
if (!fields[field]) throw new AnonymousError("invalid_sort", { httpStatus: 400 });
const direction = sort.startsWith("-") ? -1 : 1;
const limit = Math.min(100, Math.max(1, Math.floor(Number(query.limit)) || 50));
const owner = new Types.ObjectId(user.id);
function project(type: string, id: string, source: unknown, label: unknown, extra: Record<string, unknown>) {
return [
{ $project: { _id: 1, status: 1, statusMessage: 1, anonymizeDate: 1, lastView: 1, pageView: { $ifNull: ["$pageView", 0] },
options: 1, conference: 1, ...extra, _type: { $literal: type }, _anonymousId: { $ifNull: [`$${id}`, ""] },
_source: source, _fallbackLabel: label,
} },
];
}
const repoProject = project("repo", "repoId", "$source.repositoryName", "$source.repositoryName", {
repoId: 1, source: { fullName: "$source.repositoryName", commit: "$source.commit" }, coauthors: 1, size: 1,
role: { $cond: [{ $eq: ["$owner", owner] }, "owner", "coauthor"] },
});
const prProject = project("pr", "pullRequestId", { $concat: [{ $ifNull: ["$source.repositoryFullName", ""] }, "#", { $toString: "$source.pullRequestId" }] },
{ $concat: [{ $ifNull: ["$source.repositoryFullName", ""] }, "#", { $toString: "$source.pullRequestId" }] },
{ pullRequestId: 1, source: { repositoryFullName: "$source.repositoryFullName", pullRequestId: "$source.pullRequestId" } });
const gistProject = project("gist", "gistId", "$source.gistId", { $concat: ["Gist ", { $ifNull: ["$gistId", "$source.gistId"] }] },
{ gistId: 1, source: { gistId: "$source.gistId" }, isPublic: "$gist.isPublic" });
const pipeline: PipelineStage[] = [
{ $match: { $or: user.repositoryMembership().$or.map(clause => clause.owner ? { owner } : clause) } }, ...repoProject as PipelineStage[],
{ $unionWith: { coll: PullRequest.collection.name, pipeline: [{ $match: { owner } }, ...prProject] } },
{ $unionWith: { coll: Gist.collection.name, pipeline: [{ $match: { owner } }, ...gistProject] } },
{ $lookup: { from: DashboardName.collection.name, let: { type: "$_type", id: "$_anonymousId" },
pipeline: [{ $match: { owner, $expr: { $and: [{ $eq: ["$type", "$$type"] }, { $eq: ["$artifactId", "$$id"] }] } } },
{ $project: { name: 1 } }], as: "_privateName" } },
{ $set: { projectName: { $ifNull: [{ $arrayElemAt: ["$_privateName.name", 0] }, ""] } } },
{ $unset: "_privateName" },
{ $set: { status: { $cond: [{ $and: [{ $eq: ["$status", "ready"] }, { $ne: ["$options.expirationMode", "never"] },
{ $ne: [{ $ifNull: ["$options.expirationDate", null] }, null] }, { $lte: ["$options.expirationDate", now] }] }, "expired", "$status"] },
_label: { $cond: [{ $ne: ["$projectName", ""] }, "$projectName", { $ifNull: ["$_fallbackLabel", "$_anonymousId"] }] } } },
{ $set: { _statusKey: { $switch: { branches: [
{ case: { $in: ["$status", ["ready", "error"]] }, then: "$status" },
{ case: { $in: ["$status", ["expired", "expiring"]] }, then: "expired" },
{ case: { $in: ["$status", ["removed", "removing"]] }, then: "removed" },
], default: "progress" } } } },
{ $set: { _attention: { $or: [{ $eq: ["$status", "error"] }, { $eq: ["$_anonymousId", ""] },
{ $and: [{ $eq: ["$_statusKey", "progress"] }, { $ne: [{ $ifNull: ["$anonymizeDate", "$lastView"] }, null] },
{ $lt: [{ $ifNull: ["$anonymizeDate", "$lastView"] }, new Date(now.getTime() - 2 * 3600_000)] }] }] },
_sort: { $ifNull: [`$${fields[field]}`, ["pageView", "anonymizeDate", "lastView", "options.expirationDate"].includes(field)
? (["pageView"].includes(field) ? 0 : new Date(0)) : ""] } } },
];
if (["_name", "_label", "status"].includes(field)) pipeline.push({ $set: { _sort: { $toLower: "$_sort" } } });
const filter: Record<string, unknown> = {};
if (["repo", "pr", "gist"].includes(String(query.type))) filter._type = query.type;
if (query.attention === "true") filter._attention = true;
const statuses = String(query.statuses || "ready,progress,error,expired").split(",").filter(s => ["ready", "progress", "error", "expired", "removed"].includes(s));
filter._statusKey = { $in: statuses };
const search = String(query.q || "").trim().slice(0, 256);
if (search) {
const pattern = search.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
filter.$or = ["_anonymousId", "_label", "_source", "conference"].map(key => ({ [key]: { $regex: pattern, $options: "i" } }));
}
const page: PipelineStage.FacetPipelineStage[] = [{ $match: filter }];
if (query.cursor) {
try {
if (String(query.cursor).length > 2048) throw new Error();
const cursor = JSON.parse(Buffer.from(String(query.cursor), "base64url").toString());
if (cursor.sort !== sort || typeof cursor.type !== "string" || !Types.ObjectId.isValid(cursor.id)) throw new Error();
const value = ["anonymizeDate", "lastView", "options.expirationDate"].includes(field) ? new Date(cursor.value) : cursor.value;
if ((value instanceof Date && isNaN(value.getTime())) || !(typeof value === "string" || typeof value === "number" || value instanceof Date)) throw new Error();
page.push({ $match: { $or: [ { _sort: { [direction === 1 ? "$gt" : "$lt"]: value } },
{ _sort: value, _type: { $gt: cursor.type } }, { _sort: value, _type: cursor.type, _id: { $gt: new Types.ObjectId(cursor.id) } },
] } });
} catch { throw new AnonymousError("invalid_cursor", { httpStatus: 400 }); }
}
page.push({ $sort: { _sort: direction, _type: 1, _id: 1 } }, { $limit: limit + 1 });
pipeline.push({ $facet: { totals: [{ $group: { _id: null, total: { $sum: 1 }, attention: { $sum: { $cond: ["$_attention", 1, 0] } } } }],
filtered: [{ $match: filter }, { $count: "count" }], items: page } });
return pipeline;
}
export async function dashboardSummary(user: User, query: Record<string, unknown>) {
await ensureDashboardNames(user);
const [result] = await Repo.aggregate(dashboardPipeline(user, query));
const limit = Math.min(100, Math.max(1, Math.floor(Number(query.limit)) || 50));
const hasMore = result.items.length > limit;
const items = result.items.slice(0, limit);
const last = items[items.length - 1];
const cursor = hasMore ? Buffer.from(JSON.stringify({ sort: String(query.sort || "-anonymizeDate"),
value: last._sort, type: last._type, id: last._id })).toString("base64url") : null;
for (const item of items) { delete item._sort; delete item._id; }
return { items, cursor, total: result.totals[0]?.total || 0, attention: result.totals[0]?.attention || 0,
filtered: result.filtered[0]?.count || 0 };
}
src/server/routes/file.ts
+12 -1
View File
@@ -3,6 +3,9 @@ import AnonymizedFile from "../../core/AnonymizedFile";
import AnonymousError from "../../core/AnonymousError";
import { getRepo, handleError } from "./route-utils";
import { fileETag } from "./file-etag";
import config from "../../config";
import { lookup } from "mime-types";
import { isTextFile } from "../../core/anonymize-utils";
export const router = express.Router();
@@ -142,9 +145,17 @@ router.get(
// anonymization term list left old anonymizations cached under the
// same URL.
res.header("Cache-Control", "private, no-cache, must-revalidate");
if (req.headers["if-none-match"] === etag) {
if (req.fresh) {
return res.status(304).end();
}
if (req.method === "HEAD") {
const size = await f.size();
if (size != null && size > config.MAX_FILE_SIZE) throw new AnonymousError("file_too_big", { httpStatus: 413 });
const mime = lookup(originalPath);
if (mime && !originalPath.endsWith(".ts")) res.type(mime);
else if (isTextFile(originalPath)) res.type("text/plain");
res.end(); return;
}
await f.send(res);
await repo.countView();
} catch (error) {
src/server/routes/github-app.ts
+36 -13
View File
@@ -5,6 +5,7 @@ import UserModel from "../../core/model/users/users.model";
import CredentialModel from "../../core/model/credentials/credentials.model";
import InstallationModel from "../../core/model/github-installation";
import RepositoryModel from "../../core/model/anonymizedRepositories/anonymizedRepositories.model";
import CachedRepositoryModel from "../../core/model/repositories/repositories.model";
import PullRequestModel from "../../core/model/anonymizedPullRequests/anonymizedPullRequests.model";
import GistModel from "../../core/model/anonymizedGists/anonymizedGists.model";
import { getCredentialToken } from "../../core/credentials";
@@ -163,14 +164,15 @@ router.get("/connections", async (req, res) => {
}
catch (error) { appErrorCode = error instanceof Error ? error.message : "github_app_reconnect_required"; }
}
const repos = await RepositoryModel.find({ owner: user.id, status: { $ne: "removed" } }).select("repoId source.repositoryName githubAccess status").lean();
const prs = await PullRequestModel.find({ owner: user.id, status: { $ne: "removed" } }).select("pullRequestId source.repositoryFullName githubAccess status").lean();
const repos = await RepositoryModel.find({ owner: user.id, status: { $ne: "removed" } }).select("repoId source.repositoryName githubAccess status statusDate").lean();
const prs = await PullRequestModel.find({ owner: user.id, status: { $ne: "removed" } }).select("pullRequestId source.repositoryFullName githubAccess status statusDate").lean();
const gistCount = await GistModel.countDocuments({ owner: user.id, status: { $ne: "removed" } });
res.json({ csrf: req.session.githubConnectionCSRF, appEnabled: config.GITHUB_APP_ENABLED && config.GITHUB_APP_NEW_CONNECTIONS,
gistConnection: config.GITHUB_APP_ENABLED && credentials.some(c => c.provider === APP_PROVIDER) ? "github-app" : "oauth",
oauthEnabled: config.GITHUB_OAUTH_ENABLED, oauthConnected: !!(await getCredentialToken(user.id)), appConnected, appError: appErrorCode,
installations, gistCount, resources: [
...repos.map(r => ({ type: "repository", id: r.repoId, name: r.source.repositoryName, connection: r.githubAccess?.kind || "oauth", status: r.status })),
...prs.map(r => ({ type: "pull-request", id: r.pullRequestId, name: r.source.repositoryFullName, connection: r.githubAccess?.kind || "oauth", status: r.status })),
...repos.map(r => ({ type: "repository", id: r.repoId, name: r.source.repositoryName, connection: r.githubAccess?.kind || "oauth", status: r.status, statusDate: r.statusDate })),
...prs.map(r => ({ type: "pull-request", id: r.pullRequestId, name: r.source.repositoryFullName, connection: r.githubAccess?.kind || "oauth", status: r.status, statusDate: r.statusDate })),
] });
} catch (error) { handleError(error, res, req); }
});
@@ -191,16 +193,36 @@ router.post("/connections/migrate", async (req, res) => {
const isRepo = type === "repository";
const model = isRepo ? await RepositoryModel.findOne({ repoId: id, owner: user.id }) : await PullRequestModel.findOne({ pullRequestId: id, owner: user.id });
if (!model || ["removed", "archived"].includes(model.status || "")) throw appError("repo_not_found", 404);
if (["preparing", "removing", "expiring"].includes(model.status || "")) throw appError("repository_busy", 409);
const source = model.source as { repositoryName?: string; repositoryFullName?: string; commit?: string; pullRequestId?: number };
const stalePullRequest = !isRepo && model.status === "download" &&
(!model.statusDate || model.statusDate.getTime() <= Date.now() - 5 * 60 * 1000);
if (!stalePullRequest && ["preparing", "queue", "download", "removing", "expiring"].includes(model.status || "")) throw appError("repository_busy", 409);
const source = model.source as { repositoryName?: string; repositoryFullName?: string; repositoryId?: string; commit?: string; pullRequestId?: number };
const name = source.repositoryName || source.repositoryFullName || "";
const selected = await selectRepositoryAccess(user.id, name, connection);
const parts = name.split("/").map(encodeURIComponent).join("/");
await githubRequest(`/repos/${parts}/${isRepo ? `commits/${encodeURIComponent(source.commit || "")}` : `pulls/${source.pullRequestId}`}`, selected.token);
const repositoryId = model.githubAccess?.kind === "github-app" && connection === "github-app"
? model.githubAccess.repositoryId : undefined;
const selected = await selectRepositoryAccess(user.id, name, connection, repositoryId);
// Repair an installation binding without silently adopting a repository
// recreated at the same name. Replacing a source requires its own preview.
if (model.githubAccess?.kind === "github-app" && selected.binding.kind === "github-app" &&
model.githubAccess.repositoryId !== selected.binding.repositoryId) throw appError("connection_changed", 409);
const parts = (selected.fullName || name).split("/").map(encodeURIComponent).join("/");
const repositoryPath = `/repos/${parts}`;
const token = selected.binding.publicRead ? await appUserToken(user.id) : selected.token;
await githubRequest(`${repositoryPath}/${isRepo ? `commits/${encodeURIComponent(source.commit || "")}` : `pulls/${source.pullRequestId}`}`, token);
if (preview === true) return res.json({ eligible: true, connection });
const filter = { _id: model._id, owner: user.id, status: model.status, source: model.source,
// This shared cache stores GitHub identity metadata, independently of the
// resource binding. Match the numeric ID so a replacement is never renamed.
if (isRepo && source.repositoryId && selected.fullName && selected.binding.kind === "github-app") {
await CachedRepositoryModel.updateOne({ _id: source.repositoryId, externalId: `gh_${selected.binding.repositoryId}` },
{ $set: { name: selected.fullName, url: `https://github.com/${selected.fullName}` } });
}
const filter = { _id: model._id, owner: user.id, status: model.status,
...(stalePullRequest ? { statusDate: model.statusDate ?? null } : {}), source: model.source,
githubAccess: model.githubAccess ? model.githubAccess : { $exists: false } };
const change = { $set: { githubAccess: selected.binding } };
const change = { $set: { githubAccess: selected.binding,
...(selected.fullName && selected.fullName !== name
? { [isRepo ? "source.repositoryName" : "source.repositoryFullName"]: selected.fullName } : {}),
} };
const result = isRepo ? await RepositoryModel.updateOne(filter, change) : await PullRequestModel.updateOne(filter, change);
if (!result.modifiedCount) throw appError("connection_changed", 409);
res.json({ connection });
@@ -225,9 +247,10 @@ router.post("/connections/disconnect-oauth", async (req, res) => {
try {
const user = await getUser(req);
if (!(await CredentialModel.exists({ ownerId: user.id, provider: APP_PROVIDER, revoked: { $ne: true } }))) throw appError("another_login_required", 409);
await appUserToken(user.id);
// Confirm the remaining login still works on GitHub before revoking OAuth.
await userInstallations(user.id);
const active = { owner: user.id, status: { $ne: "removed" }, "githubAccess.kind": { $ne: "github-app" } };
if (await RepositoryModel.exists(active) || await PullRequestModel.exists(active) || await GistModel.exists({ owner: user.id, status: { $ne: "removed" } })) {
if (await RepositoryModel.exists(active) || await PullRequestModel.exists(active)) {
throw appError("oauth_resources_remaining", 409);
}
await revokeGrant(user.id, "github");
src/server/routes/project-names.ts
+44
View File
@@ -0,0 +1,44 @@
import { isConnected } from "../database";
import DashboardName from "../../core/model/dashboard-name";
import { invalidateDashboardNames } from "./dashboard-names";
import { createHash } from "crypto";
import AnonymousError from "../../core/AnonymousError";
import User from "../../core/User";
import UserModel from "../../core/model/users/users.model";
export function projectNameKey(type: string, id: string): string {
return createHash("sha256").update(`${type}:${id}`).digest("hex");
}
// Names belong to the user's dashboard, rather than the shared artifact.
export async function saveProjectName(user: User, input: unknown): Promise<string> {
const body = input as { type?: unknown; id?: unknown; name?: unknown } | null;
if (!body || typeof body.type !== "string" || !["repo", "pr", "gist"].includes(body.type)
|| typeof body.id !== "string" || !body.id || body.id.length > 250
|| typeof body.name !== "string" || body.name.trim().length > 100
|| Array.from(body.name).some(character => {
const code = character.charCodeAt(0);
return code < 32 || code === 127;
})) {
throw new AnonymousError("invalid_project_name", { httpStatus: 400 });
}
const type = String(body.type);
const records = type === "repo" ? await user.getRepositories()
: type === "pr" ? await user.getPullRequests() : await user.getGists();
const field = type === "repo" ? "repoId" : type === "pr" ? "pullRequestId" : "gistId";
if (!records.some(record => (record as unknown as Record<string, unknown>)[field] === body.id)) {
throw new AnonymousError("not_authorized", { httpStatus: 403 });
}
const key = projectNameKey(type, body.id);
const name = body.name.trim();
const path = `projectNames.${key}`;
await UserModel.updateOne({ _id: user.model._id }, name
? { $set: { [path]: name } } : { $unset: { [path]: "" } });
invalidateDashboardNames();
if (name) user.model.set(path, name);
else user.model.projectNames?.delete(key);
if (isConnected && name) await DashboardName.updateOne({ owner: user.model._id, type, artifactId: body.id },
{ $set: { name } }, { upsert: true });
else if (isConnected) await DashboardName.deleteOne({ owner: user.model._id, type, artifactId: body.id });
return name;
}
src/server/routes/repository-private.ts
+46 -14
View File
@@ -38,6 +38,12 @@ async function previewToken(req: express.Request) {
const resource = await db.getRepository(req.query.anonymizedRepoId);
isOwnerCoauthorOrAdmin(resource, user);
if (resource.model.source.repositoryName?.toLowerCase() !== `${req.params.owner}/${req.params.repo}`.toLowerCase()) throw appError("repo_not_found", 404);
if (req.query.reconnect === "1") {
if (user.id !== resource.owner.id) throw appError("not_owner", 403);
resource.assertNotArchived();
return (await selectRepositoryAccess(user.id, resource.model.source.repositoryName,
resource.model.githubAccess?.kind || "oauth")).token;
}
return getToken(resource);
}
return (await selectRepositoryAccess(user.id, `${req.params.owner}/${req.params.repo}`, req.query.connection)).token;
@@ -186,10 +192,11 @@ router.post(
repo.model.anonymizeDate = new Date();
updates.anonymizeDate = repo.model.anonymizeDate;
}
await AnonymizedRepositoryModel.updateOne(
{ _id: repo.model._id },
const saved = await AnonymizedRepositoryModel.updateOne(
{ _id: repo.model._id, status: repo.status, statusDate: repo.model.statusDate },
{ $set: updates }
).exec();
if (saved && saved.matchedCount === 0) throw new AnonymousError("invalid_status", { httpStatus: 409 });
if (reactivating) {
// Expiration removes the cached files. Rebuild the saved commit
@@ -247,8 +254,8 @@ router.get(
owner: req.params.owner,
repo: req.params.repo,
accessToken: token,
repositoryID: req.query.repositoryID as string,
force: req.query.force == "1",
repositoryID: req.query.reconnect === "1" ? undefined : req.query.repositoryID as string,
force: req.query.reconnect === "1" || req.query.force == "1",
});
res.json(repo.toJSON());
} catch (error) {
@@ -266,13 +273,13 @@ router.get(
accessToken: token,
owner: req.params.owner,
repo: req.params.repo,
repositoryID: req.query.repositoryID as string,
force: req.query.force == "1",
repositoryID: req.query.reconnect === "1" ? undefined : req.query.repositoryID as string,
force: req.query.reconnect === "1" || req.query.force == "1",
});
return res.json(
await repository.branches({
accessToken: token,
force: req.query.force == "1",
force: req.query.reconnect === "1" || req.query.force == "1",
})
);
} catch (error) {
@@ -291,8 +298,8 @@ router.get(
owner: req.params.owner,
repo: req.params.repo,
accessToken: token,
repositoryID: req.query.repositoryID as string,
force: req.query.force == "1",
repositoryID: req.query.reconnect === "1" ? undefined : req.query.repositoryID as string,
force: req.query.reconnect === "1" || req.query.force == "1",
});
if (!repo) {
throw new AnonymousError("repo_not_found", {
@@ -303,7 +310,7 @@ router.get(
return res.send(
await repo.readme({
accessToken: token,
force: req.query.force == "1",
force: req.query.reconnect === "1" || req.query.force == "1",
branch: req.query.branch as string,
})
);
@@ -475,6 +482,10 @@ router.post(
const repoUpdate = req.body;
if (repo.status === RepositoryStatus.EXPIRING || repo.status === RepositoryStatus.REMOVING) {
throw appError("invalid_status", 409);
}
validateNewRepo(repoUpdate);
// Only the source repository/commit/branch backs the cached FileModel —
@@ -482,7 +493,17 @@ router.post(
// the fly per request. Re-running the download queue is therefore only
// needed when the underlying snapshot moves. Other edits (e.g. turning
// off auto-update — see #360) just persist and return.
const sourceChanged = hasRepositorySourceChanged(repo.model, repoUpdate);
const reconnecting = repoUpdate.reconnectRepositoryId !== undefined;
if (reconnecting) {
if (user.id !== repo.owner.id) throw appError("not_owner", 403);
if (typeof repoUpdate.reconnectRepositoryId !== "string" ||
!/^gh_[1-9][0-9]*$/.test(repoUpdate.reconnectRepositoryId) ||
repoUpdate.fullName !== repo.model.source.repositoryName) throw appError("repo_not_found", 400);
if (repo.status && [RepositoryStatus.PREPARING, RepositoryStatus.DOWNLOAD, RepositoryStatus.REMOVING,
RepositoryStatus.EXPIRING, RepositoryStatus.QUEUE].includes(repo.status)) throw appError("invalid_status", 409);
repo.assertNotArchived();
}
const sourceChanged = reconnecting || hasRepositorySourceChanged(repo.model, repoUpdate);
const previousAccessRevision = repo.model.githubAccess?.revision;
updateRepoModel(repo.model, repoUpdate);
@@ -501,13 +522,14 @@ router.post(
});
}
if (repoUpdate.fullName !== repo.model.source.repositoryName && user.id !== repo.owner.id) throw appError("not_owner", 403);
const sourceAccess = repo.model.githubAccess?.kind === "github-app" && repoUpdate.fullName === repo.model.source.repositoryName
const sourceAccess = !reconnecting && repo.model.githubAccess?.kind === "github-app" && repoUpdate.fullName === repo.model.source.repositoryName
? { token: await boundAppToken(repo.owner.id, repo.model.githubAccess, repo.model.source.repositoryName), binding: repo.model.githubAccess }
: await selectRepositoryAccess(repo.owner.id, `${parsedRepository.owner}/${parsedRepository.name}`, repo.model.githubAccess?.kind || "oauth");
const repository = await getRepositoryFromGitHub({
accessToken: sourceAccess.token,
owner: parsedRepository.owner,
repo: parsedRepository.name,
force: reconnecting,
});
if (!repository) {
throw new AnonymousError("repo_not_found", {
@@ -515,9 +537,18 @@ router.post(
httpStatus: 404,
});
}
await repository.getCommitInfo(repoUpdate.source.commit, {
// Pin the owner's preview to a GitHub identity, even if the name is
// deleted and recreated again before Save.
if (reconnecting && (repository.id !== repoUpdate.reconnectRepositoryId ||
(sourceAccess.binding.kind === "github-app" &&
repository.id !== `gh_${sourceAccess.binding.repositoryId}`))) throw appError("connection_changed", 409);
if (reconnecting && !(await repository.branches({ accessToken: sourceAccess.token, force: true }))
.some(branch => branch.name === repoUpdate.source.branch)) throw appError("branch_not_specified", 400);
const commit = await repository.getCommitInfo(repoUpdate.source.commit, {
accessToken: sourceAccess.token,
});
if (reconnecting) repo.model.source.commitDate = commit.commit.committer?.date
? new Date(commit.commit.committer.date) : undefined;
repo.model.githubAccess = { ...sourceAccess.binding, revision: randomUUID() };
repo.model.source.repositoryId = repository.model.id;
repo.model.source.repositoryName =
@@ -576,7 +607,8 @@ router.post(
}
repo.model.conference = repoUpdate.conference;
const saved = await AnonymizedRepositoryModel.updateOne(
{ _id: repo.model._id, "githubAccess.revision": previousAccessRevision || { $exists: false } },
{ _id: repo.model._id, status: repo.status, statusDate: repo.model.statusDate,
"githubAccess.revision": previousAccessRevision || { $exists: false } },
{
$set: {
settingsSavedAt: new Date(),
src/server/routes/repository-public.ts
+22 -40
View File
@@ -14,6 +14,7 @@ import FileModel from "../../core/model/files/files.model";
import { createLogger, serializeError } from "../../core/logger";
import { githubTokenForStreamer } from "../../core/github-token-context";
import gh = require("parse-github-url");
import { requestHeaders, startStage } from "../../core/request-monitoring";
const logger = createLogger("repository-public");
@@ -52,16 +53,20 @@ router.get(
});
}
res.attachment(`${repo.repoId}.zip`);
if (req.method === "HEAD") { res.end(); return; }
await repo.countView();
if (config.STREAMER_ENTRYPOINT) {
// use the streamer service
const token = await repo.getToken();
const anonymizer = repo.generateAnonymizeTransformer("");
const headersDone = startStage("streamer_headers");
res.attachment(`${repo.repoId}.zip`);
const reqStream = got
.stream(join(config.STREAMER_ENTRYPOINT, "api/download"), {
method: "POST",
headers: requestHeaders(),
json: {
token: await githubTokenForStreamer(token, repo.model.source.repositoryName),
repoFullName: repo.model.source.repositoryName,
@@ -76,6 +81,7 @@ router.get(
},
})
.on("error", (err: Error & { response?: { statusCode?: number; body?: unknown } }) => {
headersDone(true);
const upstreamStatus = err?.response?.statusCode;
let upstreamBody: string | undefined;
let errCode = "zip_not_available";
@@ -112,10 +118,11 @@ router.get(
upstreamStatus,
upstreamBody: upstreamBody?.slice(0, 500),
url: config.STREAMER_ENTRYPOINT
? join(config.STREAMER_ENTRYPOINT, "api/zip")
? join(config.STREAMER_ENTRYPOINT, "api/download")
: undefined,
err: serializeError(err),
});
const streaming = res.headersSent;
handleError(
new AnonymousError(errCode, {
url: req.originalUrl,
@@ -124,7 +131,13 @@ router.get(
}),
res
);
// Once ZIP bytes have been sent, handleError cannot send JSON.
// Terminate the truncated response so the client reports failure
// instead of waiting forever for the ZIP's central directory.
if (streaming && !res.writableEnded) res.destroy();
});
reqStream.once("response", () => headersDone());
reqStream.once("close", () => headersDone());
reqStream.pipe(res);
res.on("close", () => {
reqStream.destroy();
@@ -197,7 +210,7 @@ router.get(
try {
const repoId = repo.repoId;
const results = await FileModel.aggregate([
{ $match: { repoId, size: { $ne: null } } },
{ $match: { repoId, treeGeneration: repo.model.treeGeneration || { $exists: false }, size: { $ne: null } } },
{ $project: { _id: 0, path: 1 } },
{ $group: { _id: "$path", count: { $sum: 1 } } },
]).exec();
@@ -251,45 +264,8 @@ router.get(
if (!query || query.length < 2) {
return res.json([]);
}
const allFiles = await repo.anonymizedFiles({
includeSha: false,
recursive: true,
});
const q = query.toLowerCase();
res.json(await repo.searchFiles(query));
// Collect folder paths whose name segment matches the query
const matchingFolders = new Set<string>();
for (const f of allFiles) {
const segments = (f.path || "").split("/").filter(Boolean);
let accumulated = "";
for (const seg of segments) {
accumulated = accumulated ? `${accumulated}/${seg}` : seg;
if (seg.toLowerCase().includes(q)) {
matchingFolders.add(accumulated);
}
}
}
const matches = allFiles.filter((f) => {
// File name matches
if (f.name?.toLowerCase().includes(q)) return true;
// File is inside a matching folder
const fullPath = f.path ? `${f.path}/${f.name}` : f.name;
let found = false;
matchingFolders.forEach((folder) => {
if (fullPath?.startsWith(folder + "/") || fullPath === folder) found = true;
})
if (found) return true;
return false;
});
res.json(
matches.slice(0, 500).map((f) => ({
name: f.name,
path: f.path,
size: f.size,
}))
);
} catch (error) {
handleError(error, res, req);
}
@@ -395,6 +371,11 @@ router.get(
sourceCommitDate: repo.model.source.commitDate,
settingsSavedAt: repo.model.settingsSavedAt,
publishedAt: repo.model.publishedAt,
// Lets reviewers see how long the link stays available.
expirationDate:
repo.options.expirationMode !== "never"
? repo.options.expirationDate || null
: null,
isAdmin: user?.isAdmin === true,
isOwner: user?.id == repo.model.owner,
hasWebsite: !!repo.options.page && !!repo.options.pageSource,
@@ -407,6 +388,7 @@ router.get(
hasSubmodules:
(await FileModel.exists({
repoId: repo.repoId,
treeGeneration: repo.model.treeGeneration || { $exists: false },
name: ".gitmodules",
path: "",
})) != null,
src/server/routes/route-utils.ts
+3 -1
View File
@@ -8,6 +8,7 @@ import { HTTPError } from "got";
import { RepositoryStatus } from "../../core/types";
import { createLogger, serializeError } from "../../core/logger";
import { isDisabledAccount } from "./auth-utils";
import { measureStage } from "../../core/request-monitoring";
const logger = createLogger("route");
@@ -81,7 +82,8 @@ export async function getRepo(
if (typeof req.params.repoId !== "string") {
throw new AnonymousError("invalid_path", { httpStatus: 400 });
}
const repo = await db.getRepository(req.params.repoId);
const repoId = req.params.repoId;
const repo = await measureStage("repository", () => db.getRepository(repoId));
if (opt.nocheck !== true) {
// redirect if the repository is expired
if (
src/server/routes/user.ts
+31 -12
View File
@@ -1,3 +1,5 @@
import DashboardName from "../../core/model/dashboard-name";
import { dashboardSummary } from "./dashboard-summary";
import { revokeGrant } from "./github-app";
import CredentialModel from "../../core/model/credentials/credentials.model";
import * as express from "express";
@@ -13,6 +15,7 @@ import FileModel from "../../core/model/files/files.model";
import { isConnected } from "../database";
import { octokit } from "../../core/GitHubUtils";
import { createLogger, serializeError } from "../../core/logger";
import { projectNameKey, saveProjectName } from "./project-names";
const logger = createLogger("user");
@@ -47,20 +50,22 @@ router.get("/", async (req, res) => {
}
});
router.get("/dashboard", async (req, res) => {
try { res.json(await dashboardSummary(await getUser(req), req.query)); }
catch (error) { handleError(error, res, req); }
});
router.get("/quota", async (req, res) => {
try {
const user = await getUser(req);
const repositories = (await user.getRepositories()).filter(
(r) => r.owner.id === user.model.id
);
const ready = repositories.filter((r) => r.status == "ready");
const ready = await user.quotaRepositories();
let totalStorage = 0;
let totalFiles = 0;
const uncachedIds: string[] = [];
for (const r of ready) {
const cached = r.model.size;
if (cached && cached.file) {
const cached = r.size;
if (cached && (cached.file > 0 || r.sizeComputedAt) && cached.storage != null) {
totalStorage += cached.storage;
totalFiles += cached.file;
} else {
@@ -71,7 +76,7 @@ router.get("/quota", async (req, res) => {
if (uncachedIds.length) {
const uncachedSet = new Set(uncachedIds);
const agg = await FileModel.aggregate([
{ $match: { repoId: { $in: uncachedIds } } },
{ $match: { $or: ready.filter(r => uncachedSet.has(r.repoId)).map(r => ({ repoId: r.repoId, treeGeneration: r.treeGeneration || { $exists: false } })) } },
{
$group: {
_id: "$repoId",
@@ -89,7 +94,8 @@ router.get("/quota", async (req, res) => {
const size = byId.get(r.repoId) || { storage: 0, file: 0 };
totalStorage += size.storage;
totalFiles += size.file;
r.model.size = size;
r.size = size;
r.sizeComputedAt = new Date();
}
if (isConnected) {
await Promise.all(
@@ -97,8 +103,9 @@ router.get("/quota", async (req, res) => {
.filter((r) => uncachedSet.has(r.repoId))
.map((r) =>
AnonymizedRepositoryModel.updateOne(
{ _id: r.model._id },
{ $set: { size: r.model.size } }
{ _id: r._id, status: "ready", treeGeneration: r.treeGeneration || { $exists: false },
fileMetadataRevision: r.fileMetadataRevision || { $exists: false } },
{ $set: { size: r.size, sizeComputedAt: r.sizeComputedAt } }
).exec()
)
);
@@ -124,6 +131,15 @@ router.get("/quota", async (req, res) => {
}
});
router.post("/project-name", async (req, res) => {
try {
const user = await getUser(req);
res.json({ name: await saveProjectName(user, req.body) });
} catch (error) {
handleError(error, res, req);
}
});
router.get("/default", async (req, res) => {
try {
const user = await getUser(req);
@@ -197,6 +213,7 @@ router.delete("/", async (req, res) => {
}
await CredentialModel.deleteMany({ ownerId: user.model._id });
if (isConnected) await DashboardName.deleteMany({ owner: user.model._id });
await UserModel.updateOne(
{ _id: user.model._id },
@@ -214,6 +231,7 @@ router.delete("/", async (req, res) => {
externalIDs: "",
photo: "",
default: "",
projectNames: "",
},
}
).exec();
@@ -239,6 +257,7 @@ router.get(
(await user.getRepositories()).map((x) => {
const json = x.toJSON() as Record<string, unknown>;
json.role = x.owner.id === user.model.id ? "owner" : "coauthor";
json.projectName = user.model.projectNames?.get(projectNameKey("repo", x.repoId));
return json;
})
);
@@ -254,7 +273,7 @@ router.get(
const user = await getUser(req);
res.json(
(await user.getGists()).map((x) => {
return x.toJSON();
return { ...x.toJSON(), projectName: user.model.projectNames?.get(projectNameKey("gist", x.gistId)) };
})
);
} catch (error) {
@@ -269,7 +288,7 @@ router.get(
const user = await getUser(req);
res.json(
(await user.getPullRequests()).map((x) => {
return x.toJSON();
return { ...x.toJSON(), projectName: user.model.projectNames?.get(projectNameKey("pr", x.pullRequestId)) };
})
);
} catch (error) {
src/server/routes/webview.ts
+20 -2
View File
@@ -5,8 +5,11 @@ import AnonymizedFile from "../../core/AnonymizedFile";
import AnonymousError from "../../core/AnonymousError";
import * as marked from "marked";
import * as sanitizeHtml from "sanitize-html";
import { streamToString } from "../../core/anonymize-utils";
import { streamToString, isTextFile } from "../../core/anonymize-utils";
import { IFile } from "../../core/model/files/files.types";
import { fileETag } from "./file-etag";
import { lookup } from "mime-types";
import config from "../../config";
function escapeHtml(str: string): string {
return str
@@ -170,13 +173,28 @@ async function webView(
object: f,
});
}
res.set("Cache-Control", "private, no-cache, must-revalidate");
const pageETag = fileETag(JSON.stringify(["web-v1", repo.model.source.commit, repo.model.treeGeneration, repo.model.contentCacheRevision,
await f.sha(), repo.model.source.repositoryName, repo.model.source.branch, config.APP_HOSTNAME, config.ANONYMIZATION_MASK]),
requestPath, repo.options);
res.set("ETag", pageETag);
if (req.fresh) { res.status(304).end(); return; }
if (req.method === "HEAD") {
const size = await f.size();
if (size != null && size > config.MAX_FILE_SIZE) throw new AnonymousError("file_too_big", { httpStatus: 413 });
const mime = lookup(requestPath);
res.type(f.extension() === "md" ? "text/html" : mime && !requestPath.endsWith(".ts") ? mime : isTextFile(requestPath) ? "text/plain" : "application/octet-stream");
res.end(); return;
}
if (f.extension() == "md") {
try {
const content = await streamToString(await f.anonymizedContent());
const body = sanitizeHtml(marked.marked(content, { headerIds: false, mangle: false }), sanitizeOptions);
const body = sanitizeHtml(marked.marked.parse(content, { async: false }), sanitizeOptions);
const html = `<!DOCTYPE html><html><head><title>Content</title></head><link rel="stylesheet" href="/css/all.min.css" /><body><div class="container p-3 file-content markdown-body">${body}</div></body></html>`;
res.contentType("text/html").send(html);
} catch {
// Raw Markdown must re-enter rendering when a transient failure clears.
res.set("ETag", fileETag(`web-raw:${pageETag}`, requestPath, repo.options));
await f.send(res);
}
} else {
src/server/schedule.ts
+69 -5
View File
@@ -1,8 +1,14 @@
import { Model, Document } from "mongoose";
import * as schedule from "node-schedule";
import AnonymizedGistModel from "../core/model/anonymizedGists/anonymizedGists.model";
import AnonymizedPullRequestModel from "../core/model/anonymizedPullRequests/anonymizedPullRequests.model";
import Gist from "../core/Gist";
import PullRequest from "../core/PullRequest";
import Conference from "../core/Conference";
import AnonymizedRepositoryModel from "../core/model/anonymizedRepositories/anonymizedRepositories.model";
import ConferenceModel from "../core/model/conference/conferences.model";
import Repository from "../core/Repository";
import FileModel from "../core/model/files/files.model";
import { createLogger, serializeError } from "../core/logger";
import { RepositoryStatus } from "../core/types";
import { computeAndStoreDailyStats } from "./dailyStatsSnapshot";
@@ -28,9 +34,12 @@ export function conferenceStatusCheck() {
}
export function repositoryStatusCheck() {
// check every 6 hours the status of the repositories
schedule.scheduleJob("0 */6 * * *", async () => {
await runRepositoryStatusCheck();
// Claim persisted expiration work every five minutes, without overlapping runs.
let running = false;
schedule.scheduleJob("*/5 * * * *", async () => {
if (running) return;
running = true;
try { await runRepositoryStatusCheck(); } finally { running = false; }
});
}
@@ -126,8 +135,7 @@ export async function runRepositoryStatusCheck(now = new Date()) {
(async () => {
const repo = new Repository(data);
try {
await repo.resetSate();
await repo.updateStatus(RepositoryStatus.EXPIRED);
await repo.expire();
logger.info("recovered expired repository cleanup", {
repoId: repo.repoId,
});
@@ -144,6 +152,62 @@ export async function runRepositoryStatusCheck(now = new Date()) {
}
}
await flushBatch();
await expireContent(AnonymizedGistModel, data => new Gist(data), now);
await expireContent(AnonymizedPullRequestModel, data => new PullRequest(data), now);
const retiredCursor = AnonymizedRepositoryModel.find({ $or: [
{ retiredTreeGenerations: { $exists: true } }, { retiredContentPrefixes: { $exists: true } },
{ legacyContentCleanupPending: true, status: RepositoryStatus.READY },
{ "stagedFileTrees.until": { $lte: now } },
] }).cursor();
for await (const data of retiredCursor) {
batch.push(new Repository(data).cleanupRetiredFileTrees(now).catch(error => {
logger.error("retired tree cleanup failed", { ...serializeError(error), repoId: data.repoId });
}));
if (batch.length >= 10) await flushBatch();
}
await flushBatch();
const stagedCursor = FileModel.aggregate<{ _id: { repoId: string; generation: string } }>([
{ $match: { treeStaged: true } }, { $group: { _id: { repoId: "$repoId", generation: "$treeGeneration" } } },
]).cursor();
for await (const stage of stagedCursor) {
batch.push(Repository.cleanupStagedFileTree(stage._id.repoId, stage._id.generation, now).catch(error => {
logger.error("staged tree cleanup failed", { ...serializeError(error), repoId: stage._id.repoId });
}));
if (batch.length >= 10) await flushBatch();
}
await flushBatch();
const pendingCursor = FileModel.find({ metadataPending: true }).cursor();
for await (const file of pendingCursor) {
batch.push((async () => {
try {
const data = await AnonymizedRepositoryModel.findOne({ repoId: file.repoId }).exec();
if (!data || data.treeGeneration !== file.treeGeneration ||
(data.status && [RepositoryStatus.EXPIRING, RepositoryStatus.EXPIRED, RepositoryStatus.REMOVING, RepositoryStatus.REMOVED].includes(data.status))) {
await FileModel.deleteOne({ _id: file._id, metadataPending: true }).exec();
return;
}
await new Repository(data).completeRecoveredFileMetadata(file);
} catch (error) {
logger.error("recovered file invalidation failed", { ...serializeError(error), repoId: file.repoId });
}
})());
if (batch.length >= 10) await flushBatch();
}
await flushBatch();
}
async function expireContent<T extends Document>(model: Model<T>, create: (data: T) => Gist | PullRequest, now: Date) {
const cursor = model.find(contentMaintenanceQuery(now)).cursor();
for await (const data of cursor) {
try { await create(data).expire(); } catch (error) { logger.error("content expiration failed", serializeError(error)); }
}
}
export function contentMaintenanceQuery(now: Date) {
return { $or: [{ status: RepositoryStatus.EXPIRING },
{ status: RepositoryStatus.READY, "options.expirationMode": { $ne: "never" }, "options.expirationDate": { $lte: now } }] };
}
export function dailyStatsSnapshot() {