Merge main and preserve email and log investigation styles

This commit is contained in:
Thomas Durieux committed 2026-11-14 17:24:13 +00:00
commit 35e69b3291
33 files changed
+1142 -69

No files matched your search

+11 -9
View File
@@ -1,3 +1,4 @@
import { rejectStreamerRedirects } from "./streamer-response";
import { isResponseCancellation } from "./response-cancellation";
import { canReadCachedRepository } from "./cached-repository-access";
import { probeStreamerCache } from "./streamer-cache-probe";
@@ -412,7 +413,8 @@ export default class AnonymizedFile {
}
// use the streamer service
const content = got.stream(join(config.STREAMER_ENTRYPOINT, "api"), {
const content = rejectStreamerRedirects(got.stream(join(config.STREAMER_ENTRYPOINT, "api"), {
followRedirect: false,
method: "POST",
headers: requestHeaders(),
json: {
@@ -429,7 +431,7 @@ export default class AnonymizedFile {
size: await this.size(),
anonymizerOptions: anonymizer.opt,
},
});
}));
content.on("error", err => {
const { error } = streamerErrorToAnonymous(err, { repoId: this.repository.repoId, filePath: this.anonymizedPath });
void notifyOwnerAccessProblem(this.repository.owner.id, error, { kind: "repository", id: String(this.repository.model._id) });
@@ -504,17 +506,17 @@ export default class AnonymizedFile {
} catch (error) { cached.stream.destroy(); throw error; }
}
if (res.destroyed || res.writableEnded) { cached?.stream.destroy(); headersDone(); resolve(); return; }
const resStream = cached?.stream || got.stream(url, {
const resStream = cached?.stream || rejectStreamerRedirects(got.stream(url, {
followRedirect: false,
method: "POST", headers, decompress: false,
json: { ...payload, token: await githubTokenForStreamer(await this.repository.getToken(), this.repository.model.source.repositoryName) },
});
}));
// Forward Content-Type from the streamer's upstream response.
// got.stream(...).pipe(res) forwards body bytes only — without
// this, the parent response has no Content-Type and the browser
// guesses (text renders as download, images as octet-stream).
const forwardHeaders = (upstream: { headers: Record<string, string | string[] | undefined> }) => {
// streamResponse isolates the body from Got's implicit header forwarding.
// Only approved content headers may reach the public response.
const forwardHeaders = (upstream: { statusCode?: number; headers: Record<string, string | string[] | undefined> }) => {
headersDone();
if (res.headersSent) return;
if (res.headersSent || (upstream.statusCode !== undefined && upstream.statusCode !== 200)) return;
for (const header of ["content-encoding", "content-length", "vary", "accept-ranges"]) {
const value = upstream.headers[header];
if (typeof value === "string") res.setHeader(header, value);
+2
View File
@@ -5,6 +5,7 @@ import { hasAutomaticUpdates } from "./billing/expiration";
import { FILE_TREE_BATCH_SIZE, FILE_LISTING_LIMIT, deleteRowsInBatches, clearStagedRowsInBatches } from "./file-tree";
import { SourceBase, sourceStorageSize } from "./source/Source";
import { notifyOwnerAccessProblem, withOwnerAccessRecovery } from "./owner-notifications";
import { notifyRepositoryExpired } from "./repository-expiration-email";
import AnonymizedPathModel from "./model/anonymized-path";
import storage from "./storage";
import { contentGenerationPrefix, contentRetirementMarker } from "./content-generation";
@@ -1048,6 +1049,7 @@ export default class Repository {
statusDate: new Date(), isReseted: true, size: { storage: 0, file: 0 } },
$unset: { cleanupToken: "", cleanupUntil: "", pathIndexKey: "", pathIndexBuiltAt: "", sizeComputedAt: "", emptyTreeGeneration: "", retiredTreeGenerations: "", retiredPathIndexKeys: "", retiredContentPrefixes: "", legacyContentCleanupPending: "", stagedFileTrees: "" } }).exec();
if (result.matchedCount) { this.model.status = RepositoryStatus.EXPIRED; this.model.isReseted = true; this.model.size = { storage: 0, file: 0 }; this.model.sizeComputedAt = undefined; this.model.emptyTreeGeneration = undefined; }
if (result.matchedCount) await notifyRepositoryExpired(this.model._id);
} finally {
clearInterval(heartbeat);
await AnonymizedRepositoryModel.updateOne({ _id: this.model._id, cleanupToken: token }, { $unset: { cleanupToken: "", cleanupUntil: "" } }).exec();
+60
View File
@@ -0,0 +1,60 @@
import { randomUUID } from "crypto";
import { mongo } from "mongoose";
import UserModel from "./model/users/users.model";
import { createLogger } from "./logger";
const logger = createLogger("email-history");
export const EMAIL_HISTORY = "email_delivery_history";
export interface EmailHistoryRecord {
_id: string;
kind: "repository-expiration" | "owner-access";
recipient: string;
ownerId: string;
resourceId: string;
resourceKind: string;
createdAt: Date;
completedAt?: Date;
state: "sending" | "accepted" | "rejected" | "unknown";
httpStatus?: number;
}
export const emailHistoryCollection = () => UserModel.db.collection(EMAIL_HISTORY) as unknown as mongo.Collection<EmailHistoryRecord>;
let indexed: Promise<unknown> | undefined;
export async function ensureEmailHistoryIndexes() {
if (!indexed) indexed = Promise.all([
emailHistoryCollection().createIndex({ createdAt: -1, _id: -1 }),
emailHistoryCollection().createIndex({ state: 1, createdAt: -1, _id: -1 }),
emailHistoryCollection().createIndex({ kind: 1, createdAt: -1, _id: -1 }),
]).catch(error => { indexed = undefined; throw error; });
await indexed;
}
/** Store metadata only: never API keys, message bodies, or provider error bodies. */
export async function sendTrackedEmail(metadata: Pick<EmailHistoryRecord, "kind" | "recipient" | "ownerId" | "resourceId" | "resourceKind">,
url: string, options: RequestInit): Promise<Response> {
const id = randomUUID(), createdAt = new Date();
let recorded = false;
const recording = (async () => {
try {
if (UserModel.db.readyState === 1) {
await ensureEmailHistoryIndexes();
await emailHistoryCollection().insertOne({ _id: id, ...metadata, createdAt, state: "sending" });
recorded = true;
}
} catch { logger.warn("email history write unavailable"); }
})();
// Start the provider request immediately after the caller commits dispatch.
// Auditing must not add a preparation gap to owner lifecycle fencing.
const complete = async (state: EmailHistoryRecord["state"], httpStatus?: number) => {
const completedAt = new Date();
await recording;
if (!recorded) return;
try {
await emailHistoryCollection().updateOne({ _id: id }, { $set: { state, completedAt, ...(httpStatus ? { httpStatus } : {}) } });
} catch { logger.warn("email history completion unavailable", { id }); }
};
let response: Response;
try { response = await fetch(url, options); }
catch (error) { await complete("unknown"); throw error; }
await complete(response.ok ? "accepted" : "rejected", response.status);
return response;
}
@@ -5,6 +5,7 @@ import { Schema } from "mongoose";
const AnonymizedRepositorySchema = new Schema({
billingManaged: Boolean,
billingNeverExpiresUntil: Date,
expirationEmailClaimedFor: { type: Date, select: false },
accessAlertGeneration: { type: String, select: false },
accessAlertClaimedAt: { type: Date, select: false },
accessAlertAttemptId: { type: String, select: false },
+3 -1
View File
@@ -1,3 +1,4 @@
import { sendTrackedEmail } from "./email-history";
import { createHash, randomUUID } from "crypto";
import { mongo, Types } from "mongoose";
import { isGitHubRateLimitError } from "./github-rate-limit";
@@ -577,7 +578,8 @@ export async function notifyOwnerAccessProblem(ownerId: string, error: unknown,
let httpStatus: number | undefined;
try {
providerStarted = true;
const response = await fetch("https://api.resend.com/emails", {
const response = await sendTrackedEmail({ kind: "owner-access", recipient: to, ownerId,
resourceId: resource.id, resourceKind: resource.kind }, "https://api.resend.com/emails", {
method: "POST",
headers: { Authorization: `Bearer ${config.RESEND_API_KEY}`, "Content-Type": "application/json",
"Idempotency-Key": `owner-access-alert/${attemptId}` },
+48
View File
@@ -0,0 +1,48 @@
import { sendTrackedEmail } from "./email-history";
import config from "../config";
import RepositoryModel from "./model/anonymizedRepositories/anonymizedRepositories.model";
import UserModel from "./model/users/users.model";
import { notificationEmail } from "./owner-notifications";
import { createLogger } from "./logger";
import { RepositoryStatus } from "./types";
const logger = createLogger("repository-expiration-email");
/** Attempt delivery once per repository expiration date, across all workers. */
export async function notifyRepositoryExpired(id: unknown): Promise<void> {
if (!config.RESEND_API_KEY || !config.EMAIL_FROM) return;
try {
const repo = await RepositoryModel.findOne({ _id: id, status: RepositoryStatus.EXPIRED }).lean();
const expirationDate = repo?.options.expirationDate;
if (!repo || repo.options.expirationMode === "never" || !expirationDate || expirationDate > new Date()) return;
const owner = await UserModel.findOne({ _id: repo.owner, status: { $nin: ["removed", "banned"] } }).lean();
const to = owner && notificationEmail(owner.emails, owner.notificationEmail);
if (!to) return;
const base = /^https?:\/\//.test(config.APP_HOSTNAME) ? config.APP_HOSTNAME : `https://${config.APP_HOSTNAME}`;
const url = new URL("/dashboard", base).href;
const claim = await RepositoryModel.updateOne({
_id: repo._id, owner: repo.owner, status: RepositoryStatus.EXPIRED, statusDate: repo.statusDate,
"options.expirationMode": repo.options.expirationMode,
"options.expirationDate": expirationDate,
expirationEmailClaimedFor: { $ne: expirationDate },
}, { $set: { expirationEmailClaimedFor: expirationDate } }).exec();
if (!claim.matchedCount) return;
// Keep the claim even after a timeout: the provider may have accepted the email.
const response = await sendTrackedEmail({ kind: "repository-expiration", recipient: to, ownerId: String(repo.owner),
resourceId: String(repo._id), resourceKind: "repository" }, "https://api.resend.com/emails", {
method: "POST",
headers: { Authorization: `Bearer ${config.RESEND_API_KEY}`, "Content-Type": "application/json",
"Idempotency-Key": `repository-expired/${repo._id}/${expirationDate.getTime()}` },
signal: AbortSignal.timeout(15000),
body: JSON.stringify({
from: config.EMAIL_FROM, to: [to], subject: "Your Anonymous GitHub repository has expired",
text: `Your anonymized repository "${repo.repoId}" expired on ${expirationDate.toISOString().slice(0, 10)} (UTC).\n\nIts anonymized content is no longer available.${repo.options.expirationMode === "redirect" ? " Its link now redirects to the original repository." : ""}\n\nOpen your dashboard to manage your repositories:\n${url}\n\nYou are receiving this email because you own this repository on Anonymous GitHub. You can change your alert address on the connections page.\n`,
}),
});
if (response.ok) logger.info("repository expiration email accepted", { repoId: repo.repoId });
else logger.warn("repository expiration email delivery failed", { repoId: repo.repoId, httpStatus: response.status });
} catch {
// Email delivery must not interrupt expiration or expose addresses in logs.
logger.warn("repository expiration email unavailable");
}
}
+8 -3
View File
@@ -1,15 +1,20 @@
import { Readable, Writable } from "stream";
import { PassThrough, Readable, Writable } from "stream";
/** Cancel upstream on every response terminal event, including a pre-header close. */
export function streamResponse(source: Readable, response: Writable,
onError: (error: Error) => void): Promise<void> {
return new Promise((resolve, reject) => {
// Got copies every upstream header/status when piped to ServerResponse.
// A plain stream keeps response policy with the caller's explicit allowlist.
const body = new PassThrough();
const cleanup = () => {
response.removeListener("close", done);
response.removeListener("finish", done);
response.removeListener("error", failed);
source.removeListener("error", upstreamFailed);
source.unpipe(response);
source.unpipe(body);
body.unpipe(response);
body.destroy();
source.destroy();
};
const done = () => { cleanup(); resolve(); };
@@ -22,6 +27,6 @@ export function streamResponse(source: Readable, response: Writable,
if (response.destroyed || response.writableEnded) done();
else if (source.errored) upstreamFailed(source.errored);
else if (source.destroyed || source.readableEnded) upstreamFailed(new Error("upstream_stream_closed"));
else source.pipe(response);
else { body.pipe(response); source.pipe(body); }
});
}
+4 -2
View File
@@ -1,9 +1,10 @@
import { rejectStreamerRedirects } from "./streamer-response";
import got, { Response } from "got";
/** Wait only for headers; the body remains a backpressured stream. */
export async function probeStreamerCache(url: string, json: Record<string, unknown>, headers: Record<string, string>, signal?: AbortSignal) {
const stream = got.stream(url, { method: "POST", json: { ...json, token: "", cacheOnly: true },
headers, timeout: { response: 15000 }, decompress: false });
const stream = rejectStreamerRedirects(got.stream(url, { followRedirect: false, method: "POST", json: { ...json, token: "", cacheOnly: true },
headers, timeout: { response: 15000 }, decompress: false }));
const responseReady = new Promise<Response>((resolve, reject) => {
stream.once("response", resolve);
stream.once("error", reject);
@@ -14,6 +15,7 @@ export async function probeStreamerCache(url: string, json: Record<string, unkno
if (signal?.aborted) abort();
try {
const response = await responseReady;
if (stream.errored) throw stream.errored;
return { stream, response };
} catch (error) {
stream.destroy();
+12
View File
@@ -0,0 +1,12 @@
import { Readable } from "stream";
import AnonymousError from "./AnonymousError";
/** Internal streamer endpoints return content, never browser redirects. */
export function rejectStreamerRedirects<T extends Readable>(stream: T): T {
stream.on("response", (response: { statusCode: number }) => {
if (response.statusCode >= 300 && response.statusCode < 400) {
stream.destroy(new AnonymousError("streamer_upstream_error", { httpStatus: 502 }));
}
});
return stream;
}
+102
View File
@@ -0,0 +1,102 @@
import { Router } from "express";
import { mongo } from "mongoose";
import { EmailHistoryRecord, emailHistoryCollection, ensureEmailHistoryIndexes } from "../../core/email-history";
const router = Router();
const QUERY_TIMEOUT_MS = 5000;
const emptyCounts = () => ({ accepted: 0, rejected: 0, unknown: 0, sending: 0 });
export function emailHistoryPeriod(period: unknown, now = new Date()) {
if (period !== undefined && (typeof period !== "string" || !["7d", "30d", "90d", "all"].includes(period))) throw new Error("Invalid period");
const selected = (period || "all") as string;
const chartDays = selected === "all" ? 30 : parseInt(selected, 10);
const start = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate() - chartDays + 1));
return { period: selected, chartDays, start };
}
export function emailHistoryQuery(query: Record<string, unknown>, now = new Date()) {
const filter: mongo.Filter<EmailHistoryRecord> = {};
if (query.state) {
if (typeof query.state !== "string" || !["sending", "accepted", "rejected", "unknown"].includes(String(query.state))) throw new Error("Invalid status");
filter.state = query.state as EmailHistoryRecord["state"];
}
if (query.kind) {
if (typeof query.kind !== "string" || !["repository-expiration", "owner-access"].includes(String(query.kind))) throw new Error("Invalid email type");
filter.kind = query.kind as EmailHistoryRecord["kind"];
}
const period = emailHistoryPeriod(query.period, now);
if (period.period !== "all") filter.createdAt = { $gte: period.start };
if (query.search !== undefined) {
if (typeof query.search !== "string" || query.search.length > 200) throw new Error("Invalid search");
const search = query.search.trim();
if (search) filter.$or = [
{ recipient: { $regex: search.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), $options: "i" } },
{ _id: search }, { ownerId: search }, { resourceId: search },
];
}
if (query.before) {
if (typeof query.before !== "string" || query.before.length > 200) throw new Error("Invalid cursor");
const [date, id, extra] = query.before.split("|");
const time = new Date(date);
if (extra || !id || !/^[0-9a-f-]{36}$/.test(id) || !Number.isFinite(time.getTime())) throw new Error("Invalid cursor");
const cursor = [{ createdAt: { $lt: time } }, { createdAt: time, _id: { $lt: id } }];
if (filter.$or) filter.$and = [{ $or: cursor }];
else filter.$or = cursor;
}
return filter;
}
export function emailHistorySummary(
countsRows: Array<{ _id: string; count: number }>,
dailyRows: Array<{ _id: { date: string; state: string }; count: number }>,
period: ReturnType<typeof emailHistoryPeriod>,
) {
const counts = emptyCounts();
for (const row of countsRows) if (Object.hasOwn(counts, row._id)) counts[row._id as keyof typeof counts] = row.count;
const series = Array.from({ length: period.chartDays }, (_, index) => ({
date: new Date(period.start.getTime() + index * 86400000).toISOString().slice(0, 10), ...emptyCounts(),
}));
for (const row of dailyRows) {
const day = series.find(point => point.date === row._id.date);
if (day && Object.hasOwn(counts, row._id.state)) day[row._id.state as keyof typeof counts] = row.count;
}
return { counts, total: Object.values(counts).reduce((sum, count) => sum + count, 0), series, chartDays: period.chartDays, period: period.period };
}
// Mounted after the shared authentication and administrator checks.
router.get("/summary", async (req, res) => {
res.set("Cache-Control", "no-store");
let filter: mongo.Filter<EmailHistoryRecord>, period: ReturnType<typeof emailHistoryPeriod>;
try {
const now = new Date();
filter = emailHistoryQuery({ period: req.query.period, search: req.query.search, kind: req.query.kind }, now);
period = emailHistoryPeriod(req.query.period, now);
} catch { res.status(400).json({ error: "Invalid email history filters." }); return; }
try {
await ensureEmailHistoryIndexes();
const [result] = await emailHistoryCollection().aggregate([
{ $match: filter },
{ $facet: {
counts: [{ $group: { _id: "$state", count: { $sum: 1 } } }],
daily: [
{ $match: { createdAt: { $gte: period.start } } },
{ $group: { _id: { date: { $dateToString: { format: "%Y-%m-%d", date: "$createdAt", timezone: "UTC" } }, state: "$state" }, count: { $sum: 1 } } },
],
} },
], { maxTimeMS: QUERY_TIMEOUT_MS }).toArray();
res.json(emailHistorySummary(result?.counts || [], result?.daily || [], period));
} catch { res.status(503).json({ error: "Email history summary is unavailable. Try again." }); }
});
router.get("/", async (req, res) => {
res.set("Cache-Control", "no-store");
let filter: mongo.Filter<EmailHistoryRecord>;
try { filter = emailHistoryQuery(req.query); }
catch { res.status(400).json({ error: "Invalid email history filters." }); return; }
try {
await ensureEmailHistoryIndexes();
const rows = await emailHistoryCollection().find(filter, { maxTimeMS: QUERY_TIMEOUT_MS, projection: {
_id: 1, kind: 1, recipient: 1, ownerId: 1, resourceId: 1, resourceKind: 1,
createdAt: 1, completedAt: 1, state: 1, httpStatus: 1,
} }).sort({ createdAt: -1, _id: -1 }).limit(51).toArray();
const entries = rows.slice(0, 50);
const last = entries.at(-1);
res.json({ entries, nextCursor: rows.length > 50 && last ? `${last.createdAt.toISOString()}|${last._id}` : null });
} catch { res.status(503).json({ error: "Email history is unavailable. Try again." }); }
});
export default router;
+2
View File
@@ -24,6 +24,7 @@ import { ensureAuthenticated } from "./connection";
import { handleError, getUser, isOwnerOrAdmin, getRepo } from "./route-utils";
import adminTokensRouter from "./admin-tokens";
import adminBillingRouter from "./admin-billing";
import adminEmailsRouter from "./admin-emails";
import { octokit, getToken } from "../../core/GitHubUtils";
import {
createLogger,
@@ -114,6 +115,7 @@ for (const [path, correlation] of [["/errors/investigation", false], ["/errors/c
router.use("/tokens", adminTokensRouter);
router.use("/billing", adminBillingRouter);
router.use("/emails", adminEmailsRouter);
router.get("/performance", async (req, res) => {
res.set("Cache-Control", "no-store");