mirror of
https://github.com/tdurieux/anonymous_github.git
synced 2026-11-18 01:54:23 +01:00
fix: bound metadata work and distinguish cancelled requests (#951)
This commit is contained in:
1 parent
ed07ecde73
commit
d508ab1754
14 files changed
+202
-16
No files matched your search
@@ -1,3 +1,4 @@
|
||||
import { isResponseCancellation } from "./response-cancellation";
|
||||
import { canReadCachedRepository } from "./cached-repository-access";
|
||||
import { probeStreamerCache } from "./streamer-cache-probe";
|
||||
import { notifyOwnerAccessProblem } from "./owner-notifications";
|
||||
@@ -485,6 +486,7 @@ export default class AnonymizedFile {
|
||||
if (res.destroyed || res.writableEnded) cancel();
|
||||
try { return await probeStreamerCache(url, payload, headers, cancellation.signal); }
|
||||
catch (err) {
|
||||
if (isResponseCancellation(err, res)) { headersDone(); return undefined; }
|
||||
headersDone(true);
|
||||
const { error } = streamerErrorToAnonymous(err as Error & {
|
||||
response?: { statusCode?: number; body?: unknown };
|
||||
@@ -531,6 +533,7 @@ export default class AnonymizedFile {
|
||||
resStream.once("error", () => headersDone(true));
|
||||
resStream.once("close", () => headersDone());
|
||||
await streamResponse(resStream, res, (err: Error) => {
|
||||
if (isResponseCancellation(err, res)) return;
|
||||
const { error } = streamerErrorToAnonymous(
|
||||
err as Error & {
|
||||
response?: { statusCode?: number; body?: unknown };
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { InvalidCredentialBackoff } from "./invalid-credential-backoff";
|
||||
import { isGitHubRateLimitError, OctokitRequestErrorLike } from "./github-rate-limit";
|
||||
import { CredentialValidation } from "./credential-validation";
|
||||
import { refreshLegacyToken } from "./legacy-token-refresh";
|
||||
@@ -202,6 +203,8 @@ export async function getRedisGateResetAt(tokenKey: string): Promise<number> {
|
||||
}
|
||||
}
|
||||
|
||||
const invalidCredentials = new InvalidCredentialBackoff();
|
||||
|
||||
export function octokit(token: string) {
|
||||
const context = githubTokenContext(token);
|
||||
const oct = new ThrottledOctokit({
|
||||
@@ -272,7 +275,11 @@ export function octokit(token: string) {
|
||||
}
|
||||
throw err;
|
||||
});
|
||||
oct.hook.wrap("request", (request, options) => measureStage("upstream", async () => request(options)));
|
||||
oct.hook.wrap("request", (request, options) => {
|
||||
const run = () => measureStage("upstream", async () => request(options));
|
||||
// Managed handles may rotate during renewal; never poison a handle with an old token's denial.
|
||||
return githubTokenContext(token) ? run() : invalidCredentials.run(token, run);
|
||||
});
|
||||
return oct;
|
||||
}
|
||||
|
||||
@@ -311,7 +318,7 @@ const checkedRepositoryTokens = new WeakMap<Repository, string>();
|
||||
|
||||
export async function getToken(repository: Repository) {
|
||||
try {
|
||||
return await measureStage("authorization", () => resolveRepositoryToken(repository));
|
||||
return await measureStage("credential_resolution", () => resolveRepositoryToken(repository));
|
||||
} catch (error) {
|
||||
void notifyOwnerAccessProblem(repository.owner.id, error, { kind: "repository", id: String(repository.model._id) });
|
||||
throw error;
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { REPOSITORY_ID_COLLATION } from "./repository-id";
|
||||
import { reserveRepository } from "./billing/quotas";
|
||||
import { hasAutomaticUpdates } from "./billing/expiration";
|
||||
import { FILE_TREE_BATCH_SIZE, FILE_LISTING_LIMIT, deleteRowsInBatches } from "./file-tree";
|
||||
import { FILE_TREE_BATCH_SIZE, FILE_LISTING_LIMIT, deleteRowsInBatches, clearStagedRowsInBatches } from "./file-tree";
|
||||
import { SourceBase, sourceStorageSize } from "./source/Source";
|
||||
import { notifyOwnerAccessProblem, withOwnerAccessRecovery } from "./owner-notifications";
|
||||
import AnonymizedPathModel from "./model/anonymized-path";
|
||||
@@ -341,8 +341,7 @@ export default class Repository {
|
||||
this.model.treeGeneration = generation;
|
||||
if (isConnected) {
|
||||
// A crash here is safe: maintenance recognizes the active generation.
|
||||
await FileModel.updateMany({ repoId: this.repoId, treeGeneration: generation, treeStaged: true },
|
||||
{ $unset: { treeStaged: "" } }).exec().catch(error => logger.warn("staged tree completion deferred", serializeError(error)));
|
||||
await clearStagedRowsInBatches(FileModel.collection, this.repoId, generation).catch(error => logger.warn("staged tree completion deferred", serializeError(error)));
|
||||
}
|
||||
this.model.emptyTreeGeneration = fileCount ? undefined : generation;
|
||||
this.model.truncatedFolders = truncatedFolders;
|
||||
@@ -434,7 +433,7 @@ export default class Repository {
|
||||
const terminal = current?.status && [RepositoryStatus.EXPIRING, RepositoryStatus.EXPIRED,
|
||||
RepositoryStatus.REMOVING, RepositoryStatus.REMOVED].includes(current.status);
|
||||
if (current?.treeGeneration === generation && !terminal) {
|
||||
await FileModel.updateMany({ repoId, treeGeneration: generation, treeStaged: true }, { $unset: { treeStaged: "" } }).exec();
|
||||
await clearStagedRowsInBatches(FileModel.collection, repoId, generation);
|
||||
} else if (terminal || !current?.stagedFileTrees?.some(stage => stage.generation === generation && stage.until > now)) {
|
||||
await deleteRowsInBatches(FileModel.collection, { repoId, treeGeneration: generation, treeStaged: true });
|
||||
}
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
import FileModel from "./model/files/files.model";
|
||||
|
||||
export async function hasVisibleSubmodules(repoId: string, generation?: string): Promise<boolean> {
|
||||
const scope = { repoId, treeGeneration: generation || { $exists: false } };
|
||||
// Separate probes prevent the OR planner choosing a scan of every generation row.
|
||||
if (await FileModel.exists({ ...scope, name: ".gitmodules", path: "" })) return true;
|
||||
const filter = { ...scope, submodule: { $exists: true, $ne: "included" } };
|
||||
try { return !!await FileModel.exists(filter).hint("file_submodule_status"); }
|
||||
catch (error) {
|
||||
// Rolling upgrades may receive requests before the additive index is installed.
|
||||
const failure = error as { code?: number; message?: string };
|
||||
if (failure.code !== 2 || !/hint.*(?:index|existing)/i.test(failure.message || "")) throw error;
|
||||
return !!await FileModel.exists(filter);
|
||||
}
|
||||
}
|
||||
@@ -30,3 +30,18 @@ export async function deleteRowsInBatches(
|
||||
logger.info("metadata cleanup deferred", { repoId: filter.repoId, deleted, ms: Date.now() - started });
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Staging markers are bookkeeping; leave the remainder discoverable by maintenance. */
|
||||
export async function clearStagedRowsInBatches(
|
||||
collection: Pick<mongo.Collection, "find" | "updateMany">,
|
||||
repoId: string, treeGeneration: string, maxBatches = 10
|
||||
): Promise<void> {
|
||||
const filter = { repoId, treeGeneration, treeStaged: true };
|
||||
for (let batch = 0; batch < maxBatches; batch++) {
|
||||
const rows = await collection.find(filter, { projection: { _id: 1 }, limit: FILE_TREE_BATCH_SIZE }).toArray();
|
||||
if (!rows.length) return;
|
||||
await collection.updateMany({ ...filter, _id: { $in: rows.map(row => row._id) } },
|
||||
{ $unset: { treeStaged: "" } }, { hint: "_id_" });
|
||||
await setImmediate();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import { createHash } from "crypto";
|
||||
import { ExpiringMap } from "./expiring-map";
|
||||
|
||||
/** Cache only denials, keyed by the exact credential. Never cache successful access. */
|
||||
export class InvalidCredentialBackoff {
|
||||
private readonly rejected: ExpiringMap<boolean>;
|
||||
constructor(now = () => Date.now()) { this.rejected = new ExpiringMap<boolean>(2000, now); }
|
||||
async run<T>(token: string, request: () => Promise<T>): Promise<T> {
|
||||
const key = createHash("sha256").update(token).digest("hex");
|
||||
if (this.rejected.get(key)) throw Object.assign(new Error("token_expired"), { status: 401 });
|
||||
try { return await request(); }
|
||||
catch (error) {
|
||||
if ((error as { status?: number })?.status === 401) this.rejected.set(key, true, 60_000);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -19,6 +19,8 @@ const FileSchema = new Schema({
|
||||
},
|
||||
});
|
||||
|
||||
FileSchema.index({ repoId: 1, treeGeneration: 1, submodule: 1 },
|
||||
{ name: "file_submodule_status", partialFilterExpression: { submodule: { $exists: true } } });
|
||||
FileSchema.index({ metadataPending: 1, repoId: 1, treeGeneration: 1 },
|
||||
{ partialFilterExpression: { metadataPending: true } });
|
||||
FileSchema.index({ treeStaged: 1, repoId: 1, treeGeneration: 1 },
|
||||
|
||||
@@ -6,7 +6,7 @@ import { createLogger } from "./logger";
|
||||
|
||||
export const LATENCY_BUCKETS = [10, 50, 100, 250, 500, 1000, 2500, 5000, 10000, 30000, 60000];
|
||||
export type Service = "api" | "streamer";
|
||||
export type Stage = "repository" | "authorization" | "upstream" | "cache_lookup" | "cache_fill" | "worker_wait" | "anonymize" | "streamer_headers";
|
||||
export type Stage = "repository" | "credential_resolution" | "upstream" | "cache_lookup" | "cache_fill" | "worker_wait" | "anonymize" | "streamer_headers";
|
||||
export interface Metric {
|
||||
metric: string; route: string; method: string; count: number; sumMs: number;
|
||||
bytes: number; aborted: number; errors: number; slow: number; buckets: number[];
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
/** Suppress only transport cancellation after the downstream has gone away. */
|
||||
export function isResponseCancellation(error: unknown, response?: { destroyed?: boolean; req?: { aborted?: boolean } }): boolean {
|
||||
if (!response?.destroyed) return false;
|
||||
const seen = new Set<unknown>();
|
||||
for (let value = error; value && typeof value === "object" && !seen.has(value);) {
|
||||
seen.add(value);
|
||||
const item = value as { code?: string; message?: string; cause?: unknown };
|
||||
if (item.message === "response_cancelled") return true;
|
||||
// pipeline also destroys the response on upstream failure. A destroyed
|
||||
// response alone does not prove that a premature close was client-driven.
|
||||
if (response.req?.aborted && (item.code === "ERR_STREAM_PREMATURE_CLOSE" ||
|
||||
item.code === "ABORT_ERR")) return true;
|
||||
value = item.cause;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
Reference in new issue
Block a user