From 530e388e463ef278add24cb19b04b010acb1440b Mon Sep 17 00:00:00 2001 From: Thomas Durieux <5577568+tdurieux@users.noreply.github.com> Date: Tue, 8 Sep 2026 02:03:09 -1000 Subject: [PATCH] fix: archive ownerless repositories without assigning ownership (#800) --- docs/credential-encryption.md | 56 +++++- public/i18n/locale-en.json | 1 + src/core/GitHubUtils.ts | 1 + src/core/Repository.ts | 19 +- src/core/migrate-credentials.ts | 2 + .../anonymizedRepositories.schema.ts | 3 + .../anonymizedRepositories.types.ts | 3 + src/core/recover-repository-owners.ts | 169 +++++++++++++----- src/core/types.ts | 1 + src/queue/processes/downloadRepository.ts | 2 +- src/scripts/recover-repository-owners.ts | 12 +- src/server/routes/repository-public.ts | 1 + test/recover-repository-owners.test.js | 147 +++++++++++++++ 13 files changed, 364 insertions(+), 53 deletions(-) diff --git a/docs/credential-encryption.md b/docs/credential-encryption.md index 4080cb1..1ded306 100644 --- a/docs/credential-encryption.md +++ b/docs/credential-encryption.md @@ -315,7 +315,10 @@ identify different accounts, it leaves the repository untouched. Conditional updates avoid overwriting a repository whose owner or tokens changed after it was read. Keep application writers stopped for apply runs. -Requests are sequential and spaced one second apart, with a 15-second timeout. +Processing uses five workers by default. Set `--concurrency=1..32` to adjust +the bound. GitHub requests share a global pacing gate of at most four new requests +per second, with a 15-second timeout. In-flight work may finish after a rate limit +is detected, but no new work is scheduled. Repeated tokens share a bounded in-memory lookup cache. HTTP 403/429 responses, other unexpected HTTP errors, and network failures stop the scan; fix the issue or wait for GitHub's limit to reset, then rerun. Existing assignments are skipped on @@ -325,3 +328,54 @@ records remain or the scan halted; successful assignments are retained. After recovery, rerun the credential migration with `--prefer-owner-token`. Recovery leaves legacy tokens in place so the migration can still encrypt them. + + +## Archive all ownerless repositories + +A token may belong to a shared administrator account rather than the original +repository creator. To avoid assigning those repositories to the administrator, +use `--archive-all-ownerless`. This mode makes no GitHub calls and never assigns +owners. It processes missing owners and references to deleted users, while +preserving every repository with an existing database owner. + +Keep API instances, workers, streamers, and other writers stopped. Build the new +image and preview the archive actions: + +```bash +docker compose build anonymous_github +docker compose run --rm --no-deps -T --entrypoint node anonymous_github \ + build/scripts/recover-repository-owners.js \ + --archive-all-ownerless --concurrency=10 +``` + +Apply the same operation: + +```bash +docker compose run --rm --no-deps -T --entrypoint node anonymous_github \ + build/scripts/recover-repository-owners.js \ + --archive-all-ownerless --concurrency=10 --apply --maintenance +``` + +Each archive sets `status=archived`, records the reason/date, disables source +updates, and removes both plaintext token fields. It deletes cached file content +from the configured filesystem or S3 storage. MongoDB repository records and file +metadata remain. Archived URLs return HTTP 410, and download workers do not +reactivate them. Existing owner assignments from earlier apply runs are not +undone. Review those separately if owner recovery was previously applied. + +File deletion is intentional. The status change happens first, with +`archiveCachePending=true`. Successful deletion clears the marker. Failed or +interrupted cleanup is retried by the same apply command without GitHub access. +Migration verification also refuses to finish while any archive cleanup remains +pending. A clean rerun reports no issues and only already-archived records for +previously completed work. Review `unsafe_or_missing_repo_id` failures manually; +the script will not construct a storage deletion path from an unsafe ID. + +For the narrower policy of archiving only missing or entirely revoked tokens, +use `--archive-unrecoverable` instead. Valid-token owner recovery still runs in +that mode. Mixed valid/invalid tokens, unexpected GitHub errors, and ambiguous +identities never trigger automatic archiving. + +Once archival completes, rerun credential migration with `--prefer-owner-token`, +then follow the verification/enforcement steps above. Start only a release that +understands the archived status. diff --git a/public/i18n/locale-en.json b/public/i18n/locale-en.json index 71aeea6..8e6877a 100644 --- a/public/i18n/locale-en.json +++ b/public/i18n/locale-en.json @@ -16,6 +16,7 @@ "repo_not_found": "The repository was not found on GitHub. Check the URL and spelling, make sure you are signed in to the account that can see it, and confirm the repo isn't hidden under an org that restricts third-party app access.", "repo_empty": "The selected branch has no commits on GitHub. Push at least one commit, or pick a different branch, then retry.", "repo_not_accessible": "Anonymous GitHub cannot access this repository. Verify the repository exists and that Anonymous GitHub has been authorized for the owning organization.", + "repository_archived": "This repository has been archived and its files are no longer available.", "repository_expired": "The repository is expired.", "invalid_status": "This action cannot be performed while the resource is in its current state.", "repository_not_ready": "Anonymous GitHub is still processing the repository, it can take several minutes.", diff --git a/src/core/GitHubUtils.ts b/src/core/GitHubUtils.ts index 2ecb3a5..ad4cebe 100644 --- a/src/core/GitHubUtils.ts +++ b/src/core/GitHubUtils.ts @@ -274,6 +274,7 @@ export async function checkToken(token: string) { const checkedRepositoryTokens = new WeakMap(); export async function getToken(repository: Repository) { + repository.assertNotArchived(); logger.debug("getToken", { repoId: repository.repoId }); const credential = await getCredential(repository.owner.id); const ownerAccessToken = credential?.token; diff --git a/src/core/Repository.ts b/src/core/Repository.ts index 5303d4c..98b77c9 100644 --- a/src/core/Repository.ts +++ b/src/core/Repository.ts @@ -71,10 +71,12 @@ export default class Repository { } async getToken() { + this.assertNotArchived(); return getToken(this); } get source() { + this.assertNotArchived(); const ghRepo = new GitHubRepository({ name: this.model.source.repositoryName, }); @@ -130,6 +132,7 @@ export default class Repository { force: false, } ): Promise { + this.assertNotArchived(); const terms = this._model.options.terms || []; let hasFile = await FileModel.exists({ repoId: this.repoId }).exec(); // Files created by GitHubDownload don't carry a valid 40-char GitHub @@ -208,7 +211,14 @@ export default class Repository { /** * Check the status of the repository */ + assertNotArchived() { + if (this.status === RepositoryStatus.ARCHIVED) { + throw new AnonymousError("repository_archived", { httpStatus: 410 }); + } + } + async check() { + this.assertNotArchived(); if ( this._model.options.expirationMode !== "never" && this.status == RepositoryStatus.READY && @@ -258,6 +268,7 @@ export default class Repository { * @returns A stream of anonymized repository compressed */ zip(): Promise { + this.assertNotArchived(); return storage.archive(this.repoId, "", { format: "zip", fileTransformer: (filename: string) => @@ -293,6 +304,7 @@ export default class Repository { * @returns void */ async updateIfNeeded(opt?: { force: boolean }): Promise { + this.assertNotArchived(); if ( this._model.options.expirationMode !== "never" && this.status != RepositoryStatus.EXPIRED && @@ -415,6 +427,7 @@ export default class Repository { * @returns void */ async anonymize(progress?: (status: string) => void) { + this.assertNotArchived(); if (this.status === RepositoryStatus.READY) { return; } @@ -459,6 +472,7 @@ export default class Repository { public protectLifecycle = false; async updateStatus(status: RepositoryStatus, statusMessage?: string) { + if (status !== RepositoryStatus.ARCHIVED) this.assertNotArchived(); if (!status) return this.model; const statusDate = new Date(); if (isConnected) { @@ -466,7 +480,7 @@ export default class Repository { { _id: this._model._id, ...(this.protectLifecycle ? { - status: { $nin: [RepositoryStatus.REMOVING, RepositoryStatus.REMOVED, + status: { $nin: [RepositoryStatus.ARCHIVED, RepositoryStatus.REMOVING, RepositoryStatus.REMOVED, RepositoryStatus.EXPIRING, RepositoryStatus.EXPIRED] }, anonymizeDate: this._model.anonymizeDate, } : {}), @@ -504,6 +518,7 @@ export default class Repository { * Reset/delete the state of the repository */ async resetSate(status?: RepositoryStatus, statusMessage?: string) { + this.assertNotArchived(); // remove attribute this._model.size = { storage: 0, file: 0 }; if (status) { @@ -522,6 +537,8 @@ export default class Repository { * @returns */ async removeCache() { + // Archive cleanup is handled by the resumable recovery script. Preserve DB metadata. + if (this.status === RepositoryStatus.ARCHIVED) return; await storage.rm(this.repoId); this.model.isReseted = true; this.model.size = { storage: 0, file: 0 }; diff --git a/src/core/migrate-credentials.ts b/src/core/migrate-credentials.ts index 290b92d..30a7edd 100644 --- a/src/core/migrate-credentials.ts +++ b/src/core/migrate-credentials.ts @@ -106,6 +106,8 @@ export async function verifyCredentials(db: mongo.Db, cipher: Cipher) { } let legacy = await db.collection("users").countDocuments({ accessTokens: { $exists: true } }); for (const name of resources) legacy += await db.collection(name).countDocuments(legacyQuery); + const pendingArchiveCleanup = await db.collection("anonymizedrepositories").countDocuments({ archiveCachePending: true }); + if (pendingArchiveCleanup) throw new Error("Archived repository cache cleanup is pending"); return { checked, legacy }; } diff --git a/src/core/model/anonymizedRepositories/anonymizedRepositories.schema.ts b/src/core/model/anonymizedRepositories/anonymizedRepositories.schema.ts index 6f91aa8..c432d51 100644 --- a/src/core/model/anonymizedRepositories/anonymizedRepositories.schema.ts +++ b/src/core/model/anonymizedRepositories/anonymizedRepositories.schema.ts @@ -10,6 +10,9 @@ const AnonymizedRepositorySchema = new Schema({ default: "preparing", }, statusDate: Date, + archivedAt: Date, + archiveReason: String, + archiveCachePending: Boolean, statusMessage: String, anonymizeDate: Date, lastView: Date, diff --git a/src/core/model/anonymizedRepositories/anonymizedRepositories.types.ts b/src/core/model/anonymizedRepositories/anonymizedRepositories.types.ts index 5406a80..0dba0ca 100644 --- a/src/core/model/anonymizedRepositories/anonymizedRepositories.types.ts +++ b/src/core/model/anonymizedRepositories/anonymizedRepositories.types.ts @@ -6,6 +6,9 @@ export interface IAnonymizedRepository { status?: RepositoryStatus; statusMessage?: string; statusDate: Date; + archivedAt?: Date; + archiveReason?: string; + archiveCachePending?: boolean; anonymizeDate: Date; source: { type: "GitHubDownload" | "GitHubStream" | "Zip"; diff --git a/src/core/recover-repository-owners.ts b/src/core/recover-repository-owners.ts index ed204d9..b185227 100644 --- a/src/core/recover-repository-owners.ts +++ b/src/core/recover-repository-owners.ts @@ -38,82 +38,159 @@ export async function identifyGitHubToken(token: string): Promise Promise; pause?: () => Promise; + deleteCache?: (repoId: string) => Promise; report?: (event: { collection: string; id: string; issue?: string; - ownerId?: string; githubId?: string; action?: string }) => void; + ownerId?: string; githubId?: string; action?: string; reason?: string }) => void; } export async function recoverRepositoryOwners(db: mongo.Db, options: RecoveryOptions = {}) { + const concurrency = options.concurrency ?? 5; + if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > 32) { + throw new Error("Concurrency must be an integer between 1 and 32"); + } const repositories = db.collection("anonymizedrepositories"); const users = db.collection("users"); const identify = options.identify || identifyGitHubToken; - const pause = options.pause || (() => new Promise(resolve => setTimeout(resolve, 1000))); - // Hashes deduplicate HTTP requests without retaining plaintext as cache keys. - const identities = new Map(); - const counts = { scanned: 0, candidates: 0, matched: 0, updated: 0, issues: 0, halted: false }; + const deleteCache = options.deleteCache || (async (repoId: string) => { + const storage = (await import("./storage")).default; + await storage.rm(repoId); + }); + // Share one pacing gate across workers. At most four new HTTP requests/second. + let gate: Promise = Promise.resolve(); + const pause = options.pause || (() => { + const next = gate.then(() => new Promise(resolve => setTimeout(resolve, 250))); + gate = next; + return next; + }); + const counts = { scanned: 0, candidates: 0, matched: 0, updated: 0, + archiveCandidates: 0, archived: 0, cacheDeleted: 0, alreadyArchived: 0, issues: 0, halted: false }; const report = options.report || (() => {}); - const query = options.repositoryId ? { _id: options.repositoryId } : {}; - for await (const row of repositories.find(query, { - projection: { owner: 1, "source.accessToken": 1, accessToken: 1 }, - }).batchSize(100)) { + // In-flight lookups are shared too; parallel workers never request the same token twice. + const identities = new Map>(); + const lookup = (token: string) => { + const hash = createHash("sha256").update(token).digest("hex"); + let result = identities.get(hash); + if (!result) { + result = (async () => { + await pause(); + if (counts.halted) return { issue: "scan_halted", halt: true }; + const identity = await identify(token); + if ("issue" in identity && identity.halt) counts.halted = true; + return identity; + })(); + identities.set(hash, result); + // Evict settled entries only, retaining deduplication of requests in flight. + void result.then(() => { + if (identities.size > 10000) identities.delete(hash); + }, () => {}); + } + return result; + }; + const processRow = async (row: mongo.WithId) => { counts.scanned++; - if (row.owner && await users.findOne({ _id: row.owner }, { projection: { _id: 1 } })) continue; - counts.candidates++; + if (row.owner && await users.findOne({ _id: row.owner }, { projection: { _id: 1 } })) return; const event = { collection: "anonymizedrepositories", id: String(row._id) }; const fail = (issue: string) => { counts.issues++; report({ ...event, issue }); }; - const values: unknown[] = [row.source?.accessToken, row.accessToken]; - if (values.some(value => value != null && typeof value !== "string")) { - fail("malformed_token"); continue; - } - const tokens = [...new Set(values.filter((value): value is string => typeof value === "string" && value.length > 0))]; - if (!tokens.length) { fail("missing_token"); continue; } - const ids = new Set(); - let failed = false; - for (const token of tokens) { - const hash = createHash("sha256").update(token).digest("hex"); - let identity = identities.get(hash); - if (!identity) { - await pause(); - identity = await identify(token); - if (identities.size >= 10000) identities.clear(); - identities.set(hash, identity); + const unchanged = (value: unknown) => value === undefined ? { $exists: false } : { $eq: value, $exists: true }; + const original = { _id: row._id, repoId: unchanged(row.repoId), owner: unchanged(row.owner), status: unchanged(row.status), + "source.accessToken": unchanged(row.source?.accessToken), accessToken: unchanged(row.accessToken) }; + const safeRepoId = () => typeof row.repoId === "string" && /^[a-zA-Z0-9_.-]+$/.test(row.repoId) && ![".", ".."].includes(row.repoId); + const cleanup = async () => { + if (!safeRepoId()) { fail("unsafe_or_missing_repo_id"); return; } + try { + await deleteCache(row.repoId); + const result = await repositories.updateOne({ _id: row._id, status: "archived", archiveCachePending: true }, { + $set: { archiveCachePending: false, isReseted: true }, + }); + if (!result.modifiedCount) { fail("repository_changed_retry"); return; } + counts.cacheDeleted++; + report({ ...event, action: "archive_cache_deleted" }); + } catch { + fail("archive_cache_cleanup_failed"); } + }; + if (row.status === "archived" && row.source?.accessToken === undefined && row.accessToken === undefined) { + counts.alreadyArchived++; + if (row.archiveCachePending && (options.archiveUnrecoverable || options.archiveAllOwnerless)) { + if (options.apply) await cleanup(); + else report({ ...event, action: "would_delete_archive_cache" }); + } + return; + } + counts.candidates++; + const archive = async (reason: string) => { + if (!options.archiveUnrecoverable && !options.archiveAllOwnerless) { fail(reason); return; } + if (!safeRepoId()) { fail("unsafe_or_missing_repo_id"); return; } + counts.archiveCandidates++; + if (options.apply) { + const now = new Date(); + const result = await repositories.updateOne(original, { + $set: { status: "archived", statusDate: now, archivedAt: now, + archiveReason: reason, archiveCachePending: true, "options.update": false }, + $unset: { "source.accessToken": "", accessToken: "" }, + }); + if (!result.modifiedCount) { fail("repository_changed_retry"); return; } + counts.archived++; + report({ ...event, reason, action: "archived" }); + await cleanup(); + } else report({ ...event, reason, action: "would_archive" }); + }; + if (options.archiveAllOwnerless) { await archive("missing_owner"); return; } + const values: unknown[] = [row.source?.accessToken, row.accessToken]; + if (values.some(value => value != null && typeof value !== "string")) { fail("malformed_token"); return; } + const tokens = [...new Set(values.filter((value): value is string => typeof value === "string" && value.length > 0))]; + if (!tokens.length) { await archive("missing_token"); return; } + const ids = new Set(); + let invalidTokens = 0; + for (const token of tokens) { + const identity = await lookup(token); if ("issue" in identity) { + if (identity.issue === "invalid_or_revoked_token") { invalidTokens++; continue; } fail(identity.issue); - counts.halted = !!identity.halt; - failed = true; - break; + return; } ids.add(identity.githubId); } - if (counts.halted) break; - if (failed) continue; - if (ids.size !== 1) { fail("conflicting_token_identities"); continue; } + if (invalidTokens === tokens.length) { await archive("invalid_or_revoked_token"); return; } + if (invalidTokens) { fail("mixed_token_validity"); return; } + if (ids.size !== 1) { fail("conflicting_token_identities"); return; } const githubId = [...ids][0]; - // Match immutable GitHub IDs only. Include historical numeric storage. const matches = await users.find({ "externalIDs.github": { $in: [githubId, Number(githubId)] } }, { projection: { _id: 1, status: 1 }, }).limit(2).toArray(); - if (matches.length !== 1) { - fail(matches.length ? "ambiguous_user" : "user_not_found"); continue; - } + if (matches.length !== 1) { fail(matches.length ? "ambiguous_user" : "user_not_found"); return; } const user = matches[0]; - if (user.status === "removed" || user.status === "banned") { fail("disabled_user"); continue; } + if (user.status === "removed" || user.status === "banned") { fail("disabled_user"); return; } counts.matched++; if (options.apply) { - const unchanged = (value: unknown) => value === undefined ? { $exists: false } : { $eq: value, $exists: true }; - const result = await repositories.updateOne({ - _id: row._id, - owner: unchanged(row.owner), - "source.accessToken": unchanged(row.source?.accessToken), - accessToken: unchanged(row.accessToken), - }, { $set: { owner: user._id } }); - if (!result.modifiedCount) { fail("repository_changed_retry"); continue; } + const result = await repositories.updateOne(original, { $set: { owner: user._id } }); + if (!result.modifiedCount) { fail("repository_changed_retry"); return; } counts.updated++; } report({ ...event, githubId, ownerId: String(user._id), action: options.apply ? "owner_assigned" : "would_assign_owner" }); + }; + const pending = new Set>(); + let failed = false; + const cursor = repositories.find(options.repositoryId ? { _id: options.repositoryId } : {}, { + projection: { owner: 1, repoId: 1, status: 1, archiveCachePending: 1, "source.accessToken": 1, accessToken: 1 }, + }).batchSize(100); + try { + for await (const row of cursor) { + if (counts.halted || failed) break; + const task = processRow(row).catch(() => { failed = true; counts.halted = true; }); + pending.add(task); + void task.then(() => pending.delete(task)); + if (pending.size >= concurrency) await Promise.race(pending); + } + } finally { + await Promise.all(pending); } + if (failed) throw new Error("Owner recovery database operation failed"); return counts; } diff --git a/src/core/types.ts b/src/core/types.ts index ddd3836..562d7a3 100644 --- a/src/core/types.ts +++ b/src/core/types.ts @@ -9,6 +9,7 @@ export enum RepositoryStatus { PREPARING = "preparing", DOWNLOAD = "download", READY = "ready", + ARCHIVED = "archived", EXPIRED = "expired", EXPIRING = "expiring", REMOVED = "removed", diff --git a/src/queue/processes/downloadRepository.ts b/src/queue/processes/downloadRepository.ts index 1d93ac4..b4922d4 100644 --- a/src/queue/processes/downloadRepository.ts +++ b/src/queue/processes/downloadRepository.ts @@ -24,7 +24,7 @@ export default async function (job: SandboxedJob) { await connect(); const repo = await getRepository(job.data.repoId); - if ([RepositoryStatus.REMOVING, RepositoryStatus.REMOVED, + if ([RepositoryStatus.ARCHIVED, RepositoryStatus.REMOVING, RepositoryStatus.REMOVED, RepositoryStatus.EXPIRING, RepositoryStatus.EXPIRED].some((status) => status === repo.status)) return; repo.protectLifecycle = true; const token = await getToken(repo); diff --git a/src/scripts/recover-repository-owners.ts b/src/scripts/recover-repository-owners.ts index 2c58222..cb4e1d7 100644 --- a/src/scripts/recover-repository-owners.ts +++ b/src/scripts/recover-repository-owners.ts @@ -6,12 +6,15 @@ import { recoverRepositoryOwners } from "../core/recover-repository-owners"; async function main() { const args = process.argv.slice(2); const apply = args.includes("--apply"); + let concurrency = 5; let repositoryId: mongoose.mongo.ObjectId | undefined; for (const arg of args) { - if (arg.startsWith("--id=") && /^[a-f0-9]{24}$/i.test(arg.slice(5))) { + if (/^--concurrency=([1-9]|[12][0-9]|3[0-2])$/.test(arg)) { + concurrency = Number(arg.split("=")[1]); + } else if (arg.startsWith("--id=") && /^[a-f0-9]{24}$/i.test(arg.slice(5))) { repositoryId = new mongoose.mongo.ObjectId(arg.slice(5)); - } else if (arg !== "--apply" && arg !== "--maintenance") { - process.stderr.write("Usage: recover-repository-owners.js [--id=] [--apply --maintenance]\n"); + } else if (arg !== "--apply" && arg !== "--maintenance" && arg !== "--archive-unrecoverable" && arg !== "--archive-all-ownerless") { + process.stderr.write("Usage: recover-repository-owners.js [--id=] [--apply --maintenance] [--archive-unrecoverable | --archive-all-ownerless] [--concurrency=1..32]\n"); process.exitCode = 1; return; } @@ -24,7 +27,8 @@ async function main() { const uri = config.MONGODB_URI || `mongodb://${config.DB_USERNAME}:${config.DB_PASSWORD}@${config.DB_HOSTNAME}:27017/production`; await mongoose.connect(uri, { ...(config.MONGODB_URI ? {} : { authSource: "admin" }), autoIndex: false }); const counts = await recoverRepositoryOwners(mongoose.connection.db, { - apply, repositoryId, + apply, repositoryId, concurrency, archiveUnrecoverable: args.includes("--archive-unrecoverable"), + archiveAllOwnerless: args.includes("--archive-all-ownerless"), report: event => process.stdout.write(JSON.stringify(event) + "\n"), }); process.stdout.write(JSON.stringify(counts) + "\n"); diff --git a/src/server/routes/repository-public.ts b/src/server/routes/repository-public.ts index 56c31d3..d47c9f5 100644 --- a/src/server/routes/repository-public.ts +++ b/src/server/routes/repository-public.ts @@ -309,6 +309,7 @@ router.get( try { user = await getUser(req); } catch { /* not logged in */ } + repo.assertNotArchived(); const canEdit = !!user && (user.isAdmin || diff --git a/test/recover-repository-owners.test.js b/test/recover-repository-owners.test.js index e65792a..3971d17 100644 --- a/test/recover-repository-owners.test.js +++ b/test/recover-repository-owners.test.js @@ -1,3 +1,4 @@ +const { setTimeout } = require("timers"); const { expect } = require("chai"); require("ts-node/register/transpile-only"); const { recoverRepositoryOwners, identifyGitHubToken } = require("../src/core/recover-repository-owners"); @@ -121,3 +122,149 @@ describe("GitHub token identification", () => { } }); }); + +describe("archive ownerless repositories", () => { + it("previews all ownerless records without GitHub calls or storage deletion", async () => { + const f = fixture([{ _id: "repo", repoId: "legacy-repo", accessToken: "admin-token" }]); + const result = await recoverRepositoryOwners(f.db, { ...f.options, archiveAllOwnerless: true, + identify: async () => { throw new Error("must not call GitHub"); }, + deleteCache: async () => { throw new Error("must not delete in dry run"); } }); + expect(result.archiveCandidates).to.equal(1); + expect(result.archived).to.equal(0); + expect(result.issues).to.equal(0); + expect(f.writes).to.have.length(0); + expect(f.events[0]).to.include({ action: "would_archive", reason: "missing_owner" }); + }); + it("marks archived before deleting cache and retains database records", async () => { + const f = fixture([{ _id: "repo", repoId: "legacy-repo", source: { accessToken: "admin-token" } }]); + const deleted = []; + const result = await recoverRepositoryOwners(f.db, { ...f.options, apply: true, archiveAllOwnerless: true, + identify: async () => { throw new Error("must not call GitHub"); }, + deleteCache: async id => { + expect(f.writes[0].update.$set.status).to.equal("archived"); + expect(f.writes[0].update.$set.archiveCachePending).to.equal(true); + deleted.push(id); + } }); + expect(result.archived).to.equal(1); + expect(result.cacheDeleted).to.equal(1); + expect(deleted).to.deep.equal(["legacy-repo"]); + expect(f.writes[0].update.$unset).to.deep.equal({ "source.accessToken": "", accessToken: "" }); + expect(f.writes[0].update.$set["options.update"]).to.equal(false); + expect(f.writes[1].update.$set.archiveCachePending).to.equal(false); + }); + it("never archives a repository with an existing owner", async () => { + const f = fixture([{ _id: "repo", owner: "admin", repoId: "repo" }], [{ _id: "admin" }]); + const result = await recoverRepositoryOwners(f.db, { ...f.options, apply: true, archiveAllOwnerless: true }); + expect(result.archiveCandidates).to.equal(0); + expect(f.writes).to.have.length(0); + }); + it("keeps failed cache cleanup pending and resumes without calling GitHub", async () => { + const f = fixture([{ _id: "repo", repoId: "legacy-repo" }]); + const result = await recoverRepositoryOwners(f.db, { ...f.options, apply: true, archiveAllOwnerless: true, + deleteCache: async () => { throw new Error("storage unavailable"); } }); + expect(result.issues).to.equal(1); + expect(f.writes).to.have.length(1); + const resumed = fixture([{ _id: "repo", repoId: "legacy-repo", status: "archived", archiveCachePending: true }]); + const retry = await recoverRepositoryOwners(resumed.db, { ...resumed.options, apply: true, archiveAllOwnerless: true, + identify: async () => { throw new Error("must not call GitHub"); }, deleteCache: async () => {} }); + expect(retry.cacheDeleted).to.equal(1); + expect(retry.issues).to.equal(0); + }); + it("skips completed archives on reruns", async () => { + const f = fixture([{ _id: "repo", repoId: "legacy-repo", status: "archived", archiveCachePending: false }]); + expect((await recoverRepositoryOwners(f.db, { ...f.options, apply: true, archiveAllOwnerless: true })).alreadyArchived).to.equal(1); + expect(f.writes).to.have.length(0); + }); + it("rejects unsafe storage paths before changing the record", async () => { + for (const repoId of [undefined, "", ".", "..", "../other", "a/b", "/etc"]) { + const f = fixture([{ _id: "repo", repoId }]); + await recoverRepositoryOwners(f.db, { ...f.options, apply: true, archiveAllOwnerless: true }); + expect(f.events[0].issue).to.equal("unsafe_or_missing_repo_id"); + expect(f.writes).to.have.length(0); + } + }); + it("does not delete files after a conditional update loses a race", async () => { + const f = fixture([{ _id: "repo", repoId: "legacy-repo" }]); + const original = f.db.collection; + f.db.collection = name => name === "users" ? original(name) : { ...original(name), updateOne: async () => ({ modifiedCount: 0 }) }; + const result = await recoverRepositoryOwners(f.db, { ...f.options, apply: true, archiveAllOwnerless: true, + deleteCache: async () => { throw new Error("must not delete"); } }); + expect(result.cacheDeleted).to.equal(0); + expect(f.events[0].issue).to.equal("repository_changed_retry"); + }); + it("archives only missing or entirely revoked tokens in selective mode", async () => { + const f = fixture([{ _id: "missing", repoId: "missing" }, { _id: "revoked", repoId: "revoked", accessToken: "bad" }, + { _id: "mixed", repoId: "mixed", accessToken: "bad", source: { accessToken: "good" } }]); + const result = await recoverRepositoryOwners(f.db, { ...f.options, archiveUnrecoverable: true, + identify: async token => token === "bad" ? { issue: "invalid_or_revoked_token" } : { githubId: "42" } }); + expect(result.archiveCandidates).to.equal(2); + expect(f.events.find(e => e.id === "mixed").issue).to.equal("mixed_token_validity"); + }); + it("runs work concurrently within the configured bound", async () => { + const f = fixture(Array.from({ length: 9 }, (_, i) => ({ _id: String(i), accessToken: `token-${i}` })), [{ _id: "owner", githubId: "42" }]); + let active = 0, maximum = 0; + await recoverRepositoryOwners(f.db, { ...f.options, concurrency: 3, identify: async () => { + active++; maximum = Math.max(maximum, active); + await new Promise(resolve => setTimeout(resolve, 5)); + active--; return { githubId: "42" }; + } }); + expect(maximum).to.equal(3); + expect(f.events).to.have.length(9); + }); + it("validates concurrency", async () => { + const f = fixture([]); + for (const concurrency of [0, 33, 1.5]) { + try { await recoverRepositoryOwners(f.db, { ...f.options, concurrency }); throw new Error("expected failure"); } + catch (error) { expect(error.message).to.equal("Concurrency must be an integer between 1 and 32"); } + } + }); +}); + +describe("archived repository access", () => { + const Repository = require("../src/core/Repository").default; + const UserModel = require("../src/core/model/users/users.model").default; + function archived() { + return new Repository({ owner: new UserModel()._id, repoId: "archived-test", status: "archived", + source: {}, options: { update: false }, size: { file: 9, storage: 123 } }); + } + it("rejects public access and source fetching without querying storage", async () => { + const repo = archived(); + for (const action of [() => repo.check(), () => repo.getToken(), () => repo.files(), () => repo.updateIfNeeded({ force: true }), () => repo.anonymize(), () => repo.resetSate()]) { + try { await action(); throw new Error("expected archive rejection"); } + catch (error) { expect(error.message).to.equal("repository_archived"); } + } + expect(() => repo.source).to.throw("repository_archived"); + expect(() => repo.zip()).to.throw("repository_archived"); + expect(repo.model.size).to.deep.equal({ file: 9, storage: 123 }); + }); + it("cannot reactivate an archive through a status update or old cache job", async () => { + const repo = archived(); + try { await repo.updateStatus("ready"); throw new Error("expected rejection"); } + catch (error) { expect(error.message).to.equal("repository_archived"); } + await repo.removeCache(); + expect(repo.status).to.equal("archived"); + expect(repo.model.size.file).to.equal(9); + }); + it("deletes only the target cache directory using filesystem storage", async () => { + const fs = require("fs"); + const os = require("os"); + const path = require("path"); + const config = require("../src/config").default; + const FileSystem = require("../src/core/storage/FileSystem").default; + const root = fs.mkdtempSync(path.join(os.tmpdir(), "archive-cache-test-")); + const before = config.FOLDER; + try { + config.FOLDER = root; + for (const id of ["archive-target", "keep-sibling"]) { + fs.mkdirSync(path.join(root, id, "original"), { recursive: true }); + fs.writeFileSync(path.join(root, id, "original", "file.txt"), "cached content"); + } + const f = fixture([{ _id: "repo", repoId: "archive-target" }]); + await recoverRepositoryOwners(f.db, { ...f.options, apply: true, archiveAllOwnerless: true, + deleteCache: id => new FileSystem().rm(id) }); + expect(fs.existsSync(path.join(root, "archive-target", "original"))).to.equal(false); + expect(fs.existsSync(path.join(root, "keep-sibling", "original", "file.txt"))).to.equal(true); + expect(f.writes).to.have.length(2); + } finally { config.FOLDER = before; fs.rmSync(root, { recursive: true, force: true }); } + }); +});