mirror of
https://github.com/tdurieux/anonymous_github.git
synced 2026-09-12 13:48:58 +02:00
fix: archive ownerless repositories without assigning ownership (#800)
This commit is contained in:
@@ -315,7 +315,10 @@ identify different accounts, it leaves the repository untouched. Conditional
|
||||
updates avoid overwriting a repository whose owner or tokens changed after it was
|
||||
read. Keep application writers stopped for apply runs.
|
||||
|
||||
Requests are sequential and spaced one second apart, with a 15-second timeout.
|
||||
Processing uses five workers by default. Set `--concurrency=1..32` to adjust
|
||||
the bound. GitHub requests share a global pacing gate of at most four new requests
|
||||
per second, with a 15-second timeout. In-flight work may finish after a rate limit
|
||||
is detected, but no new work is scheduled.
|
||||
Repeated tokens share a bounded in-memory lookup cache. HTTP 403/429 responses,
|
||||
other unexpected HTTP errors, and network failures stop the scan; fix the issue or
|
||||
wait for GitHub's limit to reset, then rerun. Existing assignments are skipped on
|
||||
@@ -325,3 +328,54 @@ records remain or the scan halted; successful assignments are retained.
|
||||
|
||||
After recovery, rerun the credential migration with `--prefer-owner-token`.
|
||||
Recovery leaves legacy tokens in place so the migration can still encrypt them.
|
||||
|
||||
|
||||
## Archive all ownerless repositories
|
||||
|
||||
A token may belong to a shared administrator account rather than the original
|
||||
repository creator. To avoid assigning those repositories to the administrator,
|
||||
use `--archive-all-ownerless`. This mode makes no GitHub calls and never assigns
|
||||
owners. It processes missing owners and references to deleted users, while
|
||||
preserving every repository with an existing database owner.
|
||||
|
||||
Keep API instances, workers, streamers, and other writers stopped. Build the new
|
||||
image and preview the archive actions:
|
||||
|
||||
```bash
|
||||
docker compose build anonymous_github
|
||||
docker compose run --rm --no-deps -T --entrypoint node anonymous_github \
|
||||
build/scripts/recover-repository-owners.js \
|
||||
--archive-all-ownerless --concurrency=10
|
||||
```
|
||||
|
||||
Apply the same operation:
|
||||
|
||||
```bash
|
||||
docker compose run --rm --no-deps -T --entrypoint node anonymous_github \
|
||||
build/scripts/recover-repository-owners.js \
|
||||
--archive-all-ownerless --concurrency=10 --apply --maintenance
|
||||
```
|
||||
|
||||
Each archive sets `status=archived`, records the reason/date, disables source
|
||||
updates, and removes both plaintext token fields. It deletes cached file content
|
||||
from the configured filesystem or S3 storage. MongoDB repository records and file
|
||||
metadata remain. Archived URLs return HTTP 410, and download workers do not
|
||||
reactivate them. Existing owner assignments from earlier apply runs are not
|
||||
undone. Review those separately if owner recovery was previously applied.
|
||||
|
||||
File deletion is intentional. The status change happens first, with
|
||||
`archiveCachePending=true`. Successful deletion clears the marker. Failed or
|
||||
interrupted cleanup is retried by the same apply command without GitHub access.
|
||||
Migration verification also refuses to finish while any archive cleanup remains
|
||||
pending. A clean rerun reports no issues and only already-archived records for
|
||||
previously completed work. Review `unsafe_or_missing_repo_id` failures manually;
|
||||
the script will not construct a storage deletion path from an unsafe ID.
|
||||
|
||||
For the narrower policy of archiving only missing or entirely revoked tokens,
|
||||
use `--archive-unrecoverable` instead. Valid-token owner recovery still runs in
|
||||
that mode. Mixed valid/invalid tokens, unexpected GitHub errors, and ambiguous
|
||||
identities never trigger automatic archiving.
|
||||
|
||||
Once archival completes, rerun credential migration with `--prefer-owner-token`,
|
||||
then follow the verification/enforcement steps above. Start only a release that
|
||||
understands the archived status.
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
"repo_not_found": "The repository was not found on GitHub. Check the URL and spelling, make sure you are signed in to the account that can see it, and confirm the repo isn't hidden under an org that restricts third-party app access.",
|
||||
"repo_empty": "The selected branch has no commits on GitHub. Push at least one commit, or pick a different branch, then retry.",
|
||||
"repo_not_accessible": "Anonymous GitHub cannot access this repository. Verify the repository exists and that Anonymous GitHub has been authorized for the owning organization.",
|
||||
"repository_archived": "This repository has been archived and its files are no longer available.",
|
||||
"repository_expired": "The repository is expired.",
|
||||
"invalid_status": "This action cannot be performed while the resource is in its current state.",
|
||||
"repository_not_ready": "Anonymous GitHub is still processing the repository, it can take several minutes.",
|
||||
|
||||
@@ -274,6 +274,7 @@ export async function checkToken(token: string) {
|
||||
const checkedRepositoryTokens = new WeakMap<Repository, string>();
|
||||
|
||||
export async function getToken(repository: Repository) {
|
||||
repository.assertNotArchived();
|
||||
logger.debug("getToken", { repoId: repository.repoId });
|
||||
const credential = await getCredential(repository.owner.id);
|
||||
const ownerAccessToken = credential?.token;
|
||||
|
||||
+18
-1
@@ -71,10 +71,12 @@ export default class Repository {
|
||||
}
|
||||
|
||||
async getToken() {
|
||||
this.assertNotArchived();
|
||||
return getToken(this);
|
||||
}
|
||||
|
||||
get source() {
|
||||
this.assertNotArchived();
|
||||
const ghRepo = new GitHubRepository({
|
||||
name: this.model.source.repositoryName,
|
||||
});
|
||||
@@ -130,6 +132,7 @@ export default class Repository {
|
||||
force: false,
|
||||
}
|
||||
): Promise<IFile[]> {
|
||||
this.assertNotArchived();
|
||||
const terms = this._model.options.terms || [];
|
||||
let hasFile = await FileModel.exists({ repoId: this.repoId }).exec();
|
||||
// Files created by GitHubDownload don't carry a valid 40-char GitHub
|
||||
@@ -208,7 +211,14 @@ export default class Repository {
|
||||
/**
|
||||
* Check the status of the repository
|
||||
*/
|
||||
assertNotArchived() {
|
||||
if (this.status === RepositoryStatus.ARCHIVED) {
|
||||
throw new AnonymousError("repository_archived", { httpStatus: 410 });
|
||||
}
|
||||
}
|
||||
|
||||
async check() {
|
||||
this.assertNotArchived();
|
||||
if (
|
||||
this._model.options.expirationMode !== "never" &&
|
||||
this.status == RepositoryStatus.READY &&
|
||||
@@ -258,6 +268,7 @@ export default class Repository {
|
||||
* @returns A stream of anonymized repository compressed
|
||||
*/
|
||||
zip(): Promise<Readable> {
|
||||
this.assertNotArchived();
|
||||
return storage.archive(this.repoId, "", {
|
||||
format: "zip",
|
||||
fileTransformer: (filename: string) =>
|
||||
@@ -293,6 +304,7 @@ export default class Repository {
|
||||
* @returns void
|
||||
*/
|
||||
async updateIfNeeded(opt?: { force: boolean }): Promise<void> {
|
||||
this.assertNotArchived();
|
||||
if (
|
||||
this._model.options.expirationMode !== "never" &&
|
||||
this.status != RepositoryStatus.EXPIRED &&
|
||||
@@ -415,6 +427,7 @@ export default class Repository {
|
||||
* @returns void
|
||||
*/
|
||||
async anonymize(progress?: (status: string) => void) {
|
||||
this.assertNotArchived();
|
||||
if (this.status === RepositoryStatus.READY) {
|
||||
return;
|
||||
}
|
||||
@@ -459,6 +472,7 @@ export default class Repository {
|
||||
public protectLifecycle = false;
|
||||
|
||||
async updateStatus(status: RepositoryStatus, statusMessage?: string) {
|
||||
if (status !== RepositoryStatus.ARCHIVED) this.assertNotArchived();
|
||||
if (!status) return this.model;
|
||||
const statusDate = new Date();
|
||||
if (isConnected) {
|
||||
@@ -466,7 +480,7 @@ export default class Repository {
|
||||
{
|
||||
_id: this._model._id,
|
||||
...(this.protectLifecycle ? {
|
||||
status: { $nin: [RepositoryStatus.REMOVING, RepositoryStatus.REMOVED,
|
||||
status: { $nin: [RepositoryStatus.ARCHIVED, RepositoryStatus.REMOVING, RepositoryStatus.REMOVED,
|
||||
RepositoryStatus.EXPIRING, RepositoryStatus.EXPIRED] },
|
||||
anonymizeDate: this._model.anonymizeDate,
|
||||
} : {}),
|
||||
@@ -504,6 +518,7 @@ export default class Repository {
|
||||
* Reset/delete the state of the repository
|
||||
*/
|
||||
async resetSate(status?: RepositoryStatus, statusMessage?: string) {
|
||||
this.assertNotArchived();
|
||||
// remove attribute
|
||||
this._model.size = { storage: 0, file: 0 };
|
||||
if (status) {
|
||||
@@ -522,6 +537,8 @@ export default class Repository {
|
||||
* @returns
|
||||
*/
|
||||
async removeCache() {
|
||||
// Archive cleanup is handled by the resumable recovery script. Preserve DB metadata.
|
||||
if (this.status === RepositoryStatus.ARCHIVED) return;
|
||||
await storage.rm(this.repoId);
|
||||
this.model.isReseted = true;
|
||||
this.model.size = { storage: 0, file: 0 };
|
||||
|
||||
@@ -106,6 +106,8 @@ export async function verifyCredentials(db: mongo.Db, cipher: Cipher) {
|
||||
}
|
||||
let legacy = await db.collection("users").countDocuments({ accessTokens: { $exists: true } });
|
||||
for (const name of resources) legacy += await db.collection(name).countDocuments(legacyQuery);
|
||||
const pendingArchiveCleanup = await db.collection("anonymizedrepositories").countDocuments({ archiveCachePending: true });
|
||||
if (pendingArchiveCleanup) throw new Error("Archived repository cache cleanup is pending");
|
||||
return { checked, legacy };
|
||||
}
|
||||
|
||||
|
||||
@@ -10,6 +10,9 @@ const AnonymizedRepositorySchema = new Schema({
|
||||
default: "preparing",
|
||||
},
|
||||
statusDate: Date,
|
||||
archivedAt: Date,
|
||||
archiveReason: String,
|
||||
archiveCachePending: Boolean,
|
||||
statusMessage: String,
|
||||
anonymizeDate: Date,
|
||||
lastView: Date,
|
||||
|
||||
@@ -6,6 +6,9 @@ export interface IAnonymizedRepository {
|
||||
status?: RepositoryStatus;
|
||||
statusMessage?: string;
|
||||
statusDate: Date;
|
||||
archivedAt?: Date;
|
||||
archiveReason?: string;
|
||||
archiveCachePending?: boolean;
|
||||
anonymizeDate: Date;
|
||||
source: {
|
||||
type: "GitHubDownload" | "GitHubStream" | "Zip";
|
||||
|
||||
@@ -38,82 +38,159 @@ export async function identifyGitHubToken(token: string): Promise<IdentityResult
|
||||
|
||||
export interface RecoveryOptions {
|
||||
apply?: boolean;
|
||||
archiveUnrecoverable?: boolean;
|
||||
archiveAllOwnerless?: boolean;
|
||||
concurrency?: number;
|
||||
repositoryId?: mongo.ObjectId;
|
||||
identify?: (token: string) => Promise<IdentityResult>;
|
||||
pause?: () => Promise<void>;
|
||||
deleteCache?: (repoId: string) => Promise<void>;
|
||||
report?: (event: { collection: string; id: string; issue?: string;
|
||||
ownerId?: string; githubId?: string; action?: string }) => void;
|
||||
ownerId?: string; githubId?: string; action?: string; reason?: string }) => void;
|
||||
}
|
||||
|
||||
export async function recoverRepositoryOwners(db: mongo.Db, options: RecoveryOptions = {}) {
|
||||
const concurrency = options.concurrency ?? 5;
|
||||
if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > 32) {
|
||||
throw new Error("Concurrency must be an integer between 1 and 32");
|
||||
}
|
||||
const repositories = db.collection("anonymizedrepositories");
|
||||
const users = db.collection("users");
|
||||
const identify = options.identify || identifyGitHubToken;
|
||||
const pause = options.pause || (() => new Promise<void>(resolve => setTimeout(resolve, 1000)));
|
||||
// Hashes deduplicate HTTP requests without retaining plaintext as cache keys.
|
||||
const identities = new Map<string, IdentityResult>();
|
||||
const counts = { scanned: 0, candidates: 0, matched: 0, updated: 0, issues: 0, halted: false };
|
||||
const deleteCache = options.deleteCache || (async (repoId: string) => {
|
||||
const storage = (await import("./storage")).default;
|
||||
await storage.rm(repoId);
|
||||
});
|
||||
// Share one pacing gate across workers. At most four new HTTP requests/second.
|
||||
let gate: Promise<void> = Promise.resolve();
|
||||
const pause = options.pause || (() => {
|
||||
const next = gate.then(() => new Promise<void>(resolve => setTimeout(resolve, 250)));
|
||||
gate = next;
|
||||
return next;
|
||||
});
|
||||
const counts = { scanned: 0, candidates: 0, matched: 0, updated: 0,
|
||||
archiveCandidates: 0, archived: 0, cacheDeleted: 0, alreadyArchived: 0, issues: 0, halted: false };
|
||||
const report = options.report || (() => {});
|
||||
const query = options.repositoryId ? { _id: options.repositoryId } : {};
|
||||
for await (const row of repositories.find(query, {
|
||||
projection: { owner: 1, "source.accessToken": 1, accessToken: 1 },
|
||||
}).batchSize(100)) {
|
||||
// In-flight lookups are shared too; parallel workers never request the same token twice.
|
||||
const identities = new Map<string, Promise<IdentityResult>>();
|
||||
const lookup = (token: string) => {
|
||||
const hash = createHash("sha256").update(token).digest("hex");
|
||||
let result = identities.get(hash);
|
||||
if (!result) {
|
||||
result = (async () => {
|
||||
await pause();
|
||||
if (counts.halted) return { issue: "scan_halted", halt: true };
|
||||
const identity = await identify(token);
|
||||
if ("issue" in identity && identity.halt) counts.halted = true;
|
||||
return identity;
|
||||
})();
|
||||
identities.set(hash, result);
|
||||
// Evict settled entries only, retaining deduplication of requests in flight.
|
||||
void result.then(() => {
|
||||
if (identities.size > 10000) identities.delete(hash);
|
||||
}, () => {});
|
||||
}
|
||||
return result;
|
||||
};
|
||||
const processRow = async (row: mongo.WithId<mongo.Document>) => {
|
||||
counts.scanned++;
|
||||
if (row.owner && await users.findOne({ _id: row.owner }, { projection: { _id: 1 } })) continue;
|
||||
counts.candidates++;
|
||||
if (row.owner && await users.findOne({ _id: row.owner }, { projection: { _id: 1 } })) return;
|
||||
const event = { collection: "anonymizedrepositories", id: String(row._id) };
|
||||
const fail = (issue: string) => { counts.issues++; report({ ...event, issue }); };
|
||||
const values: unknown[] = [row.source?.accessToken, row.accessToken];
|
||||
if (values.some(value => value != null && typeof value !== "string")) {
|
||||
fail("malformed_token"); continue;
|
||||
}
|
||||
const tokens = [...new Set(values.filter((value): value is string => typeof value === "string" && value.length > 0))];
|
||||
if (!tokens.length) { fail("missing_token"); continue; }
|
||||
const ids = new Set<string>();
|
||||
let failed = false;
|
||||
for (const token of tokens) {
|
||||
const hash = createHash("sha256").update(token).digest("hex");
|
||||
let identity = identities.get(hash);
|
||||
if (!identity) {
|
||||
await pause();
|
||||
identity = await identify(token);
|
||||
if (identities.size >= 10000) identities.clear();
|
||||
identities.set(hash, identity);
|
||||
const unchanged = (value: unknown) => value === undefined ? { $exists: false } : { $eq: value, $exists: true };
|
||||
const original = { _id: row._id, repoId: unchanged(row.repoId), owner: unchanged(row.owner), status: unchanged(row.status),
|
||||
"source.accessToken": unchanged(row.source?.accessToken), accessToken: unchanged(row.accessToken) };
|
||||
const safeRepoId = () => typeof row.repoId === "string" && /^[a-zA-Z0-9_.-]+$/.test(row.repoId) && ![".", ".."].includes(row.repoId);
|
||||
const cleanup = async () => {
|
||||
if (!safeRepoId()) { fail("unsafe_or_missing_repo_id"); return; }
|
||||
try {
|
||||
await deleteCache(row.repoId);
|
||||
const result = await repositories.updateOne({ _id: row._id, status: "archived", archiveCachePending: true }, {
|
||||
$set: { archiveCachePending: false, isReseted: true },
|
||||
});
|
||||
if (!result.modifiedCount) { fail("repository_changed_retry"); return; }
|
||||
counts.cacheDeleted++;
|
||||
report({ ...event, action: "archive_cache_deleted" });
|
||||
} catch {
|
||||
fail("archive_cache_cleanup_failed");
|
||||
}
|
||||
};
|
||||
if (row.status === "archived" && row.source?.accessToken === undefined && row.accessToken === undefined) {
|
||||
counts.alreadyArchived++;
|
||||
if (row.archiveCachePending && (options.archiveUnrecoverable || options.archiveAllOwnerless)) {
|
||||
if (options.apply) await cleanup();
|
||||
else report({ ...event, action: "would_delete_archive_cache" });
|
||||
}
|
||||
return;
|
||||
}
|
||||
counts.candidates++;
|
||||
const archive = async (reason: string) => {
|
||||
if (!options.archiveUnrecoverable && !options.archiveAllOwnerless) { fail(reason); return; }
|
||||
if (!safeRepoId()) { fail("unsafe_or_missing_repo_id"); return; }
|
||||
counts.archiveCandidates++;
|
||||
if (options.apply) {
|
||||
const now = new Date();
|
||||
const result = await repositories.updateOne(original, {
|
||||
$set: { status: "archived", statusDate: now, archivedAt: now,
|
||||
archiveReason: reason, archiveCachePending: true, "options.update": false },
|
||||
$unset: { "source.accessToken": "", accessToken: "" },
|
||||
});
|
||||
if (!result.modifiedCount) { fail("repository_changed_retry"); return; }
|
||||
counts.archived++;
|
||||
report({ ...event, reason, action: "archived" });
|
||||
await cleanup();
|
||||
} else report({ ...event, reason, action: "would_archive" });
|
||||
};
|
||||
if (options.archiveAllOwnerless) { await archive("missing_owner"); return; }
|
||||
const values: unknown[] = [row.source?.accessToken, row.accessToken];
|
||||
if (values.some(value => value != null && typeof value !== "string")) { fail("malformed_token"); return; }
|
||||
const tokens = [...new Set(values.filter((value): value is string => typeof value === "string" && value.length > 0))];
|
||||
if (!tokens.length) { await archive("missing_token"); return; }
|
||||
const ids = new Set<string>();
|
||||
let invalidTokens = 0;
|
||||
for (const token of tokens) {
|
||||
const identity = await lookup(token);
|
||||
if ("issue" in identity) {
|
||||
if (identity.issue === "invalid_or_revoked_token") { invalidTokens++; continue; }
|
||||
fail(identity.issue);
|
||||
counts.halted = !!identity.halt;
|
||||
failed = true;
|
||||
break;
|
||||
return;
|
||||
}
|
||||
ids.add(identity.githubId);
|
||||
}
|
||||
if (counts.halted) break;
|
||||
if (failed) continue;
|
||||
if (ids.size !== 1) { fail("conflicting_token_identities"); continue; }
|
||||
if (invalidTokens === tokens.length) { await archive("invalid_or_revoked_token"); return; }
|
||||
if (invalidTokens) { fail("mixed_token_validity"); return; }
|
||||
if (ids.size !== 1) { fail("conflicting_token_identities"); return; }
|
||||
const githubId = [...ids][0];
|
||||
// Match immutable GitHub IDs only. Include historical numeric storage.
|
||||
const matches = await users.find({ "externalIDs.github": { $in: [githubId, Number(githubId)] } }, {
|
||||
projection: { _id: 1, status: 1 },
|
||||
}).limit(2).toArray();
|
||||
if (matches.length !== 1) {
|
||||
fail(matches.length ? "ambiguous_user" : "user_not_found"); continue;
|
||||
}
|
||||
if (matches.length !== 1) { fail(matches.length ? "ambiguous_user" : "user_not_found"); return; }
|
||||
const user = matches[0];
|
||||
if (user.status === "removed" || user.status === "banned") { fail("disabled_user"); continue; }
|
||||
if (user.status === "removed" || user.status === "banned") { fail("disabled_user"); return; }
|
||||
counts.matched++;
|
||||
if (options.apply) {
|
||||
const unchanged = (value: unknown) => value === undefined ? { $exists: false } : { $eq: value, $exists: true };
|
||||
const result = await repositories.updateOne({
|
||||
_id: row._id,
|
||||
owner: unchanged(row.owner),
|
||||
"source.accessToken": unchanged(row.source?.accessToken),
|
||||
accessToken: unchanged(row.accessToken),
|
||||
}, { $set: { owner: user._id } });
|
||||
if (!result.modifiedCount) { fail("repository_changed_retry"); continue; }
|
||||
const result = await repositories.updateOne(original, { $set: { owner: user._id } });
|
||||
if (!result.modifiedCount) { fail("repository_changed_retry"); return; }
|
||||
counts.updated++;
|
||||
}
|
||||
report({ ...event, githubId, ownerId: String(user._id), action: options.apply ? "owner_assigned" : "would_assign_owner" });
|
||||
};
|
||||
const pending = new Set<Promise<void>>();
|
||||
let failed = false;
|
||||
const cursor = repositories.find(options.repositoryId ? { _id: options.repositoryId } : {}, {
|
||||
projection: { owner: 1, repoId: 1, status: 1, archiveCachePending: 1, "source.accessToken": 1, accessToken: 1 },
|
||||
}).batchSize(100);
|
||||
try {
|
||||
for await (const row of cursor) {
|
||||
if (counts.halted || failed) break;
|
||||
const task = processRow(row).catch(() => { failed = true; counts.halted = true; });
|
||||
pending.add(task);
|
||||
void task.then(() => pending.delete(task));
|
||||
if (pending.size >= concurrency) await Promise.race(pending);
|
||||
}
|
||||
} finally {
|
||||
await Promise.all(pending);
|
||||
}
|
||||
if (failed) throw new Error("Owner recovery database operation failed");
|
||||
return counts;
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ export enum RepositoryStatus {
|
||||
PREPARING = "preparing",
|
||||
DOWNLOAD = "download",
|
||||
READY = "ready",
|
||||
ARCHIVED = "archived",
|
||||
EXPIRED = "expired",
|
||||
EXPIRING = "expiring",
|
||||
REMOVED = "removed",
|
||||
|
||||
@@ -24,7 +24,7 @@ export default async function (job: SandboxedJob<RepoJobData, void>) {
|
||||
await connect();
|
||||
|
||||
const repo = await getRepository(job.data.repoId);
|
||||
if ([RepositoryStatus.REMOVING, RepositoryStatus.REMOVED,
|
||||
if ([RepositoryStatus.ARCHIVED, RepositoryStatus.REMOVING, RepositoryStatus.REMOVED,
|
||||
RepositoryStatus.EXPIRING, RepositoryStatus.EXPIRED].some((status) => status === repo.status)) return;
|
||||
repo.protectLifecycle = true;
|
||||
const token = await getToken(repo);
|
||||
|
||||
@@ -6,12 +6,15 @@ import { recoverRepositoryOwners } from "../core/recover-repository-owners";
|
||||
async function main() {
|
||||
const args = process.argv.slice(2);
|
||||
const apply = args.includes("--apply");
|
||||
let concurrency = 5;
|
||||
let repositoryId: mongoose.mongo.ObjectId | undefined;
|
||||
for (const arg of args) {
|
||||
if (arg.startsWith("--id=") && /^[a-f0-9]{24}$/i.test(arg.slice(5))) {
|
||||
if (/^--concurrency=([1-9]|[12][0-9]|3[0-2])$/.test(arg)) {
|
||||
concurrency = Number(arg.split("=")[1]);
|
||||
} else if (arg.startsWith("--id=") && /^[a-f0-9]{24}$/i.test(arg.slice(5))) {
|
||||
repositoryId = new mongoose.mongo.ObjectId(arg.slice(5));
|
||||
} else if (arg !== "--apply" && arg !== "--maintenance") {
|
||||
process.stderr.write("Usage: recover-repository-owners.js [--id=<MongoDB document ID>] [--apply --maintenance]\n");
|
||||
} else if (arg !== "--apply" && arg !== "--maintenance" && arg !== "--archive-unrecoverable" && arg !== "--archive-all-ownerless") {
|
||||
process.stderr.write("Usage: recover-repository-owners.js [--id=<MongoDB document ID>] [--apply --maintenance] [--archive-unrecoverable | --archive-all-ownerless] [--concurrency=1..32]\n");
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
@@ -24,7 +27,8 @@ async function main() {
|
||||
const uri = config.MONGODB_URI || `mongodb://${config.DB_USERNAME}:${config.DB_PASSWORD}@${config.DB_HOSTNAME}:27017/production`;
|
||||
await mongoose.connect(uri, { ...(config.MONGODB_URI ? {} : { authSource: "admin" }), autoIndex: false });
|
||||
const counts = await recoverRepositoryOwners(mongoose.connection.db, {
|
||||
apply, repositoryId,
|
||||
apply, repositoryId, concurrency, archiveUnrecoverable: args.includes("--archive-unrecoverable"),
|
||||
archiveAllOwnerless: args.includes("--archive-all-ownerless"),
|
||||
report: event => process.stdout.write(JSON.stringify(event) + "\n"),
|
||||
});
|
||||
process.stdout.write(JSON.stringify(counts) + "\n");
|
||||
|
||||
@@ -309,6 +309,7 @@ router.get(
|
||||
try {
|
||||
user = await getUser(req);
|
||||
} catch { /* not logged in */ }
|
||||
repo.assertNotArchived();
|
||||
const canEdit =
|
||||
!!user &&
|
||||
(user.isAdmin ||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
const { setTimeout } = require("timers");
|
||||
const { expect } = require("chai");
|
||||
require("ts-node/register/transpile-only");
|
||||
const { recoverRepositoryOwners, identifyGitHubToken } = require("../src/core/recover-repository-owners");
|
||||
@@ -121,3 +122,149 @@ describe("GitHub token identification", () => {
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("archive ownerless repositories", () => {
|
||||
it("previews all ownerless records without GitHub calls or storage deletion", async () => {
|
||||
const f = fixture([{ _id: "repo", repoId: "legacy-repo", accessToken: "admin-token" }]);
|
||||
const result = await recoverRepositoryOwners(f.db, { ...f.options, archiveAllOwnerless: true,
|
||||
identify: async () => { throw new Error("must not call GitHub"); },
|
||||
deleteCache: async () => { throw new Error("must not delete in dry run"); } });
|
||||
expect(result.archiveCandidates).to.equal(1);
|
||||
expect(result.archived).to.equal(0);
|
||||
expect(result.issues).to.equal(0);
|
||||
expect(f.writes).to.have.length(0);
|
||||
expect(f.events[0]).to.include({ action: "would_archive", reason: "missing_owner" });
|
||||
});
|
||||
it("marks archived before deleting cache and retains database records", async () => {
|
||||
const f = fixture([{ _id: "repo", repoId: "legacy-repo", source: { accessToken: "admin-token" } }]);
|
||||
const deleted = [];
|
||||
const result = await recoverRepositoryOwners(f.db, { ...f.options, apply: true, archiveAllOwnerless: true,
|
||||
identify: async () => { throw new Error("must not call GitHub"); },
|
||||
deleteCache: async id => {
|
||||
expect(f.writes[0].update.$set.status).to.equal("archived");
|
||||
expect(f.writes[0].update.$set.archiveCachePending).to.equal(true);
|
||||
deleted.push(id);
|
||||
} });
|
||||
expect(result.archived).to.equal(1);
|
||||
expect(result.cacheDeleted).to.equal(1);
|
||||
expect(deleted).to.deep.equal(["legacy-repo"]);
|
||||
expect(f.writes[0].update.$unset).to.deep.equal({ "source.accessToken": "", accessToken: "" });
|
||||
expect(f.writes[0].update.$set["options.update"]).to.equal(false);
|
||||
expect(f.writes[1].update.$set.archiveCachePending).to.equal(false);
|
||||
});
|
||||
it("never archives a repository with an existing owner", async () => {
|
||||
const f = fixture([{ _id: "repo", owner: "admin", repoId: "repo" }], [{ _id: "admin" }]);
|
||||
const result = await recoverRepositoryOwners(f.db, { ...f.options, apply: true, archiveAllOwnerless: true });
|
||||
expect(result.archiveCandidates).to.equal(0);
|
||||
expect(f.writes).to.have.length(0);
|
||||
});
|
||||
it("keeps failed cache cleanup pending and resumes without calling GitHub", async () => {
|
||||
const f = fixture([{ _id: "repo", repoId: "legacy-repo" }]);
|
||||
const result = await recoverRepositoryOwners(f.db, { ...f.options, apply: true, archiveAllOwnerless: true,
|
||||
deleteCache: async () => { throw new Error("storage unavailable"); } });
|
||||
expect(result.issues).to.equal(1);
|
||||
expect(f.writes).to.have.length(1);
|
||||
const resumed = fixture([{ _id: "repo", repoId: "legacy-repo", status: "archived", archiveCachePending: true }]);
|
||||
const retry = await recoverRepositoryOwners(resumed.db, { ...resumed.options, apply: true, archiveAllOwnerless: true,
|
||||
identify: async () => { throw new Error("must not call GitHub"); }, deleteCache: async () => {} });
|
||||
expect(retry.cacheDeleted).to.equal(1);
|
||||
expect(retry.issues).to.equal(0);
|
||||
});
|
||||
it("skips completed archives on reruns", async () => {
|
||||
const f = fixture([{ _id: "repo", repoId: "legacy-repo", status: "archived", archiveCachePending: false }]);
|
||||
expect((await recoverRepositoryOwners(f.db, { ...f.options, apply: true, archiveAllOwnerless: true })).alreadyArchived).to.equal(1);
|
||||
expect(f.writes).to.have.length(0);
|
||||
});
|
||||
it("rejects unsafe storage paths before changing the record", async () => {
|
||||
for (const repoId of [undefined, "", ".", "..", "../other", "a/b", "/etc"]) {
|
||||
const f = fixture([{ _id: "repo", repoId }]);
|
||||
await recoverRepositoryOwners(f.db, { ...f.options, apply: true, archiveAllOwnerless: true });
|
||||
expect(f.events[0].issue).to.equal("unsafe_or_missing_repo_id");
|
||||
expect(f.writes).to.have.length(0);
|
||||
}
|
||||
});
|
||||
it("does not delete files after a conditional update loses a race", async () => {
|
||||
const f = fixture([{ _id: "repo", repoId: "legacy-repo" }]);
|
||||
const original = f.db.collection;
|
||||
f.db.collection = name => name === "users" ? original(name) : { ...original(name), updateOne: async () => ({ modifiedCount: 0 }) };
|
||||
const result = await recoverRepositoryOwners(f.db, { ...f.options, apply: true, archiveAllOwnerless: true,
|
||||
deleteCache: async () => { throw new Error("must not delete"); } });
|
||||
expect(result.cacheDeleted).to.equal(0);
|
||||
expect(f.events[0].issue).to.equal("repository_changed_retry");
|
||||
});
|
||||
it("archives only missing or entirely revoked tokens in selective mode", async () => {
|
||||
const f = fixture([{ _id: "missing", repoId: "missing" }, { _id: "revoked", repoId: "revoked", accessToken: "bad" },
|
||||
{ _id: "mixed", repoId: "mixed", accessToken: "bad", source: { accessToken: "good" } }]);
|
||||
const result = await recoverRepositoryOwners(f.db, { ...f.options, archiveUnrecoverable: true,
|
||||
identify: async token => token === "bad" ? { issue: "invalid_or_revoked_token" } : { githubId: "42" } });
|
||||
expect(result.archiveCandidates).to.equal(2);
|
||||
expect(f.events.find(e => e.id === "mixed").issue).to.equal("mixed_token_validity");
|
||||
});
|
||||
it("runs work concurrently within the configured bound", async () => {
|
||||
const f = fixture(Array.from({ length: 9 }, (_, i) => ({ _id: String(i), accessToken: `token-${i}` })), [{ _id: "owner", githubId: "42" }]);
|
||||
let active = 0, maximum = 0;
|
||||
await recoverRepositoryOwners(f.db, { ...f.options, concurrency: 3, identify: async () => {
|
||||
active++; maximum = Math.max(maximum, active);
|
||||
await new Promise(resolve => setTimeout(resolve, 5));
|
||||
active--; return { githubId: "42" };
|
||||
} });
|
||||
expect(maximum).to.equal(3);
|
||||
expect(f.events).to.have.length(9);
|
||||
});
|
||||
it("validates concurrency", async () => {
|
||||
const f = fixture([]);
|
||||
for (const concurrency of [0, 33, 1.5]) {
|
||||
try { await recoverRepositoryOwners(f.db, { ...f.options, concurrency }); throw new Error("expected failure"); }
|
||||
catch (error) { expect(error.message).to.equal("Concurrency must be an integer between 1 and 32"); }
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("archived repository access", () => {
|
||||
const Repository = require("../src/core/Repository").default;
|
||||
const UserModel = require("../src/core/model/users/users.model").default;
|
||||
function archived() {
|
||||
return new Repository({ owner: new UserModel()._id, repoId: "archived-test", status: "archived",
|
||||
source: {}, options: { update: false }, size: { file: 9, storage: 123 } });
|
||||
}
|
||||
it("rejects public access and source fetching without querying storage", async () => {
|
||||
const repo = archived();
|
||||
for (const action of [() => repo.check(), () => repo.getToken(), () => repo.files(), () => repo.updateIfNeeded({ force: true }), () => repo.anonymize(), () => repo.resetSate()]) {
|
||||
try { await action(); throw new Error("expected archive rejection"); }
|
||||
catch (error) { expect(error.message).to.equal("repository_archived"); }
|
||||
}
|
||||
expect(() => repo.source).to.throw("repository_archived");
|
||||
expect(() => repo.zip()).to.throw("repository_archived");
|
||||
expect(repo.model.size).to.deep.equal({ file: 9, storage: 123 });
|
||||
});
|
||||
it("cannot reactivate an archive through a status update or old cache job", async () => {
|
||||
const repo = archived();
|
||||
try { await repo.updateStatus("ready"); throw new Error("expected rejection"); }
|
||||
catch (error) { expect(error.message).to.equal("repository_archived"); }
|
||||
await repo.removeCache();
|
||||
expect(repo.status).to.equal("archived");
|
||||
expect(repo.model.size.file).to.equal(9);
|
||||
});
|
||||
it("deletes only the target cache directory using filesystem storage", async () => {
|
||||
const fs = require("fs");
|
||||
const os = require("os");
|
||||
const path = require("path");
|
||||
const config = require("../src/config").default;
|
||||
const FileSystem = require("../src/core/storage/FileSystem").default;
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "archive-cache-test-"));
|
||||
const before = config.FOLDER;
|
||||
try {
|
||||
config.FOLDER = root;
|
||||
for (const id of ["archive-target", "keep-sibling"]) {
|
||||
fs.mkdirSync(path.join(root, id, "original"), { recursive: true });
|
||||
fs.writeFileSync(path.join(root, id, "original", "file.txt"), "cached content");
|
||||
}
|
||||
const f = fixture([{ _id: "repo", repoId: "archive-target" }]);
|
||||
await recoverRepositoryOwners(f.db, { ...f.options, apply: true, archiveAllOwnerless: true,
|
||||
deleteCache: id => new FileSystem().rm(id) });
|
||||
expect(fs.existsSync(path.join(root, "archive-target", "original"))).to.equal(false);
|
||||
expect(fs.existsSync(path.join(root, "keep-sibling", "original", "file.txt"))).to.equal(true);
|
||||
expect(f.writes).to.have.length(2);
|
||||
} finally { config.FOLDER = before; fs.rmSync(root, { recursive: true, force: true }); }
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user