import * as express from "express"; import { ensureAuthenticated } from "./connection"; import * as db from "../database"; import { getRepo, getUser, handleError, isOwnerOrAdmin, isOwnerCoauthorOrAdmin, extendExpirationDate, } from "./route-utils"; import { getRepositoryFromGitHub } from "../../core/source/GitHubRepository"; import gh = require("parse-github-url"); import AnonymizedRepositoryModel from "../../core/model/anonymizedRepositories/anonymizedRepositories.model"; import { IAnonymizedRepositoryDocument } from "../../core/model/anonymizedRepositories/anonymizedRepositories.types"; import UserModel from "../../core/model/users/users.model"; import ConferenceModel from "../../core/model/conference/conferences.model"; import AnonymousError from "../../core/AnonymousError"; import { downloadQueue, removeQueue } from "../../queue"; import RepositoryModel from "../../core/model/repositories/repositories.model"; import User from "../../core/User"; import { RepositoryStatus } from "../../core/types"; import { IUserDocument } from "../../core/model/users/users.types"; import { checkToken, octokit, getRedisGateResetAt, getToken } from "../../core/GitHubUtils"; import { createLogger, serializeError } from "../../core/logger"; const logger = createLogger("route:repo"); const router = express.Router(); // user needs to be connected for all user API router.use(ensureAuthenticated); async function getTokenForAdmin(user: User, req: express.Request) { if (user.isAdmin) { try { const existingRepo = await AnonymizedRepositoryModel.findOne( { "source.repositoryName": `${req.params.owner}/${req.params.repo}`, }, { "source.accessToken": 1, owner: 1, } ).populate({ path: "owner", model: UserModel, }); // eslint-disable-next-line @typescript-eslint/no-explicit-any const user: IUserDocument = existingRepo?.owner as any; if (user instanceof UserModel) { const check = await checkToken(user.accessTokens.github); if (check) { return user.accessTokens.github; } } if (existingRepo) { return existingRepo.source.accessToken; } } catch (error) { logger.warn("getToken lookup failed", serializeError(error)); } } } // claim a repository router.post("/claim", async (req: express.Request, res: express.Response) => { const user = await getUser(req); try { if (!req.body.repoId) { throw new AnonymousError("repoId_not_defined", { object: req.body, httpStatus: 400, }); } if (!req.body.repoUrl) { throw new AnonymousError("repoUrl_not_defined", { object: req.body, httpStatus: 400, }); } const repoConfig = await db.getRepository(req.body.repoId); if (repoConfig == null) { throw new AnonymousError("repo_not_found", { object: req.body, httpStatus: 404, }); } const r = gh(req.body.repoUrl); if (!r?.owner || !r?.name) { throw new AnonymousError("repo_not_found", { object: req.body, httpStatus: 404, }); } const repo = await getRepositoryFromGitHub({ owner: r.owner, repo: r.name, repositoryID: req.query.repositoryID as string, accessToken: user.accessToken, }); if (!repo) { throw new AnonymousError("repo_not_found", { object: req.body, httpStatus: 404, }); } const dbRepo = await RepositoryModel.findById( repoConfig.model.source.repositoryId ); if (!dbRepo || dbRepo.externalId != repo.id) { throw new AnonymousError("repo_not_found", { object: req.body, httpStatus: 404, }); } logger.info("repo claimed", { user: user.username, repo: r.repository, }); repoConfig.owner = user; await AnonymizedRepositoryModel.updateOne( { repoId: repoConfig.repoId }, { $set: { owner: user.model.id } } ).collation({ locale: "en", strength: 2 }); return res.send("Ok"); } catch (error) { handleError(error, res, req); } }); // refresh repository router.post( "/:repoId/refresh", async (req: express.Request, res: express.Response) => { try { const repo = await getRepo(req, res, { nocheck: true, }); if (!repo) return; if ( repo.status == "preparing" || repo.status == "removing" || repo.status == "expiring" ) return; const user = await getUser(req); isOwnerCoauthorOrAdmin(repo, user); await repo.updateIfNeeded({ force: true }); res.json({ status: repo.status }); } catch (error) { handleError(error, res, req); } } ); // extend the expiration of a repository (default +6 months) and bring it back // online if it had expired router.post( "/:repoId/extend", async (req: express.Request, res: express.Response) => { try { const repo = await getRepo(req, res, { nocheck: true }); if (!repo) return; if ( repo.status == RepositoryStatus.PREPARING || repo.status == RepositoryStatus.REMOVING || repo.status == RepositoryStatus.EXPIRING || repo.status == RepositoryStatus.REMOVED ) { throw new AnonymousError("invalid_status", { object: repo, httpStatus: 409, }); } const user = await getUser(req); isOwnerCoauthorOrAdmin(repo, user); const newExpiration = extendExpirationDate( repo.model.options.expirationDate ); repo.model.options.expirationDate = newExpiration; await AnonymizedRepositoryModel.updateOne( { _id: repo.model._id }, { $set: { "options.expirationDate": newExpiration } } ).exec(); // Re-anonymize so an expired repository comes back online, mirroring the // refresh flow. await repo.updateIfNeeded({ force: true }); res.json({ status: repo.status, expirationDate: newExpiration }); } catch (error) { handleError(error, res, req); } } ); // delete a repository router.delete( "/:repoId/", async (req: express.Request, res: express.Response) => { const repo = await getRepo(req, res, { nocheck: true, }); if (!repo) return; // if (repo.status == "removing") return res.json({ status: repo.status }); try { if (repo.status == "removed") throw new AnonymousError("is_removed", { object: req.params.repoId, httpStatus: 410, }); const user = await getUser(req); isOwnerOrAdmin([repo.owner.id], user); await repo.updateStatus(RepositoryStatus.REMOVING); await removeQueue.add(repo.repoId, { repoId: repo.repoId }, { jobId: `repo-${repo.repoId}` }); return res.json({ status: repo.status }); } catch (error) { handleError(error, res, req); } } ); router.get( "/:owner/:repo/", async (req: express.Request, res: express.Response) => { const user = await getUser(req); let token = user.accessToken; if (user.isAdmin) { token = (await getTokenForAdmin(user, req)) || token; } try { const repo = await getRepositoryFromGitHub({ owner: req.params.owner, repo: req.params.repo, accessToken: token, repositoryID: req.query.repositoryID as string, force: req.query.force == "1", }); res.json(repo.toJSON()); } catch (error) { handleError(error, res, req); } } ); router.get( "/:owner/:repo/branches", async (req: express.Request, res: express.Response) => { const user = await getUser(req); let token = user.accessToken; if (user.isAdmin) { token = (await getTokenForAdmin(user, req)) || token; } try { const repository = await getRepositoryFromGitHub({ accessToken: token, owner: req.params.owner, repo: req.params.repo, repositoryID: req.query.repositoryID as string, force: req.query.force == "1", }); return res.json( await repository.branches({ accessToken: token, force: req.query.force == "1", }) ); } catch (error) { handleError(error, res, req); } } ); router.get( "/:owner/:repo/readme", async (req: express.Request, res: express.Response) => { try { const user = await getUser(req); let token = user.accessToken; if (user.isAdmin) { token = (await getTokenForAdmin(user, req)) || token; } const repo = await getRepositoryFromGitHub({ owner: req.params.owner, repo: req.params.repo, accessToken: token, repositoryID: req.query.repositoryID as string, force: req.query.force == "1", }); if (!repo) { throw new AnonymousError("repo_not_found", { object: req.params.repoId, httpStatus: 404, }); } return res.send( await repo.readme({ accessToken: token, force: req.query.force == "1", branch: req.query.branch as string, }) ); } catch (error) { handleError(error, res, req); } } ); // get repository information router.get("/:repoId/", async (req: express.Request, res: express.Response) => { try { const repo = await getRepo(req, res, { nocheck: true, }); if (!repo) return; const user = await getUser(req); isOwnerCoauthorOrAdmin(repo, user); const fullRepo = await db.getRepository(req.params.repoId); const json = fullRepo.toJSON() as Record; json.ownerId = fullRepo.owner.id; json.role = user.isAdmin && fullRepo.owner.id !== user.model.id ? "admin" : fullRepo.owner.id === user.model.id ? "owner" : "coauthor"; const repoToken = await getToken(fullRepo); const gateResetAt = await getRedisGateResetAt(repoToken.slice(-8)); if (gateResetAt > 0) { json.rateLimitResetAt = gateResetAt; } res.json(json); } catch (error) { handleError(error, res, req); } }); // eslint-disable-next-line @typescript-eslint/no-explicit-any function validateNewRepo(repoUpdate: any): void { const validCharacters = /^[0-9a-zA-Z\-_]+$/; if ( typeof repoUpdate.repoId !== "string" || !repoUpdate.repoId.match(validCharacters) || repoUpdate.repoId.length < 3 ) { throw new AnonymousError("invalid_repoId", { object: repoUpdate, httpStatus: 400, }); } if (!repoUpdate.source) { throw new AnonymousError("source_not_provided", { object: repoUpdate, httpStatus: 400, }); } if (!repoUpdate.source.branch) { throw new AnonymousError("branch_not_specified", { object: repoUpdate, httpStatus: 400, }); } if (!repoUpdate.source.commit) { throw new AnonymousError("commit_not_specified", { object: repoUpdate, httpStatus: 400, }); } if (!repoUpdate.options) { throw new AnonymousError("options_not_provided", { object: repoUpdate, httpStatus: 400, }); } if (!Array.isArray(repoUpdate.terms)) { throw new AnonymousError("invalid_terms_format", { object: repoUpdate, httpStatus: 400, }); } if (!/^[a-fA-F0-9]+$/.test(repoUpdate.source.commit)) { throw new AnonymousError("invalid_commit_format", { object: repoUpdate, httpStatus: 400, }); } } function updateRepoModel( model: IAnonymizedRepositoryDocument, // eslint-disable-next-line @typescript-eslint/no-explicit-any repoUpdate: any ) { model.source.type = "GitHubStream"; model.source.commit = repoUpdate.source.commit; model.source.branch = repoUpdate.source.branch; model.options = { terms: repoUpdate.terms, expirationMode: repoUpdate.options.expirationMode, expirationDate: repoUpdate.options.expirationDate ? new Date(repoUpdate.options.expirationDate) : undefined, update: repoUpdate.options.update, image: repoUpdate.options.image, pdf: repoUpdate.options.pdf, notebook: repoUpdate.options.notebook, link: repoUpdate.options.link, page: repoUpdate.options.page, pageSource: repoUpdate.options.pageSource, }; } // eslint-disable-next-line @typescript-eslint/no-explicit-any export function hasRepositorySourceChanged( model: IAnonymizedRepositoryDocument, repoUpdate: any ): boolean { return ( repoUpdate.source.commit != model.source.commit || repoUpdate.source.branch != model.source.branch || repoUpdate.fullName != model.source.repositoryName ); } // update a repository router.post( "/:repoId/", async (req: express.Request, res: express.Response) => { try { const repo = await getRepo(req, res, { nocheck: true, }); if (!repo) return; const user = await getUser(req); isOwnerCoauthorOrAdmin(repo, user); const repoUpdate = req.body; validateNewRepo(repoUpdate); // Only the source repository/commit/branch backs the cached FileModel — // anonymization options (terms, image/link toggles, etc.) are applied on // the fly per request. Re-running the download queue is therefore only // needed when the underlying snapshot moves. Other edits (e.g. turning // off auto-update — see #360) just persist and return. const sourceChanged = hasRepositorySourceChanged(repo.model, repoUpdate); updateRepoModel(repo.model, repoUpdate); if (sourceChanged) { const parsedRepository = gh(repoUpdate.fullName); if (!parsedRepository?.owner || !parsedRepository?.name) { await repo.resetSate(RepositoryStatus.ERROR, "repo_not_found"); throw new AnonymousError("repo_not_found", { object: req.body, httpStatus: 404, }); } const repository = await getRepositoryFromGitHub({ accessToken: user.accessToken, owner: parsedRepository.owner, repo: parsedRepository.name, }); if (!repository) { await repo.resetSate(RepositoryStatus.ERROR, "repo_not_found"); throw new AnonymousError("repo_not_found", { object: req.body, httpStatus: 404, }); } await repository.getCommitInfo(repoUpdate.source.commit, { accessToken: user.accessToken, }); repo.model.source.repositoryId = repository.model.id; repo.model.source.repositoryName = repository.fullName || repoUpdate.fullName; repo.model.anonymizeDate = new Date(); await repo.remove(); } const removeRepoFromConference = async (conferenceID: string) => { const conf = await ConferenceModel.findOne({ conferenceID, }); if (conf) { const r = conf.repositories.filter((r) => r.id == repo.model.id); if (r.length == 1) r[0].removeDate = new Date(); await conf.save(); } }; if (!repoUpdate.conference) { // remove conference if (repo.model.conference) { await removeRepoFromConference(repo.model.conference); } } else if (repoUpdate.conference != repo.model.conference) { // update/add conference const conf = await ConferenceModel.findOne({ conferenceID: repoUpdate.conference, }); if (conf) { if ( new Date() < conf.startDate || new Date() > conf.endDate || conf.status !== "ready" ) { throw new AnonymousError("conf_not_activated", { object: conf, httpStatus: 400, }); } const f = conf.repositories.filter((r) => r.id == repo.model.id); if (f.length) { // the repository already referenced the conference f[0].addDate = new Date(); f[0].removeDate = undefined; } else { conf.repositories.push({ id: repo.model.id, addDate: new Date(), }); } if (repo.model.conference) { await removeRepoFromConference(repo.model.conference); } await conf.save(); } } repo.model.conference = repoUpdate.conference; await AnonymizedRepositoryModel.updateOne( { _id: repo.model._id }, { $set: { options: repo.model.options, source: repo.model.source, conference: repo.model.conference, anonymizeDate: repo.model.anonymizeDate, }, } ).exec(); if (!sourceChanged) { return res.json({ status: repo.status }); } await repo.updateStatus(RepositoryStatus.PREPARING); res.json({ status: repo.status }); await downloadQueue.add( repo.repoId, { repoId: repo.repoId }, { jobId: `repo-${repo.repoId}` } ); } catch (error) { return handleError(error, res, req); } } ); // add repository router.post("/", async (req: express.Request, res: express.Response) => { const user = await getUser(req); const repoUpdate = req.body; try { try { await db.getRepository(repoUpdate.repoId); throw new AnonymousError("repoId_already_used", { httpStatus: 400, object: repoUpdate, }); // eslint-disable-next-line @typescript-eslint/no-explicit-any } catch (error: any) { if (error.message == "repo_not_found") { // the repository does not exist yet } else { throw error; } } validateNewRepo(repoUpdate); const r = gh(repoUpdate.fullName); if (!r?.owner || !r?.name) { throw new AnonymousError("repo_not_found", { object: req.body, httpStatus: 404, }); } const repository = await getRepositoryFromGitHub({ accessToken: user.accessToken, owner: r.owner, repo: r.name, }); if (!repository) { throw new AnonymousError("repo_not_found", { object: req.body, httpStatus: 404, }); } await repository.getCommitInfo(repoUpdate.source.commit, { accessToken: user.accessToken, }); const repo = new AnonymizedRepositoryModel(); repo.repoId = repoUpdate.repoId; repo.anonymizeDate = new Date(); repo.owner = user.id; updateRepoModel(repo, repoUpdate); repo.source.type = "GitHubStream"; repo.source.accessToken = user.accessToken; repo.source.repositoryId = repository.model.id; repo.source.repositoryName = repoUpdate.fullName; repo.conference = repoUpdate.conference; await repo.save(); if (repoUpdate.conference) { const conf = await ConferenceModel.findOne({ conferenceID: repoUpdate.conference, }); if (conf) { if ( new Date() < conf.startDate || new Date() > conf.endDate || conf.status !== "ready" ) { await repo.deleteOne(); throw new AnonymousError("conf_not_activated", { object: conf, httpStatus: 400, }); } conf.repositories.push({ id: repo.id, addDate: new Date(), }); await conf.save(); } } res.send({ status: repo.status }); downloadQueue.add(repo.repoId, { repoId: repo.repoId }, { jobId: `repo-${repo.repoId}`, attempts: 3, }); } catch (error) { if ( error instanceof Error && error.message?.indexOf(" duplicate key") > -1 ) { return handleError( new AnonymousError("repoId_already_used", { httpStatus: 400, cause: error, object: repoUpdate, }), res, req ); } return handleError(error, res, req); } }); // list coauthors router.get( "/:repoId/coauthors", async (req: express.Request, res: express.Response) => { try { const repo = await getRepo(req, res, { nocheck: true }); if (!repo) return; const user = await getUser(req); isOwnerCoauthorOrAdmin(repo, user); res.json(repo.coauthors); } catch (error) { handleError(error, res, req); } } ); // add a coauthor (owner/admin only) router.post( "/:repoId/coauthors", async (req: express.Request, res: express.Response) => { try { const repo = await getRepo(req, res, { nocheck: true }); if (!repo) return; const user = await getUser(req); isOwnerOrAdmin([repo.owner.id], user); const username = (req.body.username || "").trim(); if (!username) { throw new AnonymousError("username_not_defined", { object: req.body, httpStatus: 400, }); } // verify the GitHub user exists and capture identity fields const oct = octokit(user.accessToken); let ghUser; try { const r = await oct.users.getByUsername({ username }); ghUser = r.data; } catch { throw new AnonymousError("github_user_not_found", { object: { username }, httpStatus: 404, }); } if (ghUser.login.toLowerCase() === user.username.toLowerCase()) { throw new AnonymousError("cannot_coauthor_self", { httpStatus: 400, }); } const list = repo.model.coauthors || []; if ( list.some( (c) => c.username.toLowerCase() === ghUser.login.toLowerCase() ) ) { return res.json(list); } list.push({ username: ghUser.login, githubId: String(ghUser.id), photo: ghUser.avatar_url, addedAt: new Date(), }); repo.model.coauthors = list; await AnonymizedRepositoryModel.updateOne( { _id: repo.model._id }, { $set: { coauthors: list } } ).exec(); res.json(repo.model.coauthors); } catch (error) { handleError(error, res, req); } } ); // remove a coauthor (owner/admin only, or the coauthor themselves) router.delete( "/:repoId/coauthors/:username", async (req: express.Request, res: express.Response) => { try { const repo = await getRepo(req, res, { nocheck: true }); if (!repo) return; const user = await getUser(req); const target = req.params.username; const isOwner = repo.owner.id === user.model.id; const isSelf = !!user.username && user.username.toLowerCase() === target.toLowerCase(); if (!isOwner && !isSelf && !user.isAdmin) { throw new AnonymousError("not_authorized", { httpStatus: 401 }); } repo.model.coauthors = (repo.model.coauthors || []).filter( (c) => c.username.toLowerCase() !== target.toLowerCase() ); await AnonymizedRepositoryModel.updateOne( { _id: repo.model._id }, { $set: { coauthors: repo.model.coauthors } } ).exec(); res.json(repo.model.coauthors); } catch (error) { handleError(error, res, req); } } ); export default router;