mirror of
https://github.com/tdurieux/anonymous_github.git
synced 2026-02-13 02:42:45 +00:00
feat: introduce streamers that handle the stream and anonymization from github
This commit is contained in:
tdurieux
75
src/server/database.ts
Normal file
75
src/server/database.ts
Normal file
@@ -0,0 +1,75 @@
|
||||
import mongoose, { ConnectOptions } from "mongoose";
|
||||
import Repository from "../core/Repository";
|
||||
import config from "../config";
|
||||
import AnonymizedRepositoryModel from "../core/model/anonymizedRepositories/anonymizedRepositories.model";
|
||||
import AnonymousError from "../core/AnonymousError";
|
||||
import AnonymizedPullRequestModel from "../core/model/anonymizedPullRequests/anonymizedPullRequests.model";
|
||||
import PullRequest from "../core/PullRequest";
|
||||
|
||||
const MONGO_URL = `mongodb://${config.DB_USERNAME}:${config.DB_PASSWORD}@${config.DB_HOSTNAME}:27017/`;
|
||||
|
||||
export const database = mongoose.connection;
|
||||
|
||||
export let isConnected = false;
|
||||
|
||||
export async function connect() {
|
||||
mongoose.set("strictQuery", false);
|
||||
await mongoose.connect(MONGO_URL + "production", {
|
||||
authSource: "admin",
|
||||
appName: "Anonymous GitHub Server",
|
||||
compressors: "zstd",
|
||||
} as ConnectOptions);
|
||||
isConnected = true;
|
||||
|
||||
return database;
|
||||
}
|
||||
|
||||
export async function getRepository(
|
||||
repoId: string,
|
||||
opts: {
|
||||
includeFiles: boolean;
|
||||
} = {
|
||||
includeFiles: true,
|
||||
}
|
||||
) {
|
||||
if (!repoId || repoId == "undefined") {
|
||||
throw new AnonymousError("repo_not_found", {
|
||||
object: repoId,
|
||||
httpStatus: 404,
|
||||
});
|
||||
}
|
||||
const project: any = {};
|
||||
if (!opts.includeFiles) {
|
||||
project.originalFiles = 0;
|
||||
}
|
||||
const data = await AnonymizedRepositoryModel.findOne(
|
||||
{ repoId },
|
||||
project
|
||||
).collation({
|
||||
locale: "en",
|
||||
strength: 2,
|
||||
});
|
||||
if (!data)
|
||||
throw new AnonymousError("repo_not_found", {
|
||||
object: repoId,
|
||||
httpStatus: 404,
|
||||
});
|
||||
return new Repository(data);
|
||||
}
|
||||
export async function getPullRequest(pullRequestId: string) {
|
||||
if (!pullRequestId || pullRequestId == "undefined") {
|
||||
throw new AnonymousError("pull_request_not_found", {
|
||||
object: pullRequestId,
|
||||
httpStatus: 404,
|
||||
});
|
||||
}
|
||||
const data = await AnonymizedPullRequestModel.findOne({
|
||||
pullRequestId,
|
||||
});
|
||||
if (!data)
|
||||
throw new AnonymousError("pull_request_not_found", {
|
||||
object: pullRequestId,
|
||||
httpStatus: 404,
|
||||
});
|
||||
return new PullRequest(data);
|
||||
}
|
||||
220
src/server/index.ts
Normal file
220
src/server/index.ts
Normal file
@@ -0,0 +1,220 @@
|
||||
import { config as dotenv } from "dotenv";
|
||||
dotenv();
|
||||
|
||||
import { createClient } from "redis";
|
||||
import { resolve, join } from "path";
|
||||
import { existsSync } from "fs";
|
||||
import rateLimit from "express-rate-limit";
|
||||
import { slowDown } from "express-slow-down";
|
||||
import RedisStore from "rate-limit-redis";
|
||||
import * as express from "express";
|
||||
import * as compression from "compression";
|
||||
import * as passport from "passport";
|
||||
import { connect } from "./database";
|
||||
import { initSession, router as connectionRouter } from "./routes/connection";
|
||||
import router from "./routes";
|
||||
import AnonymizedRepositoryModel from "../core/model/anonymizedRepositories/anonymizedRepositories.model";
|
||||
import { conferenceStatusCheck, repositoryStatusCheck } from "./schedule";
|
||||
import { startWorker } from "../queue";
|
||||
import AnonymizedPullRequestModel from "../core/model/anonymizedPullRequests/anonymizedPullRequests.model";
|
||||
import { getUser } from "./routes/route-utils";
|
||||
import config from "../config";
|
||||
|
||||
function indexResponse(req: express.Request, res: express.Response) {
|
||||
if (
|
||||
req.path.startsWith("/script") ||
|
||||
req.path.startsWith("/style") ||
|
||||
req.path.startsWith("/favicon") ||
|
||||
req.path.startsWith("/api")
|
||||
) {
|
||||
return res.status(404).send("Not found");
|
||||
}
|
||||
if (
|
||||
req.params.repoId &&
|
||||
req.headers["accept"] &&
|
||||
req.headers["accept"].indexOf("text/html") == -1
|
||||
) {
|
||||
const repoId = req.path.split("/")[2];
|
||||
// if it is not an html request, it assumes that the browser try to load a different type of resource
|
||||
return res.redirect(
|
||||
`/api/repo/${repoId}/file/${req.path.substring(
|
||||
req.path.indexOf(repoId) + repoId.length + 1
|
||||
)}`
|
||||
);
|
||||
}
|
||||
res.sendFile(resolve("public", "index.html"));
|
||||
}
|
||||
|
||||
export default async function start() {
|
||||
const app = express();
|
||||
app.use(express.json());
|
||||
|
||||
app.use(compression());
|
||||
app.set("etag", "strong");
|
||||
|
||||
// handle session and connection
|
||||
app.use(initSession());
|
||||
app.use(passport.initialize());
|
||||
app.use(passport.session());
|
||||
|
||||
startWorker();
|
||||
|
||||
const redisClient = createClient({
|
||||
socket: {
|
||||
host: config.REDIS_HOSTNAME,
|
||||
port: config.REDIS_PORT,
|
||||
},
|
||||
});
|
||||
redisClient.on("error", (err) => console.log("Redis Client Error", err));
|
||||
|
||||
await redisClient.connect();
|
||||
|
||||
function keyGenerator(
|
||||
request: express.Request,
|
||||
_response: express.Response
|
||||
): string {
|
||||
if (request.headers["cf-connecting-ip"]) {
|
||||
return request.headers["cf-connecting-ip"] as string;
|
||||
}
|
||||
if (!request.ip && request.socket.remoteAddress) {
|
||||
console.error("Warning: request.ip is missing!");
|
||||
return request.socket.remoteAddress;
|
||||
}
|
||||
// remove port number from IPv4 addresses
|
||||
return (request.ip || "").replace(/:\d+[^:]*$/, "");
|
||||
}
|
||||
|
||||
const rate = rateLimit({
|
||||
store: new RedisStore({
|
||||
sendCommand: (...args: string[]) => redisClient.sendCommand(args),
|
||||
}),
|
||||
windowMs: 15 * 60 * 1000, // 15 minutes
|
||||
skip: async (request: express.Request, response: express.Response) => {
|
||||
try {
|
||||
const user = await getUser(request);
|
||||
if (user && user.isAdmin) return true;
|
||||
} catch (_) {
|
||||
// ignore: user not connected
|
||||
}
|
||||
return false;
|
||||
},
|
||||
max: async (request: express.Request, response: express.Response) => {
|
||||
try {
|
||||
const user = await getUser(request);
|
||||
if (user) return config.RATE_LIMIT;
|
||||
} catch (_) {
|
||||
// ignore: user not connected
|
||||
}
|
||||
// if not logged in, limit to half the rate
|
||||
return config.RATE_LIMIT / 2;
|
||||
},
|
||||
keyGenerator,
|
||||
standardHeaders: true,
|
||||
legacyHeaders: false,
|
||||
message: (request: express.Request, response: express.Response) => {
|
||||
return `You can only make ${config.RATE_LIMIT} requests every 15min. Please try again later.`;
|
||||
},
|
||||
});
|
||||
const speedLimiter = slowDown({
|
||||
windowMs: 15 * 60 * 1000, // 15 minutes
|
||||
delayAfter: 50,
|
||||
delayMs: () => 150,
|
||||
maxDelayMs: 5000,
|
||||
keyGenerator,
|
||||
});
|
||||
const webViewSpeedLimiter = slowDown({
|
||||
windowMs: 15 * 60 * 1000, // 15 minutes
|
||||
delayAfter: 200,
|
||||
delayMs: () => 150,
|
||||
maxDelayMs: 5000,
|
||||
keyGenerator,
|
||||
});
|
||||
|
||||
app.use("/github", rate, speedLimiter, connectionRouter);
|
||||
|
||||
// api routes
|
||||
const apiRouter = express.Router();
|
||||
app.use("/api", rate, apiRouter);
|
||||
|
||||
apiRouter.use("/admin", router.admin);
|
||||
apiRouter.use("/options", router.option);
|
||||
apiRouter.use("/conferences", router.conference);
|
||||
apiRouter.use("/user", router.user);
|
||||
apiRouter.use("/repo", router.repositoryPublic);
|
||||
apiRouter.use("/repo", speedLimiter, router.file);
|
||||
apiRouter.use("/repo", speedLimiter, router.repositoryPrivate);
|
||||
apiRouter.use("/pr", speedLimiter, router.pullRequestPublic);
|
||||
apiRouter.use("/pr", speedLimiter, router.pullRequestPrivate);
|
||||
|
||||
apiRouter.get("/message", async (_, res) => {
|
||||
if (existsSync("./message.txt")) {
|
||||
return res.sendFile(resolve("message.txt"));
|
||||
}
|
||||
res.sendStatus(404);
|
||||
});
|
||||
|
||||
let stat: any = {};
|
||||
|
||||
setInterval(() => {
|
||||
stat = {};
|
||||
}, 1000 * 60 * 60);
|
||||
|
||||
apiRouter.get("/stat", async (_, res) => {
|
||||
if (stat.nbRepositories) {
|
||||
res.json(stat);
|
||||
return;
|
||||
}
|
||||
const [nbRepositories, users, nbPageViews, nbPullRequests] =
|
||||
await Promise.all([
|
||||
AnonymizedRepositoryModel.estimatedDocumentCount(),
|
||||
AnonymizedRepositoryModel.distinct("owner"),
|
||||
AnonymizedRepositoryModel.collection
|
||||
.aggregate([
|
||||
{
|
||||
$group: { _id: null, total: { $sum: "$pageView" } },
|
||||
},
|
||||
])
|
||||
.toArray(),
|
||||
AnonymizedPullRequestModel.estimatedDocumentCount(),
|
||||
]);
|
||||
|
||||
stat = {
|
||||
nbRepositories,
|
||||
nbUsers: users.length,
|
||||
nbPageViews: nbPageViews[0]?.total || 0,
|
||||
nbPullRequests,
|
||||
};
|
||||
|
||||
res.json(stat);
|
||||
});
|
||||
|
||||
// web view
|
||||
app.use("/w/", rate, webViewSpeedLimiter, router.webview);
|
||||
|
||||
app
|
||||
.get("/", indexResponse)
|
||||
.get("/404", indexResponse)
|
||||
.get("/anonymize", indexResponse)
|
||||
.get("/r/:repoId/?*", indexResponse)
|
||||
.get("/repository/:repoId/?*", indexResponse);
|
||||
|
||||
app.use(
|
||||
express.static(join("public"), {
|
||||
etag: true,
|
||||
lastModified: true,
|
||||
maxAge: 3600, // 1h
|
||||
})
|
||||
);
|
||||
|
||||
app.get("*", indexResponse);
|
||||
|
||||
// start schedules
|
||||
conferenceStatusCheck();
|
||||
repositoryStatusCheck();
|
||||
|
||||
await connect();
|
||||
app.listen(config.PORT);
|
||||
console.log("Database connected and Server started on port: " + config.PORT);
|
||||
}
|
||||
|
||||
start();
|
||||
281
src/server/routes/admin.ts
Normal file
281
src/server/routes/admin.ts
Normal file
@@ -0,0 +1,281 @@
|
||||
import { Queue } from "bullmq";
|
||||
import * as express from "express";
|
||||
import AnonymousError from "../../core/AnonymousError";
|
||||
import AnonymizedRepositoryModel from "../../core/model/anonymizedRepositories/anonymizedRepositories.model";
|
||||
import ConferenceModel from "../../core/model/conference/conferences.model";
|
||||
import UserModel from "../../core/model/users/users.model";
|
||||
import { cacheQueue, downloadQueue, removeQueue } from "../../queue";
|
||||
import Repository from "../../core/Repository";
|
||||
import User from "../../core/User";
|
||||
import { ensureAuthenticated } from "./connection";
|
||||
import { handleError, getUser, isOwnerOrAdmin, getRepo } from "./route-utils";
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
// user needs to be connected for all user API
|
||||
router.use(ensureAuthenticated);
|
||||
router.use(
|
||||
async (
|
||||
req: express.Request,
|
||||
res: express.Response,
|
||||
next: express.NextFunction
|
||||
) => {
|
||||
const user = await getUser(req);
|
||||
try {
|
||||
// only admins are allowed here
|
||||
isOwnerOrAdmin([], user);
|
||||
next();
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
router.post("/queue/:name/:repo_id", async (req, res) => {
|
||||
let queue: Queue<Repository, void>;
|
||||
if (req.params.name == "download") {
|
||||
queue = downloadQueue;
|
||||
} else if (req.params.name == "remove") {
|
||||
queue = removeQueue;
|
||||
} else {
|
||||
return res.status(404).json({ error: "queue_not_found" });
|
||||
}
|
||||
const job = await queue.getJob(req.params.repo_id);
|
||||
if (!job) {
|
||||
return res.status(404).json({ error: "job_not_found" });
|
||||
}
|
||||
try {
|
||||
await job.retry();
|
||||
res.send("ok");
|
||||
} catch (error) {
|
||||
try {
|
||||
await job.remove();
|
||||
queue.add(job.name, job.data, job.opts);
|
||||
res.send("ok");
|
||||
} catch (error) {
|
||||
res.status(500).send("error_retrying_job");
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
router.delete("/queue/:name/:repo_id", async (req, res) => {
|
||||
let queue: Queue;
|
||||
if (req.params.name == "download") {
|
||||
queue = downloadQueue;
|
||||
} else if (req.params.name == "remove") {
|
||||
queue = removeQueue;
|
||||
} else {
|
||||
return res.status(404).json({ error: "queue_not_found" });
|
||||
}
|
||||
const job = await queue.getJob(req.params.repo_id);
|
||||
if (!job) {
|
||||
return res.status(404).json({ error: "job_not_found" });
|
||||
}
|
||||
await job.remove();
|
||||
res.send("ok");
|
||||
});
|
||||
|
||||
router.get("/queues", async (req, res) => {
|
||||
const out = await Promise.all([
|
||||
downloadQueue.getJobs([
|
||||
"waiting",
|
||||
"active",
|
||||
"completed",
|
||||
"failed",
|
||||
"delayed",
|
||||
]),
|
||||
removeQueue.getJobs([
|
||||
"waiting",
|
||||
"active",
|
||||
"completed",
|
||||
"failed",
|
||||
"delayed",
|
||||
]),
|
||||
cacheQueue.getJobs(["waiting", "active", "completed", "failed", "delayed"]),
|
||||
]);
|
||||
res.json({
|
||||
downloadQueue: out[0],
|
||||
removeQueue: out[1],
|
||||
cacheQueue: out[2],
|
||||
});
|
||||
});
|
||||
|
||||
router.get("/repos", async (req, res) => {
|
||||
const page = parseInt(req.query.page as string) || 1;
|
||||
const limit = parseInt(req.query.limit as string) || 10;
|
||||
const ready = req.query.ready == "true";
|
||||
const error = req.query.error == "true";
|
||||
const preparing = req.query.preparing == "true";
|
||||
const remove = req.query.removed == "true";
|
||||
const expired = req.query.expired == "true";
|
||||
|
||||
let sort: any = { _id: 1 };
|
||||
if (req.query.sort) {
|
||||
sort = {};
|
||||
sort[req.query.sort as string] = -1;
|
||||
}
|
||||
let query = [];
|
||||
if (req.query.search) {
|
||||
query.push({ repoId: { $regex: req.query.search } });
|
||||
}
|
||||
const status: { status: string }[] = [];
|
||||
query.push({ $or: status });
|
||||
if (ready) {
|
||||
status.push({ status: "ready" });
|
||||
}
|
||||
if (error) {
|
||||
status.push({ status: "error" });
|
||||
}
|
||||
if (expired) {
|
||||
status.push({ status: "expiring" });
|
||||
status.push({ status: "expired" });
|
||||
}
|
||||
if (remove) {
|
||||
status.push({ status: "removing" });
|
||||
status.push({ status: "removed" });
|
||||
}
|
||||
if (preparing) {
|
||||
status.push({ status: "preparing" });
|
||||
status.push({ status: "download" });
|
||||
}
|
||||
const skipIndex = (page - 1) * limit;
|
||||
const [total, results] = await Promise.all([
|
||||
AnonymizedRepositoryModel.find(
|
||||
{
|
||||
$and: query,
|
||||
},
|
||||
{ originalFiles: 0 }
|
||||
).countDocuments(),
|
||||
AnonymizedRepositoryModel.find({ $and: query }, { originalFiles: 0 })
|
||||
.skip(skipIndex)
|
||||
.sort(sort)
|
||||
.limit(limit)
|
||||
.exec(),
|
||||
]);
|
||||
res.json({
|
||||
query: { $and: query },
|
||||
page,
|
||||
total,
|
||||
sort,
|
||||
results,
|
||||
});
|
||||
});
|
||||
|
||||
// delete a repository
|
||||
router.delete(
|
||||
"/repos/:repoId/",
|
||||
async (req: express.Request, res: express.Response) => {
|
||||
const repo = await getRepo(req, res, { nocheck: true });
|
||||
if (!repo) return;
|
||||
try {
|
||||
await cacheQueue.add(repo.repoId, repo, { jobId: repo.repoId });
|
||||
return res.json({ status: repo.status });
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
router.get("/users", async (req, res) => {
|
||||
const page = parseInt(req.query.page as string) || 1;
|
||||
const limit = parseInt(req.query.limit as string) || 10;
|
||||
const skipIndex = (page - 1) * limit;
|
||||
|
||||
let sort: any = { _id: 1 };
|
||||
if (req.query.sort) {
|
||||
sort = {};
|
||||
sort[req.query.sort as string] = -1;
|
||||
}
|
||||
let query = {};
|
||||
if (req.query.search) {
|
||||
query = { username: { $regex: req.query.search } };
|
||||
}
|
||||
|
||||
res.json({
|
||||
query: query,
|
||||
page,
|
||||
total: await UserModel.find(query).countDocuments(),
|
||||
sort,
|
||||
results: await UserModel.find(query)
|
||||
.sort(sort)
|
||||
.limit(limit)
|
||||
.skip(skipIndex),
|
||||
});
|
||||
});
|
||||
router.get(
|
||||
"/users/:username",
|
||||
async (req: express.Request, res: express.Response) => {
|
||||
try {
|
||||
const model = await UserModel.findOne({
|
||||
username: req.params.username,
|
||||
}).populate({
|
||||
path: "repositories",
|
||||
model: "Repository",
|
||||
foreignField: "_id",
|
||||
localField: "repositories",
|
||||
});
|
||||
if (!model) {
|
||||
req.logout((error) => console.error(error));
|
||||
throw new AnonymousError("user_not_found", {
|
||||
httpStatus: 404,
|
||||
});
|
||||
}
|
||||
const user = new User(model);
|
||||
res.json(user);
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
}
|
||||
}
|
||||
);
|
||||
router.get(
|
||||
"/users/:username/repos",
|
||||
async (req: express.Request, res: express.Response) => {
|
||||
try {
|
||||
const model = await UserModel.findOne({ username: req.params.username });
|
||||
if (!model) {
|
||||
req.logout((error) => console.error(error));
|
||||
throw new AnonymousError("user_not_found", {
|
||||
httpStatus: 404,
|
||||
});
|
||||
}
|
||||
const user = new User(model);
|
||||
const repos = await user.getRepositories();
|
||||
res.json(repos);
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
}
|
||||
}
|
||||
);
|
||||
router.get("/conferences", async (req, res) => {
|
||||
const page = parseInt(req.query.page as string) || 1;
|
||||
const limit = parseInt(req.query.limit as string) || 10;
|
||||
const skipIndex = (page - 1) * limit;
|
||||
|
||||
let sort: any = { _id: 1 };
|
||||
if (req.query.sort) {
|
||||
sort = {};
|
||||
sort[req.query.sort as string] = -1;
|
||||
}
|
||||
let query = {};
|
||||
if (req.query.search) {
|
||||
query = {
|
||||
$or: [
|
||||
{ name: { $regex: req.query.search } },
|
||||
{ conferenceID: { $regex: req.query.search } },
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
res.json({
|
||||
query: query,
|
||||
page,
|
||||
total: await ConferenceModel.find(query).estimatedDocumentCount(),
|
||||
sort,
|
||||
results: await ConferenceModel.find(query)
|
||||
.sort(sort)
|
||||
.limit(limit)
|
||||
.skip(skipIndex),
|
||||
});
|
||||
});
|
||||
|
||||
export default router;
|
||||
284
src/server/routes/conference.ts
Normal file
284
src/server/routes/conference.ts
Normal file
@@ -0,0 +1,284 @@
|
||||
import * as express from "express";
|
||||
import AnonymousError from "../../core/AnonymousError";
|
||||
import Conference from "../../core/Conference";
|
||||
import ConferenceModel from "../../core/model/conference/conferences.model";
|
||||
import { ensureAuthenticated } from "./connection";
|
||||
import { handleError, getUser, isOwnerOrAdmin } from "./route-utils";
|
||||
import { IConferenceDocument } from "../../core/model/conference/conferences.types";
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
// user needs to be connected for all user API
|
||||
router.use(ensureAuthenticated);
|
||||
|
||||
const plans = [
|
||||
{
|
||||
id: "free_conference",
|
||||
name: "Free",
|
||||
pricePerRepo: 0,
|
||||
storagePerRepo: -1,
|
||||
description: `<li><strong>Quota is deducted from user account</strong></li>
|
||||
<li>No-download</li>
|
||||
<li>Conference dashboard</li>`,
|
||||
},
|
||||
{
|
||||
id: "premium_conference",
|
||||
name: "Premium",
|
||||
pricePerRepo: 0.5,
|
||||
storagePerRepo: 500 * 8 * 1024,
|
||||
description: `<li>500Mo / repository</li>
|
||||
<li>Repository download</li>
|
||||
<li>Conference dashboard</li>`,
|
||||
},
|
||||
{
|
||||
id: "unlimited_conference",
|
||||
name: "Unlimited",
|
||||
pricePerRepo: 3,
|
||||
storagePerRepo: 0,
|
||||
description: `<li><strong>Unlimited</strong> repository size</li>
|
||||
<li>Repository download</li>
|
||||
<li>Conference dashboard</li>`,
|
||||
},
|
||||
];
|
||||
|
||||
router.get("/plans", async (req: express.Request, res: express.Response) => {
|
||||
res.json(plans);
|
||||
});
|
||||
|
||||
router.get("/", async (req: express.Request, res: express.Response) => {
|
||||
try {
|
||||
const user = await getUser(req);
|
||||
const conferences = await Promise.all(
|
||||
(
|
||||
await ConferenceModel.find({
|
||||
owners: { $in: user.model.id },
|
||||
})
|
||||
).map(async (data) => {
|
||||
const conf = new Conference(data);
|
||||
if (data.endDate < new Date() && data.status == "ready") {
|
||||
await conf.updateStatus("expired");
|
||||
}
|
||||
return conf;
|
||||
})
|
||||
);
|
||||
res.json(conferences.map((conf) => conf.toJSON()));
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
}
|
||||
});
|
||||
|
||||
function validateConferenceForm(conf: any) {
|
||||
if (!conf.name)
|
||||
throw new AnonymousError("conf_name_missing", {
|
||||
object: conf,
|
||||
httpStatus: 400,
|
||||
});
|
||||
if (!conf.conferenceID)
|
||||
throw new AnonymousError("conf_id_missing", {
|
||||
object: conf,
|
||||
httpStatus: 400,
|
||||
});
|
||||
if (!conf.startDate)
|
||||
throw new AnonymousError("conf_start_date_missing", {
|
||||
object: conf,
|
||||
httpStatus: 400,
|
||||
});
|
||||
if (!conf.endDate)
|
||||
throw new AnonymousError("conf_end_date_missing", {
|
||||
object: conf,
|
||||
httpStatus: 400,
|
||||
});
|
||||
if (new Date(conf.startDate) > new Date(conf.endDate))
|
||||
throw new AnonymousError("conf_start_date_invalid", {
|
||||
object: conf,
|
||||
httpStatus: 400,
|
||||
});
|
||||
if (new Date() > new Date(conf.endDate))
|
||||
throw new AnonymousError("conf_end_date_invalid", {
|
||||
object: conf,
|
||||
httpStatus: 400,
|
||||
});
|
||||
if (plans.filter((p) => p.id == conf.plan.planID).length != 1)
|
||||
throw new AnonymousError("invalid_plan", {
|
||||
object: conf,
|
||||
httpStatus: 400,
|
||||
});
|
||||
const plan = plans.filter((p) => p.id == conf.plan.planID)[0];
|
||||
if (plan.pricePerRepo > 0) {
|
||||
const billing = conf.billing;
|
||||
if (!billing)
|
||||
throw new AnonymousError("billing_missing", {
|
||||
object: conf,
|
||||
httpStatus: 400,
|
||||
});
|
||||
if (!billing.name)
|
||||
throw new AnonymousError("billing_name_missing", {
|
||||
object: conf,
|
||||
httpStatus: 400,
|
||||
});
|
||||
if (!billing.email)
|
||||
throw new AnonymousError("billing_email_missing", {
|
||||
object: conf,
|
||||
httpStatus: 400,
|
||||
});
|
||||
if (!billing.address)
|
||||
throw new AnonymousError("billing_address_missing", {
|
||||
object: conf,
|
||||
httpStatus: 400,
|
||||
});
|
||||
if (!billing.city)
|
||||
throw new AnonymousError("billing_city_missing", {
|
||||
object: conf,
|
||||
httpStatus: 400,
|
||||
});
|
||||
if (!billing.zip)
|
||||
throw new AnonymousError("billing_zip_missing", {
|
||||
object: conf,
|
||||
httpStatus: 400,
|
||||
});
|
||||
if (!billing.country)
|
||||
throw new AnonymousError("billing_country_missing", {
|
||||
object: conf,
|
||||
httpStatus: 400,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
router.post(
|
||||
"/:conferenceID?",
|
||||
async (req: express.Request, res: express.Response) => {
|
||||
try {
|
||||
const user = await getUser(req);
|
||||
let model: IConferenceDocument = new ConferenceModel();
|
||||
if (req.params.conferenceID) {
|
||||
const queryModel = await ConferenceModel.findOne({
|
||||
conferenceID: req.params.conferenceID,
|
||||
});
|
||||
if (!queryModel) {
|
||||
throw new AnonymousError("conference_not_found", {
|
||||
httpStatus: 404,
|
||||
});
|
||||
}
|
||||
model = queryModel;
|
||||
isOwnerOrAdmin(model.owners, user);
|
||||
}
|
||||
validateConferenceForm(req.body);
|
||||
model.name = req.body.name;
|
||||
model.startDate = new Date(req.body.startDate);
|
||||
model.endDate = new Date(req.body.endDate);
|
||||
model.status = "ready";
|
||||
model.url = req.body.url;
|
||||
model.repositories = [];
|
||||
model.options = req.body.options;
|
||||
|
||||
if (!req.params.conferenceID) {
|
||||
model.owners.push(user.model.id);
|
||||
model.conferenceID = req.body.conferenceID;
|
||||
|
||||
model.plan = {
|
||||
planID: req.body.plan.planID,
|
||||
pricePerRepository: plans.filter(
|
||||
(p) => p.id == req.body.plan.planID
|
||||
)[0].pricePerRepo,
|
||||
quota: {
|
||||
size: plans.filter((p) => p.id == req.body.plan.planID)[0]
|
||||
.storagePerRepo,
|
||||
file: 0,
|
||||
repository: 0,
|
||||
},
|
||||
};
|
||||
|
||||
if (req.body.billing)
|
||||
model.billing = {
|
||||
name: req.body.billing.name,
|
||||
email: req.body.billing.email,
|
||||
address: req.body.billing.address,
|
||||
address2: req.body.billing.address2,
|
||||
city: req.body.billing.city,
|
||||
zip: req.body.billing.zip,
|
||||
country: req.body.billing.country,
|
||||
vat: req.body.billing.vat,
|
||||
};
|
||||
}
|
||||
await model.save();
|
||||
|
||||
res.send("ok");
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof Error &&
|
||||
error.message?.indexOf(" duplicate key") > -1
|
||||
) {
|
||||
return handleError(
|
||||
new AnonymousError("conf_id_used", {
|
||||
object: req.params.conferenceID,
|
||||
httpStatus: 400,
|
||||
}),
|
||||
res
|
||||
);
|
||||
}
|
||||
handleError(error, res, req);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
router.get(
|
||||
"/:conferenceID",
|
||||
async (req: express.Request, res: express.Response) => {
|
||||
try {
|
||||
const data = await ConferenceModel.findOne({
|
||||
conferenceID: req.params.conferenceID,
|
||||
});
|
||||
if (!data)
|
||||
throw new AnonymousError("conf_not_found", {
|
||||
object: req.params.conferenceID,
|
||||
httpStatus: 404,
|
||||
});
|
||||
const user = await getUser(req);
|
||||
const conference = new Conference(data);
|
||||
try {
|
||||
isOwnerOrAdmin(conference.ownerIDs, user);
|
||||
const o: any = conference.toJSON();
|
||||
o.repositories = (await conference.repositories()).map((r) =>
|
||||
r.toJSON()
|
||||
);
|
||||
res.json(o);
|
||||
} catch (error) {
|
||||
return res.json({
|
||||
conferenceID: conference.conferenceID,
|
||||
name: conference.name,
|
||||
url: conference.url,
|
||||
startDate: conference.startDate,
|
||||
endDate: conference.endDate,
|
||||
options: conference.options,
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
router.delete(
|
||||
"/:conferenceID",
|
||||
async (req: express.Request, res: express.Response) => {
|
||||
try {
|
||||
const user = await getUser(req);
|
||||
const data = await ConferenceModel.findOne({
|
||||
conferenceID: req.params.conferenceID,
|
||||
});
|
||||
if (!data)
|
||||
throw new AnonymousError("conf_not_found", {
|
||||
object: req.params.conferenceID,
|
||||
httpStatus: 400,
|
||||
});
|
||||
const conference = new Conference(data);
|
||||
isOwnerOrAdmin(conference.ownerIDs, user);
|
||||
await conference.remove();
|
||||
res.send("ok");
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
export default router;
|
||||
130
src/server/routes/connection.ts
Normal file
130
src/server/routes/connection.ts
Normal file
@@ -0,0 +1,130 @@
|
||||
import { createClient } from "redis";
|
||||
import * as passport from "passport";
|
||||
import * as session from "express-session";
|
||||
import RedisStore from "connect-redis";
|
||||
import * as OAuth2Strategy from "passport-oauth2";
|
||||
import { Profile, Strategy } from "passport-github2";
|
||||
import * as express from "express";
|
||||
|
||||
import config from "../../config";
|
||||
import UserModel from "../../core/model/users/users.model";
|
||||
import { IUserDocument } from "../../core/model/users/users.types";
|
||||
import AnonymousError from "../../core/AnonymousError";
|
||||
|
||||
export function ensureAuthenticated(
|
||||
req: express.Request,
|
||||
res: express.Response,
|
||||
next: express.NextFunction
|
||||
) {
|
||||
if (req.isAuthenticated()) {
|
||||
return next();
|
||||
}
|
||||
res.status(401).json({ error: "not_connected" });
|
||||
}
|
||||
|
||||
const verify = async (
|
||||
accessToken: string,
|
||||
refreshToken: string,
|
||||
profile: Profile,
|
||||
done: OAuth2Strategy.VerifyCallback
|
||||
): Promise<void> => {
|
||||
let user: IUserDocument | null = null;
|
||||
try {
|
||||
user = await UserModel.findOne({ "externalIDs.github": profile.id });
|
||||
if (user) {
|
||||
user.accessTokens.github = accessToken;
|
||||
} else {
|
||||
const photo = profile.photos ? profile.photos[0]?.value : null;
|
||||
user = new UserModel({
|
||||
username: profile.username,
|
||||
accessTokens: {
|
||||
github: accessToken,
|
||||
},
|
||||
externalIDs: {
|
||||
github: profile.id,
|
||||
},
|
||||
emails: profile.emails?.map((email) => {
|
||||
return { email: email.value, default: false };
|
||||
}),
|
||||
photo,
|
||||
});
|
||||
if (user.emails?.length) user.emails[0].default = true;
|
||||
}
|
||||
await user.save();
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
throw new AnonymousError("unable_to_connect_user", {
|
||||
httpStatus: 500,
|
||||
object: profile,
|
||||
cause: error as Error,
|
||||
});
|
||||
} finally {
|
||||
done(null, {
|
||||
username: profile.username,
|
||||
accessToken,
|
||||
refreshToken,
|
||||
profile,
|
||||
user,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
passport.use(
|
||||
new Strategy(
|
||||
{
|
||||
clientID: config.CLIENT_ID,
|
||||
clientSecret: config.CLIENT_SECRET,
|
||||
callbackURL: config.AUTH_CALLBACK,
|
||||
},
|
||||
verify
|
||||
)
|
||||
);
|
||||
|
||||
passport.serializeUser((user: Express.User, done) => {
|
||||
done(null, user);
|
||||
});
|
||||
|
||||
passport.deserializeUser((user: Express.User, done) => {
|
||||
done(null, user);
|
||||
});
|
||||
|
||||
export function initSession() {
|
||||
const redisClient = createClient({
|
||||
legacyMode: false,
|
||||
socket: {
|
||||
port: config.REDIS_PORT,
|
||||
host: config.REDIS_HOSTNAME,
|
||||
},
|
||||
});
|
||||
redisClient.on("error", (err) => console.log("Redis Client Error", err));
|
||||
redisClient.connect();
|
||||
const redisStore = new RedisStore({
|
||||
client: redisClient,
|
||||
prefix: "anoGH_session:",
|
||||
});
|
||||
|
||||
return session({
|
||||
secret: config.SESSION_SECRET,
|
||||
store: redisStore,
|
||||
saveUninitialized: false,
|
||||
resave: false,
|
||||
});
|
||||
}
|
||||
|
||||
export const router = express.Router();
|
||||
|
||||
router.get(
|
||||
"/login",
|
||||
passport.authenticate("github", { scope: ["repo"] }), // Note the scope here
|
||||
function (req: express.Request, res: express.Response) {
|
||||
res.redirect("/");
|
||||
}
|
||||
);
|
||||
|
||||
router.get(
|
||||
"/auth",
|
||||
passport.authenticate("github", { failureRedirect: "/" }),
|
||||
function (req: express.Request, res: express.Response) {
|
||||
res.redirect("/");
|
||||
}
|
||||
);
|
||||
59
src/server/routes/file.ts
Normal file
59
src/server/routes/file.ts
Normal file
@@ -0,0 +1,59 @@
|
||||
import * as express from "express";
|
||||
import AnonymizedFile from "../../core/AnonymizedFile";
|
||||
import AnonymousError from "../../core/AnonymousError";
|
||||
import { getRepo, handleError } from "./route-utils";
|
||||
|
||||
export const router = express.Router();
|
||||
|
||||
router.get(
|
||||
"/:repoId/file/:path*",
|
||||
async (req: express.Request, res: express.Response) => {
|
||||
const anonymizedPath = decodeURI(
|
||||
new URL(req.url, `${req.protocol}://${req.hostname}`).pathname.replace(
|
||||
`/${req.params.repoId}/file/`,
|
||||
""
|
||||
)
|
||||
);
|
||||
if (anonymizedPath.endsWith("/")) {
|
||||
return handleError(
|
||||
new AnonymousError("folder_not_supported", {
|
||||
httpStatus: 404,
|
||||
object: anonymizedPath,
|
||||
}),
|
||||
res
|
||||
);
|
||||
}
|
||||
|
||||
const repo = await getRepo(req, res, {
|
||||
nocheck: false,
|
||||
includeFiles: false,
|
||||
});
|
||||
if (!repo) return;
|
||||
|
||||
try {
|
||||
const f = new AnonymizedFile({
|
||||
repository: repo,
|
||||
anonymizedPath,
|
||||
});
|
||||
if (!f.isFileSupported()) {
|
||||
throw new AnonymousError("file_not_supported", {
|
||||
httpStatus: 403,
|
||||
object: f,
|
||||
});
|
||||
}
|
||||
if (req.query.download) {
|
||||
res.attachment(
|
||||
anonymizedPath.substring(anonymizedPath.lastIndexOf("/") + 1)
|
||||
);
|
||||
}
|
||||
// cache the file for 5min
|
||||
res.header("Cache-Control", "max-age=300");
|
||||
await repo.countView();
|
||||
await f.send(res);
|
||||
} catch (error) {
|
||||
return handleError(error, res, req);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
export default router;
|
||||
23
src/server/routes/index.ts
Normal file
23
src/server/routes/index.ts
Normal file
@@ -0,0 +1,23 @@
|
||||
import pullRequestPrivate from "./pullRequest-private";
|
||||
import pullRequestPublic from "./pullRequest-public";
|
||||
import repositoryPrivate from "./repository-private";
|
||||
import repositoryPublic from "./repository-public";
|
||||
import conference from "./conference";
|
||||
import file from "./file";
|
||||
import webview from "./webview";
|
||||
import user from "./user";
|
||||
import option from "./option";
|
||||
import admin from "./admin";
|
||||
|
||||
export default {
|
||||
pullRequestPrivate,
|
||||
pullRequestPublic,
|
||||
repositoryPrivate,
|
||||
repositoryPublic,
|
||||
file,
|
||||
webview,
|
||||
user,
|
||||
option,
|
||||
conference,
|
||||
admin,
|
||||
};
|
||||
14
src/server/routes/option.ts
Normal file
14
src/server/routes/option.ts
Normal file
@@ -0,0 +1,14 @@
|
||||
import * as express from "express";
|
||||
import config from "../../config";
|
||||
export const router = express.Router();
|
||||
|
||||
router.get("/", async (req: express.Request, res: express.Response) => {
|
||||
res.json({
|
||||
ENABLE_DOWNLOAD: config.ENABLE_DOWNLOAD,
|
||||
MAX_FILE_SIZE: config.MAX_FILE_SIZE,
|
||||
MAX_REPO_SIZE: config.MAX_REPO_SIZE,
|
||||
ANONYMIZATION_MASK: config.ANONYMIZATION_MASK,
|
||||
});
|
||||
});
|
||||
|
||||
export default router;
|
||||
244
src/server/routes/pullRequest-private.ts
Normal file
244
src/server/routes/pullRequest-private.ts
Normal file
@@ -0,0 +1,244 @@
|
||||
import * as express from "express";
|
||||
import { ensureAuthenticated } from "./connection";
|
||||
|
||||
import {
|
||||
getPullRequest,
|
||||
getUser,
|
||||
handleError,
|
||||
isOwnerOrAdmin,
|
||||
} from "./route-utils";
|
||||
import AnonymousError from "../../core/AnonymousError";
|
||||
import { IAnonymizedPullRequestDocument } from "../../core/model/anonymizedPullRequests/anonymizedPullRequests.types";
|
||||
import PullRequest from "../../core/PullRequest";
|
||||
import AnonymizedPullRequestModel from "../../core/model/anonymizedPullRequests/anonymizedPullRequests.model";
|
||||
import { RepositoryStatus } from "../../core/types";
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
// user needs to be connected for all user API
|
||||
router.use(ensureAuthenticated);
|
||||
|
||||
// refresh pullRequest
|
||||
router.post(
|
||||
"/:pullRequestId/refresh",
|
||||
async (req: express.Request, res: express.Response) => {
|
||||
try {
|
||||
const pullRequest = await getPullRequest(req, res, { nocheck: true });
|
||||
if (!pullRequest) return;
|
||||
|
||||
const user = await getUser(req);
|
||||
isOwnerOrAdmin([pullRequest.owner.id], user);
|
||||
await pullRequest.updateIfNeeded({ force: true });
|
||||
res.json({ status: pullRequest.status });
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
// delete a pullRequest
|
||||
router.delete(
|
||||
"/:pullRequestId/",
|
||||
async (req: express.Request, res: express.Response) => {
|
||||
const pullRequest = await getPullRequest(req, res, { nocheck: true });
|
||||
if (!pullRequest) return;
|
||||
try {
|
||||
if (pullRequest.status == "removed")
|
||||
throw new AnonymousError("is_removed", {
|
||||
object: req.params.pullRequestId,
|
||||
httpStatus: 410,
|
||||
});
|
||||
const user = await getUser(req);
|
||||
isOwnerOrAdmin([pullRequest.owner.id], user);
|
||||
await pullRequest.remove();
|
||||
return res.json({ status: pullRequest.status });
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
router.get(
|
||||
"/:owner/:repository/:pullRequestId",
|
||||
async (req: express.Request, res: express.Response) => {
|
||||
const user = await getUser(req);
|
||||
try {
|
||||
const pullRequest = new PullRequest(
|
||||
new AnonymizedPullRequestModel({
|
||||
owner: user.id,
|
||||
source: {
|
||||
pullRequestId: parseInt(req.params.pullRequestId),
|
||||
repositoryFullName: `${req.params.owner}/${req.params.repository}`,
|
||||
},
|
||||
})
|
||||
);
|
||||
pullRequest.owner = user;
|
||||
await pullRequest.download();
|
||||
res.json(pullRequest.toJSON());
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
// get pullRequest information
|
||||
router.get(
|
||||
"/:pullRequestId/",
|
||||
async (req: express.Request, res: express.Response) => {
|
||||
try {
|
||||
const pullRequest = await getPullRequest(req, res, { nocheck: true });
|
||||
if (!pullRequest) return;
|
||||
|
||||
const user = await getUser(req);
|
||||
isOwnerOrAdmin([pullRequest.owner.id], user);
|
||||
res.json(pullRequest.toJSON());
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
function validateNewPullRequest(pullRequestUpdate: any): void {
|
||||
const validCharacters = /^[0-9a-zA-Z\-\_]+$/;
|
||||
if (
|
||||
!pullRequestUpdate.pullRequestId.match(validCharacters) ||
|
||||
pullRequestUpdate.pullRequestId.length < 3
|
||||
) {
|
||||
throw new AnonymousError("invalid_pullRequestId", {
|
||||
object: pullRequestUpdate,
|
||||
httpStatus: 400,
|
||||
});
|
||||
}
|
||||
if (!pullRequestUpdate.source.repositoryFullName) {
|
||||
throw new AnonymousError("repository_not_specified", {
|
||||
object: pullRequestUpdate,
|
||||
httpStatus: 400,
|
||||
});
|
||||
}
|
||||
if (!pullRequestUpdate.source.pullRequestId) {
|
||||
throw new AnonymousError("pullRequestId_not_specified", {
|
||||
object: pullRequestUpdate,
|
||||
httpStatus: 400,
|
||||
});
|
||||
}
|
||||
if (
|
||||
parseInt(pullRequestUpdate.source.pullRequestId) !=
|
||||
pullRequestUpdate.source.pullRequestId
|
||||
) {
|
||||
throw new AnonymousError("pullRequestId_is_not_a_number", {
|
||||
object: pullRequestUpdate,
|
||||
httpStatus: 400,
|
||||
});
|
||||
}
|
||||
if (!pullRequestUpdate.options) {
|
||||
throw new AnonymousError("options_not_provided", {
|
||||
object: pullRequestUpdate,
|
||||
httpStatus: 400,
|
||||
});
|
||||
}
|
||||
if (!Array.isArray(pullRequestUpdate.terms)) {
|
||||
throw new AnonymousError("invalid_terms_format", {
|
||||
object: pullRequestUpdate,
|
||||
httpStatus: 400,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function updatePullRequestModel(
|
||||
model: IAnonymizedPullRequestDocument,
|
||||
pullRequestUpdate: any
|
||||
) {
|
||||
model.options = {
|
||||
terms: pullRequestUpdate.terms,
|
||||
expirationMode: pullRequestUpdate.options.expirationMode,
|
||||
expirationDate: pullRequestUpdate.options.expirationDate
|
||||
? new Date(pullRequestUpdate.options.expirationDate)
|
||||
: undefined,
|
||||
update: pullRequestUpdate.options.update,
|
||||
image: pullRequestUpdate.options.image,
|
||||
link: pullRequestUpdate.options.link,
|
||||
body: pullRequestUpdate.options.body,
|
||||
title: pullRequestUpdate.options.title,
|
||||
username: pullRequestUpdate.options.username,
|
||||
origin: pullRequestUpdate.options.origin,
|
||||
diff: pullRequestUpdate.options.diff,
|
||||
comments: pullRequestUpdate.options.comments,
|
||||
date: pullRequestUpdate.options.date,
|
||||
};
|
||||
}
|
||||
|
||||
// update a pullRequest
|
||||
router.post(
|
||||
"/:pullRequestId/",
|
||||
async (req: express.Request, res: express.Response) => {
|
||||
try {
|
||||
const pullRequest = await getPullRequest(req, res, { nocheck: true });
|
||||
if (!pullRequest) return;
|
||||
const user = await getUser(req);
|
||||
|
||||
isOwnerOrAdmin([pullRequest.owner.id], user);
|
||||
const pullRequestUpdate = req.body;
|
||||
validateNewPullRequest(pullRequestUpdate);
|
||||
pullRequest.model.anonymizeDate = new Date();
|
||||
|
||||
updatePullRequestModel(pullRequest.model, pullRequestUpdate);
|
||||
// TODO handle conference
|
||||
pullRequest.model.conference = pullRequestUpdate.conference;
|
||||
await pullRequest.updateStatus(RepositoryStatus.PREPARING);
|
||||
await pullRequest.updateIfNeeded({ force: true });
|
||||
res.json(pullRequest.toJSON());
|
||||
} catch (error) {
|
||||
return handleError(error, res, req);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
// add pullRequest
|
||||
router.post("/", async (req: express.Request, res: express.Response) => {
|
||||
const user = await getUser(req);
|
||||
const pullRequestUpdate = req.body;
|
||||
|
||||
try {
|
||||
validateNewPullRequest(pullRequestUpdate);
|
||||
|
||||
const pullRequest = new PullRequest(
|
||||
new AnonymizedPullRequestModel({
|
||||
owner: user.id,
|
||||
options: pullRequestUpdate.options,
|
||||
})
|
||||
);
|
||||
|
||||
pullRequest.model.pullRequestId = pullRequestUpdate.pullRequestId;
|
||||
pullRequest.model.anonymizeDate = new Date();
|
||||
pullRequest.model.owner = user.id;
|
||||
|
||||
updatePullRequestModel(pullRequest.model, pullRequestUpdate);
|
||||
pullRequest.source.accessToken = user.accessToken;
|
||||
pullRequest.source.pullRequestId = pullRequestUpdate.source.pullRequestId;
|
||||
pullRequest.source.repositoryFullName =
|
||||
pullRequestUpdate.source.repositoryFullName;
|
||||
|
||||
pullRequest.conference = pullRequestUpdate.conference;
|
||||
|
||||
await pullRequest.anonymize();
|
||||
res.send(pullRequest.toJSON());
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof Error &&
|
||||
error.message.indexOf(" duplicate key") > -1
|
||||
) {
|
||||
return handleError(
|
||||
new AnonymousError("pullRequestId_already_used", {
|
||||
httpStatus: 400,
|
||||
cause: error,
|
||||
object: pullRequestUpdate,
|
||||
}),
|
||||
res,
|
||||
req
|
||||
);
|
||||
}
|
||||
return handleError(error, res, req);
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
84
src/server/routes/pullRequest-public.ts
Normal file
84
src/server/routes/pullRequest-public.ts
Normal file
@@ -0,0 +1,84 @@
|
||||
import * as express from "express";
|
||||
|
||||
import { getPullRequest, handleError } from "./route-utils";
|
||||
import AnonymousError from "../../core/AnonymousError";
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
router.get(
|
||||
"/:pullRequestId/options",
|
||||
async (req: express.Request, res: express.Response) => {
|
||||
try {
|
||||
res.header("Cache-Control", "no-cache");
|
||||
const pr = await getPullRequest(req, res, { nocheck: true });
|
||||
if (!pr) return;
|
||||
let redirectURL = null;
|
||||
if (pr.status == "expired" && pr.options.expirationMode == "redirect") {
|
||||
redirectURL = `https://github.com/${pr.source.repositoryFullName}/pull/${pr.source.pullRequestId}`;
|
||||
} else {
|
||||
if (
|
||||
pr.status == "expired" ||
|
||||
pr.status == "expiring" ||
|
||||
pr.status == "removing" ||
|
||||
pr.status == "removed"
|
||||
) {
|
||||
throw new AnonymousError("pull_request_expired", {
|
||||
object: pr,
|
||||
httpStatus: 410,
|
||||
});
|
||||
}
|
||||
|
||||
const fiveMinuteAgo = new Date();
|
||||
fiveMinuteAgo.setMinutes(fiveMinuteAgo.getMinutes() - 5);
|
||||
if (pr.status != "ready") {
|
||||
if (
|
||||
pr.model.statusDate < fiveMinuteAgo
|
||||
// && repo.status != "preparing"
|
||||
) {
|
||||
await pr.updateIfNeeded({ force: true });
|
||||
}
|
||||
if (pr.status == "error") {
|
||||
throw new AnonymousError(
|
||||
pr.model.statusMessage
|
||||
? pr.model.statusMessage
|
||||
: "pull_request_not_available",
|
||||
{
|
||||
object: pr,
|
||||
httpStatus: 500,
|
||||
}
|
||||
);
|
||||
}
|
||||
throw new AnonymousError("pull_request_not_ready", {
|
||||
httpStatus: 404,
|
||||
object: pr,
|
||||
});
|
||||
}
|
||||
|
||||
await pr.updateIfNeeded();
|
||||
}
|
||||
|
||||
res.json({
|
||||
url: redirectURL,
|
||||
lastUpdateDate: pr.model.statusDate,
|
||||
});
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
}
|
||||
}
|
||||
);
|
||||
router.get(
|
||||
"/:pullRequestId/content",
|
||||
async (req: express.Request, res: express.Response) => {
|
||||
const pullRequest = await getPullRequest(req, res);
|
||||
if (!pullRequest) return;
|
||||
try {
|
||||
await pullRequest.countView();
|
||||
res.header("Cache-Control", "no-cache");
|
||||
res.json(pullRequest.content());
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
export default router;
|
||||
559
src/server/routes/repository-private.ts
Normal file
559
src/server/routes/repository-private.ts
Normal file
@@ -0,0 +1,559 @@
|
||||
import * as express from "express";
|
||||
import { ensureAuthenticated } from "./connection";
|
||||
|
||||
import * as db from "../database";
|
||||
import { getRepo, getUser, handleError, isOwnerOrAdmin } from "./route-utils";
|
||||
import { getRepositoryFromGitHub } from "../../core/source/GitHubRepository";
|
||||
import gh = require("parse-github-url");
|
||||
import AnonymizedRepositoryModel from "../../core/model/anonymizedRepositories/anonymizedRepositories.model";
|
||||
import { IAnonymizedRepositoryDocument } from "../../core/model/anonymizedRepositories/anonymizedRepositories.types";
|
||||
import Repository from "../../core/Repository";
|
||||
import UserModel from "../../core/model/users/users.model";
|
||||
import ConferenceModel from "../../core/model/conference/conferences.model";
|
||||
import AnonymousError from "../../core/AnonymousError";
|
||||
import { downloadQueue, removeQueue } from "../../queue";
|
||||
import RepositoryModel from "../../core/model/repositories/repositories.model";
|
||||
import User from "../../core/User";
|
||||
import { RepositoryStatus } from "../../core/types";
|
||||
import { IUserDocument } from "../../core/model/users/users.types";
|
||||
import { checkToken } from "../../core/GitHubUtils";
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
// user needs to be connected for all user API
|
||||
router.use(ensureAuthenticated);
|
||||
|
||||
async function getTokenForAdmin(user: User, req: express.Request) {
|
||||
if (user.isAdmin) {
|
||||
try {
|
||||
const existingRepo = await AnonymizedRepositoryModel.findOne(
|
||||
{
|
||||
"source.repositoryName": `${req.params.owner}/${req.params.repo}`,
|
||||
},
|
||||
{
|
||||
"source.accessToken": 1,
|
||||
owner: 1,
|
||||
}
|
||||
).populate({
|
||||
path: "owner",
|
||||
model: UserModel,
|
||||
});
|
||||
const user: IUserDocument = existingRepo?.owner as any;
|
||||
if (user instanceof UserModel) {
|
||||
const check = await checkToken(user.accessTokens.github);
|
||||
if (check) {
|
||||
return user.accessTokens.github;
|
||||
}
|
||||
}
|
||||
if (existingRepo) {
|
||||
return existingRepo.source.accessToken;
|
||||
}
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// claim a repository
|
||||
router.post("/claim", async (req: express.Request, res: express.Response) => {
|
||||
const user = await getUser(req);
|
||||
try {
|
||||
if (!req.body.repoId) {
|
||||
throw new AnonymousError("repoId_not_defined", {
|
||||
object: req.body,
|
||||
httpStatus: 400,
|
||||
});
|
||||
}
|
||||
if (!req.body.repoUrl) {
|
||||
throw new AnonymousError("repoUrl_not_defined", {
|
||||
object: req.body,
|
||||
httpStatus: 400,
|
||||
});
|
||||
}
|
||||
|
||||
const repoConfig = await db.getRepository(req.body.repoId);
|
||||
if (repoConfig == null) {
|
||||
throw new AnonymousError("repo_not_found", {
|
||||
object: req.body,
|
||||
httpStatus: 404,
|
||||
});
|
||||
}
|
||||
|
||||
const r = gh(req.body.repoUrl);
|
||||
if (!r?.owner || !r?.name) {
|
||||
throw new AnonymousError("repo_not_found", {
|
||||
object: req.body,
|
||||
httpStatus: 404,
|
||||
});
|
||||
}
|
||||
const repo = await getRepositoryFromGitHub({
|
||||
owner: r.owner,
|
||||
repo: r.name,
|
||||
accessToken: user.accessToken,
|
||||
});
|
||||
if (!repo) {
|
||||
throw new AnonymousError("repo_not_found", {
|
||||
object: req.body,
|
||||
httpStatus: 404,
|
||||
});
|
||||
}
|
||||
|
||||
const dbRepo = await RepositoryModel.findById(
|
||||
repoConfig.model.source.repositoryId
|
||||
);
|
||||
|
||||
if (!dbRepo || dbRepo.externalId != repo.id) {
|
||||
throw new AnonymousError("repo_not_found", {
|
||||
object: req.body,
|
||||
httpStatus: 404,
|
||||
});
|
||||
}
|
||||
|
||||
console.log(`${user.username} claims ${r.repository}.`);
|
||||
repoConfig.owner = user;
|
||||
|
||||
await AnonymizedRepositoryModel.updateOne(
|
||||
{ repoId: repoConfig.repoId },
|
||||
{ $set: { owner: user.model.id } }
|
||||
);
|
||||
return res.send("Ok");
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
}
|
||||
});
|
||||
|
||||
// refresh repository
|
||||
router.post(
|
||||
"/:repoId/refresh",
|
||||
async (req: express.Request, res: express.Response) => {
|
||||
try {
|
||||
const repo = await getRepo(req, res, {
|
||||
nocheck: true,
|
||||
includeFiles: false,
|
||||
});
|
||||
if (!repo) return;
|
||||
|
||||
if (
|
||||
repo.status == "preparing" ||
|
||||
repo.status == "removing" ||
|
||||
repo.status == "expiring"
|
||||
)
|
||||
return;
|
||||
|
||||
const user = await getUser(req);
|
||||
isOwnerOrAdmin([repo.owner.id], user);
|
||||
await repo.updateIfNeeded({ force: true });
|
||||
res.json({ status: repo.status });
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
// delete a repository
|
||||
router.delete(
|
||||
"/:repoId/",
|
||||
async (req: express.Request, res: express.Response) => {
|
||||
const repo = await getRepo(req, res, {
|
||||
nocheck: true,
|
||||
includeFiles: false,
|
||||
});
|
||||
if (!repo) return;
|
||||
// if (repo.status == "removing") return res.json({ status: repo.status });
|
||||
try {
|
||||
if (repo.status == "removed")
|
||||
throw new AnonymousError("is_removed", {
|
||||
object: req.params.repoId,
|
||||
httpStatus: 410,
|
||||
});
|
||||
const user = await getUser(req);
|
||||
isOwnerOrAdmin([repo.owner.id], user);
|
||||
await repo.updateStatus(RepositoryStatus.REMOVING);
|
||||
await removeQueue.add(repo.repoId, repo, { jobId: repo.repoId });
|
||||
return res.json({ status: repo.status });
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
router.get(
|
||||
"/:owner/:repo/",
|
||||
async (req: express.Request, res: express.Response) => {
|
||||
const user = await getUser(req);
|
||||
let token = user.accessToken;
|
||||
if (user.isAdmin) {
|
||||
token = (await getTokenForAdmin(user, req)) || token;
|
||||
}
|
||||
try {
|
||||
const repo = await getRepositoryFromGitHub({
|
||||
owner: req.params.owner,
|
||||
repo: req.params.repo,
|
||||
accessToken: token,
|
||||
});
|
||||
res.json(repo.toJSON());
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
router.get(
|
||||
"/:owner/:repo/branches",
|
||||
async (req: express.Request, res: express.Response) => {
|
||||
const user = await getUser(req);
|
||||
let token = user.accessToken;
|
||||
if (user.isAdmin) {
|
||||
token = (await getTokenForAdmin(user, req)) || token;
|
||||
}
|
||||
try {
|
||||
const repository = await getRepositoryFromGitHub({
|
||||
accessToken: token,
|
||||
owner: req.params.owner,
|
||||
repo: req.params.repo,
|
||||
});
|
||||
return res.json(
|
||||
await repository.branches({
|
||||
accessToken: token,
|
||||
force: req.query.force == "1",
|
||||
})
|
||||
);
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
router.get(
|
||||
"/:owner/:repo/readme",
|
||||
async (req: express.Request, res: express.Response) => {
|
||||
try {
|
||||
const user = await getUser(req);
|
||||
let token = user.accessToken;
|
||||
if (user.isAdmin) {
|
||||
token = (await getTokenForAdmin(user, req)) || token;
|
||||
}
|
||||
|
||||
const repo = await getRepositoryFromGitHub({
|
||||
owner: req.params.owner,
|
||||
repo: req.params.repo,
|
||||
accessToken: token,
|
||||
});
|
||||
if (!repo) {
|
||||
throw new AnonymousError("repo_not_found", {
|
||||
object: req.params.repoId,
|
||||
httpStatus: 404,
|
||||
});
|
||||
}
|
||||
return res.send(
|
||||
await repo.readme({
|
||||
accessToken: token,
|
||||
force: req.query.force == "1",
|
||||
branch: req.query.branch as string,
|
||||
})
|
||||
);
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
// get repository information
|
||||
router.get("/:repoId/", async (req: express.Request, res: express.Response) => {
|
||||
try {
|
||||
const repo = await getRepo(req, res, {
|
||||
nocheck: true,
|
||||
includeFiles: false,
|
||||
});
|
||||
if (!repo) return;
|
||||
|
||||
const user = await getUser(req);
|
||||
isOwnerOrAdmin([repo.owner.id], user);
|
||||
res.json((await db.getRepository(req.params.repoId)).toJSON());
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
}
|
||||
});
|
||||
|
||||
function validateNewRepo(repoUpdate: any): void {
|
||||
const validCharacters = /^[0-9a-zA-Z\-\_]+$/;
|
||||
if (
|
||||
!repoUpdate.repoId.match(validCharacters) ||
|
||||
repoUpdate.repoId.length < 3
|
||||
) {
|
||||
throw new AnonymousError("invalid_repoId", {
|
||||
object: repoUpdate,
|
||||
httpStatus: 400,
|
||||
});
|
||||
}
|
||||
if (!repoUpdate.source.branch) {
|
||||
throw new AnonymousError("branch_not_specified", {
|
||||
object: repoUpdate,
|
||||
httpStatus: 400,
|
||||
});
|
||||
}
|
||||
if (!repoUpdate.source.commit) {
|
||||
throw new AnonymousError("commit_not_specified", {
|
||||
object: repoUpdate,
|
||||
httpStatus: 400,
|
||||
});
|
||||
}
|
||||
if (!repoUpdate.options) {
|
||||
throw new AnonymousError("options_not_provided", {
|
||||
object: repoUpdate,
|
||||
httpStatus: 400,
|
||||
});
|
||||
}
|
||||
if (!Array.isArray(repoUpdate.terms)) {
|
||||
throw new AnonymousError("invalid_terms_format", {
|
||||
object: repoUpdate,
|
||||
httpStatus: 400,
|
||||
});
|
||||
}
|
||||
if (!/^[a-fA-F0-9]+$/.test(repoUpdate.source.commit)) {
|
||||
throw new AnonymousError("invalid_commit_format", {
|
||||
object: repoUpdate,
|
||||
httpStatus: 400,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function updateRepoModel(
|
||||
model: IAnonymizedRepositoryDocument,
|
||||
repoUpdate: any
|
||||
) {
|
||||
if (repoUpdate.source.type) {
|
||||
model.source.type = repoUpdate.source.type;
|
||||
if (
|
||||
model.source.type != "GitHubStream" &&
|
||||
model.source.type != "GitHubDownload"
|
||||
) {
|
||||
model.source.type = "GitHubStream";
|
||||
}
|
||||
}
|
||||
model.source.commit = repoUpdate.source.commit;
|
||||
model.source.branch = repoUpdate.source.branch;
|
||||
model.options = {
|
||||
terms: repoUpdate.terms,
|
||||
expirationMode: repoUpdate.options.expirationMode,
|
||||
expirationDate: repoUpdate.options.expirationDate
|
||||
? new Date(repoUpdate.options.expirationDate)
|
||||
: undefined,
|
||||
update: repoUpdate.options.update,
|
||||
image: repoUpdate.options.image,
|
||||
pdf: repoUpdate.options.pdf,
|
||||
notebook: repoUpdate.options.notebook,
|
||||
link: repoUpdate.options.link,
|
||||
page: repoUpdate.options.page,
|
||||
pageSource: repoUpdate.options.pageSource,
|
||||
};
|
||||
}
|
||||
|
||||
// update a repository
|
||||
router.post(
|
||||
"/:repoId/",
|
||||
async (req: express.Request, res: express.Response) => {
|
||||
try {
|
||||
const repo = await getRepo(req, res, {
|
||||
nocheck: true,
|
||||
includeFiles: false,
|
||||
});
|
||||
if (!repo) return;
|
||||
const user = await getUser(req);
|
||||
|
||||
isOwnerOrAdmin([repo.owner.id], user);
|
||||
|
||||
const repoUpdate = req.body;
|
||||
|
||||
validateNewRepo(repoUpdate);
|
||||
|
||||
if (repoUpdate.source.commit != repo.model.source.commit) {
|
||||
repo.model.anonymizeDate = new Date();
|
||||
repo.model.source.commit = repoUpdate.source.commit;
|
||||
await repo.remove();
|
||||
}
|
||||
|
||||
updateRepoModel(repo.model, repoUpdate);
|
||||
repo.source.type = "GitHubStream";
|
||||
|
||||
const removeRepoFromConference = async (conferenceID: string) => {
|
||||
const conf = await ConferenceModel.findOne({
|
||||
conferenceID,
|
||||
});
|
||||
if (conf) {
|
||||
const r = conf.repositories.filter((r) => r.id == repo.model.id);
|
||||
if (r.length == 1) r[0].removeDate = new Date();
|
||||
await conf.save();
|
||||
}
|
||||
};
|
||||
if (!repoUpdate.conference) {
|
||||
// remove conference
|
||||
if (repo.model.conference) {
|
||||
await removeRepoFromConference(repo.model.conference);
|
||||
}
|
||||
} else if (repoUpdate.conference != repo.model.conference) {
|
||||
// update/add conference
|
||||
const conf = await ConferenceModel.findOne({
|
||||
conferenceID: repoUpdate.conference,
|
||||
});
|
||||
if (conf) {
|
||||
if (
|
||||
new Date() < conf.startDate ||
|
||||
new Date() > conf.endDate ||
|
||||
conf.status !== "ready"
|
||||
) {
|
||||
throw new AnonymousError("conf_not_activated", {
|
||||
object: conf,
|
||||
httpStatus: 400,
|
||||
});
|
||||
}
|
||||
const f = conf.repositories.filter((r) => r.id == repo.model.id);
|
||||
if (f.length) {
|
||||
// the repository already referenced the conference
|
||||
f[0].addDate = new Date();
|
||||
f[0].removeDate = undefined;
|
||||
} else {
|
||||
conf.repositories.push({
|
||||
id: repo.model.id,
|
||||
addDate: new Date(),
|
||||
});
|
||||
}
|
||||
if (repo.model.conference) {
|
||||
await removeRepoFromConference(repo.model.conference);
|
||||
}
|
||||
await conf.save();
|
||||
}
|
||||
}
|
||||
repo.model.conference = repoUpdate.conference;
|
||||
await repo.updateStatus(RepositoryStatus.PREPARING);
|
||||
res.json({ status: repo.status });
|
||||
await downloadQueue.add(repo.repoId, repo, { jobId: repo.repoId });
|
||||
} catch (error) {
|
||||
return handleError(error, res, req);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
// add repository
|
||||
router.post("/", async (req: express.Request, res: express.Response) => {
|
||||
const user = await getUser(req);
|
||||
const repoUpdate = req.body;
|
||||
|
||||
try {
|
||||
try {
|
||||
await db.getRepository(repoUpdate.repoId, { includeFiles: false });
|
||||
throw new AnonymousError("repoId_already_used", {
|
||||
httpStatus: 400,
|
||||
object: repoUpdate,
|
||||
});
|
||||
} catch (error: any) {
|
||||
if (error.message == "repo_not_found") {
|
||||
// the repository does not exist yet
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
validateNewRepo(repoUpdate);
|
||||
|
||||
const r = gh(repoUpdate.fullName);
|
||||
if (!r?.owner || !r?.name) {
|
||||
throw new AnonymousError("repo_not_found", {
|
||||
object: req.body,
|
||||
httpStatus: 404,
|
||||
});
|
||||
}
|
||||
const repository = await getRepositoryFromGitHub({
|
||||
accessToken: user.accessToken,
|
||||
owner: r.owner,
|
||||
repo: r.name,
|
||||
});
|
||||
|
||||
if (!repository) {
|
||||
throw new AnonymousError("repo_not_found", {
|
||||
object: req.body,
|
||||
httpStatus: 404,
|
||||
});
|
||||
}
|
||||
|
||||
const repo = new AnonymizedRepositoryModel();
|
||||
repo.repoId = repoUpdate.repoId;
|
||||
repo.anonymizeDate = new Date();
|
||||
repo.owner = user.id;
|
||||
|
||||
updateRepoModel(repo, repoUpdate);
|
||||
repo.source.type = "GitHubStream";
|
||||
repo.source.accessToken = user.accessToken;
|
||||
repo.source.repositoryId = repository.model.id;
|
||||
repo.source.repositoryName = repoUpdate.fullName;
|
||||
|
||||
// if (repo.source.type === "GitHubDownload") {
|
||||
// // details.size is in kilobytes
|
||||
// if (
|
||||
// repository.size === undefined ||
|
||||
// repository.size > config.MAX_REPO_SIZE
|
||||
// ) {
|
||||
// throw new AnonymousError("invalid_mode", {
|
||||
// object: repository,
|
||||
// httpStatus: 400,
|
||||
// });
|
||||
// }
|
||||
// }
|
||||
// if (
|
||||
// repository.size !== undefined &&
|
||||
// repository.size < config.AUTO_DOWNLOAD_REPO_SIZE
|
||||
// ) {
|
||||
// repo.source.type = "GitHubDownload";
|
||||
// }
|
||||
repo.conference = repoUpdate.conference;
|
||||
|
||||
await repo.save();
|
||||
|
||||
if (repoUpdate.conference) {
|
||||
const conf = await ConferenceModel.findOne({
|
||||
conferenceID: repoUpdate.conference,
|
||||
});
|
||||
if (conf) {
|
||||
if (
|
||||
new Date() < conf.startDate ||
|
||||
new Date() > conf.endDate ||
|
||||
conf.status !== "ready"
|
||||
) {
|
||||
await repo.deleteOne();
|
||||
throw new AnonymousError("conf_not_activated", {
|
||||
object: conf,
|
||||
httpStatus: 400,
|
||||
});
|
||||
}
|
||||
conf.repositories.push({
|
||||
id: repo.id,
|
||||
addDate: new Date(),
|
||||
});
|
||||
await conf.save();
|
||||
}
|
||||
}
|
||||
|
||||
res.send({ status: repo.status });
|
||||
downloadQueue.add(repo.repoId, new Repository(repo), {
|
||||
jobId: repo.repoId,
|
||||
attempts: 3,
|
||||
});
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof Error &&
|
||||
error.message?.indexOf(" duplicate key") > -1
|
||||
) {
|
||||
return handleError(
|
||||
new AnonymousError("repoId_already_used", {
|
||||
httpStatus: 400,
|
||||
cause: error,
|
||||
object: repoUpdate,
|
||||
}),
|
||||
res,
|
||||
req
|
||||
);
|
||||
}
|
||||
return handleError(error, res, req);
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
166
src/server/routes/repository-public.ts
Normal file
166
src/server/routes/repository-public.ts
Normal file
@@ -0,0 +1,166 @@
|
||||
import { promisify } from "util";
|
||||
import * as express from "express";
|
||||
import * as stream from "stream";
|
||||
import config from "../../config";
|
||||
|
||||
import { getRepo, handleError } from "./route-utils";
|
||||
import AnonymousError from "../../core/AnonymousError";
|
||||
import { downloadQueue } from "../../queue";
|
||||
import { RepositoryStatus } from "../../core/types";
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
router.get(
|
||||
"/:repoId/zip",
|
||||
async (req: express.Request, res: express.Response) => {
|
||||
const pipeline = promisify(stream.pipeline);
|
||||
|
||||
try {
|
||||
if (!config.ENABLE_DOWNLOAD) {
|
||||
throw new AnonymousError("download_not_enabled", {
|
||||
httpStatus: 403,
|
||||
object: req.params.repoId,
|
||||
});
|
||||
}
|
||||
const repo = await getRepo(req, res);
|
||||
if (!repo) return;
|
||||
|
||||
let download = false;
|
||||
const conference = await repo.conference();
|
||||
if (conference) {
|
||||
download =
|
||||
conference.quota.size > -1 &&
|
||||
!!config.ENABLE_DOWNLOAD &&
|
||||
repo.source.type == "GitHubDownload";
|
||||
}
|
||||
if (
|
||||
repo.size.storage < config.FREE_DOWNLOAD_REPO_SIZE * 1024 &&
|
||||
repo.source.type == "GitHubDownload"
|
||||
) {
|
||||
download = true;
|
||||
}
|
||||
|
||||
if (!download) {
|
||||
throw new AnonymousError("download_not_enabled", {
|
||||
httpStatus: 403,
|
||||
object: req.params.repoId,
|
||||
});
|
||||
}
|
||||
|
||||
res.attachment(`${repo.repoId}.zip`);
|
||||
|
||||
// cache the file for 6 hours
|
||||
res.header("Cache-Control", "max-age=21600");
|
||||
await pipeline(await repo.zip(), res);
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
router.get(
|
||||
"/:repoId/files",
|
||||
async (req: express.Request, res: express.Response) => {
|
||||
res.header("Cache-Control", "no-cache");
|
||||
const repo = await getRepo(req, res, { includeFiles: true });
|
||||
if (!repo) return;
|
||||
try {
|
||||
res.json(await repo.anonymizedFiles({ includeSha: false }));
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
router.get(
|
||||
"/:repoId/options",
|
||||
async (req: express.Request, res: express.Response) => {
|
||||
try {
|
||||
res.header("Cache-Control", "no-cache");
|
||||
const repo = await getRepo(req, res, {
|
||||
nocheck: true,
|
||||
includeFiles: false,
|
||||
});
|
||||
if (!repo) return;
|
||||
let redirectURL = null;
|
||||
if (
|
||||
repo.status == "expired" &&
|
||||
repo.options.expirationMode == "redirect" &&
|
||||
repo.model.source.repositoryName
|
||||
) {
|
||||
redirectURL = `https://github.com/${repo.model.source.repositoryName}`;
|
||||
} else {
|
||||
if (
|
||||
repo.status == "expired" ||
|
||||
repo.status == "expiring" ||
|
||||
repo.status == "removing" ||
|
||||
repo.status == "removed"
|
||||
) {
|
||||
throw new AnonymousError("repository_expired", {
|
||||
object: repo,
|
||||
httpStatus: 410,
|
||||
});
|
||||
}
|
||||
|
||||
const fiveMinuteAgo = new Date();
|
||||
fiveMinuteAgo.setMinutes(fiveMinuteAgo.getMinutes() - 5);
|
||||
if (repo.status != "ready") {
|
||||
if (
|
||||
repo.model.statusDate < fiveMinuteAgo
|
||||
// && repo.status != "preparing"
|
||||
) {
|
||||
await repo.updateStatus(RepositoryStatus.PREPARING);
|
||||
await downloadQueue.add(repo.repoId, repo, {
|
||||
jobId: repo.repoId,
|
||||
attempts: 3,
|
||||
});
|
||||
}
|
||||
if (repo.status == "error") {
|
||||
throw new AnonymousError(
|
||||
repo.model.statusMessage
|
||||
? repo.model.statusMessage
|
||||
: "repository_not_available",
|
||||
{
|
||||
object: repo,
|
||||
httpStatus: 500,
|
||||
}
|
||||
);
|
||||
}
|
||||
throw new AnonymousError("repository_not_ready", {
|
||||
httpStatus: 404,
|
||||
object: repo,
|
||||
});
|
||||
}
|
||||
|
||||
await repo.updateIfNeeded();
|
||||
}
|
||||
|
||||
let download = false;
|
||||
const conference = await repo.conference();
|
||||
if (conference) {
|
||||
download =
|
||||
conference.quota.size > -1 &&
|
||||
!!config.ENABLE_DOWNLOAD &&
|
||||
repo.source.type == "GitHubDownload";
|
||||
}
|
||||
if (
|
||||
repo.size.storage < config.FREE_DOWNLOAD_REPO_SIZE * 1024 &&
|
||||
repo.source.type == "GitHubDownload"
|
||||
) {
|
||||
download = true;
|
||||
}
|
||||
|
||||
res.json({
|
||||
url: redirectURL,
|
||||
download,
|
||||
lastUpdateDate: repo.model.source.commitDate
|
||||
? repo.model.source.commitDate
|
||||
: repo.model.anonymizeDate,
|
||||
});
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
export default router;
|
||||
153
src/server/routes/route-utils.ts
Normal file
153
src/server/routes/route-utils.ts
Normal file
@@ -0,0 +1,153 @@
|
||||
import * as express from "express";
|
||||
import AnonymousError from "../../core/AnonymousError";
|
||||
import * as db from "../database";
|
||||
import UserModel from "../../core/model/users/users.model";
|
||||
import User from "../../core/User";
|
||||
import { HTTPError } from "got";
|
||||
|
||||
export async function getPullRequest(
|
||||
req: express.Request,
|
||||
res: express.Response,
|
||||
opt?: { nocheck?: boolean }
|
||||
) {
|
||||
try {
|
||||
const pullRequest = await db.getPullRequest(req.params.pullRequestId);
|
||||
if (opt?.nocheck == true) {
|
||||
} else {
|
||||
// redirect if the repository is expired
|
||||
if (
|
||||
pullRequest.status == "expired" &&
|
||||
pullRequest.options.expirationMode == "redirect"
|
||||
) {
|
||||
res.redirect(
|
||||
`http://github.com/${pullRequest.source.repositoryFullName}/pull/${pullRequest.source.pullRequestId}`
|
||||
);
|
||||
return null;
|
||||
}
|
||||
|
||||
pullRequest.check();
|
||||
}
|
||||
return pullRequest;
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function getRepo(
|
||||
req: express.Request,
|
||||
res: express.Response,
|
||||
opt: { nocheck?: boolean; includeFiles?: boolean } = {
|
||||
nocheck: false,
|
||||
includeFiles: false,
|
||||
}
|
||||
) {
|
||||
try {
|
||||
const repo = await db.getRepository(req.params.repoId, {
|
||||
includeFiles: opt.includeFiles === true,
|
||||
});
|
||||
if (opt.nocheck == true) {
|
||||
} else {
|
||||
// redirect if the repository is expired
|
||||
if (
|
||||
repo.status == "expired" &&
|
||||
repo.options.expirationMode == "redirect" &&
|
||||
repo.model.source.repositoryId
|
||||
) {
|
||||
res.redirect(`https://github.com/${repo.model.source.repositoryName}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
repo.check();
|
||||
}
|
||||
return repo;
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function isOwnerOrAdmin(authorizedUsers: string[], user: User) {
|
||||
if (authorizedUsers.indexOf(user.model.id) == -1 && !user.isAdmin) {
|
||||
throw new AnonymousError("not_authorized", {
|
||||
httpStatus: 401,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function printError(error: any, req?: express.Request) {
|
||||
if (error instanceof AnonymousError) {
|
||||
let message = `[ERROR] ${error.toString()} ${error.stack
|
||||
?.split("\n")[1]
|
||||
.trim()}`;
|
||||
if (req) {
|
||||
message += ` ${req.originalUrl}`;
|
||||
// ignore common error
|
||||
if (req.originalUrl === "/api/repo/undefined/options") return;
|
||||
}
|
||||
console.error(message);
|
||||
} else if (error instanceof HTTPError) {
|
||||
let message = `[ERROR] HTTP.${
|
||||
error.code
|
||||
} ${error.message.toString()} ${error.stack?.split("\n")[1].trim()}`;
|
||||
console.error(message);
|
||||
} else if (error instanceof Error) {
|
||||
console.error(error);
|
||||
} else {
|
||||
console.error(error);
|
||||
}
|
||||
}
|
||||
|
||||
export function handleError(
|
||||
error: any,
|
||||
res?: express.Response,
|
||||
req?: express.Request
|
||||
) {
|
||||
printError(error, req);
|
||||
let message = error;
|
||||
if (error instanceof Error) {
|
||||
message = error.message;
|
||||
}
|
||||
let status = 500;
|
||||
if (error.httpStatus) {
|
||||
status = error.httpStatus;
|
||||
} else if (error.$metadata?.httpStatusCode) {
|
||||
status = error.$metadata.httpStatusCode;
|
||||
} else if (
|
||||
message &&
|
||||
(message.indexOf("not_found") > -1 || message.indexOf("(Not Found)") > -1)
|
||||
) {
|
||||
status = 404;
|
||||
} else if (message && message.indexOf("not_connected") > -1) {
|
||||
status = 401;
|
||||
}
|
||||
if (res && !res.headersSent) {
|
||||
res.status(status).send({ error: message });
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
export async function getUser(req: express.Request) {
|
||||
function notConnected(): never {
|
||||
req.logout((error) => {
|
||||
if (error) {
|
||||
console.error(`[ERROR] Error while logging out: ${error}`);
|
||||
}
|
||||
});
|
||||
throw new AnonymousError("not_connected", {
|
||||
httpStatus: 401,
|
||||
});
|
||||
}
|
||||
if (!req.user) {
|
||||
notConnected();
|
||||
}
|
||||
const user = (req.user as any).user;
|
||||
if (!user) {
|
||||
notConnected();
|
||||
}
|
||||
const model = await UserModel.findById(user._id);
|
||||
if (!model) {
|
||||
notConnected();
|
||||
}
|
||||
return new User(model);
|
||||
}
|
||||
162
src/server/routes/user.ts
Normal file
162
src/server/routes/user.ts
Normal file
@@ -0,0 +1,162 @@
|
||||
import * as express from "express";
|
||||
import config from "../../config";
|
||||
import { ensureAuthenticated } from "./connection";
|
||||
import { handleError, getUser, isOwnerOrAdmin } from "./route-utils";
|
||||
import UserModel from "../../core/model/users/users.model";
|
||||
import User from "../../core/User";
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
// user needs to be connected for all user API
|
||||
router.use(ensureAuthenticated);
|
||||
|
||||
router.get("/logout", async (req: express.Request, res: express.Response) => {
|
||||
try {
|
||||
req.logout((error) => {
|
||||
if (error) {
|
||||
console.error(`[ERROR] Logout error: ${error}`);
|
||||
}
|
||||
});
|
||||
res.redirect("/");
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
}
|
||||
});
|
||||
|
||||
router.get("/", async (req: express.Request, res: express.Response) => {
|
||||
try {
|
||||
const user = await getUser(req);
|
||||
res.json({
|
||||
username: user.username,
|
||||
photo: user.photo,
|
||||
isAdmin: user.isAdmin,
|
||||
});
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
}
|
||||
});
|
||||
|
||||
router.get("/quota", async (req: express.Request, res: express.Response) => {
|
||||
try {
|
||||
const user = await getUser(req);
|
||||
const repositories = await user.getRepositories();
|
||||
const sizes = await Promise.all(
|
||||
repositories
|
||||
.filter((r) => r.status == "ready")
|
||||
.map((r) => r.computeSize())
|
||||
);
|
||||
res.json({
|
||||
storage: {
|
||||
used: sizes.reduce((sum, i) => sum + i.storage, 0),
|
||||
total: config.DEFAULT_QUOTA,
|
||||
},
|
||||
file: {
|
||||
used: sizes.reduce((sum, i) => sum + i.file, 0),
|
||||
total: 0,
|
||||
},
|
||||
repository: {
|
||||
used: repositories.filter((f) => f.status == "ready").length,
|
||||
total: 20,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
}
|
||||
});
|
||||
|
||||
router.get("/default", async (req: express.Request, res: express.Response) => {
|
||||
try {
|
||||
const user = await getUser(req);
|
||||
|
||||
res.json(user.default);
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
}
|
||||
});
|
||||
|
||||
router.post("/default", async (req: express.Request, res: express.Response) => {
|
||||
try {
|
||||
const user = await getUser(req);
|
||||
|
||||
const d = req.body;
|
||||
user.model.default = d;
|
||||
|
||||
await user.model.save();
|
||||
res.send("ok");
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
}
|
||||
});
|
||||
|
||||
router.get(
|
||||
"/anonymized_repositories",
|
||||
async (req: express.Request, res: express.Response) => {
|
||||
try {
|
||||
const user = await getUser(req);
|
||||
res.json(
|
||||
(await user.getRepositories()).map((x) => {
|
||||
return x.toJSON();
|
||||
})
|
||||
);
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
}
|
||||
}
|
||||
);
|
||||
router.get(
|
||||
"/anonymized_pull_requests",
|
||||
async (req: express.Request, res: express.Response) => {
|
||||
try {
|
||||
const user = await getUser(req);
|
||||
res.json(
|
||||
(await user.getPullRequests()).map((x) => {
|
||||
return x.toJSON();
|
||||
})
|
||||
);
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
async function getAllRepositories(user: User, force: boolean) {
|
||||
const repos = await user.getGitHubRepositories({
|
||||
force,
|
||||
});
|
||||
return repos.map((x) => {
|
||||
return {
|
||||
fullName: x.fullName,
|
||||
id: x.id,
|
||||
};
|
||||
});
|
||||
}
|
||||
router.get(
|
||||
"/all_repositories",
|
||||
async (req: express.Request, res: express.Response) => {
|
||||
try {
|
||||
const user = await getUser(req);
|
||||
res.json(await getAllRepositories(user, req.query.force == "1"));
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
}
|
||||
}
|
||||
);
|
||||
router.get(
|
||||
"/:username/all_repositories",
|
||||
async (req: express.Request, res: express.Response) => {
|
||||
try {
|
||||
const loggedUser = await getUser(req);
|
||||
isOwnerOrAdmin([req.params.username], loggedUser);
|
||||
const model = await UserModel.findOne({ username: req.params.username });
|
||||
if (!model) {
|
||||
throw new Error("User not found");
|
||||
}
|
||||
const user = new User(model);
|
||||
res.json(await getAllRepositories(user, req.query.force == "1"));
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
export default router;
|
||||
118
src/server/routes/webview.ts
Normal file
118
src/server/routes/webview.ts
Normal file
@@ -0,0 +1,118 @@
|
||||
import * as express from "express";
|
||||
import { getRepo, handleError } from "./route-utils";
|
||||
import * as path from "path";
|
||||
import AnonymizedFile from "../../core/AnonymizedFile";
|
||||
import AnonymousError from "../../core/AnonymousError";
|
||||
import { Tree, TreeElement } from "../../core/types";
|
||||
import * as marked from "marked";
|
||||
import { streamToString } from "../../core/anonymize-utils";
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
const indexPriority = [
|
||||
"index.html",
|
||||
"index.htm",
|
||||
"index.md",
|
||||
"index.txt",
|
||||
"index.org",
|
||||
"index.1st",
|
||||
"index",
|
||||
"readme.md",
|
||||
"readme.txt",
|
||||
"readme.org",
|
||||
"readme.1st",
|
||||
"readme",
|
||||
];
|
||||
|
||||
async function webView(req: express.Request, res: express.Response) {
|
||||
const repo = await getRepo(req, res);
|
||||
if (!repo) return;
|
||||
try {
|
||||
if (!repo.options.page || !repo.options.pageSource) {
|
||||
throw new AnonymousError("page_not_activated", {
|
||||
httpStatus: 400,
|
||||
object: repo,
|
||||
});
|
||||
}
|
||||
|
||||
if (repo.options.pageSource?.branch != repo.model.source.branch) {
|
||||
throw new AnonymousError("page_not_supported_on_different_branch", {
|
||||
httpStatus: 400,
|
||||
object: repo,
|
||||
});
|
||||
}
|
||||
|
||||
let requestPath = path.join(
|
||||
repo.options.pageSource?.path,
|
||||
req.path.substring(
|
||||
req.path.indexOf(req.params.repoId) + req.params.repoId.length
|
||||
)
|
||||
);
|
||||
let f = new AnonymizedFile({
|
||||
repository: repo,
|
||||
anonymizedPath: requestPath,
|
||||
});
|
||||
if (requestPath[requestPath.length - 1] == "/") {
|
||||
// find index file
|
||||
const paths = f.anonymizedPath.trim().split("/");
|
||||
|
||||
let currentAnonymized: TreeElement = await repo.anonymizedFiles({
|
||||
includeSha: true,
|
||||
});
|
||||
for (let i = 0; i < paths.length; i++) {
|
||||
const fileName = paths[i];
|
||||
if (fileName == "") {
|
||||
continue;
|
||||
}
|
||||
if (!(currentAnonymized as Tree)[fileName]) {
|
||||
throw new AnonymousError("file_not_found", {
|
||||
object: repo,
|
||||
httpStatus: 404,
|
||||
});
|
||||
}
|
||||
currentAnonymized = (currentAnonymized as Tree)[fileName];
|
||||
}
|
||||
|
||||
let best_match = null;
|
||||
indexSelector: for (const p of indexPriority) {
|
||||
for (let filename in currentAnonymized) {
|
||||
if (filename.toLowerCase() == p) {
|
||||
best_match = filename;
|
||||
break indexSelector;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (best_match) {
|
||||
requestPath = path.join(requestPath, best_match);
|
||||
f = new AnonymizedFile({
|
||||
repository: repo,
|
||||
anonymizedPath: requestPath,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (!f.isFileSupported()) {
|
||||
throw new AnonymousError("file_not_supported", {
|
||||
httpStatus: 400,
|
||||
object: f,
|
||||
});
|
||||
}
|
||||
if (f.extension() == "md") {
|
||||
const content = await streamToString(await f.anonymizedContent());
|
||||
res
|
||||
.contentType("html")
|
||||
.send(marked.marked(content, { headerIds: false, mangle: false }));
|
||||
} else {
|
||||
f.send(res);
|
||||
}
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
}
|
||||
}
|
||||
|
||||
router.get("/:repoId/*", webView);
|
||||
router.get("/:repoId", (req: express.Request, res: express.Response) => {
|
||||
res.redirect("/w" + req.url + "/");
|
||||
});
|
||||
|
||||
export default router;
|
||||
53
src/server/schedule.ts
Normal file
53
src/server/schedule.ts
Normal file
@@ -0,0 +1,53 @@
|
||||
import * as schedule from "node-schedule";
|
||||
import Conference from "../core/Conference";
|
||||
import AnonymizedRepositoryModel from "../core/model/anonymizedRepositories/anonymizedRepositories.model";
|
||||
import ConferenceModel from "../core/model/conference/conferences.model";
|
||||
import Repository from "../core/Repository";
|
||||
|
||||
export function conferenceStatusCheck() {
|
||||
// check every 6 hours the status of the conferences
|
||||
const job = schedule.scheduleJob("0 */6 * * *", async () => {
|
||||
(await ConferenceModel.find({ status: { $eq: "ready" } })).forEach(
|
||||
async (data) => {
|
||||
const conference = new Conference(data);
|
||||
if (conference.isExpired() && conference.status == "ready") {
|
||||
try {
|
||||
await conference.expire();
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
export function repositoryStatusCheck() {
|
||||
// check every 6 hours the status of the repositories
|
||||
const job = schedule.scheduleJob("0 */6 * * *", async () => {
|
||||
console.log("[schedule] Check repository status and unused repositories");
|
||||
(
|
||||
await AnonymizedRepositoryModel.find({
|
||||
status: { $eq: "ready" },
|
||||
isReseted: { $eq: false },
|
||||
})
|
||||
).forEach((data) => {
|
||||
const repo = new Repository(data);
|
||||
try {
|
||||
repo.check();
|
||||
} catch (error) {
|
||||
console.log(`Repository ${repo.repoId} is expired`);
|
||||
}
|
||||
const fourMonthAgo = new Date();
|
||||
fourMonthAgo.setMonth(fourMonthAgo.getMonth() - 4);
|
||||
|
||||
if (repo.model.lastView < fourMonthAgo) {
|
||||
repo.removeCache().then(() => {
|
||||
console.log(
|
||||
`Repository ${repo.repoId} not visited for 4 months remove the cached files`
|
||||
);
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user