multiple fixes

src/server/index.ts

@@ -12,6 +12,7 @@ import * as compression from "compression";
 import * as passport from "passport";
 import { connect } from "./database";
 import { initSession, router as connectionRouter } from "./routes/connection";
+import { bearerTokenAuth } from "./routes/token-auth";
 import router from "./routes";
 import AnonymizedRepositoryModel from "../core/model/anonymizedRepositories/anonymizedRepositories.model";
 import { conferenceStatusCheck, repositoryStatusCheck } from "./schedule";
@@ -56,6 +57,7 @@ export default async function start() {
   app.use(initSession());
   app.use(passport.initialize());
   app.use(passport.session());
+  app.use(bearerTokenAuth);
 
   startWorker();
 

src/server/routes/admin-tokens.ts

@@ -0,0 +1,77 @@
+import * as express from "express";
+import { handleError, getUser, isOwnerOrAdmin } from "./route-utils";
+import UserModel from "../../core/model/users/users.model";
+import { generateToken, hashToken } from "./token-auth";
+
+const router = express.Router();
+
+router.use(async (req, res, next) => {
+  try {
+    const user = await getUser(req);
+    isOwnerOrAdmin([], user);
+    next();
+  } catch (error) {
+    handleError(error, res, req);
+  }
+});
+
+router.get("/", async (req, res) => {
+  try {
+    const user = await getUser(req);
+    const model = await UserModel.findById(user.model.id);
+    if (!model) return res.status(404).json({ error: "user_not_found" });
+    const tokens = (model.apiTokens || []).map((t) => ({
+      id: t._id,
+      name: t.name,
+      createdAt: t.createdAt,
+      lastUsedAt: t.lastUsedAt,
+    }));
+    res.json(tokens);
+  } catch (error) {
+    handleError(error, res, req);
+  }
+});
+
+router.post("/", async (req, res) => {
+  try {
+    const user = await getUser(req);
+    const name = (req.body?.name || "").toString().trim() || "unnamed";
+    const plaintext = generateToken();
+    const tokenHash = hashToken(plaintext);
+
+    const model = await UserModel.findById(user.model.id);
+    if (!model) return res.status(404).json({ error: "user_not_found" });
+    if (!model.apiTokens) model.apiTokens = [];
+    model.apiTokens.push({
+      tokenHash,
+      name,
+      createdAt: new Date(),
+    });
+    await model.save();
+
+    const created = model.apiTokens[model.apiTokens.length - 1];
+    res.json({
+      id: created._id,
+      name: created.name,
+      createdAt: created.createdAt,
+      token: plaintext,
+    });
+  } catch (error) {
+    handleError(error, res, req);
+  }
+});
+
+router.delete("/:id", async (req, res) => {
+  try {
+    const user = await getUser(req);
+    const result = await UserModel.updateOne(
+      { _id: user.model.id },
+      { $pull: { apiTokens: { _id: req.params.id } } }
+    );
+    res.json({ removed: result.modifiedCount });
+  } catch (error) {
+    handleError(error, res, req);
+  }
+});
+
+export default router;

src/server/routes/admin.ts

@@ -9,6 +9,7 @@ import Repository from "../../core/Repository";
 import User from "../../core/User";
 import { ensureAuthenticated } from "./connection";
 import { handleError, getUser, isOwnerOrAdmin, getRepo } from "./route-utils";
+import adminTokensRouter from "./admin-tokens";
 
 const router = express.Router();
 
@@ -31,6 +32,8 @@ router.use(
   }
 );
 
+router.use("/tokens", adminTokensRouter);
+
 router.post("/queue/:name/:repo_id", async (req, res) => {
   let queue: Queue<Repository, void>;
   if (req.params.name == "download") {

src/server/routes/connection.ts

@@ -31,13 +31,23 @@ const verify = async (
 ): Promise<void> => {
   let user: IUserDocument | null;
   try {
+    const now = new Date();
     user = await UserModel.findOne({ "externalIDs.github": profile.id });
     if (user) {
-      user.accessTokens.github = accessToken;
+      await UserModel.updateOne(
+        { _id: user._id },
+        {
+          $set: {
+            "accessTokens.github": accessToken,
+            "accessTokenDates.github": now,
+          },
+        }
+      );
       await AnonymizedPullRequestModel.updateMany(
         { owner: user._id },
         { "source.accessToken": accessToken }
       );
+      user = await UserModel.findById(user._id);
     } else {
       // Check if a user with this username already exists (e.g. created
       // manually without externalIDs.github). Link the GitHub ID to the
@@ -45,8 +55,17 @@ const verify = async (
       // the isAdmin flag.
       user = await UserModel.findOne({ username: profile.username });
       if (user) {
-        user.externalIDs.github = profile.id;
-        user.accessTokens.github = accessToken;
+        await UserModel.updateOne(
+          { _id: user._id },
+          {
+            $set: {
+              "externalIDs.github": profile.id,
+              "accessTokens.github": accessToken,
+              "accessTokenDates.github": now,
+            },
+          }
+        );
+        user = await UserModel.findById(user._id);
       } else {
         const photo = profile.photos ? profile.photos[0]?.value : null;
         user = new UserModel({
@@ -54,6 +73,9 @@ const verify = async (
           accessTokens: {
             github: accessToken,
           },
+          accessTokenDates: {
+            github: now,
+          },
           externalIDs: {
             github: profile.id,
           },
@@ -63,16 +85,9 @@ const verify = async (
           photo,
         });
         if (user.emails?.length) user.emails[0].default = true;
+        await user.save();
       }
     }
-    if (!user.accessTokenDates) {
-      user.accessTokenDates = {
-        github: new Date(),
-      };
-    } else {
-      user.accessTokenDates.github = new Date();
-    }
-    await user.save();
     done(null, {
       username: profile.username,
       accessToken,

src/server/routes/repository-private.ts

@@ -17,7 +17,6 @@ import User from "../../core/User";
 import { RepositoryStatus } from "../../core/types";
 import { IUserDocument } from "../../core/model/users/users.types";
 import { checkToken } from "../../core/GitHubUtils";
-import config from "../../config";
 
 const router = express.Router();
 
@@ -404,20 +403,6 @@ router.post(
           httpStatus: 404,
         });
       }
-      if (repository.size) {
-        if (
-          repository.size > config.AUTO_DOWNLOAD_REPO_SIZE &&
-          repo.model.source.type == "GitHubDownload"
-        ) {
-          repo.model.source.type = "GitHubStream";
-        } else if (
-          repository.size < config.AUTO_DOWNLOAD_REPO_SIZE &&
-          repo.model.source.type == "GitHubStream"
-        ) {
-          repo.model.source.type = "GitHubDownload";
-        }
-      }
-
       const removeRepoFromConference = async (conferenceID: string) => {
         const conf = await ConferenceModel.findOne({
           conferenceID,
@@ -528,25 +513,6 @@ router.post("/", async (req: express.Request, res: express.Response) => {
     repo.source.accessToken = user.accessToken;
     repo.source.repositoryId = repository.model.id;
     repo.source.repositoryName = repoUpdate.fullName;
-    if (
-      repository.size !== undefined &&
-      repository.size < config.AUTO_DOWNLOAD_REPO_SIZE
-    ) {
-      repo.source.type = "GitHubDownload";
-    }
-    if (repository.size) {
-      if (
-        repository.size > config.AUTO_DOWNLOAD_REPO_SIZE &&
-        repo.source.type == "GitHubDownload"
-      ) {
-        repo.source.type = "GitHubStream";
-      } else if (
-        repository.size < config.AUTO_DOWNLOAD_REPO_SIZE &&
-        repo.source.type == "GitHubStream"
-      ) {
-        repo.source.type = "GitHubDownload";
-      }
-    }
 
     repo.conference = repoUpdate.conference;
 

src/server/routes/repository-public.ts

@@ -1,6 +1,4 @@
-import { promisify } from "util";
 import * as express from "express";
-import * as stream from "stream";
 import config from "../../config";
 import got from "got";
 import { join } from "path";
@@ -10,14 +8,14 @@ import AnonymousError from "../../core/AnonymousError";
 import { downloadQueue } from "../../queue";
 import { RepositoryStatus } from "../../core/types";
 import User from "../../core/User";
+import { streamAnonymizedZip } from "../../core/zipStream";
+import gh = require("parse-github-url");
 
 const router = express.Router();
 
 router.get(
   "/:repoId/zip",
   async (req: express.Request, res: express.Response) => {
-    const pipeline = promisify(stream.pipeline);
-
     try {
       if (!config.ENABLE_DOWNLOAD) {
         throw new AnonymousError("download_not_enabled", {
@@ -87,10 +85,28 @@ router.get(
       }
 
       res.attachment(`${repo.repoId}.zip`);
-
       // cache the file for 6 hours
       res.header("Cache-Control", "max-age=21600");
-      await pipeline(await repo.zip(), res);
+
+      const parsed = gh(repo.model.source.repositoryName || "");
+      if (!parsed?.owner || !parsed?.name) {
+        throw new AnonymousError("repo_not_found", {
+          httpStatus: 404,
+          object: repo.model.source.repositoryName,
+        });
+      }
+      const anonymizer = repo.generateAnonymizeTransformer("");
+      await streamAnonymizedZip(
+        {
+          repoId: repo.repoId,
+          organization: parsed.owner,
+          repoName: parsed.name,
+          commit: repo.model.source.commit || "HEAD",
+          getToken: () => repo.getToken(),
+          anonymizerOptions: anonymizer.opt,
+        },
+        res
+      );
     } catch (error) {
       handleError(error, res, req);
     }
@@ -197,6 +213,7 @@ router.get(
         isAdmin: user?.isAdmin === true,
         isOwner: user?.id == repo.model.owner,
         hasWebsite: !!repo.options.page && !!repo.options.pageSource,
+        truncatedFolders: repo.model.truncatedFolders || [],
       });
     } catch (error) {
       handleError(error, res, req);

src/server/routes/token-auth.ts

@@ -0,0 +1,46 @@
+import * as express from "express";
+import * as crypto from "crypto";
+import UserModel from "../../core/model/users/users.model";
+
+export function hashToken(token: string): string {
+  return crypto.createHash("sha256").update(token).digest("hex");
+}
+
+export function generateToken(): string {
+  return crypto.randomBytes(32).toString("hex");
+}
+
+export async function bearerTokenAuth(
+  req: express.Request,
+  _res: express.Response,
+  next: express.NextFunction
+): Promise<void> {
+  if (req.user) return next();
+
+  const header = req.headers["authorization"];
+  if (!header || typeof header !== "string") return next();
+  const match = header.match(/^Bearer\s+(.+)$/i);
+  if (!match) return next();
+
+  const tokenHash = hashToken(match[1].trim());
+  try {
+    const model = await UserModel.findOne({ "apiTokens.tokenHash": tokenHash });
+    if (!model) return next();
+
+    // Mirror the shape produced by passport's verify() in connection.ts
+    // so existing getUser()/route code works unchanged.
+    req.user = {
+      username: model.username,
+      user: model,
+    } as Express.User;
+
+    // fire-and-forget last-used update
+    UserModel.updateOne(
+      { _id: model._id, "apiTokens.tokenHash": tokenHash },
+      { $set: { "apiTokens.$.lastUsedAt": new Date() } }
+    ).catch((err) => console.error("[token-auth] lastUsedAt update failed", err));
+  } catch (err) {
+    console.error("[token-auth] lookup failed", err);
+  }
+  return next();
+}