repo change + daily stat improvements

src/core/source/GitHubStream.ts

@@ -457,25 +457,66 @@ export default class GitHubStream extends GitHubBase {
         });
       }
     }
+
+    const fetchSubtree = async (
+      entry: { sha: string; parentPath: string }
+    ) => {
+      const data = await this.getGHTree(oct, token, entry.sha, count, {
+        recursive: true,
+        callback: () => {
+          if (progress) {
+            progress("List file: " + count.file);
+          }
+        },
+      });
+      if (!data.truncated) {
+        return this.tree2Tree(data.tree, entry.parentPath);
+      }
+      // Subtree was truncated — break it down by fetching non-recursively
+      // and then recursing into each child subtree individually.
+      logger.info(
+        `Tree truncated for ${entry.parentPath}, breaking down into subtrees`
+      );
+      const shallow = await this.getGHTree(oct, token, entry.sha, count, {
+        recursive: false,
+        callback: () => {
+          if (progress) {
+            progress("List file: " + count.file);
+          }
+        },
+      });
+      if (shallow.truncated) {
+        this._truncatedFolders.push(entry.parentPath);
+      }
+      const files = this.tree2Tree(shallow.tree, entry.parentPath);
+      const childSubtrees = shallow.tree
+        .filter(
+          (f): f is typeof f & { sha: string; path: string } =>
+            f.type === "tree" && !!f.path && !!f.sha
+        )
+        .map((f) => ({
+          sha: f.sha,
+          parentPath: path.join(entry.parentPath, f.path),
+        }));
+      const childResults = await pMap(
+        childSubtrees,
+        (child) => fetchSubtree(child),
+        GH_API_CONCURRENCY
+      );
+      for (const childFiles of childResults) {
+        files.push(...childFiles);
+      }
+      return files;
+    };
+
     const results = await pMap(
       subtrees,
-      async (entry) =>
-        this.getGHTree(oct, token, entry.sha, count, {
-          recursive: true,
-          callback: () => {
-            if (progress) {
-              progress("List file: " + count.file);
-            }
-          },
-        }),
+      (entry) => fetchSubtree(entry),
       GH_API_CONCURRENCY
     );
-    results.forEach((data, i) => {
-      if (data.truncated) {
-        this._truncatedFolders.push(subtrees[i].parentPath);
-      }
-      output.push(...this.tree2Tree(data.tree, subtrees[i].parentPath));
-    });
+    for (const files of results) {
+      output.push(...files);
+    }
     return output;
   }
 

src/server/dailyStatsSnapshot.ts

@@ -12,6 +12,10 @@ export interface HomeStats {
   nbPullRequests: number;
 }
 
+export interface HomeStatsHistoryRow extends HomeStats {
+  date: Date;
+}
+
 export async function computeStats(): Promise<HomeStats> {
   const [nbRepositories, nbUsersAgg, nbPageViews, nbPullRequests] =
     await Promise.all([
@@ -40,6 +44,31 @@ function utcMidnight(d: Date = new Date()): Date {
   );
 }
 
+export function mergeCurrentStatsIntoHistory(
+  rows: HomeStatsHistoryRow[],
+  currentStats: HomeStats,
+  now: Date = new Date()
+): HomeStatsHistoryRow[] {
+  const today = utcMidnight(now);
+  const history = rows.map((row) => ({
+    ...row,
+    date: new Date(row.date),
+  }));
+  const currentRow = { date: today, ...currentStats };
+  const todayTime = today.getTime();
+  const todayIndex = history.findIndex(
+    (row) => utcMidnight(row.date).getTime() === todayTime
+  );
+
+  if (todayIndex >= 0) {
+    history[todayIndex] = currentRow;
+  } else {
+    history.push(currentRow);
+  }
+
+  return history.sort((a, b) => a.date.getTime() - b.date.getTime());
+}
+
 export async function computeAndStoreDailyStats(): Promise<void> {
   try {
     const stats = await computeStats();

src/server/index.ts

@@ -23,6 +23,8 @@ import { startWorker, recoverStuckPreparing } from "../queue";
 import {
   computeStats,
   ensureTodaySnapshot,
+  HomeStatsHistoryRow,
+  mergeCurrentStatsIntoHistory,
 } from "./dailyStatsSnapshot";
 import DailyStatsModel from "../core/model/dailyStats/dailyStats.model";
 import { getUser } from "./routes/route-utils";
@@ -238,7 +240,7 @@ export default async function start() {
   });
 
   let stat: Record<string, unknown> = {};
-  let history: Array<Record<string, unknown>> | null = null;
+  let history: HomeStatsHistoryRow[] | null = null;
   let historyKey: number | null = null;
 
   setInterval(() => {
@@ -274,13 +276,14 @@ export default async function start() {
     const docs = await DailyStatsModel.find({ date: { $gte: since } })
       .sort({ date: 1 })
       .lean();
-    history = docs.map((d) => ({
+    const rows = docs.map((d) => ({
       date: d.date,
       nbRepositories: d.nbRepositories,
       nbUsers: d.nbUsers,
       nbPageViews: d.nbPageViews,
       nbPullRequests: d.nbPullRequests,
     }));
+    history = mergeCurrentStatsIntoHistory(rows, await computeStats());
     historyKey = days;
     res.json(history);
   });

src/server/routes/admin.ts

@@ -8,6 +8,11 @@ import ConferenceModel from "../../core/model/conference/conferences.model";
 import UserModel from "../../core/model/users/users.model";
 import { cacheQueue, downloadQueue, removeQueue } from "../../queue";
 import { queryMetrics } from "../../queue/queueMetrics";
+import {
+  computeStats,
+  HomeStatsHistoryRow,
+  mergeCurrentStatsIntoHistory,
+} from "../dailyStatsSnapshot";
 import User from "../../core/User";
 import { ensureAuthenticated } from "./connection";
 import { handleError, getUser, isOwnerOrAdmin, getRepo } from "./route-utils";
@@ -712,7 +717,7 @@ router.get("/overview", async (req, res) => {
     }
 
     // Daily history (last 30 days) from DailyStatsModel
-    let history: Array<Record<string, unknown>> = [];
+    let history: HomeStatsHistoryRow[] = [];
     try {
       const { default: DailyStatsModel } = await import(
         "../../core/model/dailyStats/dailyStats.model"
@@ -723,12 +728,14 @@ router.get("/overview", async (req, res) => {
       const docs = await DailyStatsModel.find({ date: { $gte: since } })
         .sort({ date: 1 })
         .lean();
-      history = docs.map((d) => ({
+      const rows = docs.map((d) => ({
         date: d.date,
         nbRepositories: d.nbRepositories,
         nbUsers: d.nbUsers,
         nbPageViews: d.nbPageViews,
+        nbPullRequests: d.nbPullRequests,
       }));
+      history = mergeCurrentStatsIntoHistory(rows, await computeStats());
     } catch {
       // DailyStats collection might not exist yet
     }

src/server/routes/repository-private.ts

@@ -396,24 +396,7 @@ router.post(
 
       validateNewRepo(repoUpdate);
 
-      // Only the commit/branch backs the cached FileModel — anonymization
-      // options (terms, image/link toggles, etc.) are applied on the fly per
-      // request. Re-running the download queue is therefore only needed when
-      // the underlying snapshot moves. Other edits (e.g. turning off
-      // auto-update — see #360) just persist and return.
-      const sourceChanged =
-        repoUpdate.source.commit != repo.model.source.commit ||
-        repoUpdate.source.branch != repo.model.source.branch;
-
-      if (sourceChanged) {
-        repo.model.anonymizeDate = new Date();
-        repo.model.source.commit = repoUpdate.source.commit;
-        await repo.remove();
-      }
-
-      updateRepoModel(repo.model, repoUpdate);
-
-      const r = gh(repo.model.source.repositoryName || repoUpdate.fullName);
+      const r = gh(repoUpdate.fullName);
       if (!r?.owner || !r?.name) {
         await repo.resetSate(RepositoryStatus.ERROR, "repo_not_found");
         throw new AnonymousError("repo_not_found", {
@@ -421,11 +404,22 @@ router.post(
           httpStatus: 404,
         });
       }
+
+      // Only the source repository/commit/branch backs the cached FileModel —
+      // anonymization options (terms, image/link toggles, etc.) are applied on
+      // the fly per request. Re-running the download queue is therefore only
+      // needed when the underlying snapshot moves. Other edits (e.g. turning
+      // off auto-update — see #360) just persist and return.
+      const sourceChanged =
+        repoUpdate.source.commit != repo.model.source.commit ||
+        repoUpdate.source.branch != repo.model.source.branch ||
+        repoUpdate.fullName != repo.model.source.repositoryName;
+
+      updateRepoModel(repo.model, repoUpdate);
       const repository = await getRepositoryFromGitHub({
         accessToken: user.accessToken,
         owner: r.owner,
         repo: r.name,
-        repositoryID: repo.model.source.repositoryId,
       });
 
       if (!repository) {
@@ -439,6 +433,13 @@ router.post(
       await repository.getCommitInfo(repoUpdate.source.commit, {
         accessToken: user.accessToken,
       });
+      repo.model.source.repositoryId = repository.model.id;
+      repo.model.source.repositoryName = repository.fullName || repoUpdate.fullName;
+
+      if (sourceChanged) {
+        repo.model.anonymizeDate = new Date();
+        await repo.remove();
+      }
 
       const removeRepoFromConference = async (conferenceID: string) => {
         const conf = await ConferenceModel.findOne({