+
+ New users · 30d
+ users/day
+
+
diff --git a/public/script/admin.js b/public/script/admin.js
index b824dba..28d611c 100644
--- a/public/script/admin.js
+++ b/public/script/admin.js
@@ -1826,6 +1826,27 @@ angular
return ($scope.data.errors.severity[key] / max) * 100;
};
+ function computeDailyHistory(history) {
+ var rows = history || [];
+ return rows.map(function (d, i) {
+ var previous = rows[i - 1] || {};
+ var row = Object.assign({}, d);
+ row.dailyRepositories = i ? Math.max(0, (d.nbRepositories || 0) - (previous.nbRepositories || 0)) : 0;
+ row.dailyUsers = i ? Math.max(0, (d.nbUsers || 0) - (previous.nbUsers || 0)) : 0;
+ row.dailyPageViews = i ? Math.max(0, (d.nbPageViews || 0) - (previous.nbPageViews || 0)) : 0;
+ return row;
+ });
+ }
+
+ function todayDailyStats(history) {
+ var latest = history && history.length ? history[history.length - 1] : {};
+ return {
+ repositories: latest.dailyRepositories || 0,
+ users: latest.dailyUsers || 0,
+ pageViews: latest.dailyPageViews || 0,
+ };
+ }
+
var historyMaxes = {};
$scope.historyBarH = function (d, field) {
if (!d || !historyMaxes[field]) return 0;
@@ -1839,12 +1860,16 @@ angular
function load() {
$http.get("/api/admin/overview").then(function (r) {
+ r.data.history = computeDailyHistory(r.data.history);
+ r.data.daily = {
+ today: todayDailyStats(r.data.history),
+ };
$scope.data = r.data;
$scope.loading = false;
$scope.error = null;
historyMaxes = {};
(r.data.history || []).forEach(function (d) {
- ["nbPageViews", "nbRepositories", "nbUsers"].forEach(function (k) {
+ ["dailyPageViews", "dailyRepositories", "dailyUsers"].forEach(function (k) {
if (!historyMaxes[k] || d[k] > historyMaxes[k]) historyMaxes[k] = d[k];
});
});
diff --git a/public/script/app.js b/public/script/app.js
index f8b9d71..fddee23 100644
--- a/public/script/app.js
+++ b/public/script/app.js
@@ -1686,6 +1686,24 @@ angular
}
}
+ function parseRepoFullName(url) {
+ try {
+ const parsed = parseGithubUrl(url);
+ if (parsed && parsed.owner && parsed.repo) {
+ return parsed.owner + "/" + parsed.repo;
+ }
+ } catch (_) { /* sourceUrl not yet parseable */ }
+ return null;
+ }
+
+ function sourceRepositoryID() {
+ if (!$scope.isUpdate || !$scope._originalRepositoryID) return undefined;
+ const currentFullName = parseRepoFullName($scope.sourceUrl);
+ return currentFullName === $scope._originalFullName
+ ? $scope._originalRepositoryID
+ : undefined;
+ }
+
getDefault(() => {
// Edit mode: repo
if ($routeParams.repoId && $routeParams.repoId != "") {
@@ -1695,6 +1713,7 @@ angular
$http.get("/api/repo/" + $scope.repoId).then(
async (res) => {
$scope.sourceUrl = "https://github.com/" + res.data.source.fullName;
+ $scope._originalFullName = res.data.source.fullName;
$scope.terms = res.data.options.terms.filter((f) => f).join("\n");
$scope.source = res.data.source;
$scope.role = res.data.role || "owner";
@@ -1708,6 +1727,7 @@ angular
$scope.options = Object.assign({}, $scope.options, res.data.options);
$scope.conference = res.data.conference;
$scope.repositoryID = res.data.source.repositoryID;
+ $scope._originalRepositoryID = res.data.source.repositoryID;
if (res.data.options.expirationDate) {
$scope.options.expirationDate = new Date(res.data.options.expirationDate);
}
@@ -1719,7 +1739,6 @@ angular
);
$scope.$watch("anonymize", () => {
if ($scope.anonymize.repoId) $scope.anonymize.repoId.$$element[0].disabled = true;
- if ($scope.anonymize.sourceUrl) $scope.anonymize.sourceUrl.$$element[0].disabled = true;
});
}
// Edit mode: PR
@@ -1788,9 +1807,11 @@ angular
// URL change handler - auto-detect type
$scope.urlSelected = async () => {
$scope.terms = $scope.defaultTerms;
- $scope.repoId = "";
- $scope.pullRequestId = "";
- $scope.gistId = "";
+ if (!$scope.isUpdate) {
+ $scope.repoId = "";
+ $scope.pullRequestId = "";
+ $scope.gistId = "";
+ }
$scope.details = null;
$scope.branches = [];
$scope.source = { type: "GitHubStream", branch: "", commit: "" };
@@ -1862,7 +1883,7 @@ angular
const o = parseGithubUrl($scope.sourceUrl);
try {
const branches = await $http.get(`/api/repo/${o.owner}/${o.repo}/branches`, {
- params: { force: force === true ? "1" : "0", repositoryID: $scope.repositoryID },
+ params: { force: force === true ? "1" : "0", repositoryID: sourceRepositoryID() },
});
$scope.branches = branches.data;
$scope.sourceUnreachable = false;
@@ -1910,9 +1931,12 @@ angular
// #364) are reflected without waiting for the cached metadata to
// expire. The endpoint hits the GitHub API once.
const res = await $http.get(`/api/repo/${o.owner}/${o.repo}/`, {
- params: { repositoryID: $scope.repositoryID, force: "1" },
+ params: { repositoryID: sourceRepositoryID(), force: "1" },
});
$scope.details = res.data;
+ if ($scope.details && $scope.details.id) {
+ $scope.repositoryID = $scope.details.id;
+ }
if (!$scope.repoId) {
$scope.repoId = $scope.details.repo + "-" + generateRandomId(4);
}
@@ -1935,7 +1959,7 @@ angular
const o = parseGithubUrl($scope.sourceUrl);
try {
const res = await $http.get(`/api/repo/${o.owner}/${o.repo}/readme`, {
- params: { force: force === true ? "1" : "0", branch: $scope.source.branch, repositoryID: $scope.repositoryID },
+ params: { force: force === true ? "1" : "0", branch: $scope.source.branch, repositoryID: sourceRepositoryID() },
});
$scope.readme = res.data;
} catch (error) {
diff --git a/src/core/source/GitHubStream.ts b/src/core/source/GitHubStream.ts
index 4c28359..0e765a3 100644
--- a/src/core/source/GitHubStream.ts
+++ b/src/core/source/GitHubStream.ts
@@ -457,25 +457,66 @@ export default class GitHubStream extends GitHubBase {
});
}
}
+
+ const fetchSubtree = async (
+ entry: { sha: string; parentPath: string }
+ ) => {
+ const data = await this.getGHTree(oct, token, entry.sha, count, {
+ recursive: true,
+ callback: () => {
+ if (progress) {
+ progress("List file: " + count.file);
+ }
+ },
+ });
+ if (!data.truncated) {
+ return this.tree2Tree(data.tree, entry.parentPath);
+ }
+ // Subtree was truncated — break it down by fetching non-recursively
+ // and then recursing into each child subtree individually.
+ logger.info(
+ `Tree truncated for ${entry.parentPath}, breaking down into subtrees`
+ );
+ const shallow = await this.getGHTree(oct, token, entry.sha, count, {
+ recursive: false,
+ callback: () => {
+ if (progress) {
+ progress("List file: " + count.file);
+ }
+ },
+ });
+ if (shallow.truncated) {
+ this._truncatedFolders.push(entry.parentPath);
+ }
+ const files = this.tree2Tree(shallow.tree, entry.parentPath);
+ const childSubtrees = shallow.tree
+ .filter(
+ (f): f is typeof f & { sha: string; path: string } =>
+ f.type === "tree" && !!f.path && !!f.sha
+ )
+ .map((f) => ({
+ sha: f.sha,
+ parentPath: path.join(entry.parentPath, f.path),
+ }));
+ const childResults = await pMap(
+ childSubtrees,
+ (child) => fetchSubtree(child),
+ GH_API_CONCURRENCY
+ );
+ for (const childFiles of childResults) {
+ files.push(...childFiles);
+ }
+ return files;
+ };
+
const results = await pMap(
subtrees,
- async (entry) =>
- this.getGHTree(oct, token, entry.sha, count, {
- recursive: true,
- callback: () => {
- if (progress) {
- progress("List file: " + count.file);
- }
- },
- }),
+ (entry) => fetchSubtree(entry),
GH_API_CONCURRENCY
);
- results.forEach((data, i) => {
- if (data.truncated) {
- this._truncatedFolders.push(subtrees[i].parentPath);
- }
- output.push(...this.tree2Tree(data.tree, subtrees[i].parentPath));
- });
+ for (const files of results) {
+ output.push(...files);
+ }
return output;
}
diff --git a/src/server/dailyStatsSnapshot.ts b/src/server/dailyStatsSnapshot.ts
index 2b8aae6..675a3d2 100644
--- a/src/server/dailyStatsSnapshot.ts
+++ b/src/server/dailyStatsSnapshot.ts
@@ -12,6 +12,10 @@ export interface HomeStats {
nbPullRequests: number;
}
+export interface HomeStatsHistoryRow extends HomeStats {
+ date: Date;
+}
+
export async function computeStats(): Promise {
const [nbRepositories, nbUsersAgg, nbPageViews, nbPullRequests] =
await Promise.all([
@@ -40,6 +44,31 @@ function utcMidnight(d: Date = new Date()): Date {
);
}
+export function mergeCurrentStatsIntoHistory(
+ rows: HomeStatsHistoryRow[],
+ currentStats: HomeStats,
+ now: Date = new Date()
+): HomeStatsHistoryRow[] {
+ const today = utcMidnight(now);
+ const history = rows.map((row) => ({
+ ...row,
+ date: new Date(row.date),
+ }));
+ const currentRow = { date: today, ...currentStats };
+ const todayTime = today.getTime();
+ const todayIndex = history.findIndex(
+ (row) => utcMidnight(row.date).getTime() === todayTime
+ );
+
+ if (todayIndex >= 0) {
+ history[todayIndex] = currentRow;
+ } else {
+ history.push(currentRow);
+ }
+
+ return history.sort((a, b) => a.date.getTime() - b.date.getTime());
+}
+
export async function computeAndStoreDailyStats(): Promise {
try {
const stats = await computeStats();
diff --git a/src/server/index.ts b/src/server/index.ts
index addeca2..f83b977 100644
--- a/src/server/index.ts
+++ b/src/server/index.ts
@@ -23,6 +23,8 @@ import { startWorker, recoverStuckPreparing } from "../queue";
import {
computeStats,
ensureTodaySnapshot,
+ HomeStatsHistoryRow,
+ mergeCurrentStatsIntoHistory,
} from "./dailyStatsSnapshot";
import DailyStatsModel from "../core/model/dailyStats/dailyStats.model";
import { getUser } from "./routes/route-utils";
@@ -238,7 +240,7 @@ export default async function start() {
});
let stat: Record = {};
- let history: Array> | null = null;
+ let history: HomeStatsHistoryRow[] | null = null;
let historyKey: number | null = null;
setInterval(() => {
@@ -274,13 +276,14 @@ export default async function start() {
const docs = await DailyStatsModel.find({ date: { $gte: since } })
.sort({ date: 1 })
.lean();
- history = docs.map((d) => ({
+ const rows = docs.map((d) => ({
date: d.date,
nbRepositories: d.nbRepositories,
nbUsers: d.nbUsers,
nbPageViews: d.nbPageViews,
nbPullRequests: d.nbPullRequests,
}));
+ history = mergeCurrentStatsIntoHistory(rows, await computeStats());
historyKey = days;
res.json(history);
});
diff --git a/src/server/routes/admin.ts b/src/server/routes/admin.ts
index 59dc4f1..ea6daa0 100644
--- a/src/server/routes/admin.ts
+++ b/src/server/routes/admin.ts
@@ -8,6 +8,11 @@ import ConferenceModel from "../../core/model/conference/conferences.model";
import UserModel from "../../core/model/users/users.model";
import { cacheQueue, downloadQueue, removeQueue } from "../../queue";
import { queryMetrics } from "../../queue/queueMetrics";
+import {
+ computeStats,
+ HomeStatsHistoryRow,
+ mergeCurrentStatsIntoHistory,
+} from "../dailyStatsSnapshot";
import User from "../../core/User";
import { ensureAuthenticated } from "./connection";
import { handleError, getUser, isOwnerOrAdmin, getRepo } from "./route-utils";
@@ -712,7 +717,7 @@ router.get("/overview", async (req, res) => {
}
// Daily history (last 30 days) from DailyStatsModel
- let history: Array> = [];
+ let history: HomeStatsHistoryRow[] = [];
try {
const { default: DailyStatsModel } = await import(
"../../core/model/dailyStats/dailyStats.model"
@@ -723,12 +728,14 @@ router.get("/overview", async (req, res) => {
const docs = await DailyStatsModel.find({ date: { $gte: since } })
.sort({ date: 1 })
.lean();
- history = docs.map((d) => ({
+ const rows = docs.map((d) => ({
date: d.date,
nbRepositories: d.nbRepositories,
nbUsers: d.nbUsers,
nbPageViews: d.nbPageViews,
+ nbPullRequests: d.nbPullRequests,
}));
+ history = mergeCurrentStatsIntoHistory(rows, await computeStats());
} catch {
// DailyStats collection might not exist yet
}
diff --git a/src/server/routes/repository-private.ts b/src/server/routes/repository-private.ts
index d88dbce..70c1f5d 100644
--- a/src/server/routes/repository-private.ts
+++ b/src/server/routes/repository-private.ts
@@ -396,24 +396,7 @@ router.post(
validateNewRepo(repoUpdate);
- // Only the commit/branch backs the cached FileModel — anonymization
- // options (terms, image/link toggles, etc.) are applied on the fly per
- // request. Re-running the download queue is therefore only needed when
- // the underlying snapshot moves. Other edits (e.g. turning off
- // auto-update — see #360) just persist and return.
- const sourceChanged =
- repoUpdate.source.commit != repo.model.source.commit ||
- repoUpdate.source.branch != repo.model.source.branch;
-
- if (sourceChanged) {
- repo.model.anonymizeDate = new Date();
- repo.model.source.commit = repoUpdate.source.commit;
- await repo.remove();
- }
-
- updateRepoModel(repo.model, repoUpdate);
-
- const r = gh(repo.model.source.repositoryName || repoUpdate.fullName);
+ const r = gh(repoUpdate.fullName);
if (!r?.owner || !r?.name) {
await repo.resetSate(RepositoryStatus.ERROR, "repo_not_found");
throw new AnonymousError("repo_not_found", {
@@ -421,11 +404,22 @@ router.post(
httpStatus: 404,
});
}
+
+ // Only the source repository/commit/branch backs the cached FileModel —
+ // anonymization options (terms, image/link toggles, etc.) are applied on
+ // the fly per request. Re-running the download queue is therefore only
+ // needed when the underlying snapshot moves. Other edits (e.g. turning
+ // off auto-update — see #360) just persist and return.
+ const sourceChanged =
+ repoUpdate.source.commit != repo.model.source.commit ||
+ repoUpdate.source.branch != repo.model.source.branch ||
+ repoUpdate.fullName != repo.model.source.repositoryName;
+
+ updateRepoModel(repo.model, repoUpdate);
const repository = await getRepositoryFromGitHub({
accessToken: user.accessToken,
owner: r.owner,
repo: r.name,
- repositoryID: repo.model.source.repositoryId,
});
if (!repository) {
@@ -439,6 +433,13 @@ router.post(
await repository.getCommitInfo(repoUpdate.source.commit, {
accessToken: user.accessToken,
});
+ repo.model.source.repositoryId = repository.model.id;
+ repo.model.source.repositoryName = repository.fullName || repoUpdate.fullName;
+
+ if (sourceChanged) {
+ repo.model.anonymizeDate = new Date();
+ await repo.remove();
+ }
const removeRepoFromConference = async (conferenceID: string) => {
const conf = await ConferenceModel.findOne({
diff --git a/test/daily-stats-history.test.js b/test/daily-stats-history.test.js
new file mode 100644
index 0000000..e85b97e
--- /dev/null
+++ b/test/daily-stats-history.test.js
@@ -0,0 +1,64 @@
+require("ts-node/register/transpile-only");
+
+const { expect } = require("chai");
+const {
+ mergeCurrentStatsIntoHistory,
+} = require("../src/server/dailyStatsSnapshot");
+
+describe("daily stats history", function () {
+ const liveStats = {
+ nbRepositories: 15,
+ nbUsers: 7,
+ nbPageViews: 120,
+ nbPullRequests: 4,
+ };
+
+ it("replaces today's stored snapshot with live totals", function () {
+ const rows = [
+ {
+ date: new Date("2026-05-10T00:00:00.000Z"),
+ nbRepositories: 10,
+ nbUsers: 5,
+ nbPageViews: 100,
+ nbPullRequests: 2,
+ },
+ {
+ date: new Date("2026-05-11T00:05:00.000Z"),
+ nbRepositories: 11,
+ nbUsers: 5,
+ nbPageViews: 101,
+ nbPullRequests: 2,
+ },
+ ];
+
+ const history = mergeCurrentStatsIntoHistory(
+ rows,
+ liveStats,
+ new Date("2026-05-11T14:30:00.000Z")
+ );
+
+ expect(history).to.have.length(2);
+ expect(history[1]).to.deep.include(liveStats);
+ expect(history[1].date.toISOString()).to.equal("2026-05-11T00:00:00.000Z");
+ });
+
+ it("appends today's live totals when no snapshot exists", function () {
+ const history = mergeCurrentStatsIntoHistory(
+ [
+ {
+ date: new Date("2026-05-10T00:00:00.000Z"),
+ nbRepositories: 10,
+ nbUsers: 5,
+ nbPageViews: 100,
+ nbPullRequests: 2,
+ },
+ ],
+ liveStats,
+ new Date("2026-05-11T14:30:00.000Z")
+ );
+
+ expect(history).to.have.length(2);
+ expect(history[1]).to.deep.include(liveStats);
+ expect(history[1].date.toISOString()).to.equal("2026-05-11T00:00:00.000Z");
+ });
+});