repo change + daily stat improvements

This commit is contained in:
tdurieux
2026-05-11 12:10:17 +03:00
parent b03c4b437c
commit 03e18fd572
10 changed files with 327 additions and 57 deletions
+29
View File
@@ -12,6 +12,10 @@ export interface HomeStats {
nbPullRequests: number;
}
export interface HomeStatsHistoryRow extends HomeStats {
date: Date;
}
export async function computeStats(): Promise<HomeStats> {
const [nbRepositories, nbUsersAgg, nbPageViews, nbPullRequests] =
await Promise.all([
@@ -40,6 +44,31 @@ function utcMidnight(d: Date = new Date()): Date {
);
}
export function mergeCurrentStatsIntoHistory(
rows: HomeStatsHistoryRow[],
currentStats: HomeStats,
now: Date = new Date()
): HomeStatsHistoryRow[] {
const today = utcMidnight(now);
const history = rows.map((row) => ({
...row,
date: new Date(row.date),
}));
const currentRow = { date: today, ...currentStats };
const todayTime = today.getTime();
const todayIndex = history.findIndex(
(row) => utcMidnight(row.date).getTime() === todayTime
);
if (todayIndex >= 0) {
history[todayIndex] = currentRow;
} else {
history.push(currentRow);
}
return history.sort((a, b) => a.date.getTime() - b.date.getTime());
}
export async function computeAndStoreDailyStats(): Promise<void> {
try {
const stats = await computeStats();
+5 -2
View File
@@ -23,6 +23,8 @@ import { startWorker, recoverStuckPreparing } from "../queue";
import {
computeStats,
ensureTodaySnapshot,
HomeStatsHistoryRow,
mergeCurrentStatsIntoHistory,
} from "./dailyStatsSnapshot";
import DailyStatsModel from "../core/model/dailyStats/dailyStats.model";
import { getUser } from "./routes/route-utils";
@@ -238,7 +240,7 @@ export default async function start() {
});
let stat: Record<string, unknown> = {};
let history: Array<Record<string, unknown>> | null = null;
let history: HomeStatsHistoryRow[] | null = null;
let historyKey: number | null = null;
setInterval(() => {
@@ -274,13 +276,14 @@ export default async function start() {
const docs = await DailyStatsModel.find({ date: { $gte: since } })
.sort({ date: 1 })
.lean();
history = docs.map((d) => ({
const rows = docs.map((d) => ({
date: d.date,
nbRepositories: d.nbRepositories,
nbUsers: d.nbUsers,
nbPageViews: d.nbPageViews,
nbPullRequests: d.nbPullRequests,
}));
history = mergeCurrentStatsIntoHistory(rows, await computeStats());
historyKey = days;
res.json(history);
});
+9 -2
View File
@@ -8,6 +8,11 @@ import ConferenceModel from "../../core/model/conference/conferences.model";
import UserModel from "../../core/model/users/users.model";
import { cacheQueue, downloadQueue, removeQueue } from "../../queue";
import { queryMetrics } from "../../queue/queueMetrics";
import {
computeStats,
HomeStatsHistoryRow,
mergeCurrentStatsIntoHistory,
} from "../dailyStatsSnapshot";
import User from "../../core/User";
import { ensureAuthenticated } from "./connection";
import { handleError, getUser, isOwnerOrAdmin, getRepo } from "./route-utils";
@@ -712,7 +717,7 @@ router.get("/overview", async (req, res) => {
}
// Daily history (last 30 days) from DailyStatsModel
let history: Array<Record<string, unknown>> = [];
let history: HomeStatsHistoryRow[] = [];
try {
const { default: DailyStatsModel } = await import(
"../../core/model/dailyStats/dailyStats.model"
@@ -723,12 +728,14 @@ router.get("/overview", async (req, res) => {
const docs = await DailyStatsModel.find({ date: { $gte: since } })
.sort({ date: 1 })
.lean();
history = docs.map((d) => ({
const rows = docs.map((d) => ({
date: d.date,
nbRepositories: d.nbRepositories,
nbUsers: d.nbUsers,
nbPageViews: d.nbPageViews,
nbPullRequests: d.nbPullRequests,
}));
history = mergeCurrentStatsIntoHistory(rows, await computeStats());
} catch {
// DailyStats collection might not exist yet
}
+20 -19
View File
@@ -396,24 +396,7 @@ router.post(
validateNewRepo(repoUpdate);
// Only the commit/branch backs the cached FileModel — anonymization
// options (terms, image/link toggles, etc.) are applied on the fly per
// request. Re-running the download queue is therefore only needed when
// the underlying snapshot moves. Other edits (e.g. turning off
// auto-update — see #360) just persist and return.
const sourceChanged =
repoUpdate.source.commit != repo.model.source.commit ||
repoUpdate.source.branch != repo.model.source.branch;
if (sourceChanged) {
repo.model.anonymizeDate = new Date();
repo.model.source.commit = repoUpdate.source.commit;
await repo.remove();
}
updateRepoModel(repo.model, repoUpdate);
const r = gh(repo.model.source.repositoryName || repoUpdate.fullName);
const r = gh(repoUpdate.fullName);
if (!r?.owner || !r?.name) {
await repo.resetSate(RepositoryStatus.ERROR, "repo_not_found");
throw new AnonymousError("repo_not_found", {
@@ -421,11 +404,22 @@ router.post(
httpStatus: 404,
});
}
// Only the source repository/commit/branch backs the cached FileModel —
// anonymization options (terms, image/link toggles, etc.) are applied on
// the fly per request. Re-running the download queue is therefore only
// needed when the underlying snapshot moves. Other edits (e.g. turning
// off auto-update — see #360) just persist and return.
const sourceChanged =
repoUpdate.source.commit != repo.model.source.commit ||
repoUpdate.source.branch != repo.model.source.branch ||
repoUpdate.fullName != repo.model.source.repositoryName;
updateRepoModel(repo.model, repoUpdate);
const repository = await getRepositoryFromGitHub({
accessToken: user.accessToken,
owner: r.owner,
repo: r.name,
repositoryID: repo.model.source.repositoryId,
});
if (!repository) {
@@ -439,6 +433,13 @@ router.post(
await repository.getCommitInfo(repoUpdate.source.commit, {
accessToken: user.accessToken,
});
repo.model.source.repositoryId = repository.model.id;
repo.model.source.repositoryName = repository.fullName || repoUpdate.fullName;
if (sourceChanged) {
repo.model.anonymizeDate = new Date();
await repo.remove();
}
const removeRepoFromConference = async (conferenceID: string) => {
const conf = await ConferenceModel.findOne({