perf: bound request work and resource usage (#866)

* perf: bound request work and resource usage

* fix: address performance review regressions

* fix: guard content expiration and dashboard state

* fix: tolerate cache outages and retire derived paths

* fix: revalidate expiration claims and status pagination

* fix: retry interrupted repository metadata cleanup

* fix: retire obsolete content cache storage

* fix: activate tree metadata and recheck expiration claims

* fix: fence cache publication across expiration

* fix: recover staged trees and enforce cache lifecycle cleanup

* fix: prune expired transforms during database outages

* fix: finish conference removals and reload expiration candidates

src/core/model/anonymized-path.ts

@@ -0,0 +1,11 @@
+import { IFile } from "./files/files.types";
+import { model, Schema } from "mongoose";
+const schema = new Schema({ repoId: { type: String, required: true }, key: { type: String, required: true }, originalId: Schema.Types.ObjectId,
+  name: { type: String, required: true }, path: { type: String, default: "", required: true }, sha: String, size: Number,
+  anonymousName: String, anonymousDirectory: String, anonymousPath: String,
+  searchName: String, searchDirectory: String,
+  createdAt: { type: Date, default: Date.now } });
+schema.index({ repoId: 1, key: 1, anonymousPath: 1, path: 1, name: 1 });
+schema.index({ repoId: 1, key: 1, originalId: 1 }, { unique: true });
+schema.index({ createdAt: 1 }, { expireAfterSeconds: 30 * 24 * 3600 });
+export default model<IFile & { key: string; originalId: Schema.Types.ObjectId; anonymousName: string; anonymousDirectory: string; anonymousPath: string; searchName: string; searchDirectory: string; createdAt: Date }>("AnonymizedPath", schema);

src/core/model/anonymizedGists/anonymizedGists.schema.ts

@@ -68,4 +68,6 @@ const AnonymizedGistSchema = new Schema({
   },
 });
 
+AnonymizedGistSchema.index({ status: 1, "options.expirationDate": 1 });
+
 export default AnonymizedGistSchema;

src/core/model/anonymizedPullRequests/anonymizedPullRequests.schema.ts

@@ -65,4 +65,6 @@ const AnonymizedPullRequestSchema = new Schema({
   },
 });
 
+AnonymizedPullRequestSchema.index({ status: 1, "options.expirationDate": 1 });
+
 export default AnonymizedPullRequestSchema;

src/core/model/anonymizedRepositories/anonymizedRepositories.schema.ts

@@ -47,6 +47,19 @@ const AnonymizedRepositorySchema = new Schema({
     repositoryName: String,
     accessToken: { type: String, select: false },
   },
+  cleanupToken: { type: String, select: false },
+  cleanupUntil: { type: Date, select: false },
+  treeGeneration: String,
+  stagedFileTrees: { type: [{ generation: String, until: Date }], default: undefined },
+  retiredTreeGenerations: { type: [String], default: undefined },
+  retiredContentPrefixes: { type: [String], default: undefined },
+  legacyContentCleanupPending: Boolean,
+  contentCacheVersion: Number,
+  contentCacheRevision: String,
+  emptyTreeGeneration: String,
+  fileMetadataRevision: String,
+  pathIndexKey: String,
+  pathIndexBuiltAt: Date,
   truncatedFolders: {
     type: [String],
     default: [],
@@ -70,6 +83,7 @@ const AnonymizedRepositorySchema = new Schema({
     type: Date,
     default: Date.now,
   },
+  sizeComputedAt: Date,
   size: {
     storage: {
       type: Number,
@@ -87,10 +101,17 @@ const AnonymizedRepositorySchema = new Schema({
 });
 
 AnonymizedRepositorySchema.index({ "source.repositoryName": 1 });
+AnonymizedRepositorySchema.index({ "coauthors.githubId": 1 });
+AnonymizedRepositorySchema.index({ retiredTreeGenerations: 1 }, { sparse: true });
+AnonymizedRepositorySchema.index({ "stagedFileTrees.until": 1 }, { sparse: true });
+AnonymizedRepositorySchema.index({ retiredContentPrefixes: 1 }, { sparse: true });
+AnonymizedRepositorySchema.index({ legacyContentCleanupPending: 1, status: 1 },
+  { partialFilterExpression: { legacyContentCleanupPending: true } });
 AnonymizedRepositorySchema.index({ status: 1, statusDate: 1 });
 AnonymizedRepositorySchema.index({ lastView: 1 });
 AnonymizedRepositorySchema.index({ anonymizeDate: 1 });
 AnonymizedRepositorySchema.index({ status: 1, isReseted: 1, lastView: 1 });
+AnonymizedRepositorySchema.index({ status: 1, "options.expirationDate": 1 });
 AnonymizedRepositorySchema.index({
   status: 1,
   isReseted: 1,

src/core/model/anonymizedRepositories/anonymizedRepositories.types.ts

@@ -30,6 +30,17 @@ export interface IAnonymizedRepository {
     photo?: string;
     addedAt?: Date;
   }[];
+  treeGeneration?: string;
+  stagedFileTrees?: { generation: string; until: Date }[];
+  retiredTreeGenerations?: string[];
+  retiredContentPrefixes?: string[];
+  legacyContentCleanupPending?: boolean;
+  contentCacheVersion?: number;
+  contentCacheRevision?: string;
+  emptyTreeGeneration?: string;
+  fileMetadataRevision?: string;
+  pathIndexKey?: string;
+  pathIndexBuiltAt?: Date;
   truncatedFolders: string[];
   conference: string;
   options: {
@@ -49,6 +60,7 @@ export interface IAnonymizedRepository {
   };
   pageView: number;
   lastView: Date;
+  sizeComputedAt?: Date;
   size: {
     storage: number;
     file: number;

src/core/model/dashboard-name.ts

@@ -0,0 +1,5 @@
+import { model, Schema } from "mongoose";
+const schema = new Schema({ owner: { type: Schema.Types.ObjectId, required: true },
+  type: { type: String, required: true }, artifactId: { type: String, required: true }, name: String });
+schema.index({ owner: 1, type: 1, artifactId: 1 }, { unique: true });
+export default model("DashboardName", schema);

src/core/model/files/files.schema.ts

@@ -4,6 +4,9 @@ const FileSchema = new Schema({
   name: { type: String, index: true },
   path: { type: String, index: true },
   repoId: { type: String, index: true },
+  treeGeneration: String,
+  treeStaged: Boolean,
+  metadataPending: Boolean,
   sha: {
     type: String,
   },
@@ -13,6 +16,11 @@ const FileSchema = new Schema({
 });
 
 FileSchema.index({ path: 1, repoId: 1 });
+FileSchema.index({ metadataPending: 1, repoId: 1, treeGeneration: 1 },
+  { partialFilterExpression: { metadataPending: true } });
+FileSchema.index({ treeStaged: 1, repoId: 1, treeGeneration: 1 },
+  { partialFilterExpression: { treeStaged: true } });
+FileSchema.index({ repoId: 1, treeGeneration: 1, path: 1, name: 1 });
 FileSchema.index(
   { repoId: 1, size: 1, path: 1 },
   { name: "repoId_1_size_1_path_1" }

src/core/model/files/files.types.ts

@@ -4,6 +4,9 @@ export interface IFile {
   name: string;
   path: string;
   repoId: string;
+  treeGeneration?: string;
+  treeStaged?: boolean;
+  metadataPending?: boolean;
   sha?: string;
   size?: number;
 }