fix: fix database migration

migrateDB.ts

@@ -6,7 +6,6 @@ import * as database from "./src/database/database";
 import RepositoryModel from "./src/database/repositories/repositories.model";
 import AnonymizedRepositoryModel from "./src/database/anonymizedRepositories/anonymizedRepositories.model";
 import UserModel from "./src/database/users/users.model";
-import { IRepositoryDocument } from "./src/database/repositories/repositories.types";
 
 const MONGO_URL = `mongodb://${config.DB_USERNAME}:${config.DB_PASSWORD}@${config.DB_HOSTNAME}:27017/`;
 
@@ -42,27 +41,41 @@ async function connect(db) {
         let localResolve = null;
         const p = new Promise((r) => (localResolve = r));
         promises.push(p);
+        console.log("Import User: " + r.username);
 
-        const repositoryModels: Promise<IRepositoryDocument>[] = [];
+        const newRepos = [];
-        const finds = (
+        const allRepoIds = [];
-          await RepositoryModel.find({
+        if (r.repositories) {
+          const finds = await RepositoryModel.find({
             externalId: {
               $in: r.repositories.map((repo) => "gh_" + repo.id),
             },
-          }).select("externalId")
+          }).select("externalId");
-        ).map((m) => m.externalId);
+          finds.forEach((f) => allRepoIds.push(f.id));
-        for (const repo of r.repositories.filter(
+
-          (f) => finds.indexOf("gh_" + f.id) == -1
+          const repoIds = new Set<string>();
-        )) {
+          const toInsert = r.repositories.filter((f) => {
-          repositoryModels.push(
+            if (repoIds.has(f.id)) return false;
-            new RepositoryModel({
+            repoIds.add(f.id);
-              externalId: "gh_" + repo.id,
+            const externalId = "gh_" + f.id;
-              name: repo.full_name,
+            return finds.filter((f) => f.externalId == externalId).length == 0;
-              url: repo.html_url,
+          });
-              size: repo.size,
+
-              defaultBranch: repo.default_branch,
+          for (const repo of toInsert) {
-            }).save()
+            newRepos.push(
-          );
+              new RepositoryModel({
+                externalId: "gh_" + repo.id,
+                name: repo.full_name,
+                url: repo.html_url,
+                size: repo.size,
+                defaultBranch: repo.default_branch,
+              })
+            );
+          }
+          if (newRepos.length > 0) {
+            await RepositoryModel.insertMany(newRepos);
+          }
+          newRepos.forEach((f) => allRepoIds.push(f.id));
         }
         const user = new UserModel({
           accessTokens: {
@@ -76,10 +89,10 @@ async function connect(db) {
             return { email: email.value, default: false };
           }),
           photo: r.profile.photos[0]?.value,
-          repositories: (await Promise.all(repositoryModels)).map((d) => d._id),
+          repositories: allRepoIds,
           default: {
-            terms: r.default.terms,
+            terms: r.default?.terms,
-            options: r.default.options,
+            options: r.default?.options,
           },
         });
         if (user.emails?.length) user.emails[0].default = true;
@@ -116,14 +129,16 @@ async function connect(db) {
           defaultBranch: r.default_branch,
         });
       }
-      const branches = [...Object.values(r.branches)].map((b: any) => {
+      if (r.branches) {
-        const o: any = { name: b.name, commit: b.commit.sha };
+        const branches = [...Object.values(r.branches)].map((b: any) => {
-        if (b.name == find.defaultBranch) {
+          const o: any = { name: b.name, commit: b.commit.sha };
-          o.readme = r.readme;
+          if (b.name == find.defaultBranch) {
-        }
+            o.readme = r.readme;
-        return o;
+          }
-      });
+          return o;
-      find.branches = branches;
+        });
+        find.branches = branches;
+      }
       await find.save();
       localResolve();
     });
@@ -149,7 +164,26 @@ async function connect(db) {
           console.error(`Repository ${r.fullName} is not found (renamed)`);
         }
       }
-      const owner = await UserModel.findOne({ username: r.owner });
+      let size = 0;
+      function recursiveCount(files) {
+        let total = 0;
+        for (const name in files) {
+          const file = files[name];
+          if (file.size && file.sha && parseInt(file.size) == file.size) {
+            total += file.size as number;
+          } else if (typeof file == "object") {
+            total += recursiveCount(file);
+          }
+        }
+        return total;
+      }
+
+      if (r.originalFiles) {
+        size = recursiveCount(r.originalFiles);
+      }
+      const owner = await UserModel.findOne({ username: r.owner }).select(
+        "_id"
+      );
       await new AnonymizedRepositoryModel({
         repoId: r.repoId,
         status: r.status,
@@ -157,6 +191,7 @@ async function connect(db) {
         lastView: r.lastView,
         pageView: r.pageView,
         owner: owner?.id,
+        size,
         source: {
           accessToken: r.token,
           type: