diff --git a/migrateDB.ts b/migrateDB.ts index 800d308..b5924f9 100644 --- a/migrateDB.ts +++ b/migrateDB.ts @@ -6,7 +6,6 @@ import * as database from "./src/database/database"; import RepositoryModel from "./src/database/repositories/repositories.model"; import AnonymizedRepositoryModel from "./src/database/anonymizedRepositories/anonymizedRepositories.model"; import UserModel from "./src/database/users/users.model"; -import { IRepositoryDocument } from "./src/database/repositories/repositories.types"; const MONGO_URL = `mongodb://${config.DB_USERNAME}:${config.DB_PASSWORD}@${config.DB_HOSTNAME}:27017/`; @@ -42,27 +41,41 @@ async function connect(db) { let localResolve = null; const p = new Promise((r) => (localResolve = r)); promises.push(p); + console.log("Import User: " + r.username); - const repositoryModels: Promise[] = []; - const finds = ( - await RepositoryModel.find({ + const newRepos = []; + const allRepoIds = []; + if (r.repositories) { + const finds = await RepositoryModel.find({ externalId: { $in: r.repositories.map((repo) => "gh_" + repo.id), }, - }).select("externalId") - ).map((m) => m.externalId); - for (const repo of r.repositories.filter( - (f) => finds.indexOf("gh_" + f.id) == -1 - )) { - repositoryModels.push( - new RepositoryModel({ - externalId: "gh_" + repo.id, - name: repo.full_name, - url: repo.html_url, - size: repo.size, - defaultBranch: repo.default_branch, - }).save() - ); + }).select("externalId"); + finds.forEach((f) => allRepoIds.push(f.id)); + + const repoIds = new Set(); + const toInsert = r.repositories.filter((f) => { + if (repoIds.has(f.id)) return false; + repoIds.add(f.id); + const externalId = "gh_" + f.id; + return finds.filter((f) => f.externalId == externalId).length == 0; + }); + + for (const repo of toInsert) { + newRepos.push( + new RepositoryModel({ + externalId: "gh_" + repo.id, + name: repo.full_name, + url: repo.html_url, + size: repo.size, + defaultBranch: repo.default_branch, + }) + ); + } + if (newRepos.length > 0) { + await RepositoryModel.insertMany(newRepos); + } + newRepos.forEach((f) => allRepoIds.push(f.id)); } const user = new UserModel({ accessTokens: { @@ -76,10 +89,10 @@ async function connect(db) { return { email: email.value, default: false }; }), photo: r.profile.photos[0]?.value, - repositories: (await Promise.all(repositoryModels)).map((d) => d._id), + repositories: allRepoIds, default: { - terms: r.default.terms, - options: r.default.options, + terms: r.default?.terms, + options: r.default?.options, }, }); if (user.emails?.length) user.emails[0].default = true; @@ -116,14 +129,16 @@ async function connect(db) { defaultBranch: r.default_branch, }); } - const branches = [...Object.values(r.branches)].map((b: any) => { - const o: any = { name: b.name, commit: b.commit.sha }; - if (b.name == find.defaultBranch) { - o.readme = r.readme; - } - return o; - }); - find.branches = branches; + if (r.branches) { + const branches = [...Object.values(r.branches)].map((b: any) => { + const o: any = { name: b.name, commit: b.commit.sha }; + if (b.name == find.defaultBranch) { + o.readme = r.readme; + } + return o; + }); + find.branches = branches; + } await find.save(); localResolve(); }); @@ -149,7 +164,26 @@ async function connect(db) { console.error(`Repository ${r.fullName} is not found (renamed)`); } } - const owner = await UserModel.findOne({ username: r.owner }); + let size = 0; + function recursiveCount(files) { + let total = 0; + for (const name in files) { + const file = files[name]; + if (file.size && file.sha && parseInt(file.size) == file.size) { + total += file.size as number; + } else if (typeof file == "object") { + total += recursiveCount(file); + } + } + return total; + } + + if (r.originalFiles) { + size = recursiveCount(r.originalFiles); + } + const owner = await UserModel.findOne({ username: r.owner }).select( + "_id" + ); await new AnonymizedRepositoryModel({ repoId: r.repoId, status: r.status, @@ -157,6 +191,7 @@ async function connect(db) { lastView: r.lastView, pageView: r.pageView, owner: owner?.id, + size, source: { accessToken: r.token, type: