fix: fix database migration

This commit is contained in:
tdurieux
2021-09-06 11:37:03 +02:00
parent bcc9f5449c
commit 44c70d9b28

View File

@@ -6,7 +6,6 @@ import * as database from "./src/database/database";
import RepositoryModel from "./src/database/repositories/repositories.model";
import AnonymizedRepositoryModel from "./src/database/anonymizedRepositories/anonymizedRepositories.model";
import UserModel from "./src/database/users/users.model";
import { IRepositoryDocument } from "./src/database/repositories/repositories.types";
const MONGO_URL = `mongodb://${config.DB_USERNAME}:${config.DB_PASSWORD}@${config.DB_HOSTNAME}:27017/`;
@@ -42,27 +41,41 @@ async function connect(db) {
let localResolve = null;
const p = new Promise((r) => (localResolve = r));
promises.push(p);
console.log("Import User: " + r.username);
const repositoryModels: Promise<IRepositoryDocument>[] = [];
const finds = (
await RepositoryModel.find({
const newRepos = [];
const allRepoIds = [];
if (r.repositories) {
const finds = await RepositoryModel.find({
externalId: {
$in: r.repositories.map((repo) => "gh_" + repo.id),
},
}).select("externalId")
).map((m) => m.externalId);
for (const repo of r.repositories.filter(
(f) => finds.indexOf("gh_" + f.id) == -1
)) {
repositoryModels.push(
new RepositoryModel({
externalId: "gh_" + repo.id,
name: repo.full_name,
url: repo.html_url,
size: repo.size,
defaultBranch: repo.default_branch,
}).save()
);
}).select("externalId");
finds.forEach((f) => allRepoIds.push(f.id));
const repoIds = new Set<string>();
const toInsert = r.repositories.filter((f) => {
if (repoIds.has(f.id)) return false;
repoIds.add(f.id);
const externalId = "gh_" + f.id;
return finds.filter((f) => f.externalId == externalId).length == 0;
});
for (const repo of toInsert) {
newRepos.push(
new RepositoryModel({
externalId: "gh_" + repo.id,
name: repo.full_name,
url: repo.html_url,
size: repo.size,
defaultBranch: repo.default_branch,
})
);
}
if (newRepos.length > 0) {
await RepositoryModel.insertMany(newRepos);
}
newRepos.forEach((f) => allRepoIds.push(f.id));
}
const user = new UserModel({
accessTokens: {
@@ -76,10 +89,10 @@ async function connect(db) {
return { email: email.value, default: false };
}),
photo: r.profile.photos[0]?.value,
repositories: (await Promise.all(repositoryModels)).map((d) => d._id),
repositories: allRepoIds,
default: {
terms: r.default.terms,
options: r.default.options,
terms: r.default?.terms,
options: r.default?.options,
},
});
if (user.emails?.length) user.emails[0].default = true;
@@ -116,14 +129,16 @@ async function connect(db) {
defaultBranch: r.default_branch,
});
}
const branches = [...Object.values(r.branches)].map((b: any) => {
const o: any = { name: b.name, commit: b.commit.sha };
if (b.name == find.defaultBranch) {
o.readme = r.readme;
}
return o;
});
find.branches = branches;
if (r.branches) {
const branches = [...Object.values(r.branches)].map((b: any) => {
const o: any = { name: b.name, commit: b.commit.sha };
if (b.name == find.defaultBranch) {
o.readme = r.readme;
}
return o;
});
find.branches = branches;
}
await find.save();
localResolve();
});
@@ -149,7 +164,26 @@ async function connect(db) {
console.error(`Repository ${r.fullName} is not found (renamed)`);
}
}
const owner = await UserModel.findOne({ username: r.owner });
let size = 0;
function recursiveCount(files) {
let total = 0;
for (const name in files) {
const file = files[name];
if (file.size && file.sha && parseInt(file.size) == file.size) {
total += file.size as number;
} else if (typeof file == "object") {
total += recursiveCount(file);
}
}
return total;
}
if (r.originalFiles) {
size = recursiveCount(r.originalFiles);
}
const owner = await UserModel.findOne({ username: r.owner }).select(
"_id"
);
await new AnonymizedRepositoryModel({
repoId: r.repoId,
status: r.status,
@@ -157,6 +191,7 @@ async function connect(db) {
lastView: r.lastView,
pageView: r.pageView,
owner: owner?.id,
size,
source: {
accessToken: r.token,
type: