mirror of
https://github.com/tdurieux/anonymous_github.git
synced 2026-02-13 02:42:45 +00:00
fix: fix database migration
This commit is contained in:
95
migrateDB.ts
95
migrateDB.ts
@@ -6,7 +6,6 @@ import * as database from "./src/database/database";
|
||||
import RepositoryModel from "./src/database/repositories/repositories.model";
|
||||
import AnonymizedRepositoryModel from "./src/database/anonymizedRepositories/anonymizedRepositories.model";
|
||||
import UserModel from "./src/database/users/users.model";
|
||||
import { IRepositoryDocument } from "./src/database/repositories/repositories.types";
|
||||
|
||||
const MONGO_URL = `mongodb://${config.DB_USERNAME}:${config.DB_PASSWORD}@${config.DB_HOSTNAME}:27017/`;
|
||||
|
||||
@@ -42,27 +41,41 @@ async function connect(db) {
|
||||
let localResolve = null;
|
||||
const p = new Promise((r) => (localResolve = r));
|
||||
promises.push(p);
|
||||
console.log("Import User: " + r.username);
|
||||
|
||||
const repositoryModels: Promise<IRepositoryDocument>[] = [];
|
||||
const finds = (
|
||||
await RepositoryModel.find({
|
||||
const newRepos = [];
|
||||
const allRepoIds = [];
|
||||
if (r.repositories) {
|
||||
const finds = await RepositoryModel.find({
|
||||
externalId: {
|
||||
$in: r.repositories.map((repo) => "gh_" + repo.id),
|
||||
},
|
||||
}).select("externalId")
|
||||
).map((m) => m.externalId);
|
||||
for (const repo of r.repositories.filter(
|
||||
(f) => finds.indexOf("gh_" + f.id) == -1
|
||||
)) {
|
||||
repositoryModels.push(
|
||||
new RepositoryModel({
|
||||
externalId: "gh_" + repo.id,
|
||||
name: repo.full_name,
|
||||
url: repo.html_url,
|
||||
size: repo.size,
|
||||
defaultBranch: repo.default_branch,
|
||||
}).save()
|
||||
);
|
||||
}).select("externalId");
|
||||
finds.forEach((f) => allRepoIds.push(f.id));
|
||||
|
||||
const repoIds = new Set<string>();
|
||||
const toInsert = r.repositories.filter((f) => {
|
||||
if (repoIds.has(f.id)) return false;
|
||||
repoIds.add(f.id);
|
||||
const externalId = "gh_" + f.id;
|
||||
return finds.filter((f) => f.externalId == externalId).length == 0;
|
||||
});
|
||||
|
||||
for (const repo of toInsert) {
|
||||
newRepos.push(
|
||||
new RepositoryModel({
|
||||
externalId: "gh_" + repo.id,
|
||||
name: repo.full_name,
|
||||
url: repo.html_url,
|
||||
size: repo.size,
|
||||
defaultBranch: repo.default_branch,
|
||||
})
|
||||
);
|
||||
}
|
||||
if (newRepos.length > 0) {
|
||||
await RepositoryModel.insertMany(newRepos);
|
||||
}
|
||||
newRepos.forEach((f) => allRepoIds.push(f.id));
|
||||
}
|
||||
const user = new UserModel({
|
||||
accessTokens: {
|
||||
@@ -76,10 +89,10 @@ async function connect(db) {
|
||||
return { email: email.value, default: false };
|
||||
}),
|
||||
photo: r.profile.photos[0]?.value,
|
||||
repositories: (await Promise.all(repositoryModels)).map((d) => d._id),
|
||||
repositories: allRepoIds,
|
||||
default: {
|
||||
terms: r.default.terms,
|
||||
options: r.default.options,
|
||||
terms: r.default?.terms,
|
||||
options: r.default?.options,
|
||||
},
|
||||
});
|
||||
if (user.emails?.length) user.emails[0].default = true;
|
||||
@@ -116,14 +129,16 @@ async function connect(db) {
|
||||
defaultBranch: r.default_branch,
|
||||
});
|
||||
}
|
||||
const branches = [...Object.values(r.branches)].map((b: any) => {
|
||||
const o: any = { name: b.name, commit: b.commit.sha };
|
||||
if (b.name == find.defaultBranch) {
|
||||
o.readme = r.readme;
|
||||
}
|
||||
return o;
|
||||
});
|
||||
find.branches = branches;
|
||||
if (r.branches) {
|
||||
const branches = [...Object.values(r.branches)].map((b: any) => {
|
||||
const o: any = { name: b.name, commit: b.commit.sha };
|
||||
if (b.name == find.defaultBranch) {
|
||||
o.readme = r.readme;
|
||||
}
|
||||
return o;
|
||||
});
|
||||
find.branches = branches;
|
||||
}
|
||||
await find.save();
|
||||
localResolve();
|
||||
});
|
||||
@@ -149,7 +164,26 @@ async function connect(db) {
|
||||
console.error(`Repository ${r.fullName} is not found (renamed)`);
|
||||
}
|
||||
}
|
||||
const owner = await UserModel.findOne({ username: r.owner });
|
||||
let size = 0;
|
||||
function recursiveCount(files) {
|
||||
let total = 0;
|
||||
for (const name in files) {
|
||||
const file = files[name];
|
||||
if (file.size && file.sha && parseInt(file.size) == file.size) {
|
||||
total += file.size as number;
|
||||
} else if (typeof file == "object") {
|
||||
total += recursiveCount(file);
|
||||
}
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
if (r.originalFiles) {
|
||||
size = recursiveCount(r.originalFiles);
|
||||
}
|
||||
const owner = await UserModel.findOne({ username: r.owner }).select(
|
||||
"_id"
|
||||
);
|
||||
await new AnonymizedRepositoryModel({
|
||||
repoId: r.repoId,
|
||||
status: r.status,
|
||||
@@ -157,6 +191,7 @@ async function connect(db) {
|
||||
lastView: r.lastView,
|
||||
pageView: r.pageView,
|
||||
owner: owner?.id,
|
||||
size,
|
||||
source: {
|
||||
accessToken: r.token,
|
||||
type:
|
||||
|
||||
Reference in New Issue
Block a user