mirror of
https://github.com/tdurieux/anonymous_github.git
synced 2026-08-16 17:00:30 +02:00
fix: fix migration script
This commit is contained in:
tdurieux
+167
-178
@@ -31,198 +31,187 @@ async function connect(db) {
|
|||||||
const oldDB = await connect("anonymous_github");
|
const oldDB = await connect("anonymous_github");
|
||||||
|
|
||||||
console.log("Import Users");
|
console.log("Import Users");
|
||||||
await new Promise(async (resolve) => {
|
let index = 0;
|
||||||
const promises = [];
|
const userQuery = oldDB.collection("users").find();
|
||||||
await oldDB
|
const totalUser = await userQuery.count();
|
||||||
.collection("users")
|
|
||||||
.find()
|
|
||||||
.batchSize(1)
|
|
||||||
.forEach(async (r) => {
|
|
||||||
let localResolve = null;
|
|
||||||
const p = new Promise((r) => (localResolve = r));
|
|
||||||
promises.push(p);
|
|
||||||
console.log("Import User: " + r.username);
|
|
||||||
|
|
||||||
const newRepos = [];
|
while (await userQuery.hasNext()) {
|
||||||
const allRepoIds = [];
|
const r = await userQuery.next();
|
||||||
if (r.repositories) {
|
index++;
|
||||||
const finds = await RepositoryModel.find({
|
console.log(`Import User [${index}/${totalUser}]: ${r.username}`);
|
||||||
externalId: {
|
|
||||||
$in: r.repositories.map((repo) => "gh_" + repo.id),
|
|
||||||
},
|
|
||||||
}).select("externalId");
|
|
||||||
finds.forEach((f) => allRepoIds.push(f.id));
|
|
||||||
|
|
||||||
const repoIds = new Set<string>();
|
const newRepos = [];
|
||||||
const toInsert = r.repositories.filter((f) => {
|
const allRepoIds = [];
|
||||||
if (repoIds.has(f.id)) return false;
|
if (r.repositories) {
|
||||||
repoIds.add(f.id);
|
const finds = await RepositoryModel.find({
|
||||||
const externalId = "gh_" + f.id;
|
externalId: {
|
||||||
return finds.filter((f) => f.externalId == externalId).length == 0;
|
$in: r.repositories.map((repo) => "gh_" + repo.id),
|
||||||
});
|
},
|
||||||
|
}).select("externalId");
|
||||||
|
finds.forEach((f) => allRepoIds.push(f.id));
|
||||||
|
|
||||||
for (const repo of toInsert) {
|
const repoIds = new Set<string>();
|
||||||
newRepos.push(
|
const toInsert = r.repositories.filter((f) => {
|
||||||
new RepositoryModel({
|
if (repoIds.has(f.id)) return false;
|
||||||
externalId: "gh_" + repo.id,
|
repoIds.add(f.id);
|
||||||
name: repo.full_name,
|
const externalId = "gh_" + f.id;
|
||||||
url: repo.html_url,
|
return finds.filter((f) => f.externalId == externalId).length == 0;
|
||||||
size: repo.size,
|
|
||||||
defaultBranch: repo.default_branch,
|
|
||||||
})
|
|
||||||
);
|
|
||||||
}
|
|
||||||
if (newRepos.length > 0) {
|
|
||||||
await RepositoryModel.insertMany(newRepos);
|
|
||||||
}
|
|
||||||
newRepos.forEach((f) => allRepoIds.push(f.id));
|
|
||||||
}
|
|
||||||
const user = new UserModel({
|
|
||||||
accessTokens: {
|
|
||||||
github: r.accessToken,
|
|
||||||
},
|
|
||||||
externalIDs: {
|
|
||||||
github: r.profile.id,
|
|
||||||
},
|
|
||||||
username: r.username,
|
|
||||||
emails: r.profile.emails?.map((email) => {
|
|
||||||
return { email: email.value, default: false };
|
|
||||||
}),
|
|
||||||
photo: r.profile.photos[0]?.value,
|
|
||||||
repositories: allRepoIds,
|
|
||||||
default: {
|
|
||||||
terms: r.default?.terms,
|
|
||||||
options: r.default?.options,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
if (user.emails?.length) user.emails[0].default = true;
|
|
||||||
|
|
||||||
await user.save();
|
|
||||||
|
|
||||||
localResolve(user);
|
|
||||||
});
|
});
|
||||||
Promise.all(promises).then(resolve);
|
|
||||||
});
|
for (const repo of toInsert) {
|
||||||
|
newRepos.push(
|
||||||
|
new RepositoryModel({
|
||||||
|
externalId: "gh_" + repo.id,
|
||||||
|
name: repo.full_name,
|
||||||
|
url: repo.html_url,
|
||||||
|
size: repo.size,
|
||||||
|
defaultBranch: repo.default_branch,
|
||||||
|
})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (newRepos.length > 0) {
|
||||||
|
await RepositoryModel.insertMany(newRepos);
|
||||||
|
}
|
||||||
|
newRepos.forEach((f) => allRepoIds.push(f.id));
|
||||||
|
}
|
||||||
|
const user = new UserModel({
|
||||||
|
accessTokens: {
|
||||||
|
github: r.accessToken,
|
||||||
|
},
|
||||||
|
externalIDs: {
|
||||||
|
github: r.profile.id,
|
||||||
|
},
|
||||||
|
username: r.username,
|
||||||
|
emails: r.profile.emails?.map((email) => {
|
||||||
|
return { email: email.value, default: false };
|
||||||
|
}),
|
||||||
|
photo: r.profile.photos[0]?.value,
|
||||||
|
repositories: allRepoIds,
|
||||||
|
default: {
|
||||||
|
terms: r.default?.terms,
|
||||||
|
options: r.default?.options,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
if (user.emails?.length) user.emails[0].default = true;
|
||||||
|
|
||||||
|
await user.save();
|
||||||
|
}
|
||||||
|
|
||||||
console.log("Import Repositories");
|
console.log("Import Repositories");
|
||||||
let promises = [];
|
const repoQuery = oldDB.collection("repositories").find();
|
||||||
await oldDB
|
const totalRepository = await repoQuery.count();
|
||||||
.collection("repositories")
|
index = 0;
|
||||||
.find({})
|
while (await repoQuery.hasNext()) {
|
||||||
.batchSize(1)
|
const r = await repoQuery.next();
|
||||||
.forEach(async (r) => {
|
if (!r.id) continue;
|
||||||
if (!r.id) return;
|
index++;
|
||||||
let localResolve = null;
|
console.log(
|
||||||
const p = new Promise((r) => (localResolve = r));
|
`Import Repository [${index}/${totalRepository}]: ${r.fullName}`
|
||||||
promises.push(p);
|
);
|
||||||
|
|
||||||
let find = await RepositoryModel.findOne({
|
let find = await RepositoryModel.findOne({
|
||||||
|
externalId: "gh_" + r.id,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (find == null) {
|
||||||
|
find = new RepositoryModel({
|
||||||
externalId: "gh_" + r.id,
|
externalId: "gh_" + r.id,
|
||||||
|
name: r.fullName,
|
||||||
|
url: r.html_url,
|
||||||
|
size: r.size,
|
||||||
|
defaultBranch: r.default_branch,
|
||||||
});
|
});
|
||||||
// console.log("gh_" + r.id, find != null);
|
}
|
||||||
if (find == null) {
|
if (r.branches) {
|
||||||
find = new RepositoryModel({
|
const branches = [...Object.values(r.branches)].map((b: any) => {
|
||||||
externalId: "gh_" + r.id,
|
const o: any = { name: b.name, commit: b.commit.sha };
|
||||||
name: r.fullName,
|
if (b.name == find.defaultBranch) {
|
||||||
url: r.html_url,
|
o.readme = r.readme;
|
||||||
size: r.size,
|
}
|
||||||
defaultBranch: r.default_branch,
|
return o;
|
||||||
});
|
});
|
||||||
}
|
find.branches = branches;
|
||||||
if (r.branches) {
|
}
|
||||||
const branches = [...Object.values(r.branches)].map((b: any) => {
|
await find.save();
|
||||||
const o: any = { name: b.name, commit: b.commit.sha };
|
}
|
||||||
if (b.name == find.defaultBranch) {
|
|
||||||
o.readme = r.readme;
|
|
||||||
}
|
|
||||||
return o;
|
|
||||||
});
|
|
||||||
find.branches = branches;
|
|
||||||
}
|
|
||||||
await find.save();
|
|
||||||
localResolve();
|
|
||||||
});
|
|
||||||
await Promise.all(promises);
|
|
||||||
console.log("Import Anonymized Repositories");
|
console.log("Import Anonymized Repositories");
|
||||||
promises = [];
|
const anoQuery = oldDB.collection("anonymized_repositories").find();
|
||||||
await oldDB
|
const totalAno = await anoQuery.count();
|
||||||
.collection("anonymized_repositories")
|
index = 0;
|
||||||
.find({})
|
while (await anoQuery.hasNext()) {
|
||||||
.forEach(async (r) => {
|
const r = await anoQuery.next();
|
||||||
let localResolve = null;
|
|
||||||
const p = new Promise((r) => (localResolve = r));
|
|
||||||
promises.push(p);
|
|
||||||
|
|
||||||
let repo = await RepositoryModel.findOne({ name: r.fullName });
|
index++;
|
||||||
if (repo == null) {
|
console.log(
|
||||||
const tmp = await oldDB
|
`Import Anonymized Repository [${index}/${totalAno}]: ${r.repoId}`
|
||||||
.collection("repositories")
|
);
|
||||||
.findOne({ fullName: r.fullName });
|
|
||||||
if (tmp) {
|
let repo = await RepositoryModel.findOne({ name: r.fullName });
|
||||||
repo = await RepositoryModel.findOne({ externalId: "gh_" + tmp.id });
|
if (repo == null) {
|
||||||
} else {
|
const tmp = await oldDB
|
||||||
console.error(`Repository ${r.fullName} is not found (renamed)`);
|
.collection("repositories")
|
||||||
|
.findOne({ fullName: r.fullName });
|
||||||
|
if (tmp) {
|
||||||
|
repo = await RepositoryModel.findOne({ externalId: "gh_" + tmp.id });
|
||||||
|
} else {
|
||||||
|
console.error(`Repository ${r.fullName} is not found (renamed)`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let size = { storage: 0, file: 0 };
|
||||||
|
function recursiveCount(files) {
|
||||||
|
const out = { storage: 0, file: 0 };
|
||||||
|
for (const name in files) {
|
||||||
|
const file = files[name];
|
||||||
|
if (file.size && file.sha && parseInt(file.size) == file.size) {
|
||||||
|
out.storage += file.size as number;
|
||||||
|
out.file++;
|
||||||
|
} else if (typeof file == "object") {
|
||||||
|
const r = recursiveCount(file);
|
||||||
|
out.storage += r.storage;
|
||||||
|
out.file += r.file;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
let size = { storage: 0, file: 0 };
|
return out;
|
||||||
function recursiveCount(files) {
|
}
|
||||||
const out = { storage: 0, file: 0 };
|
|
||||||
for (const name in files) {
|
|
||||||
const file = files[name];
|
|
||||||
if (file.size && file.sha && parseInt(file.size) == file.size) {
|
|
||||||
out.storage += file.size as number;
|
|
||||||
out.file++;
|
|
||||||
} else if (typeof file == "object") {
|
|
||||||
const r = recursiveCount(file);
|
|
||||||
out.storage += r.storage;
|
|
||||||
out.file += r.file;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (r.originalFiles) {
|
if (r.originalFiles) {
|
||||||
size = recursiveCount(r.originalFiles);
|
size = recursiveCount(r.originalFiles);
|
||||||
}
|
}
|
||||||
const owner = await UserModel.findOne({ username: r.owner }).select(
|
const owner = await UserModel.findOne({ username: r.owner }).select("_id");
|
||||||
"_id"
|
await new AnonymizedRepositoryModel({
|
||||||
);
|
repoId: r.repoId,
|
||||||
await new AnonymizedRepositoryModel({
|
status: r.status,
|
||||||
repoId: r.repoId,
|
anonymizeDate: r.anonymizeDate,
|
||||||
status: r.status,
|
lastView: r.lastView,
|
||||||
anonymizeDate: r.anonymizeDate,
|
pageView: r.pageView,
|
||||||
lastView: r.lastView,
|
owner: owner?.id,
|
||||||
pageView: r.pageView,
|
size,
|
||||||
owner: owner?.id,
|
source: {
|
||||||
size,
|
accessToken: r.token,
|
||||||
source: {
|
type: r.options.mode == "download" ? "GitHubDownload" : "GitHubStream",
|
||||||
accessToken: r.token,
|
branch: r.branch,
|
||||||
type:
|
commit: r.commit,
|
||||||
r.options.mode == "download" ? "GitHubDownload" : "GitHubStream",
|
repositoryId: repo?.id,
|
||||||
branch: r.branch,
|
repositoryName: r.fullName,
|
||||||
commit: r.commit,
|
},
|
||||||
repositoryId: repo?.id,
|
options: {
|
||||||
repositoryName: r.fullName,
|
terms: r.terms,
|
||||||
},
|
expirationMode: r.options.expirationMode,
|
||||||
options: {
|
expirationDate: r.options.expirationDate
|
||||||
terms: r.terms,
|
? new Date(r.options.expirationDate)
|
||||||
expirationMode: r.options.expirationMode,
|
: null,
|
||||||
expirationDate: r.options.expirationDate
|
update: r.options.update,
|
||||||
? new Date(r.options.expirationDate)
|
image: r.options.image,
|
||||||
: null,
|
pdf: r.options.pdf,
|
||||||
update: r.options.update,
|
notebook: r.options.notebook,
|
||||||
image: r.options.image,
|
loc: r.options.loc,
|
||||||
pdf: r.options.pdf,
|
link: r.options.link,
|
||||||
notebook: r.options.notebook,
|
page: r.options.page,
|
||||||
loc: r.options.loc,
|
pageSource: r.options.pageSource,
|
||||||
link: r.options.link,
|
},
|
||||||
page: r.options.page,
|
}).save();
|
||||||
pageSource: r.options.pageSource,
|
}
|
||||||
},
|
|
||||||
}).save();
|
|
||||||
localResolve();
|
|
||||||
});
|
|
||||||
await Promise.all(promises);
|
|
||||||
console.log("Import finished!");
|
console.log("Import finished!");
|
||||||
setTimeout(() => process.exit(), 5000);
|
setTimeout(() => process.exit(), 5000);
|
||||||
})();
|
})();
|
||||||
|
|||||||
Reference in New Issue
Block a user