diff --git a/public/i18n/locale-en.json b/public/i18n/locale-en.json index 5d5aa29..0cbd056 100644 --- a/public/i18n/locale-en.json +++ b/public/i18n/locale-en.json @@ -101,7 +101,8 @@ "server_error": "An unexpected server error occurred. Please try again later.", "username_not_defined": "A username must be provided.", "github_user_not_found": "The specified GitHub user could not be found.", - "cannot_coauthor_self": "You cannot add yourself as a co-author." + "cannot_coauthor_self": "You cannot add yourself as a co-author.", + "storage_write_size_mismatch": "The downloaded file was smaller than expected. The upstream source may have returned an incomplete response — please try again." }, "WARNINGS": { "page_not_enabled_on_repo": "GitHub Pages is not enabled on this repository. Enable it in the repository's Settings → Pages on GitHub, then refresh.", diff --git a/src/core/Gist.ts b/src/core/Gist.ts index 8d0452d..f1904ab 100644 --- a/src/core/Gist.ts +++ b/src/core/Gist.ts @@ -178,7 +178,12 @@ export default class Gist { await this.updateStatus(RepositoryStatus.READY); await AnonymizedGistModel.updateOne( { _id: this._model._id }, - { $set: { anonymizeDate: this._model.anonymizeDate } } + { + $set: { + anonymizeDate: this._model.anonymizeDate, + gist: this._model.gist, + }, + } ).exec(); } } diff --git a/src/core/PullRequest.ts b/src/core/PullRequest.ts index 2486456..2d0e545 100644 --- a/src/core/PullRequest.ts +++ b/src/core/PullRequest.ts @@ -192,7 +192,12 @@ export default class PullRequest { await this.updateStatus(RepositoryStatus.READY); await AnonymizedPullRequestModel.updateOne( { _id: this._model._id }, - { $set: { anonymizeDate: this._model.anonymizeDate } } + { + $set: { + anonymizeDate: this._model.anonymizeDate, + pullRequest: this._model.pullRequest, + }, + } ).exec(); } } diff --git a/src/core/Repository.ts b/src/core/Repository.ts index 0af40b6..5f0fb92 100644 --- a/src/core/Repository.ts +++ b/src/core/Repository.ts @@ -213,15 +213,14 @@ export default class Repository { /** * Check the status of the repository */ - check() { + async check() { if ( this._model.options.expirationMode !== "never" && this.status == RepositoryStatus.READY && this._model.options.expirationDate ) { if (this._model.options.expirationDate <= new Date()) { - this._model.status = RepositoryStatus.EXPIRED; - this.expire(); + await this.expire(); } } if ( @@ -384,6 +383,18 @@ export default class Repository { commit: newCommit, }); + if (isConnected) { + await AnonymizedRepositoryModel.updateOne( + { _id: this._model._id }, + { + $set: { + "source.commit": newCommit, + "source.commitDate": this._model.source.commitDate, + anonymizeDate: this._model.anonymizeDate, + }, + } + ).exec(); + } await this.resetSate(RepositoryStatus.PREPARING); await downloadQueue.add(this.repoId, { repoId: this.repoId }, { jobId: this.repoId, diff --git a/src/server/routes/gist-private.ts b/src/server/routes/gist-private.ts index a14cf21..78d2ade 100644 --- a/src/server/routes/gist-private.ts +++ b/src/server/routes/gist-private.ts @@ -166,6 +166,16 @@ router.post( updateGistModel(gist.model, gistUpdate); gist.model.conference = gistUpdate.conference; + await AnonymizedGistModel.updateOne( + { _id: gist.model._id }, + { + $set: { + options: gist.model.options, + conference: gist.model.conference, + anonymizeDate: gist.model.anonymizeDate, + }, + } + ).exec(); await gist.updateStatus(RepositoryStatus.PREPARING); await gist.updateIfNeeded({ force: true }); res.json(gist.toJSON()); @@ -200,6 +210,7 @@ router.post("/", async (req: express.Request, res: express.Response) => { gist.model.conference = gistUpdate.conference; + await gist.model.save(); await gist.anonymize(); res.send(gist.toJSON()); } catch (error) { diff --git a/src/server/routes/pullRequest-private.ts b/src/server/routes/pullRequest-private.ts index bc841f0..d7b2050 100644 --- a/src/server/routes/pullRequest-private.ts +++ b/src/server/routes/pullRequest-private.ts @@ -186,6 +186,16 @@ router.post( updatePullRequestModel(pullRequest.model, pullRequestUpdate); // TODO handle conference pullRequest.model.conference = pullRequestUpdate.conference; + await AnonymizedPullRequestModel.updateOne( + { _id: pullRequest.model._id }, + { + $set: { + options: pullRequest.model.options, + conference: pullRequest.model.conference, + anonymizeDate: pullRequest.model.anonymizeDate, + }, + } + ).exec(); await pullRequest.updateStatus(RepositoryStatus.PREPARING); await pullRequest.updateIfNeeded({ force: true }); res.json(pullRequest.toJSON()); @@ -222,6 +232,7 @@ router.post("/", async (req: express.Request, res: express.Response) => { pullRequest.conference = pullRequestUpdate.conference; + await pullRequest.model.save(); await pullRequest.anonymize(); res.send(pullRequest.toJSON()); } catch (error) { diff --git a/src/server/routes/repository-private.ts b/src/server/routes/repository-private.ts index f15c525..760ba94 100644 --- a/src/server/routes/repository-private.ts +++ b/src/server/routes/repository-private.ts @@ -480,6 +480,17 @@ router.post( } } repo.model.conference = repoUpdate.conference; + await AnonymizedRepositoryModel.updateOne( + { _id: repo.model._id }, + { + $set: { + options: repo.model.options, + source: repo.model.source, + conference: repo.model.conference, + anonymizeDate: repo.model.anonymizeDate, + }, + } + ).exec(); await repo.updateStatus(RepositoryStatus.PREPARING); res.json({ status: repo.status }); await downloadQueue.add(repo.repoId, { repoId: repo.repoId }, { jobId: repo.repoId }); diff --git a/src/server/routes/route-utils.ts b/src/server/routes/route-utils.ts index 06e3a79..a31457a 100644 --- a/src/server/routes/route-utils.ts +++ b/src/server/routes/route-utils.ts @@ -83,7 +83,7 @@ export async function getRepo( return null; } - repo.check(); + await repo.check(); } return repo; } catch (error) { diff --git a/src/server/schedule.ts b/src/server/schedule.ts index d6352a1..14bd465 100644 --- a/src/server/schedule.ts +++ b/src/server/schedule.ts @@ -35,10 +35,10 @@ export function repositoryStatusCheck() { status: { $eq: "ready" }, isReseted: { $eq: false }, }) - ).forEach((data) => { + ).forEach(async (data) => { const repo = new Repository(data); try { - repo.check(); + await repo.check(); } catch { logger.info("repository expired", { repoId: repo.repoId }); } diff --git a/test/file-route-path.test.js b/test/file-route-path.test.js new file mode 100644 index 0000000..3b98a65 --- /dev/null +++ b/test/file-route-path.test.js @@ -0,0 +1,38 @@ +const { expect } = require("chai"); +require("ts-node/register/transpile-only"); +const { filePathFromRequestUrl } = require("../src/server/routes/file"); + +describe("file route path decoding", function () { + it("decodes Chinese file names from encoded URL segments", function () { + const path = filePathFromRequestUrl( + "/repo-id/file/V%20%E7%AB%99%E6%80%8E%E4%B9%88%E6%9C%89%E8%BF%99%E4%B9%88%E5%A4%9A%E4%BA%BA.md?v=0", + "https", + "anonymous.4open.science", + "repo-id" + ); + + expect(path).to.equal("V 站怎么有这么多人.md"); + }); + + it("decodes reserved characters inside a filename without treating them as URL syntax", function () { + const path = filePathFromRequestUrl( + "/repo-id/file/docs/a%3Fb%23c%25d.md", + "https", + "anonymous.4open.science", + "repo-id" + ); + + expect(path).to.equal("docs/a?b#c%d.md"); + }); + + it("keeps malformed percent sequences as literal filename text", function () { + const path = filePathFromRequestUrl( + "/repo-id/file/notes/100%25%20done%ZZ.md", + "https", + "anonymous.4open.science", + "repo-id" + ); + + expect(path).to.equal("notes/100%25%20done%ZZ.md"); + }); +}); diff --git a/test/github-stream-url.test.js b/test/github-stream-url.test.js new file mode 100644 index 0000000..d2c0e0c --- /dev/null +++ b/test/github-stream-url.test.js @@ -0,0 +1,31 @@ +const { expect } = require("chai"); +require("ts-node/register/transpile-only"); +const { githubRawFileUrl } = require("../src/core/source/GitHubStream"); + +describe("githubRawFileUrl", function () { + it("encodes Chinese file names for GitHub raw URLs", function () { + const url = githubRawFileUrl( + "owner", + "repo", + "abc123", + "V 站怎么有这么多人以 PC 为荣? - V2EX.md" + ); + + expect(url).to.equal( + "https://github.com/owner/repo/raw/abc123/V%20%E7%AB%99%E6%80%8E%E4%B9%88%E6%9C%89%E8%BF%99%E4%B9%88%E5%A4%9A%E4%BA%BA%E4%BB%A5%20PC%20%E4%B8%BA%E8%8D%A3%EF%BC%9F%20-%20V2EX.md" + ); + }); + + it("encodes reserved characters without escaping path separators", function () { + const url = githubRawFileUrl( + "owner", + "repo", + "abc123", + "docs/a?b#c%d.md" + ); + + expect(url).to.equal( + "https://github.com/owner/repo/raw/abc123/docs/a%3Fb%23c%25d.md" + ); + }); +});