fix persistance bugs

public/i18n/locale-en.json

@@ -101,7 +101,8 @@
     "server_error": "An unexpected server error occurred. Please try again later.",
     "username_not_defined": "A username must be provided.",
     "github_user_not_found": "The specified GitHub user could not be found.",
-    "cannot_coauthor_self": "You cannot add yourself as a co-author."
+    "cannot_coauthor_self": "You cannot add yourself as a co-author.",
+    "storage_write_size_mismatch": "The downloaded file was smaller than expected. The upstream source may have returned an incomplete response — please try again."
   },
   "WARNINGS": {
     "page_not_enabled_on_repo": "GitHub Pages is not enabled on this repository. Enable it in the repository's Settings → Pages on GitHub, then refresh.",

src/core/Gist.ts

@@ -178,7 +178,12 @@ export default class Gist {
       await this.updateStatus(RepositoryStatus.READY);
       await AnonymizedGistModel.updateOne(
         { _id: this._model._id },
-        { $set: { anonymizeDate: this._model.anonymizeDate } }
+        {
+          $set: {
+            anonymizeDate: this._model.anonymizeDate,
+            gist: this._model.gist,
+          },
+        }
       ).exec();
     }
   }

src/core/PullRequest.ts

@@ -192,7 +192,12 @@ export default class PullRequest {
       await this.updateStatus(RepositoryStatus.READY);
       await AnonymizedPullRequestModel.updateOne(
         { _id: this._model._id },
-        { $set: { anonymizeDate: this._model.anonymizeDate } }
+        {
+          $set: {
+            anonymizeDate: this._model.anonymizeDate,
+            pullRequest: this._model.pullRequest,
+          },
+        }
       ).exec();
     }
   }

src/core/Repository.ts

@@ -213,15 +213,14 @@ export default class Repository {
   /**
    * Check the status of the repository
    */
-  check() {
+  async check() {
     if (
       this._model.options.expirationMode !== "never" &&
       this.status == RepositoryStatus.READY &&
       this._model.options.expirationDate
     ) {
       if (this._model.options.expirationDate <= new Date()) {
-        this._model.status = RepositoryStatus.EXPIRED;
+        await this.expire();
-        this.expire();
       }
     }
     if (
@@ -384,6 +383,18 @@ export default class Repository {
           commit: newCommit,
         });
 
+        if (isConnected) {
+          await AnonymizedRepositoryModel.updateOne(
+            { _id: this._model._id },
+            {
+              $set: {
+                "source.commit": newCommit,
+                "source.commitDate": this._model.source.commitDate,
+                anonymizeDate: this._model.anonymizeDate,
+              },
+            }
+          ).exec();
+        }
         await this.resetSate(RepositoryStatus.PREPARING);
         await downloadQueue.add(this.repoId, { repoId: this.repoId }, {
           jobId: this.repoId,

src/server/routes/gist-private.ts

@@ -166,6 +166,16 @@ router.post(
 
       updateGistModel(gist.model, gistUpdate);
       gist.model.conference = gistUpdate.conference;
+      await AnonymizedGistModel.updateOne(
+        { _id: gist.model._id },
+        {
+          $set: {
+            options: gist.model.options,
+            conference: gist.model.conference,
+            anonymizeDate: gist.model.anonymizeDate,
+          },
+        }
+      ).exec();
       await gist.updateStatus(RepositoryStatus.PREPARING);
       await gist.updateIfNeeded({ force: true });
       res.json(gist.toJSON());
@@ -200,6 +210,7 @@ router.post("/", async (req: express.Request, res: express.Response) => {
 
     gist.model.conference = gistUpdate.conference;
 
+    await gist.model.save();
     await gist.anonymize();
     res.send(gist.toJSON());
   } catch (error) {

src/server/routes/pullRequest-private.ts

@@ -186,6 +186,16 @@ router.post(
       updatePullRequestModel(pullRequest.model, pullRequestUpdate);
       // TODO handle conference
       pullRequest.model.conference = pullRequestUpdate.conference;
+      await AnonymizedPullRequestModel.updateOne(
+        { _id: pullRequest.model._id },
+        {
+          $set: {
+            options: pullRequest.model.options,
+            conference: pullRequest.model.conference,
+            anonymizeDate: pullRequest.model.anonymizeDate,
+          },
+        }
+      ).exec();
       await pullRequest.updateStatus(RepositoryStatus.PREPARING);
       await pullRequest.updateIfNeeded({ force: true });
       res.json(pullRequest.toJSON());
@@ -222,6 +232,7 @@ router.post("/", async (req: express.Request, res: express.Response) => {
 
     pullRequest.conference = pullRequestUpdate.conference;
 
+    await pullRequest.model.save();
     await pullRequest.anonymize();
     res.send(pullRequest.toJSON());
   } catch (error) {

src/server/routes/repository-private.ts

@@ -480,6 +480,17 @@ router.post(
         }
       }
       repo.model.conference = repoUpdate.conference;
+      await AnonymizedRepositoryModel.updateOne(
+        { _id: repo.model._id },
+        {
+          $set: {
+            options: repo.model.options,
+            source: repo.model.source,
+            conference: repo.model.conference,
+            anonymizeDate: repo.model.anonymizeDate,
+          },
+        }
+      ).exec();
       await repo.updateStatus(RepositoryStatus.PREPARING);
       res.json({ status: repo.status });
       await downloadQueue.add(repo.repoId, { repoId: repo.repoId }, { jobId: repo.repoId });

src/server/routes/route-utils.ts

@@ -83,7 +83,7 @@ export async function getRepo(
         return null;
       }
 
-      repo.check();
+      await repo.check();
     }
     return repo;
   } catch (error) {

src/server/schedule.ts

@@ -35,10 +35,10 @@ export function repositoryStatusCheck() {
         status: { $eq: "ready" },
         isReseted: { $eq: false },
       })
-    ).forEach((data) => {
+    ).forEach(async (data) => {
       const repo = new Repository(data);
       try {
-        repo.check();
+        await repo.check();
       } catch {
         logger.info("repository expired", { repoId: repo.repoId });
       }

test/file-route-path.test.js

@@ -0,0 +1,38 @@
+const { expect } = require("chai");
+require("ts-node/register/transpile-only");
+const { filePathFromRequestUrl } = require("../src/server/routes/file");
+
+describe("file route path decoding", function () {
+  it("decodes Chinese file names from encoded URL segments", function () {
+    const path = filePathFromRequestUrl(
+      "/repo-id/file/V%20%E7%AB%99%E6%80%8E%E4%B9%88%E6%9C%89%E8%BF%99%E4%B9%88%E5%A4%9A%E4%BA%BA.md?v=0",
+      "https",
+      "anonymous.4open.science",
+      "repo-id"
+    );
+
+    expect(path).to.equal("V 站怎么有这么多人.md");
+  });
+
+  it("decodes reserved characters inside a filename without treating them as URL syntax", function () {
+    const path = filePathFromRequestUrl(
+      "/repo-id/file/docs/a%3Fb%23c%25d.md",
+      "https",
+      "anonymous.4open.science",
+      "repo-id"
+    );
+
+    expect(path).to.equal("docs/a?b#c%d.md");
+  });
+
+  it("keeps malformed percent sequences as literal filename text", function () {
+    const path = filePathFromRequestUrl(
+      "/repo-id/file/notes/100%25%20done%ZZ.md",
+      "https",
+      "anonymous.4open.science",
+      "repo-id"
+    );
+
+    expect(path).to.equal("notes/100%25%20done%ZZ.md");
+  });
+});

test/github-stream-url.test.js

@@ -0,0 +1,31 @@
+const { expect } = require("chai");
+require("ts-node/register/transpile-only");
+const { githubRawFileUrl } = require("../src/core/source/GitHubStream");
+
+describe("githubRawFileUrl", function () {
+  it("encodes Chinese file names for GitHub raw URLs", function () {
+    const url = githubRawFileUrl(
+      "owner",
+      "repo",
+      "abc123",
+      "V 站怎么有这么多人以 PC 为荣？ - V2EX.md"
+    );
+
+    expect(url).to.equal(
+      "https://github.com/owner/repo/raw/abc123/V%20%E7%AB%99%E6%80%8E%E4%B9%88%E6%9C%89%E8%BF%99%E4%B9%88%E5%A4%9A%E4%BA%BA%E4%BB%A5%20PC%20%E4%B8%BA%E8%8D%A3%EF%BC%9F%20-%20V2EX.md"
+    );
+  });
+
+  it("encodes reserved characters without escaping path separators", function () {
+    const url = githubRawFileUrl(
+      "owner",
+      "repo",
+      "abc123",
+      "docs/a?b#c%d.md"
+    );
+
+    expect(url).to.equal(
+      "https://github.com/owner/repo/raw/abc123/docs/a%3Fb%23c%25d.md"
+    );
+  });
+});