feat: gist & co-authors

src/core/Gist.ts

@@ -0,0 +1,284 @@
+import { RepositoryStatus } from "./types";
+import User from "./User";
+import UserModel from "./model/users/users.model";
+import Conference from "./Conference";
+import ConferenceModel from "./model/conference/conferences.model";
+import AnonymousError from "./AnonymousError";
+import { IAnonymizedGistDocument } from "./model/anonymizedGists/anonymizedGists.types";
+import config from "../config";
+import { octokit } from "./GitHubUtils";
+import { ContentAnonimizer } from "./anonymize-utils";
+
+export default class Gist {
+  private _model: IAnonymizedGistDocument;
+  owner: User;
+
+  constructor(data: IAnonymizedGistDocument) {
+    this._model = data;
+    this.owner = new User(new UserModel({ _id: data.owner }));
+    this.owner.model.isNew = false;
+  }
+
+  async getToken() {
+    let owner = this.owner.model;
+    if (owner && !owner.accessTokens.github) {
+      const temp = await UserModel.findById(owner._id);
+      if (temp) {
+        owner = temp;
+      }
+    }
+    if (owner && owner.accessTokens && owner.accessTokens.github) {
+      if (owner.accessTokens.github != this._model.source.accessToken) {
+        this._model.source.accessToken = owner.accessTokens.github;
+      }
+      return owner.accessTokens.github;
+    }
+    if (this._model.source.accessToken) {
+      try {
+        return this._model.source.accessToken;
+      } catch {
+        console.debug("[ERROR] Token is invalid", this._model.source.gistId);
+      }
+    }
+    return config.GITHUB_TOKEN;
+  }
+
+  async download() {
+    console.debug("[INFO] Downloading gist", this._model.source.gistId);
+    const oct = octokit(await this.getToken());
+
+    const gist_id = this._model.source.gistId;
+
+    const [gistInfo, comments] = await Promise.all([
+      oct.rest.gists.get({ gist_id }),
+      oct.paginate("GET /gists/{gist_id}/comments", {
+        gist_id,
+        per_page: 100,
+      }),
+    ]);
+
+    const files = Object.values(gistInfo.data.files || {})
+      .filter((f): f is NonNullable<typeof f> => !!f)
+      .map((f) => ({
+        filename: f.filename || "",
+        content: f.content || "",
+        language: f.language || undefined,
+        size: f.size || 0,
+        type: f.type || undefined,
+      }));
+
+    this._model.gist = {
+      description: gistInfo.data.description || "",
+      isPublic: gistInfo.data.public,
+      creationDate: gistInfo.data.created_at
+        ? new Date(gistInfo.data.created_at)
+        : new Date(),
+      updatedDate: gistInfo.data.updated_at
+        ? new Date(gistInfo.data.updated_at)
+        : new Date(),
+      ownerLogin: gistInfo.data.owner?.login,
+      files,
+      comments: comments.map((comment) => ({
+        body: comment.body || "",
+        creationDate: new Date(comment.created_at),
+        updatedDate: new Date(comment.updated_at),
+        author: comment.user?.login || "",
+      })),
+    };
+  }
+
+  /**
+   * Check the status of the gist
+   */
+  async check() {
+    if (
+      this._model.options.expirationMode !== "never" &&
+      this.status == "ready" &&
+      this._model.options.expirationDate
+    ) {
+      if (this._model.options.expirationDate <= new Date()) {
+        await this.expire();
+      }
+    }
+    if (
+      this.status == "expired" ||
+      this.status == "expiring" ||
+      this.status == "removing" ||
+      this.status == "removed"
+    ) {
+      throw new AnonymousError("gist_expired", {
+        object: this,
+        httpStatus: 410,
+      });
+    }
+    const fiveMinuteAgo = new Date();
+    fiveMinuteAgo.setMinutes(fiveMinuteAgo.getMinutes() - 5);
+
+    if (
+      this.status == "preparing" ||
+      (this.status == "download" && this._model.statusDate > fiveMinuteAgo)
+    ) {
+      throw new AnonymousError("gist_not_ready", {
+        object: this,
+        httpStatus: 503,
+      });
+    }
+  }
+
+  async updateIfNeeded(opt?: { force: boolean }): Promise<void> {
+    const yesterday = new Date();
+    yesterday.setDate(yesterday.getDate() - 1);
+    if (
+      opt?.force ||
+      (this._model.options.update && this._model.anonymizeDate < yesterday)
+    ) {
+      await this.updateStatus(RepositoryStatus.DOWNLOAD);
+      await this.download();
+      this._model.anonymizeDate = new Date();
+      await this.updateStatus(RepositoryStatus.READY);
+      await this._model.save();
+    }
+  }
+
+  async anonymize() {
+    if (this.status === RepositoryStatus.READY) return;
+    await this.updateStatus(RepositoryStatus.PREPARING);
+    await this.updateIfNeeded({ force: true });
+    await this.updateStatus(RepositoryStatus.READY);
+  }
+
+  async countView() {
+    this._model.lastView = new Date();
+    this._model.pageView = (this._model.pageView || 0) + 1;
+    await this._model.save();
+  }
+
+  async updateStatus(status: RepositoryStatus, statusMessage?: string) {
+    this._model.status = status;
+    this._model.statusDate = new Date();
+    this._model.statusMessage = statusMessage;
+    await this._model.save();
+  }
+
+  async expire() {
+    await this.updateStatus(RepositoryStatus.EXPIRING);
+    await this.resetSate();
+    await this.updateStatus(RepositoryStatus.EXPIRED);
+  }
+
+  async remove() {
+    await this.updateStatus(RepositoryStatus.REMOVING);
+    await this.resetSate();
+    await this.updateStatus(RepositoryStatus.REMOVED);
+  }
+
+  async resetSate(status?: RepositoryStatus, statusMessage?: string) {
+    if (status) this._model.status = status;
+    if (statusMessage) this._model.statusMessage = statusMessage;
+    this._model.gist.comments = [];
+    this._model.gist.description = "";
+    this._model.gist.files = [];
+    this._model.gist.ownerLogin = "";
+    await this._model.save();
+  }
+
+  async conference(): Promise<Conference | null> {
+    if (!this._model.conference) {
+      return null;
+    }
+    const conference = await ConferenceModel.findOne({
+      conferenceID: this._model.conference,
+    });
+    if (conference) return new Conference(conference);
+    return null;
+  }
+
+  content() {
+    const output: Record<string, unknown> = {
+      anonymizeDate: this._model.anonymizeDate,
+      isPublic: this._model.gist.isPublic,
+    };
+    const anonymizer = new ContentAnonimizer({
+      ...this.options,
+      repoId: this.gistId,
+    });
+    if (this.options.title) {
+      output.description = anonymizer.anonymize(this._model.gist.description);
+    }
+    if (this.options.username) {
+      output.ownerLogin = anonymizer.anonymize(
+        this._model.gist.ownerLogin || ""
+      );
+    }
+    if (this.options.content) {
+      output.files = (this._model.gist.files || []).map((f) => ({
+        filename: anonymizer.anonymize(f.filename),
+        content: anonymizer.anonymize(f.content),
+        language: f.language,
+        size: f.size,
+        type: f.type,
+      }));
+    }
+    if (this.options.comments) {
+      output.comments = this._model.gist.comments?.map((comment) => {
+        const o: Record<string, unknown> = {};
+        if (this.options.body) o.body = anonymizer.anonymize(comment.body);
+        if (this.options.username)
+          o.author = anonymizer.anonymize(comment.author);
+        if (this.options.date) {
+          o.updatedDate = comment.updatedDate;
+          o.creationDate = comment.creationDate;
+        }
+        return o;
+      });
+    }
+    if (this.options.origin) {
+      output.sourceGistId = this._model.source.gistId;
+    }
+    if (this.options.date) {
+      output.updatedDate = this._model.gist.updatedDate;
+      output.creationDate = this._model.gist.creationDate;
+    }
+    return output;
+  }
+
+  /***** Getters ********/
+
+  get gistId() {
+    return this._model.gistId;
+  }
+
+  get options() {
+    return this._model.options;
+  }
+
+  get source() {
+    return this._model.source;
+  }
+
+  get model() {
+    return this._model;
+  }
+
+  get status() {
+    return this._model.status;
+  }
+
+  toJSON() {
+    return {
+      gistId: this._model.gistId,
+      options: this._model.options,
+      conference: this._model.conference,
+      anonymizeDate: this._model.anonymizeDate,
+      status: this._model.status,
+      isPublic: this._model.gist.isPublic,
+      statusMessage: this._model.statusMessage,
+      source: {
+        gistId: this._model.source.gistId,
+      },
+      gist: this._model.gist,
+      lastView: this._model.lastView,
+      pageView: this._model.pageView,
+    };
+  }
+}

src/core/Repository.ts

@@ -520,6 +520,10 @@ export default class Repository {
     return this._model.options;
   }
 
+  get coauthors() {
+    return this._model.coauthors || [];
+  }
+
   get model() {
     return this._model;
   }
@@ -537,6 +541,11 @@ export default class Repository {
     return {
       repoId: this._model.repoId,
       options: this._model.options,
+      coauthors: (this._model.coauthors || []).map((c) => ({
+        username: c.username,
+        githubId: c.githubId,
+        photo: c.photo,
+      })),
       conference: this._model.conference,
       anonymizeDate: this._model.anonymizeDate,
       status: this.status,

src/core/User.ts

@@ -5,6 +5,8 @@ import Repository from "./Repository";
 import { GitHubRepository } from "./source/GitHubRepository";
 import PullRequest from "./PullRequest";
 import AnonymizedPullRequestModel from "./model/anonymizedPullRequests/anonymizedPullRequests.model";
+import Gist from "./Gist";
+import AnonymizedGistModel from "./model/anonymizedGists/anonymizedGists.model";
 import { octokit } from "./GitHubUtils";
 
 /**
@@ -119,10 +121,11 @@ export default class User {
    * @returns the list of anonymized repositories
    */
   async getRepositories() {
+    const query: Record<string, unknown> = this.username
+      ? { $or: [{ owner: this.id }, { "coauthors.username": this.username }] }
+      : { owner: this.id };
     const repositories = (
-      await AnonymizedRepositoryModel.find({
-        owner: this.id,
-      }).exec()
+      await AnonymizedRepositoryModel.find(query).exec()
     ).map((d) => new Repository(d));
     const promises = [];
     for (const repo of repositories) {
@@ -165,6 +168,28 @@ export default class User {
     return pullRequests;
   }
 
+  /**
+   * Get the list of anonymized gists
+   */
+  async getGists() {
+    const gists = (
+      await AnonymizedGistModel.find({ owner: this.id }).exec()
+    ).map((d) => new Gist(d));
+    const promises = [];
+    for (const g of gists) {
+      if (
+        g.status == "ready" &&
+        g.options.expirationMode != "never" &&
+        g.options.expirationDate != null &&
+        g.options.expirationDate < new Date()
+      ) {
+        promises.push(g.expire());
+      }
+    }
+    await Promise.all(promises);
+    return gists;
+  }
+
   get model() {
     return this._model;
   }

src/core/model/anonymizedGists/anonymizedGists.model.ts

@@ -0,0 +1,14 @@
+import { model } from "mongoose";
+
+import AnonymizedGistSchema from "./anonymizedGists.schema";
+import {
+  IAnonymizedGistDocument,
+  IAnonymizedGistModel,
+} from "./anonymizedGists.types";
+
+const AnonymizedGistModel = model<IAnonymizedGistDocument>(
+  "AnonymizedGist",
+  AnonymizedGistSchema
+) as IAnonymizedGistModel;
+
+export default AnonymizedGistModel;

src/core/model/anonymizedGists/anonymizedGists.schema.ts

@@ -0,0 +1,68 @@
+import { Schema } from "mongoose";
+
+const AnonymizedGistSchema = new Schema({
+  gistId: {
+    type: String,
+    index: { unique: true },
+  },
+  status: {
+    type: String,
+    default: "preparing",
+  },
+  statusDate: Date,
+  statusMessage: String,
+  anonymizeDate: Date,
+  lastView: Date,
+  pageView: Number,
+  owner: Schema.Types.ObjectId,
+  conference: String,
+  source: {
+    gistId: String,
+    accessToken: String,
+  },
+  options: {
+    terms: [String],
+    expirationMode: { type: String },
+    expirationDate: Date,
+    update: Boolean,
+    image: Boolean,
+    link: Boolean,
+    title: Boolean,
+    body: Boolean,
+    comments: Boolean,
+    content: Boolean,
+    origin: Boolean,
+    username: Boolean,
+    date: Boolean,
+  },
+  dateOfEntry: {
+    type: Date,
+    default: new Date(),
+  },
+  gist: {
+    description: String,
+    isPublic: Boolean,
+    creationDate: Date,
+    updatedDate: Date,
+    ownerLogin: String,
+    files: [
+      {
+        filename: String,
+        content: String,
+        language: String,
+        size: Number,
+        type: String,
+      },
+    ],
+    comments: [
+      {
+        body: String,
+        creationDate: Date,
+        updatedDate: Date,
+        author: String,
+      },
+    ],
+  },
+});
+
+export default AnonymizedGistSchema;

src/core/model/anonymizedGists/anonymizedGists.types.ts

@@ -0,0 +1,58 @@
+import { Document, Model } from "mongoose";
+import { RepositoryStatus } from "../../types";
+
+export interface IAnonymizedGist {
+  gistId: string;
+  status?: RepositoryStatus;
+  statusMessage?: string;
+  statusDate: Date;
+  anonymizeDate: Date;
+  source: {
+    gistId: string;
+    accessToken?: string;
+  };
+  owner: string;
+  conference: string;
+  options: {
+    terms: string[];
+    expirationMode: "never" | "redirect" | "remove";
+    expirationDate?: Date;
+    update: boolean;
+    image: boolean;
+    link: boolean;
+    title: boolean;
+    body: boolean;
+    comments: boolean;
+    content: boolean;
+    origin: boolean;
+    username: boolean;
+    date: boolean;
+  };
+  pageView: number;
+  lastView: Date;
+  gist: {
+    description: string;
+    isPublic?: boolean;
+    creationDate: Date;
+    updatedDate: Date;
+    ownerLogin?: string;
+    files?: {
+      filename: string;
+      content: string;
+      language?: string;
+      size?: number;
+      type?: string;
+    }[];
+    comments?: {
+      body: string;
+      creationDate: Date;
+      updatedDate: Date;
+      author: string;
+    }[];
+  };
+}
+
+export interface IAnonymizedGistDocument extends IAnonymizedGist, Document {
+  setLastUpdated: (this: IAnonymizedGistDocument) => Promise<void>;
+}
+export interface IAnonymizedGistModel extends Model<IAnonymizedGistDocument> {}

src/core/model/anonymizedRepositories/anonymizedRepositories.schema.ts

@@ -20,6 +20,14 @@ const AnonymizedRepositorySchema = new Schema({
     ref: "user",
     index: true,
   },
+  coauthors: [
+    {
+      username: { type: String, index: true },
+      githubId: { type: String },
+      photo: { type: String },
+      addedAt: { type: Date, default: Date.now },
+    },
+  ],
   conference: String,
   source: {
     type: { type: String },

src/core/model/anonymizedRepositories/anonymizedRepositories.types.ts

@@ -17,6 +17,12 @@ export interface IAnonymizedRepository {
     accessToken?: string;
   };
   owner: string;
+  coauthors?: {
+    username: string;
+    githubId?: string;
+    photo?: string;
+    addedAt?: Date;
+  }[];
   truncatedFolders: string[];
   conference: string;
   options: {