feat: introduce streamers that handle the stream and anonymization from github

src/core/model/anonymizedPullRequests/anonymizedPullRequests.model.ts

@@ -0,0 +1,14 @@
+import { model } from "mongoose";
+
+import AnonymizedPullRequestSchema from "./anonymizedPullRequests.schema";
+import {
+  IAnonymizedPullRequestDocument,
+  IAnonymizedPullRequestModel,
+} from "./anonymizedPullRequests.types";
+
+const AnonymizedPullRequestModel = model<IAnonymizedPullRequestDocument>(
+  "AnonymizedPullRequest",
+  AnonymizedPullRequestSchema
+) as IAnonymizedPullRequestModel;
+
+export default AnonymizedPullRequestModel;

src/core/model/anonymizedPullRequests/anonymizedPullRequests.schema.ts

@@ -0,0 +1,66 @@
+import { Schema } from "mongoose";
+
+const AnonymizedPullRequestSchema = new Schema({
+  pullRequestId: {
+    type: String,
+    index: { unique: true },
+  },
+  status: {
+    type: String,
+    default: "preparing",
+  },
+  statusDate: Date,
+  statusMessage: String,
+  anonymizeDate: Date,
+  lastView: Date,
+  pageView: Number,
+  owner: Schema.Types.ObjectId,
+  conference: String,
+  source: {
+    pullRequestId: Number,
+    repositoryFullName: String,
+    accessToken: String,
+  },
+  options: {
+    terms: [String],
+    expirationMode: { type: String },
+    expirationDate: Date,
+    update: Boolean,
+    image: Boolean,
+    link: Boolean,
+    title: Boolean,
+    body: Boolean,
+    comments: Boolean,
+    diff: Boolean,
+    origin: Boolean,
+    username: Boolean,
+    date: Boolean,
+  },
+  dateOfEntry: {
+    type: Date,
+    default: new Date(),
+  },
+  pullRequest: {
+    diff: String,
+    title: String,
+    body: String,
+    creationDate: Date,
+    updatedDate: Date,
+    draft: Boolean,
+    merged: Boolean,
+    mergedDate: Date,
+    state: String,
+    baseRepositoryFullName: String,
+    headRepositoryFullName: String,
+    comments: [
+      {
+        body: String,
+        creationDate: Date,
+        updatedDate: Date,
+        author: String,
+      },
+    ],
+  },
+});
+
+export default AnonymizedPullRequestSchema;

src/core/model/anonymizedPullRequests/anonymizedPullRequests.types.ts

@@ -0,0 +1,61 @@
+import { Document, Model } from "mongoose";
+import { RepositoryStatus } from "../../types";
+
+export interface IAnonymizedPullRequest {
+  pullRequestId: string;
+  status?: RepositoryStatus;
+  statusMessage?: string;
+  statusDate: Date;
+  anonymizeDate: Date;
+  source: {
+    pullRequestId: number;
+    repositoryFullName: string;
+    accessToken?: string;
+  };
+  owner: string;
+  conference: string;
+  options: {
+    terms: string[];
+    expirationMode: "never" | "redirect" | "remove";
+    expirationDate?: Date;
+    update: boolean;
+    image: boolean;
+    link: boolean;
+    title: boolean;
+    body: boolean;
+    comments: boolean;
+    diff: boolean;
+    origin: boolean;
+    username: boolean;
+    date: boolean;
+  };
+  pageView: number;
+  lastView: Date;
+  pullRequest: {
+    diff: string;
+    title: string;
+    body: string;
+    creationDate: Date;
+    updatedDate: Date;
+    draft?: boolean;
+    merged?: boolean;
+    mergedDate?: Date;
+    state?: string;
+    baseRepositoryFullName?: string;
+    headRepositoryFullName?: string;
+    comments?: {
+      body: string;
+      creationDate: Date;
+      updatedDate: Date;
+      author: string;
+    }[];
+  };
+}
+
+export interface IAnonymizedPullRequestDocument
+  extends IAnonymizedPullRequest,
+    Document {
+  setLastUpdated: (this: IAnonymizedPullRequestDocument) => Promise<void>;
+}
+export interface IAnonymizedPullRequestModel
+  extends Model<IAnonymizedPullRequestDocument> {}

src/core/model/anonymizedRepositories/anonymizedRepositories.model.ts

@@ -0,0 +1,14 @@
+import { model } from "mongoose";
+
+import {
+  IAnonymizedRepositoryDocument,
+  IAnonymizedRepositoryModel,
+} from "./anonymizedRepositories.types";
+import AnonymizedRepositorySchema from "./anonymizedRepositories.schema";
+
+const AnonymizedRepositoryModel = model<IAnonymizedRepositoryDocument>(
+  "AnonymizedRepository",
+  AnonymizedRepositorySchema
+) as IAnonymizedRepositoryModel;
+
+export default AnonymizedRepositoryModel;

src/core/model/anonymizedRepositories/anonymizedRepositories.schema.ts

@@ -0,0 +1,73 @@
+import { Schema } from "mongoose";
+
+const AnonymizedRepositorySchema = new Schema({
+  repoId: {
+    type: String,
+    index: { unique: true, collation: { locale: "en", strength: 2 } },
+  },
+  status: {
+    type: String,
+    default: "preparing",
+  },
+  statusDate: Date,
+  statusMessage: String,
+  anonymizeDate: Date,
+  lastView: Date,
+  pageView: Number,
+  accessToken: String,
+  owner: {
+    type: Schema.Types.ObjectId,
+    ref: "user",
+    index: true,
+  },
+  conference: String,
+  source: {
+    type: { type: String },
+    branch: String,
+    commit: String,
+    commitDate: Date,
+    repositoryId: String,
+    repositoryName: String,
+    accessToken: String,
+  },
+  truckedFileList: {
+    type: Boolean,
+    default: false,
+  },
+  originalFiles: Schema.Types.Mixed,
+  options: {
+    terms: [String],
+    expirationMode: { type: String },
+    expirationDate: Date,
+    update: Boolean,
+    image: Boolean,
+    pdf: Boolean,
+    notebook: Boolean,
+    link: Boolean,
+    page: Boolean,
+    pageSource: {
+      branch: String,
+      path: String,
+    },
+  },
+  dateOfEntry: {
+    type: Date,
+    default: new Date(),
+  },
+  size: {
+    storage: {
+      type: Number,
+      default: 0,
+    },
+    file: {
+      type: Number,
+      default: 0,
+    },
+  },
+  isReseted: {
+    type: Boolean,
+    default: false,
+  },
+});
+
+export default AnonymizedRepositorySchema;

src/core/model/anonymizedRepositories/anonymizedRepositories.types.ts

@@ -0,0 +1,53 @@
+import { Document, Model } from "mongoose";
+import { RepositoryStatus, Tree } from "../../types";
+
+export interface IAnonymizedRepository {
+  repoId: string;
+  status?: RepositoryStatus;
+  statusMessage?: string;
+  statusDate: Date;
+  anonymizeDate: Date;
+  source: {
+    type: "GitHubDownload" | "GitHubStream" | "Zip";
+    branch?: string;
+    commit?: string;
+    commitDate?: Date,
+    repositoryId?: string;
+    repositoryName?: string;
+    accessToken?: string;
+  };
+  owner: string;
+  truckedFileList: boolean;
+  originalFiles?: Tree;
+  conference: string;
+  options: {
+    terms: string[];
+    expirationMode: "never" | "redirect" | "remove";
+    expirationDate?: Date;
+    update: boolean;
+    image: boolean;
+    pdf: boolean;
+    notebook: boolean;
+    link: boolean;
+    page: boolean;
+    pageSource?: {
+      branch: string;
+      path: string;
+    };
+  };
+  pageView: number;
+  lastView: Date;
+  size: {
+    storage: number;
+    file: number;
+  };
+  isReseted: boolean;
+}
+
+export interface IAnonymizedRepositoryDocument
+  extends IAnonymizedRepository,
+    Document {
+  setLastUpdated: (this: IAnonymizedRepositoryDocument) => Promise<void>;
+}
+export interface IAnonymizedRepositoryModel
+  extends Model<IAnonymizedRepositoryDocument> {}

src/core/model/conference/conferences.model.ts

@@ -0,0 +1,11 @@
+import { model } from "mongoose";
+
+import { IConferenceDocument, IConferenceModel } from "./conferences.types";
+import ConferenceSchema from "./conferences.schema";
+
+const ConferenceModel = model<IConferenceDocument>(
+  "Conference",
+  ConferenceSchema
+) as IConferenceModel;
+
+export default ConferenceModel;

src/core/model/conference/conferences.schema.ts

@@ -0,0 +1,58 @@
+import { Schema } from "mongoose";
+
+const RepositorySchema = new Schema({
+  name: String,
+  conferenceID: {
+    type: String,
+    index: { unique: true },
+  },
+  url: String,
+  startDate: Date,
+  endDate: Date,
+  status: String,
+  owners: { type: [Schema.Types.ObjectId] },
+  repositories: {
+    type: [
+      {
+        id: { type: Schema.Types.ObjectId },
+        addDate: { type: Date },
+        removeDate: { type: Date },
+      },
+    ],
+  },
+  options: {
+    expirationMode: String,
+    expirationDate: Date,
+    update: Boolean,
+    image: Boolean,
+    pdf: Boolean,
+    notebook: Boolean,
+    link: Boolean,
+    page: Boolean,
+  },
+  dateOfEntry: {
+    type: Date,
+    default: new Date(),
+  },
+  plan: {
+    planID: String,
+    pricePerRepository: Number,
+    quota: {
+      repository: Number,
+      size: Number,
+      file: Number,
+    },
+  },
+  billing: {
+    name: String,
+    email: String,
+    address: String,
+    address2: String,
+    city: String,
+    zip: String,
+    country: String,
+    vat: String,
+  },
+});
+
+export default RepositorySchema;

src/core/model/conference/conferences.types.ts

@@ -0,0 +1,49 @@
+import { Document, Model } from "mongoose";
+import { ConferenceStatus } from "../../types";
+
+export interface IConference {
+  name: string;
+  conferenceID: string;
+  startDate: Date;
+  endDate: Date;
+  url: string;
+  status: ConferenceStatus;
+  owners: string[];
+  repositories: {
+    id: string;
+    addDate: Date;
+    removeDate?: Date;
+  }[];
+  options: {
+    expirationMode: "never" | "redirect" | "remove";
+    expirationDate?: Date;
+    update: boolean;
+    image: boolean;
+    pdf: boolean;
+    notebook: boolean;
+    link: boolean;
+    page: boolean;
+  };
+  plan: {
+    planID: string;
+    pricePerRepository: number;
+    quota: {
+      repository: number;
+      size: number;
+      file: number;
+    };
+  };
+  billing?: {
+    name: string;
+    email: string;
+    address: string;
+    address2?: string;
+    city: string;
+    zip: string;
+    country: string;
+    vat?: string;
+  };
+}
+
+export interface IConferenceDocument extends IConference, Document {}
+export interface IConferenceModel extends Model<IConferenceDocument> {}

src/core/model/repositories/repositories.model.ts

@@ -0,0 +1,11 @@
+import { model } from "mongoose";
+
+import { IRepositoryDocument, IRepositoryModel } from "./repositories.types";
+import RepositorySchema from "./repositories.schema";
+
+const RepositoryModel = model<IRepositoryDocument>(
+  "Repository",
+  RepositorySchema
+) as IRepositoryModel;
+
+export default RepositoryModel;

src/core/model/repositories/repositories.schema.ts

@@ -0,0 +1,41 @@
+import { Schema } from "mongoose";
+
+const RepositorySchema = new Schema({
+  externalId: {
+    type: String,
+    index: { unique: true },
+  },
+  name: {
+    type: String,
+    index: true,
+  },
+  url: String,
+  source: {
+    type: String,
+    default: "github",
+  },
+  hasPage: { type: Boolean, default: false },
+  pageSource: {
+    branch: { type: String },
+    path: String,
+  },
+  branches: [
+    {
+      name: { type: String },
+      commit: String,
+      readme: String,
+    },
+  ],
+  defaultBranch: String,
+  size: Number,
+  status: {
+    type: String,
+    default: "ready",
+  },
+  dateOfEntry: {
+    type: Date,
+    default: new Date(),
+  },
+});
+
+export default RepositorySchema;

src/core/model/repositories/repositories.types.ts

@@ -0,0 +1,25 @@
+import { Document, Model } from "mongoose";
+
+export interface IRepository {
+  externalId: string;
+  name: string;
+  url?: string;
+  source: "github";
+  size?: number;
+  defaultBranch?: string;
+  hasPage: boolean;
+  pageSource?: {
+    branch: string;
+    path: string;
+  };
+  branches?: {
+    name: string;
+    commit: string;
+    readme?: string;
+  }[];
+}
+
+export interface IRepositoryDocument extends IRepository, Document {
+  setLastUpdated: (this: IRepositoryDocument) => Promise<void>;
+}
+export interface IRepositoryModel extends Model<IRepositoryDocument> {}

src/core/model/users/users.model.ts

@@ -0,0 +1,8 @@
+import { model } from "mongoose";
+
+import { IUserDocument, IUserModel } from "./users.types";
+import UserSchema from "./users.schema";
+
+const UserModel = model<IUserDocument>("user", UserSchema) as IUserModel;
+
+export default UserModel;

src/core/model/users/users.schema.ts

@@ -0,0 +1,50 @@
+import { Schema } from "mongoose";
+
+const UserSchema = new Schema({
+  accessTokens: {
+    github: { type: String },
+  },
+  externalIDs: {
+    github: { type: String, index: true },
+  },
+  username: {
+    type: String,
+    index: { unique: true },
+  },
+  emails: [
+    {
+      email: { type: String },
+      default: Boolean,
+    },
+  ],
+  isAdmin: { type: Boolean, default: false },
+  photo: String,
+  repositories: [
+    {
+      type: String,
+      ref: "Repository",
+    },
+  ],
+  default: {
+    terms: [String],
+    options: {
+      expirationMode: { type: String },
+      update: Boolean,
+      image: Boolean,
+      pdf: Boolean,
+      notebook: Boolean,
+      link: Boolean,
+      page: { type: String },
+    },
+  },
+  status: {
+    type: String,
+    default: "active",
+  },
+  dateOfEntry: {
+    type: Date,
+    default: new Date(),
+  },
+});
+
+export default UserSchema;

src/core/model/users/users.types.ts

@@ -0,0 +1,39 @@
+import { Document, Model } from "mongoose";
+
+export interface IUser {
+  accessTokens: {
+    github: string;
+  };
+  externalIDs: {
+    github: string;
+  };
+  username: string;
+  isAdmin: boolean;
+  emails: {
+    email: string;
+    default: boolean;
+  }[];
+  photo?: string;
+
+  repositories?: number[];
+  default?: {
+    terms: string[];
+    options: {
+      expirationMode: "never" | "redirect" | "";
+      update: boolean;
+      image: boolean;
+      pdf: boolean;
+      notebook: boolean;
+      link: boolean;
+      page: string | null;
+    };
+  };
+  status?: "active" | "removed";
+  dateOfEntry?: Date;
+  lastUpdated?: Date;
+}
+
+export interface IUserDocument extends IUser, Document {
+  setLastUpdated: (this: IUserDocument) => Promise<void>;
+}
+export interface IUserModel extends Model<IUserDocument> {}