CalvinBackup/anonymous_github
1
0
Fork 0
mirror of https://github.com/tdurieux/anonymous_github.git synced 2026-02-13 02:42:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf: improve the perf of Anonymous GitHub

Browse Source
This commit is contained in:
tdurieux
2023-02-08 15:34:50 +01:00
parent 2e36b72a7f
commit 5c72f54db5
21 changed files with 529 additions and 215 deletions
Download Patch File Download Diff File Expand all files Collapse all files

125
src/AnonymizedFile.ts
View File

@@ -1,34 +1,14 @@
import { join, basename } from "path";
import { Response } from "express";
import { Readable, pipeline } from "stream";
import { promisify } from "util";
import { Readable } from "stream";
import Repository from "./Repository";
import { Tree, TreeElement, TreeFile } from "./types";
import { TreeElement, TreeFile } from "./types";
import storage from "./storage";
import config from "../config";
import { anonymizePath, anonymizeStream } from "./anonymize-utils";
import AnonymousError from "./AnonymousError";
import { handleError } from "./routes/route-utils";
function tree2sha(
tree: any,
output: { [key: string]: string } = {},
parent: string = ""
): { [key: string]: string } {
for (let i in tree) {
const sha = tree[i].sha as string;
const size = tree[i].size as number;
if (sha != null && size != null) {
output[sha] = join(parent, i);
} else if (tree[i].child) {
tree2sha(tree[i].child as Tree, output, join(parent, i));
} else {
tree2sha(tree[i] as Tree, output, join(parent, i));
}
}
return output;
}
/**
* Represent a file in a anonymized repository
*/
@@ -70,27 +50,16 @@ export default class AnonymizedFile {
});
const paths = this.anonymizedPath.trim().split("/");
let currentAnonymized: TreeElement = await this.repository.anonymizedFiles({
includeSha: true,
});
let currentOriginal: TreeElement = await this.repository.files();
let currentOriginal = (await this.repository.files({
force: false,
})) as TreeElement;
let currentOriginalPath = "";
let isAmbiguous = false;
for (let i = 0; i < paths.length; i++) {
const fileName = paths[i];
if (fileName == "") {
continue;
}
if (!currentAnonymized[fileName]) {
throw new AnonymousError("file_not_found", {
object: this,
httpStatus: 404,
});
}
currentAnonymized = currentAnonymized[fileName];
if (!isAmbiguous && !currentOriginal[fileName]) {
if (!currentOriginal[fileName]) {
// anonymize all the file in the folder and check if there is one that match the current filename
const options = [];
for (let originalFileName in currentOriginal) {
@@ -106,40 +75,55 @@ export default class AnonymizedFile {
if (options.length == 1) {
currentOriginalPath = join(currentOriginalPath, options[0]);
currentOriginal = currentOriginal[options[0]];
} else if (options.length == 0) {
throw new AnonymousError("file_not_found", {
object: this,
httpStatus: 404,
});
} else {
isAmbiguous = true;
const nextName = paths[i + 1];
if (!nextName) {
// if there is no next name we can't find the file and we return the first option
currentOriginalPath = join(currentOriginalPath, options[0]);
currentOriginal = currentOriginal[options[0]];
}
let found = false;
for (const option of options) {
const optionTree = currentOriginal[option];
if (optionTree.child) {
const optionTreeChild = optionTree.child;
if (optionTreeChild[nextName]) {
currentOriginalPath = join(currentOriginalPath, option);
currentOriginal = optionTreeChild;
found = true;
break;
}
}
}
if (!found) {
// if we didn't find the next name we return the first option
currentOriginalPath = join(currentOriginalPath, options[0]);
currentOriginal = currentOriginal[options[0]];
}
}
} else if (!isAmbiguous) {
} else {
currentOriginalPath = join(currentOriginalPath, fileName);
currentOriginal = currentOriginal[fileName];
}
}
if (
currentAnonymized.sha === undefined ||
currentAnonymized.size === undefined
currentOriginal.sha === undefined ||
currentOriginal.size === undefined
) {
throw new AnonymousError("folder_not_supported", { object: this });
}
const file: TreeFile = currentAnonymized as TreeFile;
const file = currentOriginal as TreeFile;
this.fileSize = file.size;
this._sha = file.sha;
if (isAmbiguous) {
// it should never happen
const shaTree = tree2sha(currentOriginal);
if (!currentAnonymized.sha || !shaTree[file.sha]) {
throw new AnonymousError("file_not_found", {
object: this,
httpStatus: 404,
});
}
this._originalPath = join(currentOriginalPath, shaTree[file.sha]);
} else {
this._originalPath = currentOriginalPath;
}
this._originalPath = currentOriginalPath;
return this._originalPath;
}
extension() {
@@ -193,8 +177,7 @@ export default class AnonymizedFile {
}
async anonymizedContent() {
const rs = await this.content();
return rs.pipe(anonymizeStream(this));
return (await this.content()).pipe(anonymizeStream(this));
}
get originalCachePath() {
@@ -218,14 +201,24 @@ export default class AnonymizedFile {
}
async send(res: Response): Promise<void> {
const pipe = promisify(pipeline);
try {
if (this.extension()) {
res.contentType(this.extension());
}
await pipe(await this.anonymizedContent(), res);
} catch (error) {
handleError(error, res);
if (this.extension()) {
res.contentType(this.extension());
}
if (this.fileSize) {
res.set("Content-Length", this.fileSize.toString());
}
return new Promise(async (resolve, reject) => {
try {
(await this.anonymizedContent())
.pipe(res)
.on("close", () => resolve())
.on("error", (error) => {
reject(error);
handleError(error, res);
});
} catch (error) {
handleError(error, res);
}
});
}
}

14
src/PullRequest.ts
View File

@@ -135,10 +135,10 @@ export default class PullRequest {
* @returns void
*/
async anonymize() {
if (this.status == "ready") return;
await this.updateStatus("preparing");
if (this.status === RepositoryStatus.READY) return;
await this.updateStatus(RepositoryStatus.PREPARING);
await this.updateIfNeeded({ force: true });
return this.updateStatus("ready");
return this.updateStatus(RepositoryStatus.READY);
}
/**
@@ -166,18 +166,18 @@ export default class PullRequest {
* Expire the pullRequest
*/
async expire() {
await this.updateStatus("expiring");
await this.updateStatus(RepositoryStatus.EXPIRING);
await this.resetSate();
await this.updateStatus("expired");
await this.updateStatus(RepositoryStatus.EXPIRED);
}
/**
* Remove the pullRequest
*/
async remove() {
await this.updateStatus("removing");
await this.updateStatus(RepositoryStatus.REMOVING);
await this.resetSate();
await this.updateStatus("removed");
await this.updateStatus(RepositoryStatus.REMOVED);
}
/**

67
src/Repository.ts
View File

@@ -17,6 +17,29 @@ import AnonymousError from "./AnonymousError";
import { downloadQueue } from "./queue";
import { isConnected } from "./database/database";
import AnonymizedFile from "./AnonymizedFile";
import AnonymizedRepositoryModel from "./database/anonymizedRepositories/anonymizedRepositories.model";
function anonymizeTreeRecursive(
tree: TreeElement,
terms: string[],
opt: {
/** Include the file sha in the response */
includeSha: boolean;
} = {
includeSha: false,
}
): TreeElement {
if (typeof tree.size !== "object" && tree.sha !== undefined) {
if (opt?.includeSha) return tree as TreeFile;
return { size: tree.size } as TreeFile;
}
const output: Tree = {};
Object.getOwnPropertyNames(tree).forEach((file) => {
const anonymizedPath = anonymizePath(file, terms);
output[anonymizedPath] = anonymizeTreeRecursive(tree[file], terms, opt);
});
return output;
}
export default class Repository {
private _model: IAnonymizedRepositoryDocument;
@@ -61,21 +84,7 @@ export default class Repository {
}
): Promise<Tree> {
const terms = this._model.options.terms || [];
function anonymizeTreeRecursive(tree: TreeElement): TreeElement {
if (Number.isInteger(tree.size) && tree.sha !== undefined) {
if (opt?.includeSha) return tree as TreeFile;
return { size: tree.size } as TreeFile;
}
const output: Tree = {};
for (const file in tree) {
const anonymizedPath = anonymizePath(file, terms);
output[anonymizedPath] = anonymizeTreeRecursive(tree[file]);
}
return output;
}
return anonymizeTreeRecursive(await this.files(opt)) as Tree;
return anonymizeTreeRecursive(await this.files(opt), terms, opt) as Tree;
}
/**
@@ -85,9 +94,15 @@ export default class Repository {
* @returns The file tree
*/
async files(opt: { force?: boolean } = { force: false }): Promise<Tree> {
if (!this._model.originalFiles && !opt.force) {
const res = await AnonymizedRepositoryModel.findById(this._model._id, {
originalFiles: 1,
});
this.model.originalFiles = res.originalFiles;
}
if (
this._model.originalFiles &&
Object.keys(this._model.originalFiles).length !== 0 &&
this._model.size.file !== 0 &&
!opt.force
) {
return this._model.originalFiles;
@@ -185,7 +200,7 @@ export default class Repository {
console.error(
`${branch.name} for ${this.source.githubRepository.fullName} is not found`
);
await this.updateStatus("error", "branch_not_found");
await this.updateStatus(RepositoryStatus.ERROR, "branch_not_found");
await this.resetSate();
throw new AnonymousError("branch_not_found", {
object: this,
@@ -193,7 +208,7 @@ export default class Repository {
}
this._model.anonymizeDate = new Date();
console.log(`${this._model.repoId} will be updated to ${newCommit}`);
await this.resetSate("preparing");
await this.resetSate(RepositoryStatus.PREPARING);
await downloadQueue.add(this.repoId, this, {
jobId: this.repoId,
attempts: 3,
@@ -207,19 +222,19 @@ export default class Repository {
* @returns void
*/
async anonymize() {
if (this.status == "ready") return;
await this.updateStatus("preparing");
if (this.status === RepositoryStatus.READY) return;
await this.updateStatus(RepositoryStatus.PREPARING);
await this.files();
return this.updateStatus("ready");
return this.updateStatus(RepositoryStatus.READY);
}
/**
* Update the last view and view count
*/
async countView() {
if (!isConnected) return this.model;
this._model.lastView = new Date();
this._model.pageView = (this._model.pageView || 0) + 1;
if (!isConnected) return this.model;
return this._model.save();
}
@@ -241,18 +256,18 @@ export default class Repository {
* Expire the repository
*/
async expire() {
await this.updateStatus("expiring");
await this.updateStatus(RepositoryStatus.EXPIRING);
await this.resetSate();
await this.updateStatus("expired");
await this.updateStatus(RepositoryStatus.EXPIRED);
}
/**
* Remove the repository
*/
async remove() {
await this.updateStatus("removing");
await this.updateStatus(RepositoryStatus.REMOVING);
await this.resetSate();
await this.updateStatus("removed");
await this.updateStatus(RepositoryStatus.REMOVED);
}
/**

5
src/anonymize-utils.ts
View File

@@ -1,5 +1,4 @@
import config from "../config";
import Repository from "./Repository";
import GitHubBase from "./source/GitHubBase";
import { isText } from "istextorbinary";
import { basename } from "path";
@@ -60,9 +59,9 @@ export function anonymizeStream(file: AnonymizedFile) {
ts._flush = function _flush(cb) {
if (chunks.length) {
let data: any = Buffer.concat(chunks, len);
let data = Buffer.concat(chunks, len);
if (isText(file.anonymizedPath, data)) {
data = anonymizeContent(data.toString(), file.repository);
data = Buffer.from(anonymizeContent(data.toString(), file.repository));
}
this.push(data);

5
src/database/anonymizedRepositories/anonymizedRepositories.schema.ts
View File

@@ -15,7 +15,10 @@ const AnonymizedRepositorySchema = new Schema({
lastView: Date,
pageView: Number,
accessToken: String,
owner: Schema.Types.ObjectId,
owner: {
type: Schema.Types.ObjectId,
index: true,
},
conference: String,
source: {
type: { type: String },

2
src/database/database.ts
View File

@@ -15,6 +15,8 @@ export let isConnected = false;
export async function connect() {
await mongoose.connect(MONGO_URL + "production", {
authSource: "admin",
appName: "Anonymous GitHub Server",
compressors: "zlib",
} as ConnectOptions);
isConnected = true;

18
src/processes/downloadRepository.ts
View File

@@ -2,25 +2,33 @@ import { SandboxedJob } from "bullmq";
import { config } from "dotenv";
config();
import Repository from "../Repository";
import { getRepository as getRepositoryImport } from "../database/database";
import { RepositoryStatus } from "../types";
export default async function (job: SandboxedJob<Repository, void>) {
const { connect, getRepository } = require("../database/database");
console.log(`${job.data.repoId} is going to be downloaded`);
const {
connect,
getRepository,
}: {
connect: () => Promise<void>;
getRepository: typeof getRepositoryImport;
} = require("../database/database");
console.log(`[QUEUE] ${job.data.repoId} is going to be downloaded`);
try {
await connect();
const repo = await getRepository(job.data.repoId);
job.updateProgress({ status: "get_repo" });
await repo.resetSate("preparing");
await repo.resetSate(RepositoryStatus.PREPARING, "");
job.updateProgress({ status: "resetSate" });
try {
await repo.anonymize();
} catch (error) {
await repo.updateStatus("error", error.message);
await repo.updateStatus(RepositoryStatus.ERROR, error.message);
throw error;
}
} catch (error) {
console.error(error);
} finally {
console.log(`${job.data.repoId} is downloaded`);
console.log(`[QUEUE] ${job.data.repoId} is downloaded`);
}
}

9
src/processes/removeCache.ts
View File

@@ -1,8 +1,15 @@
import { SandboxedJob } from "bullmq";
import Repository from "../Repository";
import { getRepository as getRepositoryImport } from "../database/database";
export default async function (job: SandboxedJob<Repository, void>) {
const { connect, getRepository } = require("../database/database");
const {
connect,
getRepository,
}: {
connect: () => Promise<void>;
getRepository: typeof getRepositoryImport;
} = require("../database/database");
try {
await connect();
console.log(

17
src/processes/removeRepository.ts
View File

@@ -1,21 +1,30 @@
import { SandboxedJob } from "bullmq";
import Repository from "../Repository";
import { getRepository as getRepositoryImport } from "../database/database";
import { RepositoryStatus } from "../types";
export default async function (job: SandboxedJob<Repository, void>) {
const { connect, getRepository } = require("../database/database");
const {
connect,
getRepository,
}: {
connect: () => Promise<void>;
getRepository: typeof getRepositoryImport;
} = require("../database/database");
try {
await connect();
console.log(`${job.data.repoId} is going to be removed`);
console.log(`[QUEUE] ${job.data.repoId} is going to be removed`);
const repo = await getRepository(job.data.repoId);
await repo.updateStatus(RepositoryStatus.REMOVING, "");
try {
await repo.remove();
} catch (error) {
await repo.updateStatus("error", error.message);
await repo.updateStatus(RepositoryStatus.ERROR, error.message);
throw error;
}
} catch (error) {
console.error(error);
} finally {
console.log(`${job.data.repoId} is removed`);
console.log(`[QUEUE] ${job.data.repoId} is removed`);
}
}

9
src/routes/file.ts
View File

@@ -14,12 +14,13 @@ router.get(
}
anonymizedPath = anonymizedPath;
const repo = await getRepo(req, res);
const repo = await getRepo(req, res, {
nocheck: false,
includeFiles: false,
});
if (!repo) return;
try {
await repo.countView();
const f = new AnonymizedFile({
repository: repo,
anonymizedPath,
@@ -35,7 +36,7 @@ router.get(
);
// cache the file for 5min
res.header("Cache-Control", "max-age=300");
await f.send(res);
await Promise.all([repo.countView(), f.send(res)]);
} catch (error) {
return handleError(error, res, req);
}

5
src/routes/repository-private.ts
View File

@@ -16,6 +16,7 @@ import AnonymousError from "../AnonymousError";
import { downloadQueue, removeQueue } from "../queue";
import RepositoryModel from "../database/repositories/repositories.model";
import User from "../User";
import { RepositoryStatus } from "../types";
const router = express.Router();
@@ -150,7 +151,7 @@ router.delete(
});
const user = await getUser(req);
isOwnerOrAdmin([repo.owner.id], user);
await repo.updateStatus("removing");
await repo.updateStatus(RepositoryStatus.REMOVING);
await removeQueue.add(repo.repoId, repo, { jobId: repo.repoId });
return res.json({ status: repo.status });
} catch (error) {
@@ -406,7 +407,7 @@ router.post(
}
}
repo.model.conference = repoUpdate.conference;
await repo.updateStatus("preparing");
await repo.updateStatus(RepositoryStatus.PREPARING);
res.json({ status: repo.status });
await downloadQueue.add(repo.repoId, repo, { jobId: repo.repoId });
} catch (error) {

10
src/routes/repository-public.ts
View File

@@ -6,6 +6,7 @@ import config from "../../config";
import { getRepo, handleError } from "./route-utils";
import AnonymousError from "../AnonymousError";
import { downloadQueue } from "../queue";
import { RepositoryStatus } from "../types";
const router = express.Router();
@@ -61,7 +62,7 @@ router.get(
"/:repoId/files",
async (req: express.Request, res: express.Response) => {
res.header("Cache-Control", "no-cache");
const repo = await getRepo(req, res);
const repo = await getRepo(req, res, { includeFiles: true });
if (!repo) return;
try {
res.json(await repo.anonymizedFiles({ includeSha: false }));
@@ -76,7 +77,10 @@ router.get(
async (req: express.Request, res: express.Response) => {
try {
res.header("Cache-Control", "no-cache");
const repo = await getRepo(req, res, { nocheck: true, includeFiles: false });
const repo = await getRepo(req, res, {
nocheck: true,
includeFiles: false,
});
if (!repo) return;
let redirectURL = null;
if (
@@ -105,7 +109,7 @@ router.get(
repo.model.statusDate < fiveMinuteAgo
// && repo.status != "preparing"
) {
await repo.updateStatus("preparing");
await repo.updateStatus(RepositoryStatus.PREPARING);
await downloadQueue.add(repo.repoId, repo, {
jobId: repo.repoId,
attempts: 3,

19
src/routes/route-utils.ts
View File

@@ -39,7 +39,7 @@ export async function getRepo(
res: express.Response,
opt: { nocheck?: boolean; includeFiles?: boolean } = {
nocheck: false,
includeFiles: true,
includeFiles: false,
}
) {
try {
@@ -108,7 +108,7 @@ export function handleError(
if (error.httpStatus) {
status = error.httpStatus;
} else if (message && message.indexOf("not_found") > -1) {
status = 400;
status = 404;
} else if (message && message.indexOf("not_connected") > -1) {
status = 401;
}
@@ -121,17 +121,14 @@ export function handleError(
export async function getUser(req: express.Request) {
const user = (req.user as any).user;
if (!user) {
req.logout((error) => console.error(error));
req.logout((error) => {
if (error) {
console.error(`[ERROR] Error while logging out: ${error}`);
}
});
throw new AnonymousError("not_connected", {
httpStatus: 401,
});
}
const model = await UserModel.findById(user._id);
if (!model) {
req.logout((error) => console.error(error));
throw new AnonymousError("not_connected", {
httpStatus: 401,
});
}
return new User(model);
return new User(new UserModel(user));
}

6
src/routes/user.ts
View File

@@ -10,7 +10,11 @@ router.use(ensureAuthenticated);
router.get("/logout", async (req: express.Request, res: express.Response) => {
try {
req.logout((error) => console.error(error));
req.logout((error) => {
if (error) {
console.error(`[ERROR] Logout error: ${error}`);
}
});
res.redirect("/");
} catch (error) {
handleError(error, res, req);

21
src/source/GitHubDownload.ts
View File

@@ -5,7 +5,7 @@ import Repository from "../Repository";
import GitHubBase from "./GitHubBase";
import AnonymizedFile from "../AnonymizedFile";
import { SourceBase } from "../types";
import { RepositoryStatus, SourceBase } from "../types";
import got from "got";
import { Readable } from "stream";
import { OctokitResponse } from "@octokit/types";
@@ -60,7 +60,10 @@ export default class GitHubDownload extends GitHubBase implements SourceBase {
try {
response = await this._getZipUrl(config.GITHUB_TOKEN);
} catch (error) {
await this.repository.resetSate("error", "repo_not_accessible");
await this.repository.resetSate(
RepositoryStatus.ERROR,
"repo_not_accessible"
);
throw new AnonymousError("repo_not_accessible", {
httpStatus: 404,
cause: error,
@@ -68,7 +71,10 @@ export default class GitHubDownload extends GitHubBase implements SourceBase {
});
}
} else {
await this.repository.resetSate("error", "repo_not_accessible");
await this.repository.resetSate(
RepositoryStatus.ERROR,
"repo_not_accessible"
);
throw new AnonymousError("repo_not_accessible", {
httpStatus: 404,
object: this.repository,
@@ -76,7 +82,7 @@ export default class GitHubDownload extends GitHubBase implements SourceBase {
});
}
}
await this.repository.updateStatus("download");
await this.repository.updateStatus(RepositoryStatus.DOWNLOAD);
const originalPath = this.repository.originalCachePath;
await storage.mk(originalPath);
let progress = null;
@@ -102,7 +108,10 @@ export default class GitHubDownload extends GitHubBase implements SourceBase {
downloadStream.addListener("downloadProgress", (p) => (progress = p));
await storage.extractZip(originalPath, downloadStream, null, this);
} catch (error) {
await this.repository.updateStatus("error", "unable_to_download");
await this.repository.updateStatus(
RepositoryStatus.ERROR,
"unable_to_download"
);
throw new AnonymousError("unable_to_download", {
httpStatus: 500,
cause: error,
@@ -113,7 +122,7 @@ export default class GitHubDownload extends GitHubBase implements SourceBase {
clearTimeout(progressTimeout);
}
await this.repository.updateStatus("ready");
await this.repository.updateStatus(RepositoryStatus.READY);
}
async getFileContent(file: AnonymizedFile): Promise<Readable> {

26
src/source/GitHubStream.ts
View File

@@ -3,7 +3,7 @@ import AnonymizedFile from "../AnonymizedFile";
import Repository from "../Repository";
import GitHubBase from "./GitHubBase";
import storage from "../storage";
import { SourceBase, Tree } from "../types";
import { RepositoryStatus, SourceBase, Tree } from "../types";
import * as path from "path";
import * as stream from "stream";
@@ -26,11 +26,6 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
}
async getFileContent(file: AnonymizedFile): Promise<stream.Readable> {
if (!file.sha)
throw new AnonymousError("file_sha_not_provided", {
httpStatus: 400,
object: file,
});
const octokit = new Octokit({
auth: await this.getToken(),
});
@@ -57,12 +52,12 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
} else {
content = Buffer.from("");
}
if (this.repository.status != "ready")
await this.repository.updateStatus("ready");
if (this.repository.status !== RepositoryStatus.READY)
await this.repository.updateStatus(RepositoryStatus.READY);
await storage.write(file.originalCachePath, content, file, this);
return stream.Readable.from(content);
} catch (error) {
if (error.status == 404) {
if (error.status === 404 || error.httpStatus === 404) {
throw new AnonymousError("file_not_found", {
httpStatus: error.status,
cause: error,
@@ -99,15 +94,18 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
} catch (error) {
if (error.status == 409) {
// empty tree
if (this.repository.status != "ready")
await this.repository.updateStatus("ready");
if (this.repository.status != RepositoryStatus.READY)
await this.repository.updateStatus(RepositoryStatus.READY);
// cannot be empty otherwise it would try to download it again
return { __: {} };
} else {
console.log(
`[ERROR] getTree ${this.repository.repoId}@${sha}: ${error.message}`
);
await this.repository.resetSate("error", "repo_not_accessible");
await this.repository.resetSate(
RepositoryStatus.ERROR,
"repo_not_accessible"
);
throw new AnonymousError("repo_not_accessible", {
httpStatus: error.status,
cause: error,
@@ -124,8 +122,8 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
if (ghRes.truncated) {
await this.getTruncatedTree(sha, tree, parentPath, count);
}
if (this.repository.status != "ready")
await this.repository.updateStatus("ready");
if (this.repository.status !== RepositoryStatus.READY)
await this.repository.updateStatus(RepositoryStatus.READY);
return tree;
}

9
src/storage/S3.ts
View File

@@ -29,7 +29,7 @@ export default class S3Storage implements StorageBase {
secretAccessKey: config.S3_CLIENT_SECRET,
httpOptions: {
timeout: 1000 * 60 * 60 * 2, // 2 hour
}
},
});
}
@@ -106,10 +106,7 @@ export default class S3Storage implements StorageBase {
res.set("Content-Length", headers["content-length"]);
res.set("Content-Type", headers["content-type"]);
}
pipeline(
response.httpResponse.createUnbufferedStream() as Readable,
res
);
(response.httpResponse.createUnbufferedStream() as Readable).pipe(res);
});
s.send();
@@ -139,7 +136,7 @@ export default class S3Storage implements StorageBase {
ContentType: lookup(path).toString(),
};
if (source) {
params.Tagging = `source=${source.type}`
params.Tagging = `source=${source.type}`;
}
await this.client.putObject(params).promise();
return;

38
src/types.ts
View File

@@ -6,6 +6,7 @@ import FileSystem from "./storage/FileSystem";
import AnonymizedFile from "./AnonymizedFile";
import * as stream from "stream";
import * as archiver from "archiver";
import { Response } from "express";
export interface SourceBase {
readonly type: string;
@@ -43,6 +44,8 @@ export interface StorageBase {
*/
exists(path: string): Promise<boolean>;
send(p: string, res: Response): void;
/**
* Read the content of a file
* @param path the path to the file
@@ -56,7 +59,12 @@ export interface StorageBase {
* @param file the file
* @param source the source of the file
*/
write(path: string, data: Buffer, file?: AnonymizedFile, source?: SourceBase): Promise<void>;
write(
path: string,
data: Buffer,
file?: AnonymizedFile,
source?: SourceBase
): Promise<void>;
/**
* List the files from dir
@@ -71,7 +79,12 @@ export interface StorageBase {
* @param file the file
* @param source the source of the file
*/
extractZip(dir: string, tar: stream.Readable, file?: AnonymizedFile, source?: SourceBase): Promise<void>;
extractZip(
dir: string,
tar: stream.Readable,
file?: AnonymizedFile,
source?: SourceBase
): Promise<void>;
/**
* Remove the path
@@ -113,16 +126,17 @@ export interface Branch {
readme?: string;
}
export type RepositoryStatus =
| "queue"
| "preparing"
| "download"
| "ready"
| "expired"
| "expiring"
| "removed"
| "removing"
| "error";
export enum RepositoryStatus {
QUEUE = "queue",
PREPARING = "preparing",
DOWNLOAD = "download",
READY = "ready",
EXPIRED = "expired",
EXPIRING = "expiring",
REMOVED = "removed",
REMOVING = "removing",
ERROR = "error",
}
export type ConferenceStatus = "ready" | "expired" | "removed";