From a3a52f496ce7f58278b3a9421f5e06922b92c0a8 Mon Sep 17 00:00:00 2001 From: tdurieux Date: Thu, 26 Sep 2019 16:32:51 -0400 Subject: [PATCH] new home page, add expiration date, add github login --- requirements.txt | 4 + server.py | 116 +++++++++++++--- static/css/style.css | 122 +++++++++++++++++ templates/index.html | 291 ++++++++++++++++++++++++++++++++++------- templates/newrepo.html | 66 ++++++++++ 5 files changed, 532 insertions(+), 67 deletions(-) create mode 100644 templates/newrepo.html diff --git a/requirements.txt b/requirements.txt index 4e53531..33227e2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,7 @@ flask pygithub==1.40 Flask-gzip +Authlib +loginpass +Flask-Session +pymongo \ No newline at end of file diff --git a/server.py b/server.py index 61d4718..9f6f2d1 100644 --- a/server.py +++ b/server.py @@ -3,6 +3,7 @@ import uuid import json import socket import os +from bson import json_util try: from urllib import quote # Python 2.X except ImportError: @@ -13,11 +14,24 @@ import string import base64 from datetime import datetime -# non standards, in requirements.txt -from flask import Flask, request, Markup, render_template, redirect, url_for, send_from_directory -from flask_gzip import Gzip -import github +# non standards, in requirements.txt +from flask import Flask, request, Markup, render_template, redirect, url_for, send_from_directory, session +from flask_session import Session +from flask_gzip import Gzip +import github +from authlib.flask.client import OAuth +# use loginpass to make OAuth connection simpler +from loginpass import create_flask_blueprint, GitHub + + + +def handle_authorize(remote, token, user_info): + session['user'] = { + 'user': user_info, + 'token': token + } + return redirect('/myrepo') def clean_github_repository(repo): """ @@ -54,20 +68,24 @@ def istext(s, threshold=0.30): try: binary_length = float(len(s.translate(None, TEXT_CHARACTERS))) except TypeError: - print("error") translate_table = dict((ord(char), None) for char in TEXT_CHARACTERS) binary_length = float(len(s.translate(str.maketrans(translate_table)))) # s is 'text' if less than 30% of its characters are non-text ones: return binary_length/len(s) <= threshold - class Anonymous_Github: def __init__(self, github_token, host="127.0.0.1", port=5000, - config_dir='./repositories'): + config_dir='./repositories', + secret_key=None, + client_id=None, + client_secret=None): self.github_token = github_token if github_token != "" else os.environ["GITHUB_AUTH_TOKEN"] + self.secret_key = secret_key if secret_key != "" else os.environ["SECRET_KEY"] + self.client_id = client_id if client_id != "" else os.environ["GITHUB_CLIENT_ID"] + self.client_secret = client_secret if client_secret != "" else os.environ["GITHUB_CLIENT_SECRET"] self.host = host self.port = port self.config_dir = config_dir @@ -94,13 +112,27 @@ class Anonymous_Github: application.killurl = str(uuid.uuid4()) application.jinja_env.add_extension('jinja2.ext.do') + application.config.update( + SESSION_TYPE='filesystem', + SECRET_KEY=self.secret_key, + GITHUB_CLIENT_ID=self.client_id, + GITHUB_CLIENT_SECRET=self.client_secret, + GITHUB_CLIENT_KWARGS = { + 'scope': 'repo' + } + ) + Session(application) + oauth = OAuth(application) + github_bp = create_flask_blueprint(GitHub, oauth, handle_authorize) + application.register_blueprint(github_bp, url_prefix='/github') + @application.template_filter('remove_terms', ) def remove_terms(content, repository_configuration, word_boundaries=True, whole_urls=True): """ remove the blacklisted terms from the content :param content: the content to anonymize :param repository_configuration: the configuration of the repository - :return: the anonimized content + :return: the anonymized content """ repo = repository_configuration['repository'] if repo[-1] == '/': @@ -141,8 +173,11 @@ class Anonymous_Github: if file.size > 1000000: return Markup("The file %s is too big to be anonymized (beyond 1MB, Github limit)" % (file.name)) if ".md" in file.name or file.name == file.name.upper() or "changelog" == file.name.lower(): + gh = self.github + if 'token' in repository_configuration: + gh = github.Github(repository_configuration['token']) return Markup("
%s
" % remove_terms( - self.github.render_markdown(file.decoded_content.decode('utf-8')), + gh.render_markdown(file.decoded_content.decode('utf-8')), repository_configuration)) if ".jpg" in file.name or ".png" in file.name or ".png" in file.name or ".gif" in file.name: index = file.name.index('.') @@ -175,6 +210,16 @@ class Anonymous_Github: return file, folder_content return None, folder_content + @application.route('/myrepo', methods=['GET']) + def myrepo(): + user = session.get('user', None) + g = github.Github(user['token']['access_token']) + repos = g.get_user().get_repos(sort="full_name") + for repo in repos: + repo.uuid = str(uuid.uuid4()) + return render_template('newrepo.html', repos=repos) + + @application.route('/repository//commit/', methods=['GET']) def commit(id, sha): """ @@ -186,9 +231,12 @@ class Anonymous_Github: if not os.path.exists(config_path): return render_template('404.html'), 404 with open(config_path) as f: - data = json.load(f) + data = json.load(f, object_hook=json_util.object_hook) (username, repo, branch) = clean_github_repository(data['repository']) - g_repo = self.github.get_repo("%s/%s" % (username, repo)) + gh = self.github + if 'token' in data: + gh = github.Github(data['token']) + g_repo = gh.get_repo("%s/%s" % (username, repo)) commit = g_repo.get_commit(sha) return render_template('repo.html', repository=data, @@ -292,7 +340,10 @@ class Anonymous_Github: or ".js" in current_file.name: content = remove_terms(content, repository_config) if ".md" in current_file.name: - content = remove_terms(self.github.render_markdown(content), repository_config) + gh = self.github + if 'token' in repository_config: + gh = github.Github(repository_config['token']) + content = remove_terms(gh.render_markdown(content), repository_config) else: tree = files if type(tree) != list: @@ -390,9 +441,18 @@ class Anonymous_Github: if not os.path.exists(config_path): return render_template('404.html'), 404 with open(config_path, 'r') as f: - repository_configuration = json.load(f) + repository_configuration = json.load(f, object_hook=json_util.object_hook) + if 'expiration_date' in repository_configuration: + if repository_configuration['expiration_date'] > datetime.now(repository_configuration['expiration_date'].tzinfo): + if repository_configuration['expiration'] == 'redirect': + return redirect(repository_configuration['repository']) + elif repository_configuration['expiration'] == 'remove': + return render_template('404.html'), 404 (username, repo, branch) = clean_github_repository(repository_configuration['repository']) - g_repo = self.github.get_repo("%s/%s" % (username, repo)) + gh = self.github + if 'token' in repository_configuration: + gh = github.Github(repository_configuration['token']) + g_repo = gh.get_repo("%s/%s" % (username, repo)) g_commit = None try: g_commit = g_repo.get_commit(branch) @@ -442,27 +502,40 @@ class Anonymous_Github: config_path = self.config_dir + "/" + id + "/config.json" if os.path.exists(config_path): with open(config_path) as f: - data = json.load(f) + data = json.load(f, object_hook=json_util.object_hook) if repo_name == clean_github_repository(data['repository']): repo = data - - return render_template('index.html', repo=repo) + return render_template('newversion.html', repo=repo) @application.route('/', methods=['POST']) def add_repository(): id = request.args.get('id', str(uuid.uuid4())) repo = request.form['githubRepository'] terms = request.form['terms'] + expiration_date = None + expiration = None + if 'expiration' in request.form: + expiration = request.form['expiration'] + if 'expiration_date' in request.form: + expiration_date = datetime.strptime(request.form['expiration_date'], '%Y-%m-%d') + + user = session.get('user', None) config_path = self.config_dir + "/" + str(id) if not os.path.exists(config_path): os.mkdir(config_path) with open(config_path + "/config.json", 'w') as outfile: + token = None + if user is not None: + token = user['token']['access_token'] json.dump({ "id": id, "repository": repo, - "terms": terms.splitlines() - }, outfile) + "terms": terms.splitlines(), + "token": token, + "expiration_date": expiration_date, + "expiration": expiration + }, outfile, default=json_util.default) return redirect(url_for('repository', id=id)) return application @@ -474,6 +547,9 @@ class Anonymous_Github: def initParser(): parser = argparse.ArgumentParser(description='Start Anonymous Github') parser.add_argument('-token', required=True, help='GitHub token') + parser.add_argument('-secret', required=True, help='App secret') + parser.add_argument('-client_id', required=True, help='GitHub aouth client id') + parser.add_argument('-client_secret', required=True, help='GitHub aouth client secret') parser.add_argument('-host', help='The hostname', default="127.0.0.1") parser.add_argument('-port', help='The port of the application', default=5000) parser.add_argument('-config_dir', help='The repository that will contains the configuration files', @@ -483,4 +559,4 @@ def initParser(): if __name__ == "__main__": args = initParser() - Anonymous_Github(github_token=args.token, host=args.host, port=args.port, config_dir=args.config_dir).run() + Anonymous_Github(github_token=args.token, host=args.host, port=args.port, config_dir=args.config_dir, secret_key=args.secret, client_id=args.client_id, client_secret=args.client_secret).run() diff --git a/static/css/style.css b/static/css/style.css index 70d54c8..2d07864 100644 --- a/static/css/style.css +++ b/static/css/style.css @@ -1,3 +1,110 @@ +.fadeIn { + opacity: 1; + animation-name: fadeInOpacity; + animation-iteration-count: 1; + animation-timing-function: ease-in; + animation-duration: 0.6s; +} + +@keyframes fadeInOpacity { + 0% { + opacity: 0; + } + 100% { + opacity: 1; + } +} + +textarea, select, input, button { outline: none; } + +html, +body{ + position: relative; + height: 100%; +} +header { + position: relative; + min-height: 100%; +} +.view { + width: 100%; + color: #ffffff; +} + +.top-nav-collapse { + background: #4a507b !important; +} + +.navbar { + box-shadow: 0 2px 5px 0 rgba(0,0,0,.16), 0 2px 10px 0 rgba(0,0,0,.12); + font-weight: 300; + transition: background-color 0.5s ease; +} +.navbar .nav-link.nav-icon { + padding: 0 .5rem; +} +.navbar .nav-link .fa { + font-size: 30px; + line-height: 40px; +} +.navbar:not(.top-nav-collapse) { + background: transparent !important; +} + +@media (max-width: 991px) { + .navbar:not(.top-nav-collapse) { + background: #4a507b !important; + } +} + +.rgba-gradient { + background: -moz-linear-gradient(45deg, rgba(51, 51, 51, 0.82), rgba(13, 17, 198, 0.69) 100%); + background: -webkit-linear-gradient(45deg, rgba(51, 51, 51, 0.82), rgba(13, 17, 198, 0.69) 100%); + background: linear-gradient(to 45deg, rgba(51, 51, 51, 0.82), rgba(13, 17, 198, 0.69) 100%); +} +a:hover { + text-decoration: none; +} +.white_border, .black_border { + border: 1px solid white; + padding: 7px; + background: transparent; + color: white; + min-width: 115px; +} +a.white_border, a.white_border:hover, a.black_border, a.black_border:hover { + padding: 10px; + color: white; +} +.white_border:focus, .black_border:focus { + background: #333333; + border: 1px solid #333333; +} +.white_border::placeholder, .black_border::placeholder { /* Chrome, Firefox, Opera, Safari 10.1+ */ + color: #cccccc; + opacity: 1; /* Firefox */ +} + +.black_border { + border: 1px solid #333333; +} + a.black_border, a.black_border:hover { + color: #333333; +} +.black_border::placeholder { /* Chrome, Firefox, Opera, Safari 10.1+ */ + color: #666666; + opacity: 1; /* Firefox */ +} + +.active .container { + transition: height 1s ease; +} +.add_form { + display: none; +} +.active .add_form { + display: block; +} .main { width: 80%; margin: auto; @@ -73,4 +180,19 @@ border-right: 0; border-radius: 0; } +} +.repos { + margin: 0; + padding: 0; +} +.repo { + list-style: none; + margin: 0; + padding: 3px 7px; + background: #fdfdfd; + border: 1px solid #ddd; + border-top: 0 +} +.repo .options { + float: right; } \ No newline at end of file diff --git a/templates/index.html b/templates/index.html index f47b8c5..00f43ae 100644 --- a/templates/index.html +++ b/templates/index.html @@ -4,59 +4,256 @@ - GitHub Anonymous + Anonymous GitHub + -
-
-

Anonymous GitHub for Open-Science

- Code and documentation: https://github.com/tdurieux/anonymous_github/ -

Create an Anonymous GitHub repository

-

- Fill 1. the Github repo URL and 2. the word list (which can be updated afterwards). - The anonymization of the content is done by replacing all occurrences of words in a list by "XXX". - The word list typically contains the institution name, author names, logins, etc... - The README is anonymized as well as all files of the repository. Even filenames are anonymized. - In a double-blind submission, put the anonymous repository URL (e.g. http://anonymous.4open.science/repository/840c8c57-3c32-451e-bf12-0e20be300389/). -

-
-
- - The github url to the repository that you want to anonymize. - -
-
- - One term per line (case insensitive). - -
- -
- {% if not repo %} -

Edit an Anonymous GitHub repository

-
-
- - The id of the Anonymous GitHub repository. - -
-
- - Password: the real Github url. - -
- -
- {% endif %} + +
+ + + + +
+ +
+ +
+
+ +
+

Anonymous GitHub

+
+

Double-blind your repository in 5 min!

+ + or + Login to GitHub + or + +
+ +
+
+ {% if not repo %} +
+ +
+ {% endif %} +
+
+ + One term per line (case insensitive). + +
+
+ + +
+ + ` +
+
+
+ + +
+
- - - - + +
+ + +
+
+ +
+
+

Usage

+

+

    +
  1. + Fill the Github repo URL +
    2. +
  2. + Complete the list of terms that will be anonymized. + The anonymization of the content is done by replacing all occurrences of words in a list by "XXX".The word list typically contains the institution name, author names, logins, etc... +
    3. +
  3. + Define if you want an expiration date for your anonymized repository. + You can keep it for ever, remove the repository after a specific date or redirect the user to the GitHub repository. +
    4. +
+ As result, a unique url is created with the content of your repository, for example, http://anonymous.4open.science/repository/840c8c57-3c32-451e-bf12-0e20be300389/. +

+
+ +
+

Features

+
+
+
+
+

Anonymized

+

+ Anonymous GitHub Anonymizes the content of the repository but also hide the issues, pull-requests, history. This way you are sure that your repository stays anonymized. +

+
+
+
+
+
+
+

Always up-to-date

+

+ Anonymous GitHub follows to track of the changes on your repository and updates the anonymous version automatically. +

+
+
+
+
+
+
+

Fast

+

+ With Anonymous GitHub, it requires only 5min to anonymize your repository. No more time lost to create a anonymized version of your repository. +

+
+
+
+
+
+
+

Open-source

+

+ Anonymous GitHub is open-source, you can easily deploy it for your conference and simplify the life of your authors. +

+
+
+
+
+

Metrics

+
+
+
+
+

2609 Anonymized Repositories

+

+ +

+
+
+
+ +
+ +
+
+ + + + + - + \ No newline at end of file diff --git a/templates/newrepo.html b/templates/newrepo.html new file mode 100644 index 0000000..8f95d14 --- /dev/null +++ b/templates/newrepo.html @@ -0,0 +1,66 @@ + + + + + + + GitHub Anonymous + + + + + + + +
+
+ {% for repo in repos %} +
+
+
{{ repo.full_name }}
+ {% if repo.private %} Private {% else %} Public {% endif %} +
+ {% if repo.description %} +

{{ repo.description }}

+ {% endif %} +
+ Anonymize +
+
+ {% endfor %} +
+
+ + + + + + +