mirror of
https://github.com/tdurieux/anonymous_github.git
synced 2026-05-25 18:37:47 +02:00
introduce cache
This commit is contained in:
tdurieux
@@ -5,10 +5,11 @@ import socket
|
|||||||
import os
|
import os
|
||||||
import urllib
|
import urllib
|
||||||
import re
|
import re
|
||||||
|
import shutil
|
||||||
import base64
|
import base64
|
||||||
|
|
||||||
# non standards, in requirements.txt
|
# non standards, in requirements.txt
|
||||||
from flask import Flask, request, Markup, render_template, redirect, url_for
|
from flask import Flask, request, Markup, render_template, redirect, url_for, send_from_directory
|
||||||
import requests
|
import requests
|
||||||
import github
|
import github
|
||||||
|
|
||||||
@@ -116,44 +117,56 @@ class Anonymous_Github:
|
|||||||
files=[],
|
files=[],
|
||||||
path=[])
|
path=[])
|
||||||
|
|
||||||
@application.route('/repository/<id>', methods=['GET'], defaults={'path': ''})
|
def is_up_to_date(repository_config, g_repo):
|
||||||
@application.route('/repository/<id>/', methods=['GET'], defaults={'path': ''})
|
return 'pushed_at' in repository_config and g_repo.pushed_at.strftime("%s") == repository_config["pushed_at"]
|
||||||
@application.route('/repository/<id>/<path:path>', methods=['GET'])
|
|
||||||
def repository(id, path):
|
|
||||||
config_path = self.config_dir + "/" + str(id) + "/config.json"
|
|
||||||
if not os.path.exists(config_path):
|
|
||||||
return render_template('404.html'), 404
|
|
||||||
with open(config_path) as f:
|
|
||||||
data = json.load(f)
|
|
||||||
repo = clean_github_repository(data['repository'])
|
|
||||||
g_repo = self.github.get_repo(repo)
|
|
||||||
clean_path = path
|
|
||||||
if len(clean_path) > 0 and clean_path[-1] == '/':
|
|
||||||
clean_path = clean_path[0:-1]
|
|
||||||
current_file = get_current_element(g_repo, clean_path)
|
|
||||||
files = []
|
|
||||||
if type(current_file) is not github.ContentFile.ContentFile:
|
|
||||||
files = g_repo.get_git_tree("master")
|
|
||||||
for f in current_file:
|
|
||||||
if f.name.lower() == "readme.md" or f.name.lower() == "index.html":
|
|
||||||
current_file = f
|
|
||||||
break
|
|
||||||
elif current_file.type == 'file':
|
|
||||||
if os.path.dirname(clean_path) == '':
|
|
||||||
files = g_repo.get_git_tree("master")
|
|
||||||
else:
|
|
||||||
files = g_repo.get_git_tree(get_current_element(g_repo, os.path.dirname(clean_path)).sha)
|
|
||||||
else:
|
|
||||||
if len(clean_path) > 0 and path[-1] != '/':
|
|
||||||
return redirect(url_for('repository', id=id, path=path + '/'))
|
|
||||||
files = g_repo.get_git_tree(current_file.sha)
|
|
||||||
for f in files.tree:
|
|
||||||
if f.path.lower() == "readme.md" or f.path.lower() == "index.html":
|
|
||||||
current_file = get_current_element(g_repo, path + f.path)
|
|
||||||
break
|
|
||||||
|
|
||||||
if clean_path[:4] == "docs":
|
def get_type_content(file_name, path, data, g_repo):
|
||||||
|
if is_website(path, data, g_repo):
|
||||||
content_type = 'text/plain; charset=utf-8'
|
content_type = 'text/plain; charset=utf-8'
|
||||||
|
if ".html" in file_name:
|
||||||
|
content_type = 'text/html; charset=utf-8'
|
||||||
|
if ".md" in file_name:
|
||||||
|
content_type = 'text/html; charset=utf-8'
|
||||||
|
if ".jpg" in file_name \
|
||||||
|
or ".png" in file_name \
|
||||||
|
or ".gif" in file_name:
|
||||||
|
content_type = 'image/jpeg'
|
||||||
|
if ".png" in file_name:
|
||||||
|
content_type = 'image/png'
|
||||||
|
elif ".gif" in file_name:
|
||||||
|
content_type = 'image/gif'
|
||||||
|
if ".txt" in file_name \
|
||||||
|
or ".log" in file_name \
|
||||||
|
or ".java" in file_name \
|
||||||
|
or ".py" in file_name \
|
||||||
|
or ".xml" in file_name \
|
||||||
|
or ".json" in file_name \
|
||||||
|
or ".js" in file_name:
|
||||||
|
content_type = 'text/plain; charset=utf-8'
|
||||||
|
if ".xml" in file_name:
|
||||||
|
content_type = 'application/xml; charset=utf-8'
|
||||||
|
elif ".json" in file_name:
|
||||||
|
content_type = 'application/json; charset=utf-8'
|
||||||
|
elif ".js" in file_name:
|
||||||
|
content_type = 'application/javascript; charset=utf-8'
|
||||||
|
if ".css" in file_name:
|
||||||
|
content_type = 'text/css; charset=utf-8'
|
||||||
|
return content_type
|
||||||
|
return 'text/html; charset=utf-8'
|
||||||
|
|
||||||
|
def get_content(current_file, files, path, repository_config, g_repo):
|
||||||
|
cache_path = os.path.join(self.config_dir, repository_config['id'], "cache")
|
||||||
|
file_path = path
|
||||||
|
if current_file is not None:
|
||||||
|
if current_file.type == 'dir':
|
||||||
|
file_path = os.path.join(current_file.path, "index.html")
|
||||||
|
else:
|
||||||
|
file_path = current_file.path
|
||||||
|
cached_file_path = os.path.join(cache_path, file_path)
|
||||||
|
if os.path.exists(cached_file_path):
|
||||||
|
return send_from_directory(os.path.dirname(cached_file_path), os.path.basename(cached_file_path))
|
||||||
|
content = ''
|
||||||
|
if is_website(path, repository_config, g_repo):
|
||||||
if current_file.size > 1000000:
|
if current_file.size > 1000000:
|
||||||
blob = g_repo.get_git_blob(current_file.sha)
|
blob = g_repo.get_git_blob(current_file.sha)
|
||||||
if blob.encoding == 'base64':
|
if blob.encoding == 'base64':
|
||||||
@@ -162,47 +175,102 @@ class Anonymous_Github:
|
|||||||
content = blob.content
|
content = blob.content
|
||||||
else:
|
else:
|
||||||
content = current_file.decoded_content
|
content = current_file.decoded_content
|
||||||
if ".html" in current_file.name:
|
if ".html" in current_file.name \
|
||||||
content = removeTerms(content, data)
|
or ".txt" in current_file.name \
|
||||||
content_type = 'text/html; charset=utf-8'
|
|
||||||
if ".md" in current_file.name:
|
|
||||||
content = removeTerms(self.github.render_markdown(content), data)
|
|
||||||
content_type = 'text/html; charset=utf-8'
|
|
||||||
if ".jpg" in current_file.name \
|
|
||||||
or ".png" in current_file.name \
|
|
||||||
or ".gif" in current_file.name:
|
|
||||||
content = current_file.decoded_content
|
|
||||||
content_type = 'image/jpeg'
|
|
||||||
if ".png" in current_file.name:
|
|
||||||
content_type = 'image/png'
|
|
||||||
elif".gif" in current_file.name:
|
|
||||||
content_type = 'image/gif'
|
|
||||||
if ".txt" in current_file.name \
|
|
||||||
or ".log" in current_file.name \
|
or ".log" in current_file.name \
|
||||||
or ".java" in current_file.name \
|
or ".java" in current_file.name \
|
||||||
or ".py" in current_file.name \
|
or ".py" in current_file.name \
|
||||||
or ".xml" in current_file.name \
|
or ".xml" in current_file.name \
|
||||||
or ".json" in current_file.name \
|
or ".json" in current_file.name \
|
||||||
or ".js" in current_file.name:
|
or ".js" in current_file.name:
|
||||||
content = removeTerms(content, data)
|
content = removeTerms(content, repository_config)
|
||||||
content_type = 'text/plain; charset=utf-8'
|
if ".md" in current_file.name:
|
||||||
if ".xml" in current_file.name:
|
content = removeTerms(self.github.render_markdown(content), repository_config)
|
||||||
content_type = 'application/xml; charset=utf-8'
|
|
||||||
elif".json" in current_file.name:
|
|
||||||
content_type = 'application/json; charset=utf-8'
|
|
||||||
elif ".js" in current_file.name:
|
|
||||||
content_type = 'application/javascript; charset=utf-8'
|
|
||||||
if ".css" in current_file.name:
|
|
||||||
content_type = 'text/css; charset=utf-8'
|
|
||||||
return content, {'Content-Type': content_type}
|
|
||||||
else:
|
else:
|
||||||
return render_template('repo.html',
|
content = render_template('repo.html',
|
||||||
repository=data,
|
repository=repository_config,
|
||||||
current_repository=id,
|
current_repository=repository_config['id'],
|
||||||
current_file=current_file,
|
current_file=current_file,
|
||||||
files=files.tree,
|
files=files.tree,
|
||||||
path_directory=clean_path if type(current_file) is not github.ContentFile.ContentFile or current_file.type=='dir'else os.path.dirname(clean_path),
|
path_directory=path if type(
|
||||||
path=clean_path.split("/") if clean_path != '' else [])
|
current_file) is not github.ContentFile.ContentFile or current_file.type == 'dir' else os.path.dirname(
|
||||||
|
path),
|
||||||
|
path=path.split("/") if path != '' else [])
|
||||||
|
content_cache_path = cached_file_path
|
||||||
|
if not os.path.exists(os.path.dirname(content_cache_path)):
|
||||||
|
os.makedirs(os.path.dirname(content_cache_path))
|
||||||
|
with open(content_cache_path, 'w') as f:
|
||||||
|
f.write(content)
|
||||||
|
return content
|
||||||
|
|
||||||
|
def is_website(path, repository_config, g_repo):
|
||||||
|
return path[:4] == "docs"
|
||||||
|
|
||||||
|
def get_current_folder_files(path, current_file, repository_config, g_repo):
|
||||||
|
files = []
|
||||||
|
if type(current_file) is not github.ContentFile.ContentFile:
|
||||||
|
files = g_repo.get_git_tree("master")
|
||||||
|
for f in current_file:
|
||||||
|
if f.name.lower() == "readme.md" or f.name.lower() == "index.html":
|
||||||
|
current_file = f
|
||||||
|
break
|
||||||
|
elif current_file.type == 'file':
|
||||||
|
if os.path.dirname(path) == '':
|
||||||
|
files = g_repo.get_git_tree("master")
|
||||||
|
else:
|
||||||
|
files = g_repo.get_git_tree(get_current_element(g_repo, os.path.dirname(path)).sha)
|
||||||
|
else:
|
||||||
|
files = g_repo.get_git_tree(current_file.sha)
|
||||||
|
for f in files.tree:
|
||||||
|
if f.path.lower() == "readme.md" or f.path.lower() == "index.html":
|
||||||
|
current_file = get_current_element(g_repo, os.path.join(path, f.path))
|
||||||
|
break
|
||||||
|
return files, current_file
|
||||||
|
|
||||||
|
@application.route('/repository/<id>', methods=['GET'], defaults={'path': ''})
|
||||||
|
@application.route('/repository/<id>/', methods=['GET'], defaults={'path': ''})
|
||||||
|
@application.route('/repository/<id>/<path:path>', methods=['GET'])
|
||||||
|
def repository(id, path):
|
||||||
|
repo_path = self.config_dir + "/" + str(id)
|
||||||
|
config_path = repo_path + "/config.json"
|
||||||
|
if not os.path.exists(config_path):
|
||||||
|
return render_template('404.html'), 404
|
||||||
|
with open(config_path, 'rw') as f:
|
||||||
|
data = json.load(f)
|
||||||
|
repo = clean_github_repository(data['repository'])
|
||||||
|
g_repo = self.github.get_repo(repo)
|
||||||
|
|
||||||
|
if not is_up_to_date(data, g_repo):
|
||||||
|
if os.path.exists(os.path.join(repo_path, "cache")):
|
||||||
|
shutil.rmtree(os.path.join(repo_path, "cache"))
|
||||||
|
data["pushed_at"] = g_repo.pushed_at.strftime("%s")
|
||||||
|
with open(config_path, 'w') as fa:
|
||||||
|
json.dump(data, fa)
|
||||||
|
|
||||||
|
cache_path = os.path.join(self.config_dir, id, "cache")
|
||||||
|
if os.path.isfile(os.path.join(cache_path, path)):
|
||||||
|
return send_from_directory(os.path.dirname(os.path.join(cache_path, path)),
|
||||||
|
os.path.basename(os.path.join(cache_path, path))),\
|
||||||
|
{'Content-Type': get_type_content(path, path, data, g_repo)}
|
||||||
|
elif os.path.exists(os.path.join(cache_path, path, "index.html")):
|
||||||
|
return send_from_directory(os.path.join(cache_path, path), "index.html")
|
||||||
|
elif os.path.exists(os.path.join(cache_path, path, "README.md")):
|
||||||
|
return send_from_directory(os.path.join(cache_path, path), "README.md")
|
||||||
|
|
||||||
|
clean_path = path
|
||||||
|
if len(clean_path) > 0 and clean_path[-1] == '/':
|
||||||
|
clean_path = clean_path[0:-1]
|
||||||
|
|
||||||
|
current_file = get_current_element(g_repo, clean_path)
|
||||||
|
if type(current_file) == github.ContentFile.ContentFile and current_file.type == 'dir' and len(path) > 0 and path[-1] != '/':
|
||||||
|
return redirect(url_for('repository', id=id, path=path + '/'))
|
||||||
|
|
||||||
|
files, current_file = get_current_folder_files(clean_path, current_file, data, g_repo)
|
||||||
|
|
||||||
|
content = get_content(current_file, files, clean_path, data, g_repo)
|
||||||
|
content_type = get_type_content(current_file.name, clean_path, data, g_repo)
|
||||||
|
|
||||||
|
return content, {'Content-Type': content_type}
|
||||||
|
|
||||||
@application.route('/', methods=['GET'])
|
@application.route('/', methods=['GET'])
|
||||||
def index():
|
def index():
|
||||||
|
|||||||
Reference in New Issue
Block a user