diff --git a/deploy/nginx/README.md b/deploy/nginx/README.md index dc7234d..9071be0 100644 --- a/deploy/nginx/README.md +++ b/deploy/nginx/README.md @@ -23,3 +23,21 @@ python3 scripts/test-nginx-logging.py deploy/nginx/anonymous-logging.conf ``` The test starts an isolated loopback listener and checks trusted IPv4/IPv6 peers, spoofed headers, missing-header fallback, JSON escaping and omitted sensitive data. It never changes production listeners. + +## Bulk file/listing protection + +Install `anonymous-bulk-limits.conf` in the http context, and create an empty root-owned `/etc/nginx/anonymous-bulk-blocks.map` with mode 0600. The latter can hold private temporary blocks in this format: + +```nginx +"~^2026-11-10T.*:203\\.0\\.113\\.9$" 1; +``` + +Dates use the Nginx host timezone; production uses UTC. Such a block automatically ceases matching at midnight. Remove expired entries during maintenance. Use only verified visitor IPs, never Cloudflare edge IPs. Targeted blocks apply only to the file/listing paths and return a one-hour Retry-After; they do not block ZIPs or the whole site. + +Install the location/server fragments under `/etc/nginx/snippets/`. Include `anonymous-bulk-location.conf` inside each application's proxied `location /`, and `anonymous-bulk-server.conf` in its enclosing HTTPS server. Run `nginx -t` before gracefully reloading. Keep a backup for rollback. Do not enable proxy_intercept_errors: application 429 responses should retain their existing bodies and headers. + +Limits aggregate across repositories per trusted client IP. File reads allow a burst of 60 plus a sustained four requests/minute; listing/count/search reads allow a burst of 30 plus two/minute. GET and HEAD count, including conditional requests. Other paths have empty limit keys and do not consume these budgets. Shared-NAT users share these limits, so review legitimate 429 rates and adjust if needed. Neither User-Agent nor Referer grants a bypass. This protects these API routes, not the homepage or every source of resource exhaustion. + +Rollback by removing the server/location includes, validating, and reloading; then remove the unused http config. Existing client identification and application rate limits are unchanged. + +Run `python3 scripts/test-nginx-bulk-limits.py deploy/nginx` on a host with Nginx to test these limits against an isolated mock origin. It checks budgets, per-client separation, targeted blocks and ZIP/navigation/write exclusions without sending a traffic burst to production. diff --git a/deploy/nginx/anonymous-bulk-limits.conf b/deploy/nginx/anonymous-bulk-limits.conf new file mode 100644 index 0000000..dd0a9bf --- /dev/null +++ b/deploy/nginx/anonymous-bulk-limits.conf @@ -0,0 +1,28 @@ +# http context. Requires anonymous-logging.conf's trusted client IP variable. +# Empty keys exclude ZIP downloads, navigation, writes and unrelated APIs. +map "$request_method:$uri" $anonymous_file_limit_key { + default ""; + ~*^(GET|HEAD):/api/repo/[^/]+/file/ $anonymous_client_ip; +} +map "$request_method:$uri" $anonymous_listing_limit_key { + default ""; + ~*^(GET|HEAD):/api/repo/[^/]+/files(/|$) $anonymous_client_ip; +} +limit_req_zone $anonymous_file_limit_key zone=anonymous_file_requests:10m rate=4r/m; +limit_req_zone $anonymous_listing_limit_key zone=anonymous_listing_requests:10m rate=2r/m; + +# Private operational entries are date-scoped regexes, never committed IP lists. +# An empty file disables targeted blocks. Date must match the host's log timezone. +map "$time_iso8601:$anonymous_client_ip" $anonymous_bulk_block { + default 0; + include /etc/nginx/anonymous-bulk-blocks.map; +} +map "$anonymous_bulk_block:$anonymous_file_limit_key:$anonymous_listing_limit_key" $anonymous_reject_bulk { + default 0; + ~^1:.+ 1; + "1::" 0; +} +map $anonymous_reject_bulk $anonymous_bulk_retry { + default 30; + 1 3600; +} diff --git a/deploy/nginx/anonymous-bulk-location.conf b/deploy/nginx/anonymous-bulk-location.conf new file mode 100644 index 0000000..126840c --- /dev/null +++ b/deploy/nginx/anonymous-bulk-location.conf @@ -0,0 +1,8 @@ +# Include in the application's proxied location /, before proxy_pass. +# return-only if is safe here; intercepted upstream errors remain disabled. +error_page 429 = @anonymous_bulk_limited; +if ($anonymous_reject_bulk) { return 429; } +limit_req zone=anonymous_file_requests burst=60 nodelay; +limit_req zone=anonymous_listing_requests burst=30 nodelay; +limit_req_status 429; +limit_req_log_level notice; diff --git a/deploy/nginx/anonymous-bulk-server.conf b/deploy/nginx/anonymous-bulk-server.conf new file mode 100644 index 0000000..aca9d22 --- /dev/null +++ b/deploy/nginx/anonymous-bulk-server.conf @@ -0,0 +1,7 @@ +# Include in each HTTPS application server block, outside location /. +location @anonymous_bulk_limited { + default_type application/json; + add_header Retry-After $anonymous_bulk_retry always; + add_header Cache-Control "no-store" always; + return 429 '{"error":"rate_limited","message":"Too many individual file or directory requests. Use the repository ZIP download for bulk access."}'; +} diff --git a/deploy/nginx/anonymous-logging.conf b/deploy/nginx/anonymous-logging.conf index 12bcdd9..b9a2d3a 100644 --- a/deploy/nginx/anonymous-logging.conf +++ b/deploy/nginx/anonymous-logging.conf @@ -58,7 +58,7 @@ log_format anonymous_json escape=json '"cloudflare_peer":"$anonymous_cloudflare_peer","cf_ray":"$anonymous_cf_ray",' '"edge_request_id":"$request_id","app_request_id":"$upstream_http_x_request_id",' '"method":"$request_method","route":"$anonymous_log_route","protocol":"$server_protocol",' - '"status":$status,"upstream_status":"$upstream_status",' + '"status":$status,"limit_req_status":"$limit_req_status","upstream_status":"$upstream_status",' '"request_time":$request_time,"upstream_connect_time":"$upstream_connect_time",' '"upstream_header_time":"$upstream_header_time","upstream_response_time":"$upstream_response_time",' '"request_bytes":$request_length,"response_bytes":$bytes_sent, "completion":"$request_completion",' diff --git a/scripts/test-nginx-bulk-limits.py b/scripts/test-nginx-bulk-limits.py new file mode 100644 index 0000000..1987dff --- /dev/null +++ b/scripts/test-nginx-bulk-limits.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +"""Exercise Nginx admission against a disposable origin, not production traffic.""" +import datetime +import http.server +import json +import pathlib +import socket +import subprocess +import sys +import tempfile +import threading +import time +import urllib.error +import urllib.request + +source = pathlib.Path(sys.argv[1]) +class Origin(http.server.BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.end_headers() + self.wfile.write(b"origin") + do_HEAD = do_GET + do_POST = do_GET + def log_message(self, *_args): + pass +origin = http.server.HTTPServer(("127.0.0.1", 0), Origin) +threading.Thread(target=origin.serve_forever, daemon=True).start() +try: + with tempfile.TemporaryDirectory(prefix="anonymous-nginx-bulk-test-") as temp: + root = pathlib.Path(temp) + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + date = datetime.datetime.now().astimezone().strftime("%Y-%m-%d") + blocks = root / "blocks.map" + blocks.write_text(f'"~^{date}T.*:203\\.0\\.113\\.2$" 1;\n') + logging = (source / "anonymous-logging.conf").read_text().replace("geo $remote_addr", "geo $arg_peer") + limits = (source / "anonymous-bulk-limits.conf").read_text().replace("/etc/nginx/anonymous-bulk-blocks.map", str(blocks)) + location = (source / "anonymous-bulk-location.conf").read_text() + server = (source / "anonymous-bulk-server.conf").read_text() + config = root / "nginx.conf" + config.write_text(f'pid {root}/pid; error_log {root}/error.log notice; events {{}} http {{ access_log off; ' + + logging + limits + f' server {{ listen 127.0.0.1:{port}; ' + server + + ' location / { ' + location + f' proxy_pass http://127.0.0.1:{origin.server_port}; }} }} }}') + subprocess.run(["nginx", "-t", "-p", str(root), "-c", str(config)], check=True, capture_output=True) + process = subprocess.Popen(["nginx", "-p", str(root), "-c", str(config), "-g", "daemon off;"], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + try: + for _ in range(50): + try: + with socket.create_connection(("127.0.0.1", port), timeout=.1): + break + except OSError: + time.sleep(.1) + def request(path, client="203.0.113.1", method="GET"): + req = urllib.request.Request(f"http://127.0.0.1:{port}{path}?peer=173.245.48.1", method=method, + headers={"CF-Connecting-IP": client}) + try: + response = urllib.request.urlopen(req) + except urllib.error.HTTPError as error: + response = error + with response: + return response.status, dict(response.headers), response.read() + files = [request("/api/repo/test/file/a.txt")[0] for _ in range(65)] + assert files[0] == 200 and files[-1] == 429 and files.count(200) <= 61, files + listing = [request("/api/repo/test/files/counts")[0] for _ in range(35)] + assert listing[0] == 200 and listing[-1] == 429 and listing.count(200) <= 31, listing + status, headers, body = request("/api/repo/test/file/a.txt") + assert status == 429 and headers["Retry-After"] == "30" + assert json.loads(body)["error"] == "rate_limited" and b"ZIP" in body + assert request("/api/repo/test/zip")[0] == 200 + assert request("/r/test/")[0] == 200 + assert request("/api/repo/test/file/a.txt", method="POST")[0] == 200 + assert request("/api/repo/test/file/a.txt", client="203.0.113.3")[0] == 200 + for path in ["/api/repo/test/file/a.txt", "/api/repo/test/files/counts", "/API/REPO/test/FILE/a.txt"]: + status, headers, body = request(path, client="203.0.113.2") + assert status == 429 and headers["Retry-After"] == "3600", (status, headers, body) + assert request("/api/repo/test/file/a.txt", client="203.0.113.2", method="HEAD")[0] == 429 + assert request("/api/repo/test/zip", client="203.0.113.2")[0] == 200 + assert request("/", client="203.0.113.2")[0] == 200 + print("PASS: file/listing budgets, client separation, scoped block, HEAD/case variants, JSON retry response, ZIP/navigation/write exclusions") + finally: + process.terminate() + process.wait(timeout=10) +finally: + origin.shutdown() + origin.server_close()