ops: limit bulk file fetching and repeated directory polling

deploy/nginx/README.md

@@ -23,3 +23,21 @@ python3 scripts/test-nginx-logging.py deploy/nginx/anonymous-logging.conf
 ```
 
 The test starts an isolated loopback listener and checks trusted IPv4/IPv6 peers, spoofed headers, missing-header fallback, JSON escaping and omitted sensitive data. It never changes production listeners.
+
+## Bulk file/listing protection
+
+Install `anonymous-bulk-limits.conf` in the http context, and create an empty root-owned `/etc/nginx/anonymous-bulk-blocks.map` with mode 0600. The latter can hold private temporary blocks in this format:
+
+```nginx
+"~^2026-11-10T.*:203\\.0\\.113\\.9$" 1;
+```
+
+Dates use the Nginx host timezone; production uses UTC. Such a block automatically ceases matching at midnight. Remove expired entries during maintenance. Use only verified visitor IPs, never Cloudflare edge IPs. Targeted blocks apply only to the file/listing paths and return a one-hour Retry-After; they do not block ZIPs or the whole site.
+
+Install the location/server fragments under `/etc/nginx/snippets/`. Include `anonymous-bulk-location.conf` inside each application's proxied `location /`, and `anonymous-bulk-server.conf` in its enclosing HTTPS server. Run `nginx -t` before gracefully reloading. Keep a backup for rollback. Do not enable proxy_intercept_errors: application 429 responses should retain their existing bodies and headers.
+
+Limits aggregate across repositories per trusted client IP. File reads allow a burst of 60 plus a sustained four requests/minute; listing/count/search reads allow a burst of 30 plus two/minute. GET and HEAD count, including conditional requests. Other paths have empty limit keys and do not consume these budgets. Shared-NAT users share these limits, so review legitimate 429 rates and adjust if needed. Neither User-Agent nor Referer grants a bypass. This protects these API routes, not the homepage or every source of resource exhaustion.
+
+Rollback by removing the server/location includes, validating, and reloading; then remove the unused http config. Existing client identification and application rate limits are unchanged.
+
+Run `python3 scripts/test-nginx-bulk-limits.py deploy/nginx` on a host with Nginx to test these limits against an isolated mock origin. It checks budgets, per-client separation, targeted blocks and ZIP/navigation/write exclusions without sending a traffic burst to production.

deploy/nginx/anonymous-bulk-limits.conf

@@ -0,0 +1,28 @@
+# http context. Requires anonymous-logging.conf's trusted client IP variable.
+# Empty keys exclude ZIP downloads, navigation, writes and unrelated APIs.
+map "$request_method:$uri" $anonymous_file_limit_key {
+    default "";
+    ~*^(GET|HEAD):/api/repo/[^/]+/file/ $anonymous_client_ip;
+}
+map "$request_method:$uri" $anonymous_listing_limit_key {
+    default "";
+    ~*^(GET|HEAD):/api/repo/[^/]+/files(/|$) $anonymous_client_ip;
+}
+limit_req_zone $anonymous_file_limit_key zone=anonymous_file_requests:10m rate=4r/m;
+limit_req_zone $anonymous_listing_limit_key zone=anonymous_listing_requests:10m rate=2r/m;
+
+# Private operational entries are date-scoped regexes, never committed IP lists.
+# An empty file disables targeted blocks. Date must match the host's log timezone.
+map "$time_iso8601:$anonymous_client_ip" $anonymous_bulk_block {
+    default 0;
+    include /etc/nginx/anonymous-bulk-blocks.map;
+}
+map "$anonymous_bulk_block:$anonymous_file_limit_key:$anonymous_listing_limit_key" $anonymous_reject_bulk {
+    default 0;
+    ~^1:.+ 1;
+    "1::" 0;
+}
+map $anonymous_reject_bulk $anonymous_bulk_retry {
+    default 30;
+    1 3600;
+}

deploy/nginx/anonymous-bulk-location.conf

@@ -0,0 +1,8 @@
+# Include in the application's proxied location /, before proxy_pass.
+# return-only if is safe here; intercepted upstream errors remain disabled.
+error_page 429 = @anonymous_bulk_limited;
+if ($anonymous_reject_bulk) { return 429; }
+limit_req zone=anonymous_file_requests burst=60 nodelay;
+limit_req zone=anonymous_listing_requests burst=30 nodelay;
+limit_req_status 429;
+limit_req_log_level notice;

deploy/nginx/anonymous-bulk-server.conf

@@ -0,0 +1,7 @@
+# Include in each HTTPS application server block, outside location /.
+location @anonymous_bulk_limited {
+    default_type application/json;
+    add_header Retry-After $anonymous_bulk_retry always;
+    add_header Cache-Control "no-store" always;
+    return 429 '{"error":"rate_limited","message":"Too many individual file or directory requests. Use the repository ZIP download for bulk access."}';
+}

deploy/nginx/anonymous-logging.conf

@@ -58,7 +58,7 @@ log_format anonymous_json escape=json
     '"cloudflare_peer":"$anonymous_cloudflare_peer","cf_ray":"$anonymous_cf_ray",'
     '"edge_request_id":"$request_id","app_request_id":"$upstream_http_x_request_id",'
     '"method":"$request_method","route":"$anonymous_log_route","protocol":"$server_protocol",'
-    '"status":$status,"upstream_status":"$upstream_status",'
+    '"status":$status,"limit_req_status":"$limit_req_status","upstream_status":"$upstream_status",'
     '"request_time":$request_time,"upstream_connect_time":"$upstream_connect_time",'
     '"upstream_header_time":"$upstream_header_time","upstream_response_time":"$upstream_response_time",'
     '"request_bytes":$request_length,"response_bytes":$bytes_sent, "completion":"$request_completion",'

scripts/test-nginx-bulk-limits.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+"""Exercise Nginx admission against a disposable origin, not production traffic."""
+import datetime
+import http.server
+import json
+import pathlib
+import socket
+import subprocess
+import sys
+import tempfile
+import threading
+import time
+import urllib.error
+import urllib.request
+
+source = pathlib.Path(sys.argv[1])
+class Origin(http.server.BaseHTTPRequestHandler):
+    def do_GET(self):
+        self.send_response(200)
+        self.end_headers()
+        self.wfile.write(b"origin")
+    do_HEAD = do_GET
+    do_POST = do_GET
+    def log_message(self, *_args):
+        pass
+origin = http.server.HTTPServer(("127.0.0.1", 0), Origin)
+threading.Thread(target=origin.serve_forever, daemon=True).start()
+try:
+    with tempfile.TemporaryDirectory(prefix="anonymous-nginx-bulk-test-") as temp:
+        root = pathlib.Path(temp)
+        with socket.socket() as sock:
+            sock.bind(("127.0.0.1", 0))
+            port = sock.getsockname()[1]
+        date = datetime.datetime.now().astimezone().strftime("%Y-%m-%d")
+        blocks = root / "blocks.map"
+        blocks.write_text(f'"~^{date}T.*:203\\.0\\.113\\.2$" 1;\n')
+        logging = (source / "anonymous-logging.conf").read_text().replace("geo $remote_addr", "geo $arg_peer")
+        limits = (source / "anonymous-bulk-limits.conf").read_text().replace("/etc/nginx/anonymous-bulk-blocks.map", str(blocks))
+        location = (source / "anonymous-bulk-location.conf").read_text()
+        server = (source / "anonymous-bulk-server.conf").read_text()
+        config = root / "nginx.conf"
+        config.write_text(f'pid {root}/pid; error_log {root}/error.log notice; events {{}} http {{ access_log off; '
+                          + logging + limits + f' server {{ listen 127.0.0.1:{port}; ' + server
+                          + ' location / { ' + location + f' proxy_pass http://127.0.0.1:{origin.server_port}; }} }} }}')
+        subprocess.run(["nginx", "-t", "-p", str(root), "-c", str(config)], check=True, capture_output=True)
+        process = subprocess.Popen(["nginx", "-p", str(root), "-c", str(config), "-g", "daemon off;"],
+                                   stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        try:
+            for _ in range(50):
+                try:
+                    with socket.create_connection(("127.0.0.1", port), timeout=.1):
+                        break
+                except OSError:
+                    time.sleep(.1)
+            def request(path, client="203.0.113.1", method="GET"):
+                req = urllib.request.Request(f"http://127.0.0.1:{port}{path}?peer=173.245.48.1", method=method,
+                                             headers={"CF-Connecting-IP": client})
+                try:
+                    response = urllib.request.urlopen(req)
+                except urllib.error.HTTPError as error:
+                    response = error
+                with response:
+                    return response.status, dict(response.headers), response.read()
+            files = [request("/api/repo/test/file/a.txt")[0] for _ in range(65)]
+            assert files[0] == 200 and files[-1] == 429 and files.count(200) <= 61, files
+            listing = [request("/api/repo/test/files/counts")[0] for _ in range(35)]
+            assert listing[0] == 200 and listing[-1] == 429 and listing.count(200) <= 31, listing
+            status, headers, body = request("/api/repo/test/file/a.txt")
+            assert status == 429 and headers["Retry-After"] == "30"
+            assert json.loads(body)["error"] == "rate_limited" and b"ZIP" in body
+            assert request("/api/repo/test/zip")[0] == 200
+            assert request("/r/test/")[0] == 200
+            assert request("/api/repo/test/file/a.txt", method="POST")[0] == 200
+            assert request("/api/repo/test/file/a.txt", client="203.0.113.3")[0] == 200
+            for path in ["/api/repo/test/file/a.txt", "/api/repo/test/files/counts", "/API/REPO/test/FILE/a.txt"]:
+                status, headers, body = request(path, client="203.0.113.2")
+                assert status == 429 and headers["Retry-After"] == "3600", (status, headers, body)
+            assert request("/api/repo/test/file/a.txt", client="203.0.113.2", method="HEAD")[0] == 429
+            assert request("/api/repo/test/zip", client="203.0.113.2")[0] == 200
+            assert request("/", client="203.0.113.2")[0] == 200
+            print("PASS: file/listing budgets, client separation, scoped block, HEAD/case variants, JSON retry response, ZIP/navigation/write exclusions")
+        finally:
+            process.terminate()
+            process.wait(timeout=10)
+finally:
+    origin.shutdown()
+    origin.server_close()