mirror of
https://github.com/tdurieux/anonymous_github.git
synced 2026-11-14 00:07:00 +01:00
Merge pull request #936 from tdurieux/ops/attributable-edge-logs
ops: attribute edge traffic and correlate upstream failures
This commit is contained in:
3 files changed
+156
No files matched your search
@@ -0,0 +1,25 @@
|
||||
# Production edge logging
|
||||
|
||||
Load `anonymous-logging.conf` in Nginx's **http** context, normally by installing it as `/etc/nginx/conf.d/anonymous-logging.conf`. Add this directive inside each application **server** block, including HTTP redirects and alternate hostnames:
|
||||
|
||||
```nginx
|
||||
access_log /var/log/nginx/anonymous-access.log anonymous_json buffer=64k flush=1s;
|
||||
```
|
||||
|
||||
Back up the existing configuration, run `sudo nginx -t`, then gracefully reload Nginx. On validation failure restore the backup before reloading. The server-level directive replaces the inherited combined access log for that virtual host; do not add a second combined log containing raw queries.
|
||||
|
||||
The JSON log separates visitor and Cloudflare edge addresses. Cloudflare headers are accepted only for peers in its published IP ranges. Direct requests cannot spoof the visitor IP or Ray ID through these headers. Refresh the ranges from the two URLs in the config when Cloudflare changes them. This configuration deliberately leaves `real_ip`, `X-Forwarded-For`, and application rate-limit identity unchanged.
|
||||
|
||||
Use `app_request_id` to correlate with API/streamer request logs and `cf_ray` with Cloudflare security events. `edge_request_id` exists even when the upstream never responds. Upstream statuses and connect/header/response timings separate origin failures from proxy failures. Status 499 and an empty `completion` identify incomplete requests; connection fields expose keep-alive reuse. Use the separate Nginx error log to distinguish connection-slot exhaustion from timeouts. An access event is emitted when its request finishes, so per-second counts are completion counts, not exact arrival rates.
|
||||
|
||||
Route labels omit queries, repository IDs and file paths. Cookies, authorization headers, bodies and referrers are not collected. User-agent text is JSON escaped. Treat IPs and Ray IDs as private operational data: keep logs readable only by operators. On Ubuntu, the existing `/var/log/nginx/*.log` logrotate rule covers this file; verify it creates mode 0640, compresses old logs and retains no more than 14 daily rotations. Watch disk usage during floods; daily rotation is not a hard size cap.
|
||||
|
||||
This is observability, not flood prevention. Cloudflare rate limiting/challenges and origin connection limits need a separate measured rollout. Do not block the Cloudflare edge IPs reported in `peer_ip` as if they were visitors.
|
||||
|
||||
Validate with the installed Nginx binary before rollout:
|
||||
|
||||
```sh
|
||||
python3 scripts/test-nginx-logging.py deploy/nginx/anonymous-logging.conf
|
||||
```
|
||||
|
||||
The test starts an isolated loopback listener and checks trusted IPv4/IPv6 peers, spoofed headers, missing-header fallback, JSON escaping and omitted sensitive data. It never changes production listeners.
|
||||
@@ -0,0 +1,65 @@
|
||||
# Include in nginx http context (for example /etc/nginx/conf.d/anonymous-logging.conf).
|
||||
# Logging only: do not change real_ip or the application's proxy trust chain.
|
||||
# Ranges verified against https://www.cloudflare.com/ips-v4 and /ips-v6 on 2026-11-10.
|
||||
geo $remote_addr $anonymous_cloudflare_peer {
|
||||
default 0;
|
||||
173.245.48.0/20 1;
|
||||
103.21.244.0/22 1;
|
||||
103.22.200.0/22 1;
|
||||
103.31.4.0/22 1;
|
||||
141.101.64.0/18 1;
|
||||
108.162.192.0/18 1;
|
||||
190.93.240.0/20 1;
|
||||
188.114.96.0/20 1;
|
||||
197.234.240.0/22 1;
|
||||
198.41.128.0/17 1;
|
||||
162.158.0.0/15 1;
|
||||
104.16.0.0/13 1;
|
||||
104.24.0.0/14 1;
|
||||
172.64.0.0/13 1;
|
||||
131.0.72.0/22 1;
|
||||
2400:cb00::/32 1;
|
||||
2606:4700::/32 1;
|
||||
2803:f800::/32 1;
|
||||
2405:b500::/32 1;
|
||||
2405:8100::/32 1;
|
||||
2a06:98c0::/29 1;
|
||||
2c0f:f248::/32 1;
|
||||
}
|
||||
map "$anonymous_cloudflare_peer:$http_cf_connecting_ip" $anonymous_client_ip {
|
||||
default $remote_addr;
|
||||
~^1:.+$ $http_cf_connecting_ip;
|
||||
}
|
||||
map $anonymous_cloudflare_peer $anonymous_cf_ray {
|
||||
default "";
|
||||
1 $http_cf_ray;
|
||||
}
|
||||
# Fixed route labels: no query strings, repository identifiers or file paths.
|
||||
map $uri $anonymous_log_route {
|
||||
default "other";
|
||||
/ "/";
|
||||
/api/healthcheck "/api/healthcheck";
|
||||
/api/user "/api/user";
|
||||
~^/api/repo/[^/]+/file/ "/api/repo/:id/file/:path";
|
||||
~^/api/repo/[^/]+/files/counts "/api/repo/:id/files/counts";
|
||||
~^/api/repo/[^/]+/files/search "/api/repo/:id/files/search";
|
||||
~^/api/repo/[^/]+/files "/api/repo/:id/files";
|
||||
~^/api/repo/[^/]+/zip "/api/repo/:id/zip";
|
||||
~^/api/repo/ "/api/repo/:id";
|
||||
~^/(r|w|repository)/ "/viewer/:id/:path";
|
||||
~^/github/ "/github/:action";
|
||||
~^/api/admin/ "/api/admin/:action";
|
||||
~^/api/user/ "/api/user/:action";
|
||||
~^/(css|script|fonts|partials)/ "/assets/:path";
|
||||
}
|
||||
log_format anonymous_json escape=json
|
||||
'{"time":"$time_iso8601","host":"$host","server":"$server_name",'
|
||||
'"client_ip":"$anonymous_client_ip","peer_ip":"$remote_addr",'
|
||||
'"cloudflare_peer":"$anonymous_cloudflare_peer","cf_ray":"$anonymous_cf_ray",'
|
||||
'"edge_request_id":"$request_id","app_request_id":"$upstream_http_x_request_id",'
|
||||
'"method":"$request_method","route":"$anonymous_log_route","protocol":"$server_protocol",'
|
||||
'"status":$status,"upstream_status":"$upstream_status",'
|
||||
'"request_time":$request_time,"upstream_connect_time":"$upstream_connect_time",'
|
||||
'"upstream_header_time":"$upstream_header_time","upstream_response_time":"$upstream_response_time",'
|
||||
'"request_bytes":$request_length,"response_bytes":$bytes_sent, "completion":"$request_completion",'
|
||||
'"connection":"$connection","connection_requests":$connection_requests, "user_agent":"$http_user_agent"}';
|
||||
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Exercise the logging config with an isolated local Nginx, never production listeners."""
|
||||
import json
|
||||
import pathlib
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
source = pathlib.Path(sys.argv[1]).read_text()
|
||||
with tempfile.TemporaryDirectory(prefix="anonymous-nginx-test-") as root:
|
||||
root = pathlib.Path(root)
|
||||
with socket.socket() as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
port = sock.getsockname()[1]
|
||||
# Only this isolated test substitutes the peer so both address families can
|
||||
# be exercised on loopback. Production always uses the real socket peer.
|
||||
config = source.replace("geo $remote_addr", "geo $arg_peer")
|
||||
path = root / "nginx.conf"
|
||||
path.write_text(f'pid {root}/pid; error_log {root}/error.log; events {{}} http {{\n'
|
||||
+ config + f'\nserver {{ listen 127.0.0.1:{port}; '
|
||||
f'access_log {root}/access.log anonymous_json; return 200 "ok"; }} }}')
|
||||
subprocess.run(["nginx", "-t", "-p", str(root), "-c", str(path)], check=True, capture_output=True)
|
||||
process = subprocess.Popen(["nginx", "-p", str(root), "-c", str(path), "-g", "daemon off;"],
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
try:
|
||||
for _ in range(50):
|
||||
try:
|
||||
with socket.create_connection(("127.0.0.1", port), timeout=.1):
|
||||
break
|
||||
except OSError:
|
||||
time.sleep(.1)
|
||||
cases = [
|
||||
("127.0.0.1", "203.0.113.9", "0", "127.0.0.1", ""),
|
||||
("173.245.48.1", "203.0.113.9", "1", "203.0.113.9", "test-ray"),
|
||||
("2606:4700::1", "2001:db8::9", "1", "2001:db8::9", "test-ray"),
|
||||
("173.245.48.1", "", "1", "127.0.0.1", "test-ray"),
|
||||
]
|
||||
for peer, visitor, trusted, client, ray in cases:
|
||||
req = urllib.request.Request(
|
||||
f"http://127.0.0.1:{port}/api/repo/private-repo/file/private-path?peer={peer}&code=SECRET",
|
||||
headers={"CF-Connecting-IP": visitor, "CF-Ray": "test-ray",
|
||||
"User-Agent": 'test "quoted" \\ agent', "Cookie": "session=SECRET",
|
||||
"Authorization": "Bearer SECRET"})
|
||||
with urllib.request.urlopen(req) as response:
|
||||
assert response.status == 200
|
||||
for _ in range(50):
|
||||
lines = (root / "access.log").read_text().splitlines()
|
||||
if len(lines) >= cases.index((peer, visitor, trusted, client, ray)) + 1:
|
||||
break
|
||||
time.sleep(.01)
|
||||
row = json.loads(lines[-1])
|
||||
assert row["client_ip"] == client, row
|
||||
assert row["cloudflare_peer"] == trusted, row
|
||||
assert row["cf_ray"] == ray, row
|
||||
assert row["peer_ip"] == "127.0.0.1", row
|
||||
assert row["route"] == "/api/repo/:id/file/:path", row
|
||||
assert len(row["edge_request_id"]) == 32, row
|
||||
assert row["user_agent"] == 'test "quoted" \\ agent', row
|
||||
assert not any(secret in lines[-1] for secret in ["SECRET", "private-repo", "private-path"])
|
||||
print("PASS: trusted IPv4/IPv6, spoof rejection, missing-header fallback, JSON escaping and sensitive-data omission")
|
||||
finally:
|
||||
process.terminate()
|
||||
process.wait(timeout=10)
|
||||
Reference in new issue
Block a user