mirror of
https://github.com/tdurieux/anonymous_github.git
synced 2026-11-17 17:44:09 +01:00
245 lines
14 KiB
Python
Executable File
245 lines
14 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Isolated local Docker integration test. Never reads production logs or .env.
|
|
Requires Docker Compose, Python 3 and PyYAML. Removes only its own project.
|
|
"""
|
|
import json
|
|
import base64
|
|
import os
|
|
from pathlib import Path
|
|
import secrets
|
|
import sys
|
|
import subprocess
|
|
import tempfile
|
|
import time
|
|
import urllib.parse
|
|
import urllib.request
|
|
import uuid
|
|
import yaml
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
CONFIG = ROOT / 'deploy/logging'
|
|
PROJECT = 'logging-test-' + secrets.token_hex(4)
|
|
|
|
def run(args, **kw):
|
|
return subprocess.run(args, check=True, text=True, capture_output=True, **kw).stdout.strip()
|
|
|
|
def wait_for(fn, timeout=60):
|
|
end = time.monotonic() + timeout
|
|
last = None
|
|
while time.monotonic() < end:
|
|
try:
|
|
result = fn()
|
|
if result:
|
|
return result
|
|
except Exception as e:
|
|
last = e
|
|
time.sleep(1)
|
|
raise AssertionError('Timed out: ' + str(last))
|
|
|
|
with tempfile.TemporaryDirectory(prefix=PROJECT) as temp:
|
|
temp = Path(temp)
|
|
docker = temp / 'docker'
|
|
nginx = temp / 'nginx'
|
|
docker.mkdir(); nginx.mkdir()
|
|
(docker / 'fixture').mkdir()
|
|
log = docker / 'fixture/fixture-json.log'
|
|
log.touch()
|
|
edge = nginx / 'anonymous.access.json'
|
|
edge.touch()
|
|
password = temp / 'password'
|
|
password.write_text(secrets.token_urlsafe(32))
|
|
password.chmod(0o644) # temp directory is 0700; Grafana uid must read the bind mount
|
|
env = {**os.environ, 'GRAFANA_ADMIN_USER': 'local-test', 'GRAFANA_PASSWORD_FILE': str(password),
|
|
'DOCKER_LOG_DIR': str(docker), 'NGINX_LOG_DIR': str(nginx), 'GRAFANA_PORT': '0'}
|
|
override = temp / 'override.yml'
|
|
override.write_text(yaml.safe_dump({'services': {
|
|
'victorialogs': {'ports': ['127.0.0.1::9428']},
|
|
'prometheus': {'ports': ['127.0.0.1::9090']},
|
|
'vector': {'ports': ['127.0.0.1::9598']},
|
|
'vmalert': {'ports': ['127.0.0.1::8880']},
|
|
}, 'networks': {'private': {'internal': False}}}))
|
|
cmd = ['docker', 'compose', '-p', PROJECT, '-f', str(CONFIG / 'compose.yml'), '-f', str(override)]
|
|
def compose(*args):
|
|
return run(cmd + list(args), env=env)
|
|
def url(service, port):
|
|
return 'http://' + compose('port', service, str(port))
|
|
def get(base, path, params=None):
|
|
address = base + path + ('?' + urllib.parse.urlencode(params) if params else '')
|
|
return urllib.request.urlopen(address, timeout=10).read().decode()
|
|
def query(expression):
|
|
return [json.loads(line) for line in get(vl, '/select/logsql/query', {'query': expression, 'limit': 50000}).splitlines() if line]
|
|
def append(service, record):
|
|
with log.open('a') as out:
|
|
out.write(json.dumps({'log': json.dumps(record) + '\n', 'attrs': {'tag': service}, 'stream': 'stdout'}) + '\n')
|
|
def count():
|
|
return int(query('* | stats count() as n')[0]['n'])
|
|
try:
|
|
compose('up', '-d')
|
|
vl = url('victorialogs', 9428)
|
|
prom = url('prometheus', 9090)
|
|
vector = url('vector', 9598)
|
|
alerts = url('vmalert', 8880)
|
|
grafana = url('grafana', 3000)
|
|
wait_for(lambda: get(vl, '/health') is not None)
|
|
wait_for(lambda: get(vector, '/metrics'))
|
|
wait_for(lambda: get(grafana, '/api/health'))
|
|
wait_for(lambda: 'application-logs' in get(alerts, '/api/v1/rules'))
|
|
time.sleep(2) # file watcher must discover empty files before append
|
|
request_id, job_id = str(uuid.uuid4()), str(uuid.uuid4())
|
|
sentinel = 'PRIVATE_SENTINEL_owner_repository_cookie_oauth_state'
|
|
for service in ['api', 'streamer']:
|
|
append(service, {'event': 'request_completed', 'severity': 'info', 'requestId': request_id,
|
|
'route': '/api/repo/:repoId/file/:path', 'status': 200, 'outcome': 'completed',
|
|
'firstByteMs': 40, 'responseMs': 600, 'ms': 640, 'stage_upstream_ms': 20,
|
|
'message': sentinel, 'url': 'https://example.invalid/' + sentinel, 'cookie': sentinel,
|
|
'authorization': sentinel, 'requestBody': {'secret': sentinel}, 'release': 'abcdef1'})
|
|
append('api', {'service': 'worker', 'event': 'job_failed', 'severity': 'error', 'requestId': request_id,
|
|
'jobId': job_id, 'attempt': 2, 'queue': 'download', 'errorCode': 'ETIMEDOUT', 'raw': sentinel})
|
|
append('mongodb', {'id': 51803, 's': 'E', 'msg': sentinel, 'attr': {'command': sentinel}})
|
|
append('redis', {'message': sentinel})
|
|
append('mongodb-backup', {'message': sentinel})
|
|
append('unrelated-container', {'message': sentinel})
|
|
# Malformed and malicious field values are represented without their raw content.
|
|
append('api', {'event': sentinel, 'route': '/' + sentinel, 'requestId': sentinel, 'severity': sentinel, 'module': sentinel})
|
|
with log.open('a') as out:
|
|
out.write(json.dumps({'log': sentinel, 'attrs': {'tag': 'api'}}) + '\n')
|
|
with edge.open('a') as out:
|
|
out.write(json.dumps({'status': 429, 'request_time': 0.1, 'upstream_header_time': '0.05',
|
|
'app_request_id': request_id, 'edge_request_id': 'f' * 32, 'route': '/api/repo/:id',
|
|
'limit_req_status': 'REJECTED', 'client_ip': '192.0.2.1', 'user_agent': sentinel,
|
|
'host': sentinel, 'request': sentinel}) + '\n')
|
|
wait_for(lambda: count() >= 9)
|
|
rows = query('*')
|
|
stored = json.dumps(rows)
|
|
assert sentinel not in stored and '192.0.2.1' not in stored
|
|
correlated = query('requestId:="' + request_id + '"')
|
|
assert {r['service'] for r in correlated} == {'api', 'streamer', 'worker', 'nginx'}
|
|
assert len(query('jobId:="' + job_id + '"')) == 1
|
|
assert all('requestId' not in r['_stream'] and 'instance' not in r['_stream'] for r in rows)
|
|
print('PASS ingestion, privacy allowlist, malformed input, API/streamer/job/edge correlation')
|
|
# Query every shipped dashboard and alert expression against the real server.
|
|
dashboard = json.loads((CONFIG / 'grafana/dashboards/operations.json').read_text())
|
|
for panel in dashboard['panels']:
|
|
for target in panel['targets']:
|
|
expr = target['expr'].replace('$request_id', request_id).replace('$job_id', job_id).replace('$edge_id', 'f' * 32)
|
|
if panel['datasource']['uid'] == 'anonymous-logs':
|
|
query(expr)
|
|
else:
|
|
get(prom, '/api/v1/query', {'query': expr})
|
|
for group in yaml.safe_load((CONFIG / 'log-alerts.yml').read_text())['groups']:
|
|
for rule in group['rules']:
|
|
get(vl, '/select/logsql/stats_query', {'query': rule['expr']})
|
|
auth = base64.b64encode(('local-test:' + password.read_text()).encode()).decode()
|
|
def grafana_request(path, payload=None):
|
|
request = urllib.request.Request(grafana + path,
|
|
data=json.dumps(payload).encode() if payload is not None else None,
|
|
headers={'Authorization': 'Basic ' + auth, 'Content-Type': 'application/json'})
|
|
return json.loads(urllib.request.urlopen(request, timeout=20).read())
|
|
assert grafana_request('/api/dashboards/uid/anonymous-operations')['dashboard']['uid'] == 'anonymous-operations'
|
|
assert grafana_request('/api/datasources/uid/anonymous-logs/health')['status'] == 'OK'
|
|
result = grafana_request('/api/ds/query', {'from': str(int((time.time() - 3600) * 1000)),
|
|
'to': str(int(time.time() * 1000)), 'queries': [{'refId': 'A',
|
|
'datasource': {'type': 'victoriametrics-logs-datasource', 'uid': 'anonymous-logs'},
|
|
'expr': 'requestId:="' + request_id + '"', 'queryType': 'instant', 'maxLines': 100}]})
|
|
assert not result['results']['A'].get('error'), result['results']['A'].get('error')
|
|
assert result['results']['A']['frames']
|
|
print('PASS dashboard and alert queries, provisioned Grafana datasource query')
|
|
# Rename rotation, then collector restart, preserving the spool/checkpoint volume.
|
|
before = count()
|
|
log.rename(log.with_suffix('.log.1'))
|
|
log.touch()
|
|
append('api', {'event': 'process_started', 'severity': 'info'})
|
|
wait_for(lambda: count() > before)
|
|
compose('restart', 'vector')
|
|
vector = url('vector', 9598)
|
|
wait_for(lambda: get(vector, '/metrics'))
|
|
before = count()
|
|
append('api', {'event': 'process_heartbeat', 'severity': 'info'})
|
|
wait_for(lambda: count() > before)
|
|
print('PASS rename rotation and collector checkpoint restart')
|
|
# Backend failure: sanitized disk spool remains bounded; source writer stays independent.
|
|
before = count()
|
|
compose('stop', 'victorialogs')
|
|
started = time.monotonic()
|
|
for _ in range(2000):
|
|
append('api', {'event': 'request_completed', 'severity': 'info', 'requestId': str(uuid.uuid4()), 'status': 200})
|
|
assert time.monotonic() - started < 10
|
|
wait_for(lambda: 'vector_buffer_size_events' in get(vector, '/metrics'))
|
|
time.sleep(3)
|
|
compose('restart', 'vector')
|
|
compose('start', 'victorialogs')
|
|
vl = url('victorialogs', 9428)
|
|
wait_for(lambda: count() >= before + 2000, 120)
|
|
print('PASS unavailable backend, spool persistence, recovery of 2000 records')
|
|
# Reject data older than retention rather than resurrecting it.
|
|
old = json.dumps({'timestamp': '2000-01-01T00:00:00Z', 'event': 'expired_fixture'}).encode()
|
|
req = urllib.request.Request(vl + '/insert/jsonline?_time_field=timestamp', data=old, headers={'Content-Type': 'application/stream+json'})
|
|
urllib.request.urlopen(req, timeout=10).read()
|
|
time.sleep(2)
|
|
assert not query('event:expired_fixture')
|
|
assert 'reason="too_small_timestamp"' in get(vl, '/metrics')
|
|
print('PASS retention rejection')
|
|
# Compression observation, synthetic workload only, including random trace IDs.
|
|
raw_bytes = 0
|
|
for _ in range(10000):
|
|
record = {'event': 'request_completed', 'severity': 'info', 'requestId': str(uuid.uuid4()),
|
|
'status': 200, 'ms': 50, 'firstByteMs': 20, 'responseMs': 30, 'route': '/api/repo/:repoId/file/:path'}
|
|
raw_bytes += len(json.dumps(record).encode()) + 1
|
|
append('api', record)
|
|
wait_for(lambda: count() >= before + 12000, 120)
|
|
compose('restart', 'victorialogs')
|
|
vl = url('victorialogs', 9428)
|
|
wait_for(lambda: get(vl, '/health') is not None)
|
|
metrics_text = get(vl, '/metrics')
|
|
size = sum(float(line.split()[-1]) for line in metrics_text.splitlines() if line.startswith('vl_data_size_bytes{'))
|
|
print(f'MEASURED fixture JSON bytes={raw_bytes}; total store data+index bytes={int(size)}; store also contains earlier fixtures; no production ratio claimed')
|
|
# Check fixed limits and no accidental public ingress.
|
|
config = json.loads(compose('config', '--format', 'json'))
|
|
for service, spec in config['services'].items():
|
|
assert int(spec['mem_limit']) > 0, service
|
|
for port in spec.get('ports', []):
|
|
assert port['host_ip'] == '127.0.0.1', service
|
|
print('PASS configured memory/rotation/disk bounds and loopback-only test ports')
|
|
if '--stress' in sys.argv:
|
|
compose('stop', 'victorialogs')
|
|
vector = url('vector', 9598)
|
|
numeric_fields = ['status', 'httpStatus', 'ms', 'firstByteMs', 'responseMs', 'bytes', 'bytesPerSecond',
|
|
'attempt', 'rss', 'memoryLimitBytes', 'memoryRatio', 'uptime', 'activeRequests', 'waiting',
|
|
'active', 'delayed', 'failed', 'oldestWaitingMs', 'droppedLogs', 'loopP95Ms', 'upstreamStatus',
|
|
'stage_repository_ms', 'stage_credential_resolution_ms', 'stage_upstream_ms', 'stage_cache_lookup_ms',
|
|
'stage_cache_fill_ms', 'stage_worker_wait_ms', 'stage_anonymize_ms', 'stage_streamer_headers_ms']
|
|
stress = {field: 123456789 for field in numeric_fields}
|
|
stress.update(event='request_completed', severity='info', requestId=request_id)
|
|
line = json.dumps({'log': json.dumps(stress), 'attrs': {'tag': 'api'}}) + '\n'
|
|
with log.open('a') as output:
|
|
for _ in range(500):
|
|
output.write(line * 1000)
|
|
def metric_value(name):
|
|
lines = get(vector, '/metrics').splitlines()
|
|
return sum(float(line.split()[1]) for line in lines if line.startswith(name + '{') and (name.startswith('vector_buffer_') or 'component_id="logs"' in line))
|
|
wait_for(lambda: metric_value('vector_buffer_discarded_events_total') > 0 or metric_value('vector_component_discarded_events_total') > 0, 600)
|
|
buffered = metric_value('vector_buffer_size_bytes')
|
|
# Memory-stage metrics inherit buffer_type=disk from the pipeline;
|
|
# select its stage explicitly instead of that misleading label.
|
|
memory_events = sum(float(line.split()[1]) for line in get(vector, '/metrics').splitlines()
|
|
if line.startswith('vector_buffer_size_events{') and 'buffer_id="logs"' in line and 'stage="1"' in line)
|
|
assert memory_events <= 1000, memory_events
|
|
assert buffered <= 268435488 + 8 * 1048576, buffered
|
|
inspection = json.loads(run(['docker', 'inspect', compose('ps', '-q', 'vector')]))[0]
|
|
assert inspection['RestartCount'] == 0 and not inspection['State']['OOMKilled']
|
|
print(f'PASS full-spool drop-newest stress: 500000 events; buffered={int(buffered)} bytes; collector stayed alive')
|
|
# Wait through the production two-minute 'for', rather than merely
|
|
# checking PromQL syntax or that a counter exists.
|
|
wait_for(lambda: json.loads(get(prom, '/api/v1/query', {'query':
|
|
'ALERTS{alertname="CollectorDrops",alertstate="firing"}'}))['data']['result'], 210)
|
|
print('PASS sustained CollectorDrops alert fires on full-spool loss')
|
|
except Exception:
|
|
if '--stress' in sys.argv:
|
|
print('\n'.join(line for line in get(url('vector', 9598), '/metrics').splitlines()
|
|
if line.startswith(('vector_buffer_', 'vector_component_discarded')) and not '_bucket' in line))
|
|
# Only synthetic fixture logs exist in this project. No production reads.
|
|
print(compose('logs', '--tail', '30', 'vector', 'victorialogs', 'vmalert', 'grafana'))
|
|
raise
|
|
finally:
|
|
compose('down', '-v', '--remove-orphans')
|