fix: make Grafana operations understandable and query-correct

This commit is contained in:
Thomas Durieux committed 2026-11-15 13:34:03 +00:00
1 parent 1e617d2df0
commit 7f3cf71bc3
6 files changed
+1771 -325

No files matched your search

+92
View File
@@ -0,0 +1,92 @@
#!/usr/bin/env python3
"""Create/stop an isolated dashboard fixture stack. Never reads production data.
python3 scripts/grafana-fixtures.py start
python3 scripts/grafana-fixtures.py stop
State and a generated password stay in /tmp/grafana-dashboard-fixtures (0700).
Only the grafana-dashboard-fixtures Compose project is managed by this script.
"""
import datetime
import json
import os
from pathlib import Path
import secrets
import subprocess
import sys
import time
import urllib.request
import uuid
import yaml
ROOT = Path(__file__).resolve().parents[1]
STATE = Path('/tmp/grafana-dashboard-fixtures')
PROJECT = 'grafana-dashboard-fixtures'
CMD = ['docker', 'compose', '-p', PROJECT, '-f', str(ROOT / 'deploy/logging/compose.yml'), '-f', str(STATE / 'override.yml')]
def compose(*args):
env = {**os.environ, **json.loads((STATE / 'environment.json').read_text())}
return subprocess.run(CMD + list(args), env=env, check=True, capture_output=True, text=True).stdout.strip()
if sys.argv[1:] == ['stop']:
compose('down', '-v', '--remove-orphans')
print('Removed fixture containers and volumes')
sys.exit(0)
if sys.argv[1:] != ['start']:
sys.exit('Use start or stop')
if STATE.exists():
sys.exit('Fixture directory exists. Stop the fixture stack and remove ' + str(STATE) + ' before recreating it.')
STATE.mkdir(mode=0o700)
for directory in ['docker/fixture', 'nginx']:
(STATE / directory).mkdir(parents=True)
log = STATE / 'docker/fixture/fixture-json.log'; log.touch()
edge = STATE / 'nginx/anonymous.access.json'; edge.touch(); edge.chmod(0o640)
password = STATE / 'password'; password.write_text(secrets.token_urlsafe(32)); password.chmod(0o644)
(STATE / 'environment.json').write_text(json.dumps({'GRAFANA_ADMIN_USER':'local-test', 'GRAFANA_PASSWORD_FILE':str(password),
'NGINX_LOG_GID':str(os.getgid()), 'DOCKER_LOG_DIR':str(STATE / 'docker'), 'NGINX_LOG_DIR':str(STATE / 'nginx'), 'GRAFANA_PORT':'0'}))
# Published ports on an internal Docker bridge are not reachable on all hosts.
# This override is local-test-only; production compose and auth stay unchanged.
(STATE / 'override.yml').write_text(yaml.safe_dump({'networks':{'private':{'internal':False}}}))
try:
compose('up', '-d')
url = 'http://' + compose('port', 'grafana', '3000')
(STATE / 'url').write_text(url)
deadline = time.monotonic() + 180
while True:
try:
with urllib.request.urlopen(url + '/api/health', timeout=5) as response:
if response.status == 200: break
except Exception:
if time.monotonic() > deadline: raise
time.sleep(2)
now = time.time(); rid = str(uuid.uuid4()); jid = str(uuid.uuid4())
instances = [str(uuid.uuid4()) for _ in range(5)]
def stamp(t): return datetime.datetime.fromtimestamp(t, datetime.timezone.utc).isoformat()
with log.open('a') as out, edge.open('a') as nginx:
def emit(service, event, t, **fields):
record = dict(event=event, time=stamp(t), severity='info', **fields)
out.write(json.dumps({'log':json.dumps(record)+'\n', 'attrs':{'tag':service}, 'stream':'stdout'})+'\n')
for minute in range(60):
t = now - minute * 60
for i in range(15 + minute % 8):
status = 503 if minute < 12 and i == 0 else 200
emit('api', 'request_completed', t, requestId=rid, route='/api/repo/:repoId/file/:path', status=status,
outcome='completed', firstByteMs=40+minute*2+i*3, responseMs=500+minute*5,
release='abcdef1' if minute<30 else 'abcdef0')
nginx.write(json.dumps({'time':stamp(t), 'status':status, 'request_time':.8,
'upstream_header_time':'0.06', 'app_request_id':rid, 'route':'/api/repo/:id'})+'\n')
if minute < 6: emit('api','request_cancelled',t,requestId=rid,status=200,outcome='interrupted')
for i, instance in enumerate(instances):
emit('api' if i==0 else 'streamer','process_heartbeat',t,instance=instance,memoryRatio=.31+i*.09,rss=100000000,droppedLogs=0)
for queue in ['download','cache','remove']:
emit('api','queue_heartbeat',t,queue=queue,waiting=3 if queue=='download' else 0,active=1,oldestWaitingMs=45000 if queue=='download' else 0)
for event,outcome,status,latency in [('request_completed','completed',200,40),('request_slow','completed',503,1200),('request_cancelled','interrupted',503,None)]:
emit('api',event,now,requestId=rid,release='fff0001',outcome=outcome,status=status,firstByteMs=latency)
emit('api','job_failed',now,jobId=jid,requestId=rid,errorCode='ETIMEDOUT',queue='download')
(STATE / 'ids.json').write_text(json.dumps({'request_id':rid,'job_id':jid}))
time.sleep(3)
print('Fixture Grafana: ' + url)
print('Username: local-test; password file: ' + str(password))
print('Run GRAFANA_TEST_URL="$(cat ' + str(STATE / 'url') + ')" GRAFANA_TEST_PASSWORD_FILE=' + str(password) + ' python3 scripts/test-grafana-dashboard.py')
except Exception:
compose('down','-v','--remove-orphans')
raise
+109
View File
@@ -0,0 +1,109 @@
#!/usr/bin/env python3
"""Check dashboard query frames against an isolated, running pinned logging stack.
Use only synthetic data. GRAFANA_TEST_URL, GRAFANA_TEST_PASSWORD_FILE and
GRAFANA_TEST_USER identify that local stack. Never point this at production.
See deploy/logging/dashboard-validation.md for the reproducible setup.
"""
import base64
import json
import os
from pathlib import Path
import time
import urllib.request
from urllib.parse import urlparse
ROOT = Path(__file__).resolve().parents[1]
base = os.environ.get('GRAFANA_TEST_URL', 'http://localhost:13301')
assert urlparse(base).hostname in ('localhost', '127.0.0.1'), 'Local fixture stack only'
password = Path(os.environ['GRAFANA_TEST_PASSWORD_FILE']).read_text().strip()
auth = base64.b64encode((os.environ.get('GRAFANA_TEST_USER', 'local-test') + ':' + password).encode()).decode()
def request(path, payload=None):
req = urllib.request.Request(base + path, data=json.dumps(payload).encode() if payload else None,
headers={'Authorization': 'Basic ' + auth, 'Content-Type': 'application/json'})
return json.loads(urllib.request.urlopen(req, timeout=30).read())
def panels(items):
for panel in items:
yield panel
yield from panels(panel.get('panels', []))
def query(panel, target, start=None, end=None):
target = {**target, 'datasource': panel['datasource'], 'intervalMs': 60000, 'maxDataPoints': 100}
for name in ['request_id', 'job_id', 'edge_id']:
target['expr'] = target['expr'].replace('${' + name + ':json}', '"fixture-not-present"').replace('$' + name, 'fixture-not-present')
for name in ['service', 'release', 'event']:
target['expr'] = target['expr'].replace('${' + name + ':json}', '".*"').replace('$' + name, '*')
result = request('/api/ds/query', {'from': str(start or int((time.time()-3600)*1000)),
'to': str(end or int(time.time()*1000)), 'queries': [target]})['results'][target['refId']]
assert not result.get('error'), (panel['title'], result.get('error'))
return result.get('frames', [])
for path in sorted((ROOT / 'deploy/logging/grafana/dashboards').glob('*.json')):
dashboard = json.loads(path.read_text())
assert request('/api/dashboards/uid/' + dashboard['uid'])['dashboard']['uid'] == dashboard['uid']
for panel in panels(dashboard['panels']):
for target in panel.get('targets', []):
frames = query(panel, target)
if panel['type'] in ('stat', 'timeseries') and panel['datasource']['uid'] == 'anonymous-logs':
assert target['queryType'] in ('stats', 'statsRange')
assert all(any(f['type'] == 'number' for f in frame['schema']['fields']) for frame in frames), panel['title']
if panel['type'] == 'table' and panel['datasource']['uid'] == 'anonymous-logs':
assert panel['transformations'][0]['id'] == 'extractFields'
# Aggregate results must carry parseable fields, not a timestamp to display.
for frame in frames:
fields = frame['schema']['fields']; values = frame['data']['values']
labels_index = next((i for i, f in enumerate(fields) if f['name'] == 'labels'), None)
if labels_index is not None:
for value in values[labels_index]:
row = json.loads(value) if isinstance(value, str) else value
assert row, panel['title']
print('PASS', dashboard['uid'], panel['title'], target['refId'])
print('PASS all provisioned dashboard datasource queries')
# A reserved synthetic release contains one normal completion, one slow 503,
# and one interrupted 503. This proves we count each terminal event once and
# never classify the interrupted status as a completed server error.
operations = json.loads((ROOT / 'deploy/logging/grafana/dashboards/operations.json').read_text())
by_id = {p['id']: p for p in panels(operations['panels'])}
def numeric(frames):
return [v for frame in frames for field, values in zip(frame['schema']['fields'], frame['data']['values'])
if field['type'] == 'number' for v in values]
for panel_id, expected in [(101, 2), (102, 1), (104, 1), (103, 1200)]:
panel = by_id[panel_id]
target = {**panel['targets'][0]}
target['expr'] = 'release:=\"fff0001\" ' + target['expr']
assert numeric(query(panel, target)) == [expected], (panel['title'], numeric(query(panel, target)))
print('PASS exact completed/slow/interrupted counts and measured p95')
for panel_id in (10, 11, 12):
panel = by_id[panel_id]
target = {**panel['targets'][0]}
for name in ['request_id', 'job_id', 'edge_id']:
target['expr'] = target['expr'].replace('${' + name + ':json}', '\"\"')
frames = query(panel, target)
assert not any(values for frame in frames for values in frame['data']['values']), panel['title']
print('PASS blank correlation IDs never match unrelated logs')
# No measured latency must remain absent/NaN, never a fabricated healthy zero.
panel = by_id[103]; target = {**panel['targets'][0]}
target['expr'] = 'release:=\"0000000\" ' + target['expr']
values = numeric(query(panel, target))
assert not values or all(v is None or str(v).lower() == 'nan' for v in values), values
print('PASS absent latency is not zero')
fixture_dir = Path(os.environ['GRAFANA_TEST_PASSWORD_FILE']).parent
ids = json.loads((fixture_dir / 'ids.json').read_text())
fixture_end = int((fixture_dir / 'docker/fixture/fixture-json.log').stat().st_mtime * 1000) + 2000
for panel_id in (10, 11):
panel = by_id[panel_id]; target = {**panel['targets'][0]}
name = 'request_id' if panel_id == 10 else 'job_id'
target['expr'] = target['expr'].replace('${' + name + ':json}', json.dumps(ids[name]))
frames = query(panel, target, fixture_end-3600000, fixture_end)
assert any(values for frame in frames for values in frame['data']['values']), panel['title']
for panel_id in (110, 130):
panel = by_id[panel_id]
fresh = query(panel, panel['targets'][0], fixture_end-3600000, fixture_end)
stale = query(panel, panel['targets'][0], fixture_end-3600000, fixture_end+240000)
assert any(values for frame in fresh for values in frame['data']['values']), panel['title']
assert not any(values for frame in stale for values in frame['data']['values']), panel['title']
print('PASS real correlation matches and range-end freshness/staleness')