mirror of
https://github.com/tdurieux/anonymous_github.git
synced 2026-11-17 17:44:09 +01:00
fix: make Grafana operations understandable and query-correct
This commit is contained in:
1 parent
1e617d2df0
commit
7f3cf71bc3
6 files changed
+1771
-325
No files matched your search
@@ -0,0 +1,92 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create/stop an isolated dashboard fixture stack. Never reads production data.
|
||||
|
||||
python3 scripts/grafana-fixtures.py start
|
||||
python3 scripts/grafana-fixtures.py stop
|
||||
State and a generated password stay in /tmp/grafana-dashboard-fixtures (0700).
|
||||
Only the grafana-dashboard-fixtures Compose project is managed by this script.
|
||||
"""
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
import uuid
|
||||
import yaml
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
STATE = Path('/tmp/grafana-dashboard-fixtures')
|
||||
PROJECT = 'grafana-dashboard-fixtures'
|
||||
CMD = ['docker', 'compose', '-p', PROJECT, '-f', str(ROOT / 'deploy/logging/compose.yml'), '-f', str(STATE / 'override.yml')]
|
||||
|
||||
def compose(*args):
|
||||
env = {**os.environ, **json.loads((STATE / 'environment.json').read_text())}
|
||||
return subprocess.run(CMD + list(args), env=env, check=True, capture_output=True, text=True).stdout.strip()
|
||||
|
||||
if sys.argv[1:] == ['stop']:
|
||||
compose('down', '-v', '--remove-orphans')
|
||||
print('Removed fixture containers and volumes')
|
||||
sys.exit(0)
|
||||
if sys.argv[1:] != ['start']:
|
||||
sys.exit('Use start or stop')
|
||||
if STATE.exists():
|
||||
sys.exit('Fixture directory exists. Stop the fixture stack and remove ' + str(STATE) + ' before recreating it.')
|
||||
STATE.mkdir(mode=0o700)
|
||||
for directory in ['docker/fixture', 'nginx']:
|
||||
(STATE / directory).mkdir(parents=True)
|
||||
log = STATE / 'docker/fixture/fixture-json.log'; log.touch()
|
||||
edge = STATE / 'nginx/anonymous.access.json'; edge.touch(); edge.chmod(0o640)
|
||||
password = STATE / 'password'; password.write_text(secrets.token_urlsafe(32)); password.chmod(0o644)
|
||||
(STATE / 'environment.json').write_text(json.dumps({'GRAFANA_ADMIN_USER':'local-test', 'GRAFANA_PASSWORD_FILE':str(password),
|
||||
'NGINX_LOG_GID':str(os.getgid()), 'DOCKER_LOG_DIR':str(STATE / 'docker'), 'NGINX_LOG_DIR':str(STATE / 'nginx'), 'GRAFANA_PORT':'0'}))
|
||||
# Published ports on an internal Docker bridge are not reachable on all hosts.
|
||||
# This override is local-test-only; production compose and auth stay unchanged.
|
||||
(STATE / 'override.yml').write_text(yaml.safe_dump({'networks':{'private':{'internal':False}}}))
|
||||
try:
|
||||
compose('up', '-d')
|
||||
url = 'http://' + compose('port', 'grafana', '3000')
|
||||
(STATE / 'url').write_text(url)
|
||||
deadline = time.monotonic() + 180
|
||||
while True:
|
||||
try:
|
||||
with urllib.request.urlopen(url + '/api/health', timeout=5) as response:
|
||||
if response.status == 200: break
|
||||
except Exception:
|
||||
if time.monotonic() > deadline: raise
|
||||
time.sleep(2)
|
||||
now = time.time(); rid = str(uuid.uuid4()); jid = str(uuid.uuid4())
|
||||
instances = [str(uuid.uuid4()) for _ in range(5)]
|
||||
def stamp(t): return datetime.datetime.fromtimestamp(t, datetime.timezone.utc).isoformat()
|
||||
with log.open('a') as out, edge.open('a') as nginx:
|
||||
def emit(service, event, t, **fields):
|
||||
record = dict(event=event, time=stamp(t), severity='info', **fields)
|
||||
out.write(json.dumps({'log':json.dumps(record)+'\n', 'attrs':{'tag':service}, 'stream':'stdout'})+'\n')
|
||||
for minute in range(60):
|
||||
t = now - minute * 60
|
||||
for i in range(15 + minute % 8):
|
||||
status = 503 if minute < 12 and i == 0 else 200
|
||||
emit('api', 'request_completed', t, requestId=rid, route='/api/repo/:repoId/file/:path', status=status,
|
||||
outcome='completed', firstByteMs=40+minute*2+i*3, responseMs=500+minute*5,
|
||||
release='abcdef1' if minute<30 else 'abcdef0')
|
||||
nginx.write(json.dumps({'time':stamp(t), 'status':status, 'request_time':.8,
|
||||
'upstream_header_time':'0.06', 'app_request_id':rid, 'route':'/api/repo/:id'})+'\n')
|
||||
if minute < 6: emit('api','request_cancelled',t,requestId=rid,status=200,outcome='interrupted')
|
||||
for i, instance in enumerate(instances):
|
||||
emit('api' if i==0 else 'streamer','process_heartbeat',t,instance=instance,memoryRatio=.31+i*.09,rss=100000000,droppedLogs=0)
|
||||
for queue in ['download','cache','remove']:
|
||||
emit('api','queue_heartbeat',t,queue=queue,waiting=3 if queue=='download' else 0,active=1,oldestWaitingMs=45000 if queue=='download' else 0)
|
||||
for event,outcome,status,latency in [('request_completed','completed',200,40),('request_slow','completed',503,1200),('request_cancelled','interrupted',503,None)]:
|
||||
emit('api',event,now,requestId=rid,release='fff0001',outcome=outcome,status=status,firstByteMs=latency)
|
||||
emit('api','job_failed',now,jobId=jid,requestId=rid,errorCode='ETIMEDOUT',queue='download')
|
||||
(STATE / 'ids.json').write_text(json.dumps({'request_id':rid,'job_id':jid}))
|
||||
time.sleep(3)
|
||||
print('Fixture Grafana: ' + url)
|
||||
print('Username: local-test; password file: ' + str(password))
|
||||
print('Run GRAFANA_TEST_URL="$(cat ' + str(STATE / 'url') + ')" GRAFANA_TEST_PASSWORD_FILE=' + str(password) + ' python3 scripts/test-grafana-dashboard.py')
|
||||
except Exception:
|
||||
compose('down','-v','--remove-orphans')
|
||||
raise
|
||||
@@ -0,0 +1,109 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check dashboard query frames against an isolated, running pinned logging stack.
|
||||
|
||||
Use only synthetic data. GRAFANA_TEST_URL, GRAFANA_TEST_PASSWORD_FILE and
|
||||
GRAFANA_TEST_USER identify that local stack. Never point this at production.
|
||||
See deploy/logging/dashboard-validation.md for the reproducible setup.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import time
|
||||
import urllib.request
|
||||
from urllib.parse import urlparse
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
base = os.environ.get('GRAFANA_TEST_URL', 'http://localhost:13301')
|
||||
assert urlparse(base).hostname in ('localhost', '127.0.0.1'), 'Local fixture stack only'
|
||||
password = Path(os.environ['GRAFANA_TEST_PASSWORD_FILE']).read_text().strip()
|
||||
auth = base64.b64encode((os.environ.get('GRAFANA_TEST_USER', 'local-test') + ':' + password).encode()).decode()
|
||||
|
||||
def request(path, payload=None):
|
||||
req = urllib.request.Request(base + path, data=json.dumps(payload).encode() if payload else None,
|
||||
headers={'Authorization': 'Basic ' + auth, 'Content-Type': 'application/json'})
|
||||
return json.loads(urllib.request.urlopen(req, timeout=30).read())
|
||||
|
||||
def panels(items):
|
||||
for panel in items:
|
||||
yield panel
|
||||
yield from panels(panel.get('panels', []))
|
||||
|
||||
def query(panel, target, start=None, end=None):
|
||||
target = {**target, 'datasource': panel['datasource'], 'intervalMs': 60000, 'maxDataPoints': 100}
|
||||
for name in ['request_id', 'job_id', 'edge_id']:
|
||||
target['expr'] = target['expr'].replace('${' + name + ':json}', '"fixture-not-present"').replace('$' + name, 'fixture-not-present')
|
||||
for name in ['service', 'release', 'event']:
|
||||
target['expr'] = target['expr'].replace('${' + name + ':json}', '".*"').replace('$' + name, '*')
|
||||
result = request('/api/ds/query', {'from': str(start or int((time.time()-3600)*1000)),
|
||||
'to': str(end or int(time.time()*1000)), 'queries': [target]})['results'][target['refId']]
|
||||
assert not result.get('error'), (panel['title'], result.get('error'))
|
||||
return result.get('frames', [])
|
||||
|
||||
for path in sorted((ROOT / 'deploy/logging/grafana/dashboards').glob('*.json')):
|
||||
dashboard = json.loads(path.read_text())
|
||||
assert request('/api/dashboards/uid/' + dashboard['uid'])['dashboard']['uid'] == dashboard['uid']
|
||||
for panel in panels(dashboard['panels']):
|
||||
for target in panel.get('targets', []):
|
||||
frames = query(panel, target)
|
||||
if panel['type'] in ('stat', 'timeseries') and panel['datasource']['uid'] == 'anonymous-logs':
|
||||
assert target['queryType'] in ('stats', 'statsRange')
|
||||
assert all(any(f['type'] == 'number' for f in frame['schema']['fields']) for frame in frames), panel['title']
|
||||
if panel['type'] == 'table' and panel['datasource']['uid'] == 'anonymous-logs':
|
||||
assert panel['transformations'][0]['id'] == 'extractFields'
|
||||
# Aggregate results must carry parseable fields, not a timestamp to display.
|
||||
for frame in frames:
|
||||
fields = frame['schema']['fields']; values = frame['data']['values']
|
||||
labels_index = next((i for i, f in enumerate(fields) if f['name'] == 'labels'), None)
|
||||
if labels_index is not None:
|
||||
for value in values[labels_index]:
|
||||
row = json.loads(value) if isinstance(value, str) else value
|
||||
assert row, panel['title']
|
||||
print('PASS', dashboard['uid'], panel['title'], target['refId'])
|
||||
print('PASS all provisioned dashboard datasource queries')
|
||||
|
||||
# A reserved synthetic release contains one normal completion, one slow 503,
|
||||
# and one interrupted 503. This proves we count each terminal event once and
|
||||
# never classify the interrupted status as a completed server error.
|
||||
operations = json.loads((ROOT / 'deploy/logging/grafana/dashboards/operations.json').read_text())
|
||||
by_id = {p['id']: p for p in panels(operations['panels'])}
|
||||
def numeric(frames):
|
||||
return [v for frame in frames for field, values in zip(frame['schema']['fields'], frame['data']['values'])
|
||||
if field['type'] == 'number' for v in values]
|
||||
for panel_id, expected in [(101, 2), (102, 1), (104, 1), (103, 1200)]:
|
||||
panel = by_id[panel_id]
|
||||
target = {**panel['targets'][0]}
|
||||
target['expr'] = 'release:=\"fff0001\" ' + target['expr']
|
||||
assert numeric(query(panel, target)) == [expected], (panel['title'], numeric(query(panel, target)))
|
||||
print('PASS exact completed/slow/interrupted counts and measured p95')
|
||||
for panel_id in (10, 11, 12):
|
||||
panel = by_id[panel_id]
|
||||
target = {**panel['targets'][0]}
|
||||
for name in ['request_id', 'job_id', 'edge_id']:
|
||||
target['expr'] = target['expr'].replace('${' + name + ':json}', '\"\"')
|
||||
frames = query(panel, target)
|
||||
assert not any(values for frame in frames for values in frame['data']['values']), panel['title']
|
||||
print('PASS blank correlation IDs never match unrelated logs')
|
||||
# No measured latency must remain absent/NaN, never a fabricated healthy zero.
|
||||
panel = by_id[103]; target = {**panel['targets'][0]}
|
||||
target['expr'] = 'release:=\"0000000\" ' + target['expr']
|
||||
values = numeric(query(panel, target))
|
||||
assert not values or all(v is None or str(v).lower() == 'nan' for v in values), values
|
||||
print('PASS absent latency is not zero')
|
||||
|
||||
fixture_dir = Path(os.environ['GRAFANA_TEST_PASSWORD_FILE']).parent
|
||||
ids = json.loads((fixture_dir / 'ids.json').read_text())
|
||||
fixture_end = int((fixture_dir / 'docker/fixture/fixture-json.log').stat().st_mtime * 1000) + 2000
|
||||
for panel_id in (10, 11):
|
||||
panel = by_id[panel_id]; target = {**panel['targets'][0]}
|
||||
name = 'request_id' if panel_id == 10 else 'job_id'
|
||||
target['expr'] = target['expr'].replace('${' + name + ':json}', json.dumps(ids[name]))
|
||||
frames = query(panel, target, fixture_end-3600000, fixture_end)
|
||||
assert any(values for frame in frames for values in frame['data']['values']), panel['title']
|
||||
for panel_id in (110, 130):
|
||||
panel = by_id[panel_id]
|
||||
fresh = query(panel, panel['targets'][0], fixture_end-3600000, fixture_end)
|
||||
stale = query(panel, panel['targets'][0], fixture_end-3600000, fixture_end+240000)
|
||||
assert any(values for frame in fresh for values in frame['data']['values']), panel['title']
|
||||
assert not any(values for frame in stale for values in frame['data']['values']), panel['title']
|
||||
print('PASS real correlation matches and range-end freshness/staleness')
|
||||
Reference in new issue
Block a user