Files
anonymous_github/scripts/test-grafana-dashboard.py
T

110 lines
6.4 KiB
Python

#!/usr/bin/env python3
"""Check dashboard query frames against an isolated, running pinned logging stack.
Use only synthetic data. GRAFANA_TEST_URL, GRAFANA_TEST_PASSWORD_FILE and
GRAFANA_TEST_USER identify that local stack. Never point this at production.
See deploy/logging/dashboard-validation.md for the reproducible setup.
"""
import base64
import json
import os
from pathlib import Path
import time
import urllib.request
from urllib.parse import urlparse
ROOT = Path(__file__).resolve().parents[1]
base = os.environ.get('GRAFANA_TEST_URL', 'http://localhost:13301')
assert urlparse(base).hostname in ('localhost', '127.0.0.1'), 'Local fixture stack only'
password = Path(os.environ['GRAFANA_TEST_PASSWORD_FILE']).read_text().strip()
auth = base64.b64encode((os.environ.get('GRAFANA_TEST_USER', 'local-test') + ':' + password).encode()).decode()
def request(path, payload=None):
req = urllib.request.Request(base + path, data=json.dumps(payload).encode() if payload else None,
headers={'Authorization': 'Basic ' + auth, 'Content-Type': 'application/json'})
return json.loads(urllib.request.urlopen(req, timeout=30).read())
def panels(items):
for panel in items:
yield panel
yield from panels(panel.get('panels', []))
def query(panel, target, start=None, end=None):
target = {**target, 'datasource': panel['datasource'], 'intervalMs': 60000, 'maxDataPoints': 100}
for name in ['request_id', 'job_id', 'edge_id']:
target['expr'] = target['expr'].replace('${' + name + ':json}', '"fixture-not-present"').replace('$' + name, 'fixture-not-present')
for name in ['service', 'release', 'event']:
target['expr'] = target['expr'].replace('${' + name + ':json}', '".*"').replace('$' + name, '.*')
result = request('/api/ds/query', {'from': str(start or int((time.time()-3600)*1000)),
'to': str(end or int(time.time()*1000)), 'queries': [target]})['results'][target['refId']]
assert not result.get('error'), (panel['title'], result.get('error'))
return result.get('frames', [])
for path in sorted((ROOT / 'deploy/logging/grafana/dashboards').glob('*.json')):
dashboard = json.loads(path.read_text())
assert request('/api/dashboards/uid/' + dashboard['uid'])['dashboard']['uid'] == dashboard['uid']
for panel in panels(dashboard['panels']):
for target in panel.get('targets', []):
frames = query(panel, target)
if panel['type'] in ('stat', 'timeseries') and panel['datasource']['uid'] == 'anonymous-logs':
assert target['queryType'] in ('stats', 'statsRange')
assert all(any(f['type'] == 'number' for f in frame['schema']['fields']) for frame in frames), panel['title']
if panel['type'] == 'table' and panel['datasource']['uid'] == 'anonymous-logs':
assert panel['transformations'][0]['id'] == 'extractFields'
# Aggregate results must carry parseable fields, not a timestamp to display.
for frame in frames:
fields = frame['schema']['fields']; values = frame['data']['values']
labels_index = next((i for i, f in enumerate(fields) if f['name'] == 'labels'), None)
if labels_index is not None:
for value in values[labels_index]:
row = json.loads(value) if isinstance(value, str) else value
assert row, panel['title']
print('PASS', dashboard['uid'], panel['title'], target['refId'])
print('PASS all provisioned dashboard datasource queries')
# A reserved synthetic release contains one normal completion, one slow 503,
# and one interrupted 503. This proves we count each terminal event once and
# never classify the interrupted status as a completed server error.
operations = json.loads((ROOT / 'deploy/logging/grafana/dashboards/operations.json').read_text())
by_id = {p['id']: p for p in panels(operations['panels'])}
def numeric(frames):
return [v for frame in frames for field, values in zip(frame['schema']['fields'], frame['data']['values'])
if field['type'] == 'number' for v in values]
for panel_id, expected in [(101, 2), (102, 1), (104, 1), (103, 1200)]:
panel = by_id[panel_id]
target = {**panel['targets'][0]}
target['expr'] = 'release:=\"fff0001\" ' + target['expr']
assert numeric(query(panel, target)) == [expected], (panel['title'], numeric(query(panel, target)))
print('PASS exact completed/slow/interrupted counts and measured p95')
for panel_id in (10, 11, 12):
panel = by_id[panel_id]
target = {**panel['targets'][0]}
for name in ['request_id', 'job_id', 'edge_id']:
target['expr'] = target['expr'].replace('${' + name + ':json}', '\"\"')
frames = query(panel, target)
assert not any(values for frame in frames for values in frame['data']['values']), panel['title']
print('PASS blank correlation IDs never match unrelated logs')
# No measured latency must remain absent/NaN, never a fabricated healthy zero.
panel = by_id[103]; target = {**panel['targets'][0]}
target['expr'] = 'release:=\"0000000\" ' + target['expr']
values = numeric(query(panel, target))
assert not values or all(v is None or str(v).lower() == 'nan' for v in values), values
print('PASS absent latency is not zero')
fixture_dir = Path(os.environ['GRAFANA_TEST_PASSWORD_FILE']).parent
ids = json.loads((fixture_dir / 'ids.json').read_text())
fixture_end = int((fixture_dir / 'docker/fixture/fixture-json.log').stat().st_mtime * 1000) + 2000
for panel_id in (10, 11):
panel = by_id[panel_id]; target = {**panel['targets'][0]}
name = 'request_id' if panel_id == 10 else 'job_id'
target['expr'] = target['expr'].replace('${' + name + ':json}', json.dumps(ids[name]))
frames = query(panel, target, fixture_end-3600000, fixture_end)
assert any(values for frame in frames for values in frame['data']['values']), panel['title']
for panel_id in (110, 130):
panel = by_id[panel_id]
fresh = query(panel, panel['targets'][0], fixture_end-3600000, fixture_end)
stale = query(panel, panel['targets'][0], fixture_end-3600000, fixture_end+240000)
assert any(values for frame in fresh for values in frame['data']['values']), panel['title']
assert not any(values for frame in stale for values in frame['data']['values']), panel['title']
print('PASS real correlation matches and range-end freshness/staleness')