mirror of
https://github.com/tdurieux/anonymous_github.git
synced 2026-11-18 01:54:23 +01:00
110 lines
6.4 KiB
Python
110 lines
6.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Check dashboard query frames against an isolated, running pinned logging stack.
|
|
|
|
Use only synthetic data. GRAFANA_TEST_URL, GRAFANA_TEST_PASSWORD_FILE and
|
|
GRAFANA_TEST_USER identify that local stack. Never point this at production.
|
|
See deploy/logging/dashboard-validation.md for the reproducible setup.
|
|
"""
|
|
import base64
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import time
|
|
import urllib.request
|
|
from urllib.parse import urlparse
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
base = os.environ.get('GRAFANA_TEST_URL', 'http://localhost:13301')
|
|
assert urlparse(base).hostname in ('localhost', '127.0.0.1'), 'Local fixture stack only'
|
|
password = Path(os.environ['GRAFANA_TEST_PASSWORD_FILE']).read_text().strip()
|
|
auth = base64.b64encode((os.environ.get('GRAFANA_TEST_USER', 'local-test') + ':' + password).encode()).decode()
|
|
|
|
def request(path, payload=None):
|
|
req = urllib.request.Request(base + path, data=json.dumps(payload).encode() if payload else None,
|
|
headers={'Authorization': 'Basic ' + auth, 'Content-Type': 'application/json'})
|
|
return json.loads(urllib.request.urlopen(req, timeout=30).read())
|
|
|
|
def panels(items):
|
|
for panel in items:
|
|
yield panel
|
|
yield from panels(panel.get('panels', []))
|
|
|
|
def query(panel, target, start=None, end=None):
|
|
target = {**target, 'datasource': panel['datasource'], 'intervalMs': 60000, 'maxDataPoints': 100}
|
|
for name in ['request_id', 'job_id', 'edge_id']:
|
|
target['expr'] = target['expr'].replace('${' + name + ':json}', '"fixture-not-present"').replace('$' + name, 'fixture-not-present')
|
|
for name in ['service', 'release', 'event']:
|
|
target['expr'] = target['expr'].replace('${' + name + ':json}', '".*"').replace('$' + name, '.*')
|
|
result = request('/api/ds/query', {'from': str(start or int((time.time()-3600)*1000)),
|
|
'to': str(end or int(time.time()*1000)), 'queries': [target]})['results'][target['refId']]
|
|
assert not result.get('error'), (panel['title'], result.get('error'))
|
|
return result.get('frames', [])
|
|
|
|
for path in sorted((ROOT / 'deploy/logging/grafana/dashboards').glob('*.json')):
|
|
dashboard = json.loads(path.read_text())
|
|
assert request('/api/dashboards/uid/' + dashboard['uid'])['dashboard']['uid'] == dashboard['uid']
|
|
for panel in panels(dashboard['panels']):
|
|
for target in panel.get('targets', []):
|
|
frames = query(panel, target)
|
|
if panel['type'] in ('stat', 'timeseries') and panel['datasource']['uid'] == 'anonymous-logs':
|
|
assert target['queryType'] in ('stats', 'statsRange')
|
|
assert all(any(f['type'] == 'number' for f in frame['schema']['fields']) for frame in frames), panel['title']
|
|
if panel['type'] == 'table' and panel['datasource']['uid'] == 'anonymous-logs':
|
|
assert panel['transformations'][0]['id'] == 'extractFields'
|
|
# Aggregate results must carry parseable fields, not a timestamp to display.
|
|
for frame in frames:
|
|
fields = frame['schema']['fields']; values = frame['data']['values']
|
|
labels_index = next((i for i, f in enumerate(fields) if f['name'] == 'labels'), None)
|
|
if labels_index is not None:
|
|
for value in values[labels_index]:
|
|
row = json.loads(value) if isinstance(value, str) else value
|
|
assert row, panel['title']
|
|
print('PASS', dashboard['uid'], panel['title'], target['refId'])
|
|
print('PASS all provisioned dashboard datasource queries')
|
|
|
|
# A reserved synthetic release contains one normal completion, one slow 503,
|
|
# and one interrupted 503. This proves we count each terminal event once and
|
|
# never classify the interrupted status as a completed server error.
|
|
operations = json.loads((ROOT / 'deploy/logging/grafana/dashboards/operations.json').read_text())
|
|
by_id = {p['id']: p for p in panels(operations['panels'])}
|
|
def numeric(frames):
|
|
return [v for frame in frames for field, values in zip(frame['schema']['fields'], frame['data']['values'])
|
|
if field['type'] == 'number' for v in values]
|
|
for panel_id, expected in [(101, 2), (102, 1), (104, 1), (103, 1200)]:
|
|
panel = by_id[panel_id]
|
|
target = {**panel['targets'][0]}
|
|
target['expr'] = 'release:=\"fff0001\" ' + target['expr']
|
|
assert numeric(query(panel, target)) == [expected], (panel['title'], numeric(query(panel, target)))
|
|
print('PASS exact completed/slow/interrupted counts and measured p95')
|
|
for panel_id in (10, 11, 12):
|
|
panel = by_id[panel_id]
|
|
target = {**panel['targets'][0]}
|
|
for name in ['request_id', 'job_id', 'edge_id']:
|
|
target['expr'] = target['expr'].replace('${' + name + ':json}', '\"\"')
|
|
frames = query(panel, target)
|
|
assert not any(values for frame in frames for values in frame['data']['values']), panel['title']
|
|
print('PASS blank correlation IDs never match unrelated logs')
|
|
# No measured latency must remain absent/NaN, never a fabricated healthy zero.
|
|
panel = by_id[103]; target = {**panel['targets'][0]}
|
|
target['expr'] = 'release:=\"0000000\" ' + target['expr']
|
|
values = numeric(query(panel, target))
|
|
assert not values or all(v is None or str(v).lower() == 'nan' for v in values), values
|
|
print('PASS absent latency is not zero')
|
|
|
|
fixture_dir = Path(os.environ['GRAFANA_TEST_PASSWORD_FILE']).parent
|
|
ids = json.loads((fixture_dir / 'ids.json').read_text())
|
|
fixture_end = int((fixture_dir / 'docker/fixture/fixture-json.log').stat().st_mtime * 1000) + 2000
|
|
for panel_id in (10, 11):
|
|
panel = by_id[panel_id]; target = {**panel['targets'][0]}
|
|
name = 'request_id' if panel_id == 10 else 'job_id'
|
|
target['expr'] = target['expr'].replace('${' + name + ':json}', json.dumps(ids[name]))
|
|
frames = query(panel, target, fixture_end-3600000, fixture_end)
|
|
assert any(values for frame in frames for values in frame['data']['values']), panel['title']
|
|
for panel_id in (110, 130):
|
|
panel = by_id[panel_id]
|
|
fresh = query(panel, panel['targets'][0], fixture_end-3600000, fixture_end)
|
|
stale = query(panel, panel['targets'][0], fixture_end-3600000, fixture_end+240000)
|
|
assert any(values for frame in fresh for values in frame['data']['values']), panel['title']
|
|
assert not any(values for frame in stale for values in frame['data']['values']), panel['title']
|
|
print('PASS real correlation matches and range-end freshness/staleness')
|