mirror of
https://github.com/tdurieux/anonymous_github.git
synced 2026-11-17 17:44:09 +01:00
feat: show latency by API endpoint and method
This commit is contained in:
1 parent
87882c5a4b
commit
ba159878b4
4 files changed
+319
-13
No files matched your search
@@ -4,7 +4,7 @@ The overview uses the compact summary row, paired trends, and troubleshooting se
|
||||
|
||||
The default window is one hour. Heartbeat and queue summaries cover the three minutes before the selected range end. The alert table and scrape status also describe that range end. A missing row is missing telemetry. A zero count says only that no matching events were collected. Expected deployment counts are one API instance, four streamer instances, and download/cache/remove queues; adjust the expectations alongside planned scaling.
|
||||
|
||||
API counts include both normal and slow completed requests. Interrupted requests are separate regardless of their recorded HTTP status. Nginx and API counts are separate hops and must not be added. `firstByteMs` ends at Node's `writeHead`; `responseMs` runs from that boundary to finish/close. Neither measures client receipt. Queue values and container memory ratios are recent maxima, not current snapshots or sums. Release comparison uses only records observed in the selected window.
|
||||
API counts include both normal and slow completed requests. Interrupted requests are separate regardless of their recorded HTTP status. Nginx and API counts are separate hops and must not be added. `firstByteMs` ends at Node's `writeHead`; `responseMs` runs from that boundary to finish/close. Neither measures client receipt. Queue values and container memory ratios are recent maxima, not current snapshots or sums. Release comparison uses only records observed in the selected window. The endpoint latency table groups sanitized route templates and HTTP methods, shows both p95 timings and their measured sample counts, and lists up to 30 groups ordered by header p95. Completed slow requests contribute; interrupted requests and other service hops do not. Missing measurements remain unmeasured, and missing route/method labels appear as Unreported.
|
||||
|
||||
Statistics panels use the datasource's `stats` / `statsRange` modes. Aggregate tables use raw aggregate rows, pack their fields into `Line`, extract JSON, keep only named columns, then convert numeric fields. This deliberately removes the meaningless aggregate timestamp and log metadata. Raw logs retain real timestamps and map the sanitized `severity` field through datasource log-level rules.
|
||||
|
||||
|
||||
@@ -737,13 +737,272 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 122,
|
||||
"title": "Latency by API endpoint \u00b7 slowest 30 by header p95",
|
||||
"type": "table",
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 19,
|
||||
"w": 24,
|
||||
"h": 8
|
||||
},
|
||||
"description": "Selected range, grouped by sanitized route template and HTTP method. Includes normal and slow completed API requests; excludes interruptions, edge and streamer hops. Header time is request start to writeHead; post-header time ends at finish/close, not client receipt. Sample counts show measured coverage; absent timings remain Not measured. Missing route or method is Unreported. Sorted by header p95, up to 30 groups.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"noValue": "Not measured",
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"mappings": [
|
||||
{
|
||||
"type": "special",
|
||||
"options": {
|
||||
"match": "nan",
|
||||
"result": {
|
||||
"text": "Not measured",
|
||||
"color": "gray"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "Requests"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "unit",
|
||||
"value": "none"
|
||||
},
|
||||
{
|
||||
"id": "decimals",
|
||||
"value": 0
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "Header samples"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "unit",
|
||||
"value": "none"
|
||||
},
|
||||
{
|
||||
"id": "decimals",
|
||||
"value": 0
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "Header p95"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "unit",
|
||||
"value": "ms"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "Post-header samples"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "unit",
|
||||
"value": "none"
|
||||
},
|
||||
{
|
||||
"id": "decimals",
|
||||
"value": 0
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "Post-header p95"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "unit",
|
||||
"value": "ms"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "API endpoint"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "noValue",
|
||||
"value": "Unreported"
|
||||
},
|
||||
{
|
||||
"id": "mappings",
|
||||
"value": [
|
||||
{
|
||||
"type": "value",
|
||||
"options": {
|
||||
"": {
|
||||
"text": "Unreported"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "custom.width",
|
||||
"value": 320
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "Method"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "noValue",
|
||||
"value": "Unreported"
|
||||
},
|
||||
{
|
||||
"id": "mappings",
|
||||
"value": [
|
||||
{
|
||||
"type": "value",
|
||||
"options": {
|
||||
"": {
|
||||
"text": "Unreported"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"datasource": {
|
||||
"type": "victoriametrics-logs-datasource",
|
||||
"uid": "anonymous-logs"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "service:api event:request* outcome:completed | stats by (route, method) count() as requests, count() if (firstByteMs:*) as header_samples, quantile(0.95, firstByteMs) as header_p95, count() if (responseMs:*) as response_samples, quantile(0.95, responseMs) as response_p95 | sort by (header_p95) desc | limit 30",
|
||||
"queryType": "instant",
|
||||
"editorMode": "code",
|
||||
"maxLines": 30,
|
||||
"packJson": true
|
||||
}
|
||||
],
|
||||
"transformations": [
|
||||
{
|
||||
"id": "extractFields",
|
||||
"options": {
|
||||
"source": "Line",
|
||||
"format": "json",
|
||||
"replace": true,
|
||||
"keepTime": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "filterFieldsByName",
|
||||
"options": {
|
||||
"include": {
|
||||
"names": [
|
||||
"route",
|
||||
"method",
|
||||
"requests",
|
||||
"header_samples",
|
||||
"header_p95",
|
||||
"response_samples",
|
||||
"response_p95"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "convertFieldType",
|
||||
"options": {
|
||||
"conversions": [
|
||||
{
|
||||
"targetField": "requests",
|
||||
"destinationType": "number"
|
||||
},
|
||||
{
|
||||
"targetField": "header_samples",
|
||||
"destinationType": "number"
|
||||
},
|
||||
{
|
||||
"targetField": "header_p95",
|
||||
"destinationType": "number"
|
||||
},
|
||||
{
|
||||
"targetField": "response_samples",
|
||||
"destinationType": "number"
|
||||
},
|
||||
{
|
||||
"targetField": "response_p95",
|
||||
"destinationType": "number"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "organize",
|
||||
"options": {
|
||||
"indexByName": {
|
||||
"route": 0,
|
||||
"method": 1,
|
||||
"requests": 2,
|
||||
"header_samples": 3,
|
||||
"header_p95": 4,
|
||||
"response_samples": 5,
|
||||
"response_p95": 6
|
||||
},
|
||||
"renameByName": {
|
||||
"route": "API endpoint",
|
||||
"method": "Method",
|
||||
"requests": "Requests",
|
||||
"header_samples": "Header samples",
|
||||
"header_p95": "Header p95",
|
||||
"response_samples": "Post-header samples",
|
||||
"response_p95": "Post-header p95"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"showHeader": true,
|
||||
"cellHeight": "sm",
|
||||
"footer": {
|
||||
"show": false
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 130,
|
||||
"title": "Queues \u00b7 3 min before range end",
|
||||
"type": "table",
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 19,
|
||||
"y": 27,
|
||||
"w": 12,
|
||||
"h": 6
|
||||
},
|
||||
@@ -895,7 +1154,7 @@
|
||||
"type": "table",
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 19,
|
||||
"y": 27,
|
||||
"w": 12,
|
||||
"h": 6
|
||||
},
|
||||
@@ -1047,7 +1306,7 @@
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 25,
|
||||
"y": 33,
|
||||
"w": 12,
|
||||
"h": 7
|
||||
},
|
||||
@@ -1111,7 +1370,7 @@
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 25,
|
||||
"y": 33,
|
||||
"w": 12,
|
||||
"h": 7
|
||||
},
|
||||
@@ -1175,7 +1434,7 @@
|
||||
"type": "stat",
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 32,
|
||||
"y": 40,
|
||||
"w": 12,
|
||||
"h": 4
|
||||
},
|
||||
@@ -1238,7 +1497,7 @@
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 32,
|
||||
"y": 40,
|
||||
"w": 12,
|
||||
"h": 6
|
||||
},
|
||||
@@ -1303,7 +1562,7 @@
|
||||
"collapsed": true,
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 40,
|
||||
"y": 48,
|
||||
"w": 24,
|
||||
"h": 1
|
||||
},
|
||||
@@ -1318,7 +1577,7 @@
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 41,
|
||||
"y": 49,
|
||||
"w": 24,
|
||||
"h": 8
|
||||
},
|
||||
@@ -1347,7 +1606,7 @@
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 49,
|
||||
"y": 57,
|
||||
"w": 24,
|
||||
"h": 8
|
||||
},
|
||||
@@ -1376,7 +1635,7 @@
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 57,
|
||||
"y": 65,
|
||||
"w": 24,
|
||||
"h": 8
|
||||
},
|
||||
|
||||
@@ -6,6 +6,7 @@ python3 scripts/grafana-fixtures.py stop
|
||||
State and a generated password stay in /tmp/grafana-dashboard-fixtures (0700).
|
||||
Only the grafana-dashboard-fixtures Compose project is managed by this script.
|
||||
"""
|
||||
import base64
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
@@ -69,7 +70,7 @@ try:
|
||||
t = now - minute * 60
|
||||
for i in range(15 + minute % 8):
|
||||
status = 503 if minute < 12 and i == 0 else 200
|
||||
emit('api', 'request_completed', t, requestId=rid, route='/api/repo/:repoId/file/:path', status=status,
|
||||
emit('api', 'request_completed', t, requestId=rid, route='/api/repo/:repoId/file/:path', method='GET', status=status,
|
||||
outcome='completed', firstByteMs=40+minute*2+i*3, responseMs=500+minute*5,
|
||||
release='abcdef1' if minute<30 else 'abcdef0')
|
||||
nginx.write(json.dumps({'time':stamp(t), 'status':status, 'request_time':.8,
|
||||
@@ -81,9 +82,36 @@ try:
|
||||
emit('api','queue_heartbeat',t,queue=queue,waiting=3 if queue=='download' else 0,active=1,oldestWaitingMs=45000 if queue=='download' else 0)
|
||||
for event,outcome,status,latency in [('request_completed','completed',200,40),('request_slow','completed',503,1200),('request_cancelled','interrupted',503,None)]:
|
||||
emit('api',event,now,requestId=rid,release='fff0001',outcome=outcome,status=status,firstByteMs=latency)
|
||||
# Independent endpoint groups, slow completion, missing timings and excluded hops.
|
||||
for service, event, outcome, route, method, header, response in [
|
||||
('api','request_completed','completed','/api/repo/:repoId','GET',20,80),
|
||||
('api','request_slow','completed','/api/repo/:repoId','GET',1200,3000),
|
||||
('api','request_completed','completed','/api/repo/:repoId','POST',70,90),
|
||||
('api','request_completed','completed','/api/repo/:repoId/file/:path','GET',None,None),
|
||||
('api','request_cancelled','interrupted','/api/repo/:repoId','GET',9999,9999),
|
||||
('streamer','request_completed','completed','/api/repo/:repoId','GET',8888,8888),
|
||||
]:
|
||||
emit(service,event,now,requestId=rid,release='fff0002',route=route,method=method,
|
||||
outcome=outcome,status=200,firstByteMs=header,responseMs=response)
|
||||
emit('api','job_failed',now,jobId=jid,requestId=rid,errorCode='ETIMEDOUT',queue='download')
|
||||
(STATE / 'ids.json').write_text(json.dumps({'request_id':rid,'job_id':jid}))
|
||||
time.sleep(3)
|
||||
# Vector batching and VictoriaLogs visibility can take longer than startup.
|
||||
auth = base64.b64encode(('local-test:' + password.read_text()).encode()).decode()
|
||||
deadline = time.monotonic() + 60
|
||||
while True:
|
||||
req = urllib.request.Request(url + '/api/ds/query', headers={
|
||||
'Authorization':'Basic ' + auth, 'Content-Type':'application/json'},
|
||||
data=json.dumps({'from':str(int((now-3600)*1000)), 'to':str(int((now+2)*1000)),
|
||||
'queries':[{'refId':'A','datasource':{'type':'victoriametrics-logs-datasource','uid':'anonymous-logs'},
|
||||
'expr':'release:="fff0002" service:api outcome:completed | stats count() as requests',
|
||||
'queryType':'stats'}]}).encode())
|
||||
result = json.loads(urllib.request.urlopen(req, timeout=10).read())
|
||||
frames = result['results']['A'].get('frames', [])
|
||||
counts = [v for frame in frames for field, values in zip(frame['schema']['fields'], frame['data']['values'])
|
||||
if field['type']=='number' for v in values]
|
||||
if counts == [4]: break
|
||||
if time.monotonic() > deadline: raise RuntimeError('Endpoint fixtures did not become queryable')
|
||||
time.sleep(1)
|
||||
print('Fixture Grafana: ' + url)
|
||||
print('Username: local-test; password file: ' + str(password))
|
||||
print('Run GRAFANA_TEST_URL="$(cat ' + str(STATE / 'url') + ')" GRAFANA_TEST_PASSWORD_FILE=' + str(password) + ' python3 scripts/test-grafana-dashboard.py')
|
||||
|
||||
@@ -107,3 +107,22 @@ for panel_id in (110, 130):
|
||||
assert any(values for frame in fresh for values in frame['data']['values']), panel['title']
|
||||
assert not any(values for frame in stale for values in frame['data']['values']), panel['title']
|
||||
print('PASS real correlation matches and range-end freshness/staleness')
|
||||
|
||||
panel = by_id[122]
|
||||
target = {**panel['targets'][0], 'expr': 'release:="fff0002" ' + panel['targets'][0]['expr']}
|
||||
frames = query(panel, target, fixture_end-3600000, fixture_end)
|
||||
rows = []
|
||||
for frame in frames:
|
||||
for field, values in zip(frame['schema']['fields'], frame['data']['values']):
|
||||
if field['name'] == 'labels':
|
||||
rows.extend(json.loads(value) if isinstance(value, str) else value for value in values)
|
||||
groups = {(row['route'], row['method']): row for row in rows}
|
||||
assert len(groups) == 3, groups
|
||||
get_repo = groups[('/api/repo/:repoId', 'GET')]
|
||||
assert [float(get_repo[k]) for k in ['requests','header_samples','header_p95','response_samples','response_p95']] == [2,2,1200,2,3000], get_repo
|
||||
post_repo = groups[('/api/repo/:repoId', 'POST')]
|
||||
assert [float(post_repo[k]) for k in ['requests','header_p95','response_p95']] == [1,70,90], post_repo
|
||||
missing = groups[('/api/repo/:repoId/file/:path', 'GET')]
|
||||
assert [int(missing[k]) for k in ['requests','header_samples','response_samples']] == [1,0,0], missing
|
||||
assert all((k not in missing or missing[k] is None or str(missing[k]).lower() == 'nan') for k in ['header_p95','response_p95']), missing
|
||||
print('PASS endpoint/method separation, slow completion, excluded interruption/streamer, absent timing coverage')
|
||||
Reference in new issue
Block a user