feat: show latency by API endpoint and method

This commit is contained in:
Thomas Durieux committed 2026-11-15 13:50:03 +00:00
1 parent 87882c5a4b
commit ba159878b4
4 files changed
+319 -13

No files matched your search

+1 -1
View File
@@ -4,7 +4,7 @@ The overview uses the compact summary row, paired trends, and troubleshooting se
The default window is one hour. Heartbeat and queue summaries cover the three minutes before the selected range end. The alert table and scrape status also describe that range end. A missing row is missing telemetry. A zero count says only that no matching events were collected. Expected deployment counts are one API instance, four streamer instances, and download/cache/remove queues; adjust the expectations alongside planned scaling.
API counts include both normal and slow completed requests. Interrupted requests are separate regardless of their recorded HTTP status. Nginx and API counts are separate hops and must not be added. `firstByteMs` ends at Node's `writeHead`; `responseMs` runs from that boundary to finish/close. Neither measures client receipt. Queue values and container memory ratios are recent maxima, not current snapshots or sums. Release comparison uses only records observed in the selected window.
API counts include both normal and slow completed requests. Interrupted requests are separate regardless of their recorded HTTP status. Nginx and API counts are separate hops and must not be added. `firstByteMs` ends at Node's `writeHead`; `responseMs` runs from that boundary to finish/close. Neither measures client receipt. Queue values and container memory ratios are recent maxima, not current snapshots or sums. Release comparison uses only records observed in the selected window. The endpoint latency table groups sanitized route templates and HTTP methods, shows both p95 timings and their measured sample counts, and lists up to 30 groups ordered by header p95. Completed slow requests contribute; interrupted requests and other service hops do not. Missing measurements remain unmeasured, and missing route/method labels appear as Unreported.
Statistics panels use the datasource's `stats` / `statsRange` modes. Aggregate tables use raw aggregate rows, pack their fields into `Line`, extract JSON, keep only named columns, then convert numeric fields. This deliberately removes the meaningless aggregate timestamp and log metadata. Raw logs retain real timestamps and map the sanitized `severity` field through datasource log-level rules.
+269 -10
View File
@@ -737,13 +737,272 @@
}
}
},
{
"id": 122,
"title": "Latency by API endpoint \u00b7 slowest 30 by header p95",
"type": "table",
"gridPos": {
"x": 0,
"y": 19,
"w": 24,
"h": 8
},
"description": "Selected range, grouped by sanitized route template and HTTP method. Includes normal and slow completed API requests; excludes interruptions, edge and streamer hops. Header time is request start to writeHead; post-header time ends at finish/close, not client receipt. Sample counts show measured coverage; absent timings remain Not measured. Missing route or method is Unreported. Sorted by header p95, up to 30 groups.",
"fieldConfig": {
"defaults": {
"unit": "short",
"noValue": "Not measured",
"color": {
"mode": "palette-classic"
},
"mappings": [
{
"type": "special",
"options": {
"match": "nan",
"result": {
"text": "Not measured",
"color": "gray"
}
}
}
]
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "Requests"
},
"properties": [
{
"id": "unit",
"value": "none"
},
{
"id": "decimals",
"value": 0
}
]
},
{
"matcher": {
"id": "byName",
"options": "Header samples"
},
"properties": [
{
"id": "unit",
"value": "none"
},
{
"id": "decimals",
"value": 0
}
]
},
{
"matcher": {
"id": "byName",
"options": "Header p95"
},
"properties": [
{
"id": "unit",
"value": "ms"
}
]
},
{
"matcher": {
"id": "byName",
"options": "Post-header samples"
},
"properties": [
{
"id": "unit",
"value": "none"
},
{
"id": "decimals",
"value": 0
}
]
},
{
"matcher": {
"id": "byName",
"options": "Post-header p95"
},
"properties": [
{
"id": "unit",
"value": "ms"
}
]
},
{
"matcher": {
"id": "byName",
"options": "API endpoint"
},
"properties": [
{
"id": "noValue",
"value": "Unreported"
},
{
"id": "mappings",
"value": [
{
"type": "value",
"options": {
"": {
"text": "Unreported"
}
}
}
]
},
{
"id": "custom.width",
"value": 320
}
]
},
{
"matcher": {
"id": "byName",
"options": "Method"
},
"properties": [
{
"id": "noValue",
"value": "Unreported"
},
{
"id": "mappings",
"value": [
{
"type": "value",
"options": {
"": {
"text": "Unreported"
}
}
}
]
}
]
}
]
},
"datasource": {
"type": "victoriametrics-logs-datasource",
"uid": "anonymous-logs"
},
"targets": [
{
"refId": "A",
"expr": "service:api event:request* outcome:completed | stats by (route, method) count() as requests, count() if (firstByteMs:*) as header_samples, quantile(0.95, firstByteMs) as header_p95, count() if (responseMs:*) as response_samples, quantile(0.95, responseMs) as response_p95 | sort by (header_p95) desc | limit 30",
"queryType": "instant",
"editorMode": "code",
"maxLines": 30,
"packJson": true
}
],
"transformations": [
{
"id": "extractFields",
"options": {
"source": "Line",
"format": "json",
"replace": true,
"keepTime": false
}
},
{
"id": "filterFieldsByName",
"options": {
"include": {
"names": [
"route",
"method",
"requests",
"header_samples",
"header_p95",
"response_samples",
"response_p95"
]
}
}
},
{
"id": "convertFieldType",
"options": {
"conversions": [
{
"targetField": "requests",
"destinationType": "number"
},
{
"targetField": "header_samples",
"destinationType": "number"
},
{
"targetField": "header_p95",
"destinationType": "number"
},
{
"targetField": "response_samples",
"destinationType": "number"
},
{
"targetField": "response_p95",
"destinationType": "number"
}
]
}
},
{
"id": "organize",
"options": {
"indexByName": {
"route": 0,
"method": 1,
"requests": 2,
"header_samples": 3,
"header_p95": 4,
"response_samples": 5,
"response_p95": 6
},
"renameByName": {
"route": "API endpoint",
"method": "Method",
"requests": "Requests",
"header_samples": "Header samples",
"header_p95": "Header p95",
"response_samples": "Post-header samples",
"response_p95": "Post-header p95"
}
}
}
],
"options": {
"showHeader": true,
"cellHeight": "sm",
"footer": {
"show": false
}
}
},
{
"id": 130,
"title": "Queues \u00b7 3 min before range end",
"type": "table",
"gridPos": {
"x": 0,
"y": 19,
"y": 27,
"w": 12,
"h": 6
},
@@ -895,7 +1154,7 @@
"type": "table",
"gridPos": {
"x": 12,
"y": 19,
"y": 27,
"w": 12,
"h": 6
},
@@ -1047,7 +1306,7 @@
"type": "timeseries",
"gridPos": {
"x": 0,
"y": 25,
"y": 33,
"w": 12,
"h": 7
},
@@ -1111,7 +1370,7 @@
"type": "timeseries",
"gridPos": {
"x": 12,
"y": 25,
"y": 33,
"w": 12,
"h": 7
},
@@ -1175,7 +1434,7 @@
"type": "stat",
"gridPos": {
"x": 0,
"y": 32,
"y": 40,
"w": 12,
"h": 4
},
@@ -1238,7 +1497,7 @@
"type": "timeseries",
"gridPos": {
"x": 12,
"y": 32,
"y": 40,
"w": 12,
"h": 6
},
@@ -1303,7 +1562,7 @@
"collapsed": true,
"gridPos": {
"x": 0,
"y": 40,
"y": 48,
"w": 24,
"h": 1
},
@@ -1318,7 +1577,7 @@
},
"gridPos": {
"x": 0,
"y": 41,
"y": 49,
"w": 24,
"h": 8
},
@@ -1347,7 +1606,7 @@
},
"gridPos": {
"x": 0,
"y": 49,
"y": 57,
"w": 24,
"h": 8
},
@@ -1376,7 +1635,7 @@
},
"gridPos": {
"x": 0,
"y": 57,
"y": 65,
"w": 24,
"h": 8
},
+30 -2
View File
@@ -6,6 +6,7 @@ python3 scripts/grafana-fixtures.py stop
State and a generated password stay in /tmp/grafana-dashboard-fixtures (0700).
Only the grafana-dashboard-fixtures Compose project is managed by this script.
"""
import base64
import datetime
import json
import os
@@ -69,7 +70,7 @@ try:
t = now - minute * 60
for i in range(15 + minute % 8):
status = 503 if minute < 12 and i == 0 else 200
emit('api', 'request_completed', t, requestId=rid, route='/api/repo/:repoId/file/:path', status=status,
emit('api', 'request_completed', t, requestId=rid, route='/api/repo/:repoId/file/:path', method='GET', status=status,
outcome='completed', firstByteMs=40+minute*2+i*3, responseMs=500+minute*5,
release='abcdef1' if minute<30 else 'abcdef0')
nginx.write(json.dumps({'time':stamp(t), 'status':status, 'request_time':.8,
@@ -81,9 +82,36 @@ try:
emit('api','queue_heartbeat',t,queue=queue,waiting=3 if queue=='download' else 0,active=1,oldestWaitingMs=45000 if queue=='download' else 0)
for event,outcome,status,latency in [('request_completed','completed',200,40),('request_slow','completed',503,1200),('request_cancelled','interrupted',503,None)]:
emit('api',event,now,requestId=rid,release='fff0001',outcome=outcome,status=status,firstByteMs=latency)
# Independent endpoint groups, slow completion, missing timings and excluded hops.
for service, event, outcome, route, method, header, response in [
('api','request_completed','completed','/api/repo/:repoId','GET',20,80),
('api','request_slow','completed','/api/repo/:repoId','GET',1200,3000),
('api','request_completed','completed','/api/repo/:repoId','POST',70,90),
('api','request_completed','completed','/api/repo/:repoId/file/:path','GET',None,None),
('api','request_cancelled','interrupted','/api/repo/:repoId','GET',9999,9999),
('streamer','request_completed','completed','/api/repo/:repoId','GET',8888,8888),
]:
emit(service,event,now,requestId=rid,release='fff0002',route=route,method=method,
outcome=outcome,status=200,firstByteMs=header,responseMs=response)
emit('api','job_failed',now,jobId=jid,requestId=rid,errorCode='ETIMEDOUT',queue='download')
(STATE / 'ids.json').write_text(json.dumps({'request_id':rid,'job_id':jid}))
time.sleep(3)
# Vector batching and VictoriaLogs visibility can take longer than startup.
auth = base64.b64encode(('local-test:' + password.read_text()).encode()).decode()
deadline = time.monotonic() + 60
while True:
req = urllib.request.Request(url + '/api/ds/query', headers={
'Authorization':'Basic ' + auth, 'Content-Type':'application/json'},
data=json.dumps({'from':str(int((now-3600)*1000)), 'to':str(int((now+2)*1000)),
'queries':[{'refId':'A','datasource':{'type':'victoriametrics-logs-datasource','uid':'anonymous-logs'},
'expr':'release:="fff0002" service:api outcome:completed | stats count() as requests',
'queryType':'stats'}]}).encode())
result = json.loads(urllib.request.urlopen(req, timeout=10).read())
frames = result['results']['A'].get('frames', [])
counts = [v for frame in frames for field, values in zip(frame['schema']['fields'], frame['data']['values'])
if field['type']=='number' for v in values]
if counts == [4]: break
if time.monotonic() > deadline: raise RuntimeError('Endpoint fixtures did not become queryable')
time.sleep(1)
print('Fixture Grafana: ' + url)
print('Username: local-test; password file: ' + str(password))
print('Run GRAFANA_TEST_URL="$(cat ' + str(STATE / 'url') + ')" GRAFANA_TEST_PASSWORD_FILE=' + str(password) + ' python3 scripts/test-grafana-dashboard.py')
+19
View File
@@ -107,3 +107,22 @@ for panel_id in (110, 130):
assert any(values for frame in fresh for values in frame['data']['values']), panel['title']
assert not any(values for frame in stale for values in frame['data']['values']), panel['title']
print('PASS real correlation matches and range-end freshness/staleness')
panel = by_id[122]
target = {**panel['targets'][0], 'expr': 'release:="fff0002" ' + panel['targets'][0]['expr']}
frames = query(panel, target, fixture_end-3600000, fixture_end)
rows = []
for frame in frames:
for field, values in zip(frame['schema']['fields'], frame['data']['values']):
if field['name'] == 'labels':
rows.extend(json.loads(value) if isinstance(value, str) else value for value in values)
groups = {(row['route'], row['method']): row for row in rows}
assert len(groups) == 3, groups
get_repo = groups[('/api/repo/:repoId', 'GET')]
assert [float(get_repo[k]) for k in ['requests','header_samples','header_p95','response_samples','response_p95']] == [2,2,1200,2,3000], get_repo
post_repo = groups[('/api/repo/:repoId', 'POST')]
assert [float(post_repo[k]) for k in ['requests','header_p95','response_p95']] == [1,70,90], post_repo
missing = groups[('/api/repo/:repoId/file/:path', 'GET')]
assert [int(missing[k]) for k in ['requests','header_samples','response_samples']] == [1,0,0], missing
assert all((k not in missing or missing[k] is None or str(missing[k]).lower() == 'nan') for k in ['header_p95','response_p95']), missing
print('PASS endpoint/method separation, slow completion, excluded interruption/streamer, absent timing coverage')