From 7f3cf71bc3ade25017da0ca69b8d8844fdfb82c2 Mon Sep 17 00:00:00 2001 From: Thomas Durieux <5577568+tdurieux@users.noreply.github.com> Date: Sun, 15 Nov 2026 13:34:03 +0000 Subject: [PATCH 1/3] fix: make Grafana operations understandable and query-correct --- deploy/logging/dashboard-validation.md | 56 + .../grafana/dashboards/investigation.json | 237 +++ .../grafana/dashboards/operations.json | 1595 +++++++++++++---- .../grafana/provisioning/datasources/logs.yml | 7 + scripts/grafana-fixtures.py | 92 + scripts/test-grafana-dashboard.py | 109 ++ 6 files changed, 1771 insertions(+), 325 deletions(-) create mode 100644 deploy/logging/dashboard-validation.md create mode 100644 deploy/logging/grafana/dashboards/investigation.json create mode 100644 scripts/grafana-fixtures.py create mode 100644 scripts/test-grafana-dashboard.py diff --git a/deploy/logging/dashboard-validation.md b/deploy/logging/dashboard-validation.md new file mode 100644 index 0000000..2079740 --- /dev/null +++ b/deploy/logging/dashboard-validation.md @@ -0,0 +1,56 @@ +# Operations dashboard + +The overview uses the compact summary row, paired trends, and troubleshooting separation shown by the [VictoriaLogs single-node dashboard](https://grafana.com/grafana/dashboards/22084-victorialogs-single-node/) and [VictoriaLogs Internal State dashboard](https://grafana.com/grafana/dashboards/24585-victorialogs-internal-state/). Queries are adapted to this application's emitted fields. These are observed log counts, not availability or delivery guarantees. + +The default window is one hour. Heartbeat and queue summaries cover the three minutes before the selected range end. The alert table and scrape status also describe that range end. A missing row is missing telemetry. A zero count says only that no matching events were collected. Expected deployment counts are one API instance, four streamer instances, and download/cache/remove queues; adjust the expectations alongside planned scaling. + +API counts include both normal and slow completed requests. Interrupted requests are separate regardless of their recorded HTTP status. Nginx and API counts are separate hops and must not be added. `firstByteMs` ends at Node's `writeHead`; `responseMs` runs from that boundary to finish/close. Neither measures client receipt. Queue values and container memory ratios are recent maxima, not current snapshots or sums. Release comparison uses only records observed in the selected window. + +Statistics panels use the datasource's `stats` / `statsRange` modes. Aggregate tables use raw aggregate rows, pack their fields into `Line`, extract JSON, keep only named columns, then convert numeric fields. This deliberately removes the meaningless aggregate timestamp and log metadata. Raw logs retain real timestamps and map the sanitized `severity` field through datasource log-level rules. + +## Investigation links + +Use `/d/anonymous-investigation?from=&to=` with URL-encoded variables: + +- `var-request_id`, `var-job_id`, `var-edge_id`: exact IDs. Empty values match nothing in correlation panels. +- `var-service`, `var-release`, `var-event`: regular expressions. Defaults are `.*`, including absent fields. Escape regex metacharacters for exact values, and use `^...$` for exact matching. + +The overview preserves the old variables and correlation panels in a collapsed row, so existing operations-dashboard links still work. Its investigation link carries the selected time range and variables. Filtered events and ID correlation are separate panels; the service/release/event filters apply only to filtered events so an ID lookup can still show every correlated hop. + +## Responding to alerts + +Open the panel's alert-details link for rule state. Empty results are not a health guarantee: first check the telemetry scrape panel and reporting instances. The following names come from the existing alert rules; this PR does not modify those rules or silence anything. + +| Alert | Meaning and next check | +| --- | --- | +| ApplicationMemoryPressure | Container memory usage exceeded 85% of its limit for the rule duration. Compare the reporting table and investigate process heartbeats. Process RSS is a different measurement. | +| ApiTelemetryMissing / StreamerTelemetryMissing / QueueTelemetryMissing | Expected heartbeat coverage is missing. Check collector ingestion and process/queue reporting before assuming the application stopped. | +| SustainedServerErrors / WebhookFailures | Repeated completed HTTP 5xx. Investigate the affected service and request IDs in the selected window. Interrupted events are separate. | +| QueueWaitingTooLong / QueueStalled | Old runnable work or stalled jobs. Inspect the named queue, job failures and job IDs. | +| ApplicationRestartBurst | Repeated process starts. Filter investigation events to process_started or process_fatal_error. | +| ApplicationLogDrops / CollectorDrops / CollectorErrors / LogIngestionRejected | Observability may be incomplete. Inspect producer drop counters, collector buffers/errors and ingestion metrics before trusting low error counts. | +| LoggingTelemetryMissing / LogAlertEvaluationFailed | Monitoring targets or alert evaluation are failing. Restore telemetry before interpreting an empty alert table. | +| LogStoragePressure | Log data approaches the 10 GiB budget or free disk approaches the reserve. Inspect both storage charts. Do not delete application data to resolve this alert. | + +During read-only production inspection, the reported firing rule was `ApplicationMemoryPressure` for `streamer`; one instance's three-minute peak container ratio was about 90.9%. This was an observation at inspection time, not a current-health claim. No production configuration, dashboard, alert, auth, retention, or deployment was changed. + +## Local validation + +Requires Docker Compose, Python 3 and PyYAML. The fixture tool starts only its own project using the pinned images and plugin in `compose.yml`, generated credentials, loopback ports, and synthetic log files collected through the real Vector sanitizer. It overrides the private bridge only in this local test project to make published loopback ports reachable. Production networking is unchanged. + +```sh +python3 scripts/grafana-fixtures.py start +GRAFANA_TEST_URL="$(cat /tmp/grafana-dashboard-fixtures/url)" \ +GRAFANA_TEST_PASSWORD_FILE=/tmp/grafana-dashboard-fixtures/password \ +python3 scripts/test-grafana-dashboard.py +``` + +Open the printed local URL, authenticate using the generated password file without printing or committing it, then open `anonymous-operations`. The fixture contains normal requests, slow 503s, interruptions, two releases, five reporting processes, and three queues. After three minutes without new fixtures, reporting/queue rows disappear. This tests missing telemetry without replacing it with zeros. The reserved `fff0001` release contains exactly two completed requests, one completed 5xx, one interruption, and a measured p95 of 1.2 seconds. + +In the browser, verify readable aggregate columns, header/post-header legend labels, bytes in storage charts, named firing alerts when rules fire, an empty historical window, and correlation links. Backend frame tests cannot substitute for checking Grafana's frontend transformations. Verify both desktop and a narrow viewport. Screenshots must be labeled synthetic, not production health evidence. + +```sh +python3 scripts/grafana-fixtures.py stop +``` + +The password and synthetic source files remain under the private temporary directory for inspection. Remove that directory before starting another fixture run. Never point the fixture or test scripts at a production tunnel. diff --git a/deploy/logging/grafana/dashboards/investigation.json b/deploy/logging/grafana/dashboards/investigation.json new file mode 100644 index 0000000..ebea0f0 --- /dev/null +++ b/deploy/logging/grafana/dashboards/investigation.json @@ -0,0 +1,237 @@ +{ + "uid": "anonymous-investigation", + "title": "Anonymous GitHub investigation", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "time": { + "from": "now-1h", + "to": "now" + }, + "tags": [ + "operations" + ], + "templating": { + "list": [ + { + "name": "request_id", + "label": "Request UUID", + "type": "textbox", + "current": { + "text": "", + "value": "" + } + }, + { + "name": "job_id", + "label": "Job UUID", + "type": "textbox", + "current": { + "text": "", + "value": "" + } + }, + { + "name": "edge_id", + "label": "Edge request ID", + "type": "textbox", + "current": { + "text": "", + "value": "" + } + }, + { + "name": "service", + "label": "Service", + "type": "textbox", + "current": { + "text": ".*", + "value": ".*" + } + }, + { + "name": "release", + "label": "Release", + "type": "textbox", + "current": { + "text": ".*", + "value": ".*" + } + }, + { + "name": "event", + "label": "Event", + "type": "textbox", + "current": { + "text": ".*", + "value": ".*" + } + } + ] + }, + "panels": [ + { + "id": 200, + "title": "Filtered events", + "type": "logs", + "gridPos": { + "x": 0, + "y": 0, + "w": 24, + "h": 10 + }, + "description": "Regular-expression filters. .* includes missing values. Only sanitized fields are available. Expand a line for request/job IDs.", + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "No data", + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "type": "special", + "options": { + "match": "nan", + "result": { + "text": "Not measured", + "color": "gray" + } + } + } + ] + }, + "overrides": [] + }, + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "anonymous-logs" + }, + "targets": [ + { + "refId": "A", + "expr": "service:~${service:json} release:~${release:json} event:~${event:json}", + "queryType": "instant", + "editorMode": "code", + "maxLines": 200, + "packJson": true + } + ], + "options": { + "showTime": true, + "showLogLabels": false, + "showCommonLabels": false, + "sortOrder": "Descending", + "wrapLogMessage": true, + "enableLogDetails": true + } + }, + { + "id": 10, + "title": "Request correlation", + "type": "logs", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "anonymous-logs" + }, + "gridPos": { + "x": 0, + "y": 10, + "w": 24, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "requestId:* requestId:=${request_id:json}", + "queryType": "instant", + "editorMode": "code", + "maxLines": 200 + } + ], + "options": { + "showTime": true, + "showLogLabels": false, + "showCommonLabels": false, + "sortOrder": "Descending", + "wrapLogMessage": true, + "enableLogDetails": true + }, + "description": "Legacy correlation link. Open the investigation dashboard to enter IDs and filter events." + }, + { + "id": 11, + "title": "Job correlation", + "type": "logs", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "anonymous-logs" + }, + "gridPos": { + "x": 0, + "y": 18, + "w": 24, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "jobId:* jobId:=${job_id:json}", + "queryType": "instant", + "editorMode": "code", + "maxLines": 200 + } + ], + "options": { + "showTime": true, + "showLogLabels": false, + "showCommonLabels": false, + "sortOrder": "Descending", + "wrapLogMessage": true, + "enableLogDetails": true + }, + "description": "Legacy correlation link. Open the investigation dashboard to enter IDs and filter events." + }, + { + "id": 12, + "title": "Edge to application correlation", + "type": "logs", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "anonymous-logs" + }, + "gridPos": { + "x": 0, + "y": 26, + "w": 24, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "(edge_request_id:* edge_request_id:=${edge_id:json}) OR (requestId:* requestId:=${request_id:json})", + "queryType": "instant", + "editorMode": "code", + "maxLines": 200 + } + ], + "options": { + "showTime": true, + "showLogLabels": false, + "showCommonLabels": false, + "sortOrder": "Descending", + "wrapLogMessage": true, + "enableLogDetails": true + }, + "description": "Legacy correlation link. Open the investigation dashboard to enter IDs and filter events." + } + ], + "links": [ + { + "type": "link", + "title": "Operations overview", + "url": "/d/anonymous-operations", + "keepTime": true + } + ] +} diff --git a/deploy/logging/grafana/dashboards/operations.json b/deploy/logging/grafana/dashboards/operations.json index b4148ba..fbc8dde 100644 --- a/deploy/logging/grafana/dashboards/operations.json +++ b/deploy/logging/grafana/dashboards/operations.json @@ -2,7 +2,7 @@ "uid": "anonymous-operations", "title": "Anonymous GitHub operations", "schemaVersion": 39, - "version": 1, + "version": 2, "refresh": "30s", "time": { "from": "now-1h", @@ -20,7 +20,8 @@ "current": { "text": "", "value": "" - } + }, + "hide": 2 }, { "name": "job_id", @@ -29,7 +30,8 @@ "current": { "text": "", "value": "" - } + }, + "hide": 2 }, { "name": "edge_id", @@ -38,430 +40,1373 @@ "current": { "text": "", "value": "" - } + }, + "hide": 2 } ] }, "panels": [ { - "id": 1, - "title": "Failures and cancellations", - "type": "logs", - "datasource": { - "type": "victoriametrics-logs-datasource", - "uid": "anonymous-logs" - }, + "id": 100, + "title": "Observed activity, not an uptime probe", + "type": "text", "gridPos": { "x": 0, "y": 0, - "w": 12, - "h": 8 + "w": 24, + "h": 3 + }, + "description": "", + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "No data", + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "type": "special", + "options": { + "match": "nan", + "result": { + "text": "Not measured", + "color": "gray" + } + } + } + ] + }, + "overrides": [] }, - "targets": [ - { - "refId": "A", - "expr": "severity:error OR event:request_cancelled", - "queryType": "instant", - "editorMode": "code", - "maxLines": 200 - } - ], "options": { - "showTime": true, - "sortOrder": "Descending" - } + "mode": "markdown", + "content": "Selected range. Check reporting before reading zeros. Empty alerts do not prove health." + }, + "targets": [] }, { - "id": 2, - "title": "First byte versus transfer time", - "type": "table", - "datasource": { - "type": "victoriametrics-logs-datasource", - "uid": "anonymous-logs" - }, - "gridPos": { - "x": 12, - "y": 0, - "w": 12, - "h": 8 - }, - "targets": [ - { - "refId": "A", - "expr": "event:request* | stats by (service, route) quantile(0.95, firstByteMs) as first_byte_p95_ms, quantile(0.95, responseMs) as transfer_p95_ms", - "queryType": "instant", - "editorMode": "code", - "maxLines": 200 - } - ], - "options": { - "showTime": true, - "sortOrder": "Descending" - } - }, - { - "id": 3, - "title": "Request stages", - "type": "table", - "datasource": { - "type": "victoriametrics-logs-datasource", - "uid": "anonymous-logs" - }, + "id": 101, + "title": "API requests", + "type": "stat", "gridPos": { "x": 0, - "y": 8, - "w": 12, - "h": 8 + "y": 3, + "w": 6, + "h": 4 + }, + "description": "Completed API request events, including slow requests in selected range. Streamer and edge events are excluded.", + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "No data", + "color": { + "mode": "fixed", + "fixedColor": "blue" + }, + "mappings": [ + { + "type": "special", + "options": { + "match": "nan", + "result": { + "text": "Not measured", + "color": "gray" + } + } + } + ] + }, + "overrides": [] }, - "targets": [ - { - "refId": "A", - "expr": "event:request* | stats by (service) avg(stage_upstream_ms) as upstream_ms, avg(stage_worker_wait_ms) as worker_wait_ms, avg(stage_anonymize_ms) as anonymize_ms", - "queryType": "instant", - "editorMode": "code", - "maxLines": 200 - } - ], - "options": { - "showTime": true, - "sortOrder": "Descending" - } - }, - { - "id": 4, - "title": "Traffic and rate limits", - "type": "table", "datasource": { "type": "victoriametrics-logs-datasource", "uid": "anonymous-logs" }, + "targets": [ + { + "refId": "A", + "expr": "service:api event:request* outcome:completed | stats count() as requests", + "queryType": "stats", + "editorMode": "code", + "maxLines": 200, + "packJson": true + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto" + } + }, + { + "id": 102, + "title": "API server errors", + "type": "stat", + "gridPos": { + "x": 6, + "y": 3, + "w": 6, + "h": 4 + }, + "description": "Completed API requests with status 500 or higher. Interrupted requests are separate. Zero means no matching collected events; check freshness.", + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "No data", + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "special", + "options": { + "match": "nan", + "result": { + "text": "Not measured", + "color": "gray" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "anonymous-logs" + }, + "targets": [ + { + "refId": "A", + "expr": "service:api event:request* outcome:completed | stats count() if (status:>=500) as errors", + "queryType": "stats", + "editorMode": "code", + "maxLines": 200, + "packJson": true + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto" + } + }, + { + "id": 103, + "title": "API header time \u00b7 p95", + "type": "stat", "gridPos": { "x": 12, - "y": 8, - "w": 12, - "h": 8 + "y": 3, + "w": 6, + "h": 4 + }, + "description": "95th percentile of firstByteMs: request start to Node writeHead, in the selected range. Not client receipt. Missing samples are not zero latency.", + "fieldConfig": { + "defaults": { + "unit": "ms", + "noValue": "No data", + "color": { + "mode": "fixed", + "fixedColor": "blue" + }, + "mappings": [ + { + "type": "special", + "options": { + "match": "nan", + "result": { + "text": "Not measured", + "color": "gray" + } + } + } + ] + }, + "overrides": [] }, - "targets": [ - { - "refId": "A", - "expr": "service:nginx | stats by (status, limit_req_status) count() as requests", - "queryType": "instant", - "editorMode": "code", - "maxLines": 200 - } - ], - "options": { - "showTime": true, - "sortOrder": "Descending" - } - }, - { - "id": 5, - "title": "Authentication versus upstream failures", - "type": "table", "datasource": { "type": "victoriametrics-logs-datasource", "uid": "anonymous-logs" }, + "targets": [ + { + "refId": "A", + "expr": "service:api event:request* outcome:completed firstByteMs:* | stats quantile(0.95, firstByteMs) as latency", + "queryType": "stats", + "editorMode": "code", + "maxLines": 200, + "packJson": true + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto" + } + }, + { + "id": 104, + "title": "Interrupted API requests", + "type": "stat", + "gridPos": { + "x": 18, + "y": 3, + "w": 6, + "h": 4 + }, + "description": "API request_cancelled events, selected range. A disconnect does not identify the initiator; proxy, upstream and timeout causes may be involved.", + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "No data", + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "special", + "options": { + "match": "nan", + "result": { + "text": "Not measured", + "color": "gray" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + }, + { + "color": "orange", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "anonymous-logs" + }, + "targets": [ + { + "refId": "A", + "expr": "service:api event:request_cancelled | stats count() as interrupted", + "queryType": "stats", + "editorMode": "code", + "maxLines": 200, + "packJson": true + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto" + } + }, + { + "id": 110, + "title": "Reporting \u00b7 3 min before range end", + "type": "table", "gridPos": { "x": 0, - "y": 16, - "w": 12, - "h": 8 + "y": 7, + "w": 10, + "h": 5 + }, + "description": "Heartbeats in the three minutes before the selected range end. Expected API: 1; streamer: 4. Missing rows mean missing telemetry. Counts may include recently replaced instances. Memory is peak container usage / limit, not process RSS.", + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "No data", + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "type": "special", + "options": { + "match": "nan", + "result": { + "text": "Not measured", + "color": "gray" + } + } + } + ] + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Reporting instances" + }, + "properties": [ + { + "id": "unit", + "value": "short" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Peak container memory" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + } + ] + } + ] + }, + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "anonymous-logs" }, "targets": [ { "refId": "A", - "expr": "errorCode:* OR status:401 OR status:403 OR status:>=500 | stats by (service, errorCode, status) count() as events", + "expr": "_time:3m event:process_heartbeat | stats by (service) count_uniq(instance) as instances, max(memoryRatio) as memory_ratio", "queryType": "instant", "editorMode": "code", - "maxLines": 200 + "maxLines": 200, + "packJson": true + } + ], + "transformations": [ + { + "id": "extractFields", + "options": { + "source": "Line", + "format": "json", + "replace": true, + "keepTime": false + } + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "service", + "instances", + "memory_ratio" + ] + } + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "targetField": "instances", + "destinationType": "number" + }, + { + "targetField": "memory_ratio", + "destinationType": "number" + } + ] + } + }, + { + "id": "organize", + "options": { + "indexByName": { + "service": 0, + "instances": 1, + "memory_ratio": 2 + }, + "renameByName": { + "service": "Service", + "instances": "Reporting instances", + "memory_ratio": "Peak container memory" + } + } } ], "options": { - "showTime": true, - "sortOrder": "Descending" + "showHeader": true, + "cellHeight": "sm", + "footer": { + "show": false + } } }, { - "id": 6, - "title": "Release comparison", + "id": 111, + "title": "Firing alerts \u00b7 at range end", "type": "table", - "datasource": { - "type": "victoriametrics-logs-datasource", - "uid": "anonymous-logs" - }, "gridPos": { - "x": 12, - "y": 16, - "w": 12, - "h": 8 + "x": 10, + "y": 7, + "w": 14, + "h": 5 + }, + "description": "Firing alert identities at the selected range end, from Prometheus and remote-written log rules. Empty results mean no reported firing series, not proof of health. Check telemetry below. For memory pressure inspect container usage, not just RSS.", + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "No data", + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "type": "special", + "options": { + "match": "nan", + "result": { + "text": "Not measured", + "color": "gray" + } + } + } + ] + }, + "overrides": [] + }, + "datasource": { + "type": "prometheus", + "uid": "logging-health" }, "targets": [ { "refId": "A", - "expr": "service:api outcome:completed | stats by (release) count() as requests, count() if (status:>=500) as server_errors, quantile(0.95, firstByteMs) as first_byte_p95_ms", - "queryType": "instant", - "editorMode": "code", - "maxLines": 200 + "expr": "ALERTS{alertstate=\"firing\"}", + "instant": true, + "range": false, + "legendFormat": "What needs attention \u00b7 active alerts", + "format": "table" } ], - "options": { - "showTime": true, - "sortOrder": "Descending" - } + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true, + "__name__": true, + "stats_result": true, + "alertgroup": true, + "alertstate": true + }, + "renameByName": { + "alertname": "Alert", + "service": "Service", + "severity": "Severity", + "instance": "Target", + "job": "Component", + "queue": "Queue" + } + } + } + ], + "links": [ + { + "title": "Alert details", + "url": "/alerting/list" + }, + { + "title": "What each alert means and what to check", + "url": "https://github.com/tdurieux/anonymous_github/blob/main/deploy/logging/dashboard-validation.md#responding-to-alerts", + "targetBlank": true + } + ] }, { - "id": 7, - "title": "Crashes and restarts", - "type": "logs", - "datasource": { - "type": "victoriametrics-logs-datasource", - "uid": "anonymous-logs" - }, + "id": 120, + "title": "Traffic \u00b7 edge and API are separate", + "type": "timeseries", "gridPos": { "x": 0, - "y": 24, + "y": 12, "w": 12, - "h": 8 + "h": 7 + }, + "description": "Events per interval. Edge counts include requests rejected before the API. Never add these series together.", + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "No data", + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "type": "special", + "options": { + "match": "nan", + "result": { + "text": "Not measured", + "color": "gray" + } + } + } + ], + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] }, - "targets": [ - { - "refId": "A", - "expr": "event:process_fatal_error OR event:process_started", - "queryType": "instant", - "editorMode": "code", - "maxLines": 200 - } - ], - "options": { - "showTime": true, - "sortOrder": "Descending" - } - }, - { - "id": 8, - "title": "Queue stalls and failures", - "type": "logs", "datasource": { "type": "victoriametrics-logs-datasource", "uid": "anonymous-logs" }, + "targets": [ + { + "refId": "A", + "expr": "service:nginx event:edge_request | stats count() as requests", + "queryType": "statsRange", + "editorMode": "code", + "maxLines": 200, + "packJson": true, + "legendFormat": "Edge requests", + "step": "1m" + }, + { + "refId": "B", + "expr": "service:api event:request* outcome:completed | stats count() as requests", + "queryType": "statsRange", + "editorMode": "code", + "maxLines": 200, + "packJson": true, + "legendFormat": "API completed", + "step": "1m" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 121, + "title": "API latency \u00b7 headers vs post-header duration", + "type": "timeseries", "gridPos": { "x": 12, - "y": 24, + "y": 12, "w": 12, - "h": 8 + "h": 7 + }, + "description": "p95 per interval. firstByteMs measures request start to writeHead; responseMs measures writeHead to finish/close. Neither proves delivery to the client. Only measured fields contribute.", + "fieldConfig": { + "defaults": { + "unit": "ms", + "noValue": "No data", + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "type": "special", + "options": { + "match": "nan", + "result": { + "text": "Not measured", + "color": "gray" + } + } + } + ], + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "first_byte" + }, + "properties": [ + { + "id": "displayName", + "value": "Headers written \u00b7 p95" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "response_duration" + }, + "properties": [ + { + "id": "displayName", + "value": "After headers \u00b7 p95" + } + ] + } + ] }, - "targets": [ - { - "refId": "A", - "expr": "event:job_stalled OR event:job_failed OR event:queue_heartbeat", - "queryType": "instant", - "editorMode": "code", - "maxLines": 200 - } - ], - "options": { - "showTime": true, - "sortOrder": "Descending" - } - }, - { - "id": 9, - "title": "Memory and lost application logs", - "type": "table", "datasource": { "type": "victoriametrics-logs-datasource", "uid": "anonymous-logs" }, + "targets": [ + { + "refId": "A", + "expr": "service:api event:request* outcome:completed | stats quantile(0.95, firstByteMs) as first_byte, quantile(0.95, responseMs) as response_duration", + "queryType": "statsRange", + "editorMode": "code", + "maxLines": 200, + "packJson": true, + "legendFormat": "{{__name__}}", + "step": "1m" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 130, + "title": "Queues \u00b7 3 min before range end", + "type": "table", + "gridPos": { + "x": 0, + "y": 19, + "w": 12, + "h": 6 + }, + "description": "Peak values in the three minutes before the selected range end. Expected queues: download, cache, remove. Missing rows mean missing telemetry. Waiting over 10 minutes meets the age condition of QueueWaitingTooLong.", + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "No data", + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "type": "special", + "options": { + "match": "nan", + "result": { + "text": "Not measured", + "color": "gray" + } + } + } + ] + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Waiting" + }, + "properties": [ + { + "id": "unit", + "value": "short" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "unit", + "value": "short" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Oldest waiting" + }, + "properties": [ + { + "id": "unit", + "value": "ms" + } + ] + } + ] + }, + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "anonymous-logs" + }, + "targets": [ + { + "refId": "A", + "expr": "_time:3m event:queue_heartbeat | stats by (queue) max(waiting) as waiting, max(active) as active, max(oldestWaitingMs) as oldest", + "queryType": "instant", + "editorMode": "code", + "maxLines": 200, + "packJson": true + } + ], + "transformations": [ + { + "id": "extractFields", + "options": { + "source": "Line", + "format": "json", + "replace": true, + "keepTime": false + } + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "queue", + "waiting", + "active", + "oldest" + ] + } + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "targetField": "waiting", + "destinationType": "number" + }, + { + "targetField": "active", + "destinationType": "number" + }, + { + "targetField": "oldest", + "destinationType": "number" + } + ] + } + }, + { + "id": "organize", + "options": { + "indexByName": { + "queue": 0, + "waiting": 1, + "active": 2, + "oldest": 3 + }, + "renameByName": { + "queue": "Queue", + "waiting": "Waiting", + "active": "Active", + "oldest": "Oldest waiting" + } + } + } + ], + "options": { + "showHeader": true, + "cellHeight": "sm", + "footer": { + "show": false + } + } + }, + { + "id": 131, + "title": "What changed \u00b7 API releases", + "type": "table", + "gridPos": { + "x": 12, + "y": 19, + "w": 12, + "h": 6 + }, + "description": "Selected range. Compare request volume before comparing error counts. Missing release or latency is unreported, not a successful measurement.", + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "No data", + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "type": "special", + "options": { + "match": "nan", + "result": { + "text": "Not measured", + "color": "gray" + } + } + } + ] + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Requests" + }, + "properties": [ + { + "id": "unit", + "value": "short" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "unit", + "value": "short" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "First byte p95" + }, + "properties": [ + { + "id": "unit", + "value": "ms" + } + ] + } + ] + }, + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "anonymous-logs" + }, + "targets": [ + { + "refId": "A", + "expr": "service:api event:request* outcome:completed | stats by (release) count() as requests, count() if (status:>=500) as errors, quantile(0.95, firstByteMs) as latency", + "queryType": "instant", + "editorMode": "code", + "maxLines": 200, + "packJson": true + } + ], + "transformations": [ + { + "id": "extractFields", + "options": { + "source": "Line", + "format": "json", + "replace": true, + "keepTime": false + } + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "release", + "requests", + "errors", + "latency" + ] + } + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "targetField": "requests", + "destinationType": "number" + }, + { + "targetField": "errors", + "destinationType": "number" + }, + { + "targetField": "latency", + "destinationType": "number" + } + ] + } + }, + { + "id": "organize", + "options": { + "indexByName": { + "release": 0, + "requests": 1, + "errors": 2, + "latency": 3 + }, + "renameByName": { + "release": "Release", + "requests": "Requests", + "errors": "5xx", + "latency": "First byte p95" + } + } + } + ], + "options": { + "showHeader": true, + "cellHeight": "sm", + "footer": { + "show": false + } + } + }, + { + "id": 140, + "title": "Log storage \u00b7 10 GiB retention budget", + "type": "timeseries", + "gridPos": { + "x": 0, + "y": 25, + "w": 12, + "h": 7 + }, + "description": "VictoriaLogs stored data. Retention budget 10 GiB; warning rule at 8 GiB. This is log storage only.", + "fieldConfig": { + "defaults": { + "unit": "bytes", + "noValue": "No data", + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "type": "special", + "options": { + "match": "nan", + "result": { + "text": "Not measured", + "color": "gray" + } + } + } + ], + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + }, + "datasource": { + "type": "prometheus", + "uid": "logging-health" + }, + "targets": [ + { + "refId": "A", + "expr": "sum(vl_data_size_bytes)", + "instant": false, + "range": true, + "legendFormat": "Log storage \u00b7 10 GiB retention budget" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 141, + "title": "Disk available \u00b7 5 GiB ingestion reserve", + "type": "timeseries", + "gridPos": { + "x": 12, + "y": 25, + "w": 12, + "h": 7 + }, + "description": "Underlying filesystem free space. Warning at 8 GiB; ingestion reserve 5 GiB.", + "fieldConfig": { + "defaults": { + "unit": "bytes", + "noValue": "No data", + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "type": "special", + "options": { + "match": "nan", + "result": { + "text": "Not measured", + "color": "gray" + } + } + } + ], + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + }, + "datasource": { + "type": "prometheus", + "uid": "logging-health" + }, + "targets": [ + { + "refId": "A", + "expr": "min(vl_free_disk_space_bytes)", + "instant": false, + "range": true, + "legendFormat": "Disk available \u00b7 5 GiB ingestion reserve" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 142, + "title": "Telemetry scrape status", + "type": "stat", "gridPos": { "x": 0, "y": 32, "w": 12, - "h": 8 + "h": 4 + }, + "description": "Scrape status at selected range end for each configured monitoring target. 1 = reachable, 0 = scrape failed, no data = unknown. This does not prove application availability.", + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "No data", + "color": { + "mode": "fixed", + "fixedColor": "blue" + }, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "Unavailable", + "color": "red" + }, + "1": { + "text": "Reporting", + "color": "green" + } + } + } + ] + }, + "overrides": [] + }, + "datasource": { + "type": "prometheus", + "uid": "logging-health" }, "targets": [ { "refId": "A", - "expr": "(event:process_heartbeat OR event:job_completed OR event:job_failed) | stats by (service, instance) max(rss) as rss, max(memoryRatio) as memory_ratio, max(droppedLogs) as dropped_logs", - "queryType": "instant", - "editorMode": "code", - "maxLines": 200 + "expr": "up", + "instant": true, + "range": false, + "legendFormat": "{{job}}" } ], "options": { - "showTime": true, - "sortOrder": "Descending" + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto" } }, { - "id": 10, - "title": "Request correlation", - "type": "logs", - "datasource": { - "type": "victoriametrics-logs-datasource", - "uid": "anonymous-logs" - }, + "id": 143, + "title": "Log ingestion \u00b7 records per second", + "type": "timeseries", "gridPos": { "x": 12, "y": 32, "w": 12, - "h": 8 + "h": 6 + }, + "description": "VictoriaLogs ingestion rate, rolling five minutes. No data means metric unavailable. Zero means no observed ingestion, not healthy application traffic.", + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "type": "special", + "options": { + "match": "nan", + "result": { + "text": "Not measured", + "color": "gray" + } + } + } + ], + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + }, + "datasource": { + "type": "prometheus", + "uid": "logging-health" }, "targets": [ { "refId": "A", - "expr": "requestId:=\"$request_id\"", - "queryType": "instant", - "editorMode": "code", - "maxLines": 200 + "expr": "sum(rate(vl_rows_ingested_total[5m]))", + "instant": false, + "range": true, + "legendFormat": "Log ingestion \u00b7 records per second" } ], "options": { - "showTime": true, - "sortOrder": "Descending" + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } } }, { - "id": 11, - "title": "Job correlation", - "type": "logs", - "datasource": { - "type": "victoriametrics-logs-datasource", - "uid": "anonymous-logs" - }, + "id": 190, + "title": "Request and job correlation \u00b7 legacy links", + "type": "row", + "collapsed": true, "gridPos": { "x": 0, "y": 40, - "w": 12, - "h": 8 + "w": 24, + "h": 1 }, - "targets": [ + "panels": [ { - "refId": "A", - "expr": "jobId:=\"$job_id\"", - "queryType": "instant", - "editorMode": "code", - "maxLines": 200 + "id": 10, + "title": "Request correlation", + "type": "logs", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "anonymous-logs" + }, + "gridPos": { + "x": 0, + "y": 41, + "w": 24, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "requestId:* requestId:=${request_id:json}", + "queryType": "instant", + "editorMode": "code", + "maxLines": 200 + } + ], + "options": { + "showTime": true, + "sortOrder": "Descending" + }, + "description": "Legacy correlation link. Open the investigation dashboard to enter IDs and filter events." + }, + { + "id": 11, + "title": "Job correlation", + "type": "logs", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "anonymous-logs" + }, + "gridPos": { + "x": 0, + "y": 49, + "w": 24, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "jobId:* jobId:=${job_id:json}", + "queryType": "instant", + "editorMode": "code", + "maxLines": 200 + } + ], + "options": { + "showTime": true, + "sortOrder": "Descending" + }, + "description": "Legacy correlation link. Open the investigation dashboard to enter IDs and filter events." + }, + { + "id": 12, + "title": "Edge to application correlation", + "type": "logs", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "anonymous-logs" + }, + "gridPos": { + "x": 0, + "y": 57, + "w": 24, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "(edge_request_id:* edge_request_id:=${edge_id:json}) OR (requestId:* requestId:=${request_id:json})", + "queryType": "instant", + "editorMode": "code", + "maxLines": 200 + } + ], + "options": { + "showTime": true, + "sortOrder": "Descending" + }, + "description": "Legacy correlation link. Open the investigation dashboard to enter IDs and filter events." } ], - "options": { - "showTime": true, - "sortOrder": "Descending" - } - }, + "targets": [] + } + ], + "description": "Operator overview. Logs summarize observed events, not uptime. Open investigation for individual requests and jobs.", + "links": [ { - "id": 12, - "title": "Edge to application correlation", - "type": "logs", - "datasource": { - "type": "victoriametrics-logs-datasource", - "uid": "anonymous-logs" - }, - "gridPos": { - "x": 12, - "y": 40, - "w": 12, - "h": 8 - }, - "targets": [ - { - "refId": "A", - "expr": "edge_request_id:=\"$edge_id\" OR requestId:=\"$request_id\"", - "queryType": "instant", - "editorMode": "code", - "maxLines": 200 - } - ], - "options": { - "showTime": true, - "sortOrder": "Descending" - } - }, - { - "id": 20, - "title": "Storage bytes", - "type": "timeseries", - "datasource": { - "type": "prometheus", - "uid": "logging-health" - }, - "gridPos": { - "x": 0, - "y": 48, - "w": 12, - "h": 8 - }, - "targets": [ - { - "refId": "A", - "expr": "sum(vl_data_size_bytes)" - } - ] - }, - { - "id": 21, - "title": "Free disk bytes", - "type": "timeseries", - "datasource": { - "type": "prometheus", - "uid": "logging-health" - }, - "gridPos": { - "x": 12, - "y": 48, - "w": 12, - "h": 8 - }, - "targets": [ - { - "refId": "A", - "expr": "min(vl_free_disk_space_bytes)" - } - ] - }, - { - "id": 22, - "title": "Collector discarded events", - "type": "timeseries", - "datasource": { - "type": "prometheus", - "uid": "logging-health" - }, - "gridPos": { - "x": 0, - "y": 56, - "w": 12, - "h": 8 - }, - "targets": [ - { - "refId": "A", - "expr": "(sum(rate(vector_buffer_discarded_events_total[5m])) or vector(0)) + (sum(rate(vector_component_discarded_events_total{intentional!=\"true\"}[5m])) or vector(0))" - } - ] - }, - { - "id": 23, - "title": "Firing alerts", - "type": "timeseries", - "datasource": { - "type": "prometheus", - "uid": "logging-health" - }, - "gridPos": { - "x": 12, - "y": 56, - "w": 12, - "h": 8 - }, - "targets": [ - { - "refId": "A", - "expr": "ALERTS{alertstate=\"firing\"}" - } - ] + "type": "link", + "title": "Investigate requests & jobs", + "url": "/d/anonymous-investigation", + "includeVars": true, + "keepTime": true } ] } diff --git a/deploy/logging/grafana/provisioning/datasources/logs.yml b/deploy/logging/grafana/provisioning/datasources/logs.yml index ac83f59..b4b3a23 100644 --- a/deploy/logging/grafana/provisioning/datasources/logs.yml +++ b/deploy/logging/grafana/provisioning/datasources/logs.yml @@ -5,6 +5,13 @@ datasources: type: victoriametrics-logs-datasource access: proxy url: http://victorialogs:9428 + jsonData: + logLevelRules: + - {field: severity, operator: equals, value: debug, level: debug, enabled: true} + - {field: severity, operator: equals, value: info, level: info, enabled: true} + - {field: severity, operator: equals, value: warn, level: warn, enabled: true} + - {field: severity, operator: equals, value: error, level: error, enabled: true} + - {field: severity, operator: equals, value: fatal, level: fatal, enabled: true} isDefault: true editable: false - name: Logging health diff --git a/scripts/grafana-fixtures.py b/scripts/grafana-fixtures.py new file mode 100644 index 0000000..95e075d --- /dev/null +++ b/scripts/grafana-fixtures.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""Create/stop an isolated dashboard fixture stack. Never reads production data. + +python3 scripts/grafana-fixtures.py start +python3 scripts/grafana-fixtures.py stop +State and a generated password stay in /tmp/grafana-dashboard-fixtures (0700). +Only the grafana-dashboard-fixtures Compose project is managed by this script. +""" +import datetime +import json +import os +from pathlib import Path +import secrets +import subprocess +import sys +import time +import urllib.request +import uuid +import yaml + +ROOT = Path(__file__).resolve().parents[1] +STATE = Path('/tmp/grafana-dashboard-fixtures') +PROJECT = 'grafana-dashboard-fixtures' +CMD = ['docker', 'compose', '-p', PROJECT, '-f', str(ROOT / 'deploy/logging/compose.yml'), '-f', str(STATE / 'override.yml')] + +def compose(*args): + env = {**os.environ, **json.loads((STATE / 'environment.json').read_text())} + return subprocess.run(CMD + list(args), env=env, check=True, capture_output=True, text=True).stdout.strip() + +if sys.argv[1:] == ['stop']: + compose('down', '-v', '--remove-orphans') + print('Removed fixture containers and volumes') + sys.exit(0) +if sys.argv[1:] != ['start']: + sys.exit('Use start or stop') +if STATE.exists(): + sys.exit('Fixture directory exists. Stop the fixture stack and remove ' + str(STATE) + ' before recreating it.') +STATE.mkdir(mode=0o700) +for directory in ['docker/fixture', 'nginx']: + (STATE / directory).mkdir(parents=True) +log = STATE / 'docker/fixture/fixture-json.log'; log.touch() +edge = STATE / 'nginx/anonymous.access.json'; edge.touch(); edge.chmod(0o640) +password = STATE / 'password'; password.write_text(secrets.token_urlsafe(32)); password.chmod(0o644) +(STATE / 'environment.json').write_text(json.dumps({'GRAFANA_ADMIN_USER':'local-test', 'GRAFANA_PASSWORD_FILE':str(password), + 'NGINX_LOG_GID':str(os.getgid()), 'DOCKER_LOG_DIR':str(STATE / 'docker'), 'NGINX_LOG_DIR':str(STATE / 'nginx'), 'GRAFANA_PORT':'0'})) +# Published ports on an internal Docker bridge are not reachable on all hosts. +# This override is local-test-only; production compose and auth stay unchanged. +(STATE / 'override.yml').write_text(yaml.safe_dump({'networks':{'private':{'internal':False}}})) +try: + compose('up', '-d') + url = 'http://' + compose('port', 'grafana', '3000') + (STATE / 'url').write_text(url) + deadline = time.monotonic() + 180 + while True: + try: + with urllib.request.urlopen(url + '/api/health', timeout=5) as response: + if response.status == 200: break + except Exception: + if time.monotonic() > deadline: raise + time.sleep(2) + now = time.time(); rid = str(uuid.uuid4()); jid = str(uuid.uuid4()) + instances = [str(uuid.uuid4()) for _ in range(5)] + def stamp(t): return datetime.datetime.fromtimestamp(t, datetime.timezone.utc).isoformat() + with log.open('a') as out, edge.open('a') as nginx: + def emit(service, event, t, **fields): + record = dict(event=event, time=stamp(t), severity='info', **fields) + out.write(json.dumps({'log':json.dumps(record)+'\n', 'attrs':{'tag':service}, 'stream':'stdout'})+'\n') + for minute in range(60): + t = now - minute * 60 + for i in range(15 + minute % 8): + status = 503 if minute < 12 and i == 0 else 200 + emit('api', 'request_completed', t, requestId=rid, route='/api/repo/:repoId/file/:path', status=status, + outcome='completed', firstByteMs=40+minute*2+i*3, responseMs=500+minute*5, + release='abcdef1' if minute<30 else 'abcdef0') + nginx.write(json.dumps({'time':stamp(t), 'status':status, 'request_time':.8, + 'upstream_header_time':'0.06', 'app_request_id':rid, 'route':'/api/repo/:id'})+'\n') + if minute < 6: emit('api','request_cancelled',t,requestId=rid,status=200,outcome='interrupted') + for i, instance in enumerate(instances): + emit('api' if i==0 else 'streamer','process_heartbeat',t,instance=instance,memoryRatio=.31+i*.09,rss=100000000,droppedLogs=0) + for queue in ['download','cache','remove']: + emit('api','queue_heartbeat',t,queue=queue,waiting=3 if queue=='download' else 0,active=1,oldestWaitingMs=45000 if queue=='download' else 0) + for event,outcome,status,latency in [('request_completed','completed',200,40),('request_slow','completed',503,1200),('request_cancelled','interrupted',503,None)]: + emit('api',event,now,requestId=rid,release='fff0001',outcome=outcome,status=status,firstByteMs=latency) + emit('api','job_failed',now,jobId=jid,requestId=rid,errorCode='ETIMEDOUT',queue='download') + (STATE / 'ids.json').write_text(json.dumps({'request_id':rid,'job_id':jid})) + time.sleep(3) + print('Fixture Grafana: ' + url) + print('Username: local-test; password file: ' + str(password)) + print('Run GRAFANA_TEST_URL="$(cat ' + str(STATE / 'url') + ')" GRAFANA_TEST_PASSWORD_FILE=' + str(password) + ' python3 scripts/test-grafana-dashboard.py') +except Exception: + compose('down','-v','--remove-orphans') + raise diff --git a/scripts/test-grafana-dashboard.py b/scripts/test-grafana-dashboard.py new file mode 100644 index 0000000..8e1ea3a --- /dev/null +++ b/scripts/test-grafana-dashboard.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +"""Check dashboard query frames against an isolated, running pinned logging stack. + +Use only synthetic data. GRAFANA_TEST_URL, GRAFANA_TEST_PASSWORD_FILE and +GRAFANA_TEST_USER identify that local stack. Never point this at production. +See deploy/logging/dashboard-validation.md for the reproducible setup. +""" +import base64 +import json +import os +from pathlib import Path +import time +import urllib.request +from urllib.parse import urlparse + +ROOT = Path(__file__).resolve().parents[1] +base = os.environ.get('GRAFANA_TEST_URL', 'http://localhost:13301') +assert urlparse(base).hostname in ('localhost', '127.0.0.1'), 'Local fixture stack only' +password = Path(os.environ['GRAFANA_TEST_PASSWORD_FILE']).read_text().strip() +auth = base64.b64encode((os.environ.get('GRAFANA_TEST_USER', 'local-test') + ':' + password).encode()).decode() + +def request(path, payload=None): + req = urllib.request.Request(base + path, data=json.dumps(payload).encode() if payload else None, + headers={'Authorization': 'Basic ' + auth, 'Content-Type': 'application/json'}) + return json.loads(urllib.request.urlopen(req, timeout=30).read()) + +def panels(items): + for panel in items: + yield panel + yield from panels(panel.get('panels', [])) + +def query(panel, target, start=None, end=None): + target = {**target, 'datasource': panel['datasource'], 'intervalMs': 60000, 'maxDataPoints': 100} + for name in ['request_id', 'job_id', 'edge_id']: + target['expr'] = target['expr'].replace('${' + name + ':json}', '"fixture-not-present"').replace('$' + name, 'fixture-not-present') + for name in ['service', 'release', 'event']: + target['expr'] = target['expr'].replace('${' + name + ':json}', '".*"').replace('$' + name, '*') + result = request('/api/ds/query', {'from': str(start or int((time.time()-3600)*1000)), + 'to': str(end or int(time.time()*1000)), 'queries': [target]})['results'][target['refId']] + assert not result.get('error'), (panel['title'], result.get('error')) + return result.get('frames', []) + +for path in sorted((ROOT / 'deploy/logging/grafana/dashboards').glob('*.json')): + dashboard = json.loads(path.read_text()) + assert request('/api/dashboards/uid/' + dashboard['uid'])['dashboard']['uid'] == dashboard['uid'] + for panel in panels(dashboard['panels']): + for target in panel.get('targets', []): + frames = query(panel, target) + if panel['type'] in ('stat', 'timeseries') and panel['datasource']['uid'] == 'anonymous-logs': + assert target['queryType'] in ('stats', 'statsRange') + assert all(any(f['type'] == 'number' for f in frame['schema']['fields']) for frame in frames), panel['title'] + if panel['type'] == 'table' and panel['datasource']['uid'] == 'anonymous-logs': + assert panel['transformations'][0]['id'] == 'extractFields' + # Aggregate results must carry parseable fields, not a timestamp to display. + for frame in frames: + fields = frame['schema']['fields']; values = frame['data']['values'] + labels_index = next((i for i, f in enumerate(fields) if f['name'] == 'labels'), None) + if labels_index is not None: + for value in values[labels_index]: + row = json.loads(value) if isinstance(value, str) else value + assert row, panel['title'] + print('PASS', dashboard['uid'], panel['title'], target['refId']) +print('PASS all provisioned dashboard datasource queries') + +# A reserved synthetic release contains one normal completion, one slow 503, +# and one interrupted 503. This proves we count each terminal event once and +# never classify the interrupted status as a completed server error. +operations = json.loads((ROOT / 'deploy/logging/grafana/dashboards/operations.json').read_text()) +by_id = {p['id']: p for p in panels(operations['panels'])} +def numeric(frames): + return [v for frame in frames for field, values in zip(frame['schema']['fields'], frame['data']['values']) + if field['type'] == 'number' for v in values] +for panel_id, expected in [(101, 2), (102, 1), (104, 1), (103, 1200)]: + panel = by_id[panel_id] + target = {**panel['targets'][0]} + target['expr'] = 'release:=\"fff0001\" ' + target['expr'] + assert numeric(query(panel, target)) == [expected], (panel['title'], numeric(query(panel, target))) +print('PASS exact completed/slow/interrupted counts and measured p95') +for panel_id in (10, 11, 12): + panel = by_id[panel_id] + target = {**panel['targets'][0]} + for name in ['request_id', 'job_id', 'edge_id']: + target['expr'] = target['expr'].replace('${' + name + ':json}', '\"\"') + frames = query(panel, target) + assert not any(values for frame in frames for values in frame['data']['values']), panel['title'] +print('PASS blank correlation IDs never match unrelated logs') +# No measured latency must remain absent/NaN, never a fabricated healthy zero. +panel = by_id[103]; target = {**panel['targets'][0]} +target['expr'] = 'release:=\"0000000\" ' + target['expr'] +values = numeric(query(panel, target)) +assert not values or all(v is None or str(v).lower() == 'nan' for v in values), values +print('PASS absent latency is not zero') + +fixture_dir = Path(os.environ['GRAFANA_TEST_PASSWORD_FILE']).parent +ids = json.loads((fixture_dir / 'ids.json').read_text()) +fixture_end = int((fixture_dir / 'docker/fixture/fixture-json.log').stat().st_mtime * 1000) + 2000 +for panel_id in (10, 11): + panel = by_id[panel_id]; target = {**panel['targets'][0]} + name = 'request_id' if panel_id == 10 else 'job_id' + target['expr'] = target['expr'].replace('${' + name + ':json}', json.dumps(ids[name])) + frames = query(panel, target, fixture_end-3600000, fixture_end) + assert any(values for frame in frames for values in frame['data']['values']), panel['title'] +for panel_id in (110, 130): + panel = by_id[panel_id] + fresh = query(panel, panel['targets'][0], fixture_end-3600000, fixture_end) + stale = query(panel, panel['targets'][0], fixture_end-3600000, fixture_end+240000) + assert any(values for frame in fresh for values in frame['data']['values']), panel['title'] + assert not any(values for frame in stale for values in frame['data']['values']), panel['title'] +print('PASS real correlation matches and range-end freshness/staleness') From 87882c5a4b89f02b00cf4cca168aceae9190af28 Mon Sep 17 00:00:00 2001 From: Thomas Durieux <5577568+tdurieux@users.noreply.github.com> Date: Sun, 15 Nov 2026 13:36:09 +0000 Subject: [PATCH 2/3] fix: use datasource-native regex variable interpolation --- deploy/logging/grafana/dashboards/investigation.json | 4 ++-- scripts/test-grafana-dashboard.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/logging/grafana/dashboards/investigation.json b/deploy/logging/grafana/dashboards/investigation.json index ebea0f0..1ce87b7 100644 --- a/deploy/logging/grafana/dashboards/investigation.json +++ b/deploy/logging/grafana/dashboards/investigation.json @@ -110,11 +110,11 @@ "targets": [ { "refId": "A", - "expr": "service:~${service:json} release:~${release:json} event:~${event:json}", + "expr": "service:~\"$service\" release:~\"$release\" event:~\"$event\"", "queryType": "instant", "editorMode": "code", "maxLines": 200, - "packJson": true + "packJson": false } ], "options": { diff --git a/scripts/test-grafana-dashboard.py b/scripts/test-grafana-dashboard.py index 8e1ea3a..550d6d2 100644 --- a/scripts/test-grafana-dashboard.py +++ b/scripts/test-grafana-dashboard.py @@ -34,7 +34,7 @@ def query(panel, target, start=None, end=None): for name in ['request_id', 'job_id', 'edge_id']: target['expr'] = target['expr'].replace('${' + name + ':json}', '"fixture-not-present"').replace('$' + name, 'fixture-not-present') for name in ['service', 'release', 'event']: - target['expr'] = target['expr'].replace('${' + name + ':json}', '".*"').replace('$' + name, '*') + target['expr'] = target['expr'].replace('${' + name + ':json}', '".*"').replace('$' + name, '.*') result = request('/api/ds/query', {'from': str(start or int((time.time()-3600)*1000)), 'to': str(end or int(time.time()*1000)), 'queries': [target]})['results'][target['refId']] assert not result.get('error'), (panel['title'], result.get('error')) From ba159878b46b15eecccab7f557345d7328fd6dc3 Mon Sep 17 00:00:00 2001 From: Thomas Durieux <5577568+tdurieux@users.noreply.github.com> Date: Sun, 15 Nov 2026 13:50:03 +0000 Subject: [PATCH 3/3] feat: show latency by API endpoint and method --- deploy/logging/dashboard-validation.md | 2 +- .../grafana/dashboards/operations.json | 279 +++++++++++++++++- scripts/grafana-fixtures.py | 32 +- scripts/test-grafana-dashboard.py | 19 ++ 4 files changed, 319 insertions(+), 13 deletions(-) diff --git a/deploy/logging/dashboard-validation.md b/deploy/logging/dashboard-validation.md index 2079740..d43e847 100644 --- a/deploy/logging/dashboard-validation.md +++ b/deploy/logging/dashboard-validation.md @@ -4,7 +4,7 @@ The overview uses the compact summary row, paired trends, and troubleshooting se The default window is one hour. Heartbeat and queue summaries cover the three minutes before the selected range end. The alert table and scrape status also describe that range end. A missing row is missing telemetry. A zero count says only that no matching events were collected. Expected deployment counts are one API instance, four streamer instances, and download/cache/remove queues; adjust the expectations alongside planned scaling. -API counts include both normal and slow completed requests. Interrupted requests are separate regardless of their recorded HTTP status. Nginx and API counts are separate hops and must not be added. `firstByteMs` ends at Node's `writeHead`; `responseMs` runs from that boundary to finish/close. Neither measures client receipt. Queue values and container memory ratios are recent maxima, not current snapshots or sums. Release comparison uses only records observed in the selected window. +API counts include both normal and slow completed requests. Interrupted requests are separate regardless of their recorded HTTP status. Nginx and API counts are separate hops and must not be added. `firstByteMs` ends at Node's `writeHead`; `responseMs` runs from that boundary to finish/close. Neither measures client receipt. Queue values and container memory ratios are recent maxima, not current snapshots or sums. Release comparison uses only records observed in the selected window. The endpoint latency table groups sanitized route templates and HTTP methods, shows both p95 timings and their measured sample counts, and lists up to 30 groups ordered by header p95. Completed slow requests contribute; interrupted requests and other service hops do not. Missing measurements remain unmeasured, and missing route/method labels appear as Unreported. Statistics panels use the datasource's `stats` / `statsRange` modes. Aggregate tables use raw aggregate rows, pack their fields into `Line`, extract JSON, keep only named columns, then convert numeric fields. This deliberately removes the meaningless aggregate timestamp and log metadata. Raw logs retain real timestamps and map the sanitized `severity` field through datasource log-level rules. diff --git a/deploy/logging/grafana/dashboards/operations.json b/deploy/logging/grafana/dashboards/operations.json index fbc8dde..952788a 100644 --- a/deploy/logging/grafana/dashboards/operations.json +++ b/deploy/logging/grafana/dashboards/operations.json @@ -737,13 +737,272 @@ } } }, + { + "id": 122, + "title": "Latency by API endpoint \u00b7 slowest 30 by header p95", + "type": "table", + "gridPos": { + "x": 0, + "y": 19, + "w": 24, + "h": 8 + }, + "description": "Selected range, grouped by sanitized route template and HTTP method. Includes normal and slow completed API requests; excludes interruptions, edge and streamer hops. Header time is request start to writeHead; post-header time ends at finish/close, not client receipt. Sample counts show measured coverage; absent timings remain Not measured. Missing route or method is Unreported. Sorted by header p95, up to 30 groups.", + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "Not measured", + "color": { + "mode": "palette-classic" + }, + "mappings": [ + { + "type": "special", + "options": { + "match": "nan", + "result": { + "text": "Not measured", + "color": "gray" + } + } + } + ] + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Requests" + }, + "properties": [ + { + "id": "unit", + "value": "none" + }, + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Header samples" + }, + "properties": [ + { + "id": "unit", + "value": "none" + }, + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Header p95" + }, + "properties": [ + { + "id": "unit", + "value": "ms" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Post-header samples" + }, + "properties": [ + { + "id": "unit", + "value": "none" + }, + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Post-header p95" + }, + "properties": [ + { + "id": "unit", + "value": "ms" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "API endpoint" + }, + "properties": [ + { + "id": "noValue", + "value": "Unreported" + }, + { + "id": "mappings", + "value": [ + { + "type": "value", + "options": { + "": { + "text": "Unreported" + } + } + } + ] + }, + { + "id": "custom.width", + "value": 320 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Method" + }, + "properties": [ + { + "id": "noValue", + "value": "Unreported" + }, + { + "id": "mappings", + "value": [ + { + "type": "value", + "options": { + "": { + "text": "Unreported" + } + } + } + ] + } + ] + } + ] + }, + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "anonymous-logs" + }, + "targets": [ + { + "refId": "A", + "expr": "service:api event:request* outcome:completed | stats by (route, method) count() as requests, count() if (firstByteMs:*) as header_samples, quantile(0.95, firstByteMs) as header_p95, count() if (responseMs:*) as response_samples, quantile(0.95, responseMs) as response_p95 | sort by (header_p95) desc | limit 30", + "queryType": "instant", + "editorMode": "code", + "maxLines": 30, + "packJson": true + } + ], + "transformations": [ + { + "id": "extractFields", + "options": { + "source": "Line", + "format": "json", + "replace": true, + "keepTime": false + } + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "route", + "method", + "requests", + "header_samples", + "header_p95", + "response_samples", + "response_p95" + ] + } + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "targetField": "requests", + "destinationType": "number" + }, + { + "targetField": "header_samples", + "destinationType": "number" + }, + { + "targetField": "header_p95", + "destinationType": "number" + }, + { + "targetField": "response_samples", + "destinationType": "number" + }, + { + "targetField": "response_p95", + "destinationType": "number" + } + ] + } + }, + { + "id": "organize", + "options": { + "indexByName": { + "route": 0, + "method": 1, + "requests": 2, + "header_samples": 3, + "header_p95": 4, + "response_samples": 5, + "response_p95": 6 + }, + "renameByName": { + "route": "API endpoint", + "method": "Method", + "requests": "Requests", + "header_samples": "Header samples", + "header_p95": "Header p95", + "response_samples": "Post-header samples", + "response_p95": "Post-header p95" + } + } + } + ], + "options": { + "showHeader": true, + "cellHeight": "sm", + "footer": { + "show": false + } + } + }, { "id": 130, "title": "Queues \u00b7 3 min before range end", "type": "table", "gridPos": { "x": 0, - "y": 19, + "y": 27, "w": 12, "h": 6 }, @@ -895,7 +1154,7 @@ "type": "table", "gridPos": { "x": 12, - "y": 19, + "y": 27, "w": 12, "h": 6 }, @@ -1047,7 +1306,7 @@ "type": "timeseries", "gridPos": { "x": 0, - "y": 25, + "y": 33, "w": 12, "h": 7 }, @@ -1111,7 +1370,7 @@ "type": "timeseries", "gridPos": { "x": 12, - "y": 25, + "y": 33, "w": 12, "h": 7 }, @@ -1175,7 +1434,7 @@ "type": "stat", "gridPos": { "x": 0, - "y": 32, + "y": 40, "w": 12, "h": 4 }, @@ -1238,7 +1497,7 @@ "type": "timeseries", "gridPos": { "x": 12, - "y": 32, + "y": 40, "w": 12, "h": 6 }, @@ -1303,7 +1562,7 @@ "collapsed": true, "gridPos": { "x": 0, - "y": 40, + "y": 48, "w": 24, "h": 1 }, @@ -1318,7 +1577,7 @@ }, "gridPos": { "x": 0, - "y": 41, + "y": 49, "w": 24, "h": 8 }, @@ -1347,7 +1606,7 @@ }, "gridPos": { "x": 0, - "y": 49, + "y": 57, "w": 24, "h": 8 }, @@ -1376,7 +1635,7 @@ }, "gridPos": { "x": 0, - "y": 57, + "y": 65, "w": 24, "h": 8 }, diff --git a/scripts/grafana-fixtures.py b/scripts/grafana-fixtures.py index 95e075d..34bc0b6 100644 --- a/scripts/grafana-fixtures.py +++ b/scripts/grafana-fixtures.py @@ -6,6 +6,7 @@ python3 scripts/grafana-fixtures.py stop State and a generated password stay in /tmp/grafana-dashboard-fixtures (0700). Only the grafana-dashboard-fixtures Compose project is managed by this script. """ +import base64 import datetime import json import os @@ -69,7 +70,7 @@ try: t = now - minute * 60 for i in range(15 + minute % 8): status = 503 if minute < 12 and i == 0 else 200 - emit('api', 'request_completed', t, requestId=rid, route='/api/repo/:repoId/file/:path', status=status, + emit('api', 'request_completed', t, requestId=rid, route='/api/repo/:repoId/file/:path', method='GET', status=status, outcome='completed', firstByteMs=40+minute*2+i*3, responseMs=500+minute*5, release='abcdef1' if minute<30 else 'abcdef0') nginx.write(json.dumps({'time':stamp(t), 'status':status, 'request_time':.8, @@ -81,9 +82,36 @@ try: emit('api','queue_heartbeat',t,queue=queue,waiting=3 if queue=='download' else 0,active=1,oldestWaitingMs=45000 if queue=='download' else 0) for event,outcome,status,latency in [('request_completed','completed',200,40),('request_slow','completed',503,1200),('request_cancelled','interrupted',503,None)]: emit('api',event,now,requestId=rid,release='fff0001',outcome=outcome,status=status,firstByteMs=latency) + # Independent endpoint groups, slow completion, missing timings and excluded hops. + for service, event, outcome, route, method, header, response in [ + ('api','request_completed','completed','/api/repo/:repoId','GET',20,80), + ('api','request_slow','completed','/api/repo/:repoId','GET',1200,3000), + ('api','request_completed','completed','/api/repo/:repoId','POST',70,90), + ('api','request_completed','completed','/api/repo/:repoId/file/:path','GET',None,None), + ('api','request_cancelled','interrupted','/api/repo/:repoId','GET',9999,9999), + ('streamer','request_completed','completed','/api/repo/:repoId','GET',8888,8888), + ]: + emit(service,event,now,requestId=rid,release='fff0002',route=route,method=method, + outcome=outcome,status=200,firstByteMs=header,responseMs=response) emit('api','job_failed',now,jobId=jid,requestId=rid,errorCode='ETIMEDOUT',queue='download') (STATE / 'ids.json').write_text(json.dumps({'request_id':rid,'job_id':jid})) - time.sleep(3) + # Vector batching and VictoriaLogs visibility can take longer than startup. + auth = base64.b64encode(('local-test:' + password.read_text()).encode()).decode() + deadline = time.monotonic() + 60 + while True: + req = urllib.request.Request(url + '/api/ds/query', headers={ + 'Authorization':'Basic ' + auth, 'Content-Type':'application/json'}, + data=json.dumps({'from':str(int((now-3600)*1000)), 'to':str(int((now+2)*1000)), + 'queries':[{'refId':'A','datasource':{'type':'victoriametrics-logs-datasource','uid':'anonymous-logs'}, + 'expr':'release:="fff0002" service:api outcome:completed | stats count() as requests', + 'queryType':'stats'}]}).encode()) + result = json.loads(urllib.request.urlopen(req, timeout=10).read()) + frames = result['results']['A'].get('frames', []) + counts = [v for frame in frames for field, values in zip(frame['schema']['fields'], frame['data']['values']) + if field['type']=='number' for v in values] + if counts == [4]: break + if time.monotonic() > deadline: raise RuntimeError('Endpoint fixtures did not become queryable') + time.sleep(1) print('Fixture Grafana: ' + url) print('Username: local-test; password file: ' + str(password)) print('Run GRAFANA_TEST_URL="$(cat ' + str(STATE / 'url') + ')" GRAFANA_TEST_PASSWORD_FILE=' + str(password) + ' python3 scripts/test-grafana-dashboard.py') diff --git a/scripts/test-grafana-dashboard.py b/scripts/test-grafana-dashboard.py index 550d6d2..6470768 100644 --- a/scripts/test-grafana-dashboard.py +++ b/scripts/test-grafana-dashboard.py @@ -107,3 +107,22 @@ for panel_id in (110, 130): assert any(values for frame in fresh for values in frame['data']['values']), panel['title'] assert not any(values for frame in stale for values in frame['data']['values']), panel['title'] print('PASS real correlation matches and range-end freshness/staleness') + +panel = by_id[122] +target = {**panel['targets'][0], 'expr': 'release:="fff0002" ' + panel['targets'][0]['expr']} +frames = query(panel, target, fixture_end-3600000, fixture_end) +rows = [] +for frame in frames: + for field, values in zip(frame['schema']['fields'], frame['data']['values']): + if field['name'] == 'labels': + rows.extend(json.loads(value) if isinstance(value, str) else value for value in values) +groups = {(row['route'], row['method']): row for row in rows} +assert len(groups) == 3, groups +get_repo = groups[('/api/repo/:repoId', 'GET')] +assert [float(get_repo[k]) for k in ['requests','header_samples','header_p95','response_samples','response_p95']] == [2,2,1200,2,3000], get_repo +post_repo = groups[('/api/repo/:repoId', 'POST')] +assert [float(post_repo[k]) for k in ['requests','header_p95','response_p95']] == [1,70,90], post_repo +missing = groups[('/api/repo/:repoId/file/:path', 'GET')] +assert [int(missing[k]) for k in ['requests','header_samples','response_samples']] == [1,0,0], missing +assert all((k not in missing or missing[k] is None or str(missing[k]).lower() == 'nan') for k in ['header_p95','response_p95']), missing +print('PASS endpoint/method separation, slow completion, excluded interruption/streamer, absent timing coverage')