From 8ec205612c7f31ba4ac6af0289ce07c35879323f Mon Sep 17 00:00:00 2001 From: Thomas Durieux <5577568+tdurieux@users.noreply.github.com> Date: Sat, 14 Nov 2026 17:37:34 +0000 Subject: [PATCH] fix: preserve log event fields during VictoriaLogs ingestion --- deploy/logging/sanitize.vrl | 2 ++ deploy/logging/vector.yaml | 2 +- scripts/test-logging.py | 5 +++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/deploy/logging/sanitize.vrl b/deploy/logging/sanitize.vrl index ecaf197..172b62f 100644 --- a/deploy/logging/sanitize.vrl +++ b/deploy/logging/sanitize.vrl @@ -58,4 +58,6 @@ for_each(["status", "httpStatus", "ms", "firstByteMs", "responseMs", "bytes", "b } if includes(["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "OTHER"], record.method) { out.method = record.method } if includes(["/", "other", "/dashboard", "/anonymize", "/healthcheck", "/api/healthcheck", "/api", "/api/download", "/api/stat", "/api/stat/history", "/api/options", "/api/message", "/api/user", "/api/user/quota", "/api/user/dashboard", "/api/repo", "/api/anonymize-preview", "/api/admin/overview", "/api/admin/performance", "/api/admin/errors", "/api/admin/errors/stats", "/api/admin/queues", "/api/admin/queues/metrics", "/github/app/webhook", "/api/repo/:repoId", "/api/repo/:repoId/file/:path", "/api/repo/:repoId/files", "/api/repo/:repoId/files/counts", "/api/repo/:repoId/files/search", "/api/repo/:repoId/zip", "/api/repo/:repoId/options", "/api/repo/:repoId/refresh", "/api/repo/:repoId/coauthors", "/api/repo/:repoId/extend", "/api/repo/:owner/:repo/readme", "/api/repo/:owner/:repo/branches", "/w/:repoId/:path", "/r/:repoId/:path", "/api/pr/:id/:path", "/api/gist/:id/:path", "/assets/:path", "/github/:action", "/api/admin/:action", "/api/user/:action", "/api/repo/:id/file/:path", "/api/repo/:id/files/counts", "/api/repo/:id/files/search", "/api/repo/:id/files", "/api/repo/:id/zip", "/api/repo/:id", "/viewer/:id/:path"], record.route) { out.route = record.route } +# VictoriaLogs consumes its message field; keep event independently searchable. +out._msg = out.event . = out diff --git a/deploy/logging/vector.yaml b/deploy/logging/vector.yaml index 7598205..fc9b985 100644 --- a/deploy/logging/vector.yaml +++ b/deploy/logging/vector.yaml @@ -34,7 +34,7 @@ sinks: compression: gzip healthcheck: {enabled: false} query: - _msg_field: event + _msg_field: _msg _time_field: timestamp _stream_fields: service,severity batch: {max_bytes: 1048576, timeout_secs: 1} diff --git a/scripts/test-logging.py b/scripts/test-logging.py index 09355de..3cde5a9 100755 --- a/scripts/test-logging.py +++ b/scripts/test-logging.py @@ -115,6 +115,10 @@ with tempfile.TemporaryDirectory(prefix=PROJECT) as temp: correlated = query('requestId:="' + request_id + '"') assert {r['service'] for r in correlated} == {'api', 'streamer', 'worker', 'nginx'} assert len(query('jobId:="' + job_id + '"')) == 1 + assert {r['event'] for r in correlated} == {'request_completed', 'job_failed', 'edge_request'} + assert all(r['_msg'] == r['event'] for r in correlated) + assert len(query('event:request_completed')) == 2 + assert len(query('event:job_failed')) == 1 assert all('requestId' not in r['_stream'] and 'instance' not in r['_stream'] for r in rows) print('PASS ingestion, privacy allowlist, malformed input, API/streamer/job/edge correlation') # Query every shipped dashboard and alert expression against the real server. @@ -155,6 +159,7 @@ with tempfile.TemporaryDirectory(prefix=PROJECT) as temp: wait_for(lambda: get(vector, '/metrics')) before = count() append('api', {'event': 'process_heartbeat', 'severity': 'info'}) + wait_for(lambda: len(query('event:process_heartbeat')) >= 1) wait_for(lambda: count() > before) print('PASS rename rotation and collector checkpoint restart') # Backend failure: sanitized disk spool remains bounded; source writer stays independent.