From f084f5cbfc7b328a04b4e7e859ce9fa224c605c4 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 14 Nov 2026 14:01:50 +0000 Subject: [PATCH] fix: make buffer losses observable and preserve admin error codes --- deploy/logging/README.md | 23 ++++++++++- .../grafana/dashboards/operations.json | 2 +- deploy/logging/vector.yaml | 11 ++++-- scripts/test-logging.py | 14 +++++-- src/core/log-record.ts | 39 ++++++++++++------- src/core/logger.ts | 11 +++--- test/repository-reconnect.test.js | 3 +- test/structured-logging.test.js | 18 +++++++-- 8 files changed, 90 insertions(+), 31 deletions(-) diff --git a/deploy/logging/README.md b/deploy/logging/README.md index e2dff54..3d888c0 100644 --- a/deploy/logging/README.md +++ b/deploy/logging/README.md @@ -102,7 +102,10 @@ for this stack and alert at 8 GiB used or under 8 GiB free. Keep application dat on a different filesystem for protection from a logging flood. At the free-space reserve, ingest stops and Vector queues sanitized records. -Vector's persistent disk buffer is 256 MiB, with `drop_newest`. Its files and +Vector's persistent disk buffer is 256 MiB, followed by a 1,000-event memory +overflow buffer with `drop_newest`. The memory stage makes full-buffer losses +observable in Vector 0.61.0; its disk-only drop path omits fullness-loss counters. +The overflow stage is volatile and can deliver records out of order. Disk files and checkpoints survive collector restarts. Allocation and metadata add overhead; budget 512 MiB on disk. Outages cause retries with one in-flight request. Once the buffer fills, new records are lost; applications do not wait. Vector syncs @@ -195,9 +198,23 @@ privacy, correlation, query syntax, rotation, restart, backend outage and recove retention rejection and fixture storage size. It never reads production logs or uses the application `.env`. The optional `--stress` writes 500,000 synthetic records while the backend is stopped, checks the buffer bound and collector liveness, and waits for the real -CollectorDrops alert to fire. Allow 1 GiB temporary disk and several minutes. +CollectorDrops alert to fire. Allow 1 GiB temporary disk and up to 15 minutes with the 0.5 CPU collector limit. These fixtures do not establish sustained production throughput. +Local results on 2026-11-14: application build, TypeScript and lint passed. The full +suite had 1,656 passing, 441 skipped and five failures also reproduced on base +`b456ab8`: one missing `cache_miss` translation, three activity-date mocks missing +`hint()`, and one existing Vue split-preview expectation. A focused logging and +job-reconnection run passed 34 tests. Skipped database tests were not executed. +The synthetic 10,000-record batch contained 2,070,000 raw JSON bytes; the entire +VictoriaLogs fixture store occupied about 357–358 KB of data and indexes, +including earlier records. This is a fixture measurement, not a production ratio. +The backend-outage stress run submitted 500,000 records, observed 136,532,570 +buffered bytes without collector restart/OOM, and verified that `CollectorDrops` +reached firing after its sustained two-minute threshold. The overflow stage +reported its losses. A separate real query verified terminal worker drop counts +in the dashboard. + Official references checked 2026-11-14: - [VictoriaLogs retention and resource flags](https://docs.victoriametrics.com/victorialogs/) @@ -206,3 +223,5 @@ Official references checked 2026-11-14: - [VictoriaLogs metrics](https://docs.victoriametrics.com/victorialogs/metrics/) - [vmalert LogsQL integration](https://docs.victoriametrics.com/victorialogs/vmalert/) - [Grafana plugin provisioning](https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/) + +- [Pinned Vector buffer sender accounting](https://github.com/vectordotdev/vector/blob/v0.61.0/lib/vector-buffers/src/topology/channel/sender.rs) diff --git a/deploy/logging/grafana/dashboards/operations.json b/deploy/logging/grafana/dashboards/operations.json index 19b7057..b4148ba 100644 --- a/deploy/logging/grafana/dashboards/operations.json +++ b/deploy/logging/grafana/dashboards/operations.json @@ -284,7 +284,7 @@ "targets": [ { "refId": "A", - "expr": "event:process_heartbeat | stats by (service, instance) max(rss) as rss, max(memoryRatio) as memory_ratio, max(droppedLogs) as dropped_logs", + "expr": "(event:process_heartbeat OR event:job_completed OR event:job_failed) | stats by (service, instance) max(rss) as rss, max(memoryRatio) as memory_ratio, max(droppedLogs) as dropped_logs", "queryType": "instant", "editorMode": "code", "maxLines": 200 diff --git a/deploy/logging/vector.yaml b/deploy/logging/vector.yaml index 89f20b8..7598205 100644 --- a/deploy/logging/vector.yaml +++ b/deploy/logging/vector.yaml @@ -39,10 +39,15 @@ sinks: _stream_fields: service,severity batch: {max_bytes: 1048576, timeout_secs: 1} request: {timeout_secs: 5, concurrency: 1} + # Disk fullness drops lack accounting in Vector 0.61.0. Overflow into a + # bounded memory stage whose drop_newest losses have observable counters. buffer: - type: disk - max_size: 268435488 - when_full: drop_newest + - type: disk + max_size: 268435488 + when_full: overflow + - type: memory + max_events: 1000 + when_full: drop_newest metrics: type: prometheus_exporter inputs: [internal] diff --git a/scripts/test-logging.py b/scripts/test-logging.py index ae64af6..09355de 100755 --- a/scripts/test-logging.py +++ b/scripts/test-logging.py @@ -216,10 +216,15 @@ with tempfile.TemporaryDirectory(prefix=PROJECT) as temp: output.write(line * 1000) def metric_value(name): lines = get(vector, '/metrics').splitlines() - return sum(float(line.split()[1]) for line in lines if line.startswith(name + '{') and ('buffer_id="logs"' in line if name.startswith('vector_buffer_') else 'component_id="logs"' in line)) - wait_for(lambda: metric_value('vector_buffer_discarded_events_total') > 0 or metric_value('vector_component_discarded_events_total') > 0, 240) + return sum(float(line.split()[1]) for line in lines if line.startswith(name + '{') and (name.startswith('vector_buffer_') or 'component_id="logs"' in line)) + wait_for(lambda: metric_value('vector_buffer_discarded_events_total') > 0 or metric_value('vector_component_discarded_events_total') > 0, 600) buffered = metric_value('vector_buffer_size_bytes') - assert buffered <= 268435488 + 1048576, buffered + # Memory-stage metrics inherit buffer_type=disk from the pipeline; + # select its stage explicitly instead of that misleading label. + memory_events = sum(float(line.split()[1]) for line in get(vector, '/metrics').splitlines() + if line.startswith('vector_buffer_size_events{') and 'buffer_id="logs"' in line and 'stage="1"' in line) + assert memory_events <= 1000, memory_events + assert buffered <= 268435488 + 8 * 1048576, buffered inspection = json.loads(run(['docker', 'inspect', compose('ps', '-q', 'vector')]))[0] assert inspection['RestartCount'] == 0 and not inspection['State']['OOMKilled'] print(f'PASS full-spool drop-newest stress: 500000 events; buffered={int(buffered)} bytes; collector stayed alive') @@ -229,6 +234,9 @@ with tempfile.TemporaryDirectory(prefix=PROJECT) as temp: 'ALERTS{alertname="CollectorDrops",alertstate="firing"}'}))['data']['result'], 210) print('PASS sustained CollectorDrops alert fires on full-spool loss') except Exception: + if '--stress' in sys.argv: + print('\n'.join(line for line in get(url('vector', 9598), '/metrics').splitlines() + if line.startswith(('vector_buffer_', 'vector_component_discarded')) and not '_bucket' in line)) # Only synthetic fixture logs exist in this project. No production reads. print(compose('logs', '--tail', '30', 'vector', 'victorialogs', 'vmalert', 'grafana')) raise diff --git a/src/core/log-record.ts b/src/core/log-record.ts index 6f22efe..b0590b3 100644 --- a/src/core/log-record.ts +++ b/src/core/log-record.ts @@ -5,6 +5,7 @@ import { logContext, validTraceId } from "./log-context"; // No arbitrary strings: upstream errors, paths and even innocuous-looking messages // can contain a private repository name. Add diagnostics here deliberately. const instance = randomUUID(); +const allowed = (values: readonly string[], value: unknown): value is string => typeof value === "string" && values.includes(value); const events: Record = { ...LOG_EVENTS, request: "request_completed", "slow request": "request_slow", "request interrupted": "request_cancelled", @@ -25,11 +26,22 @@ const numbers = ["status", "httpStatus", "ms", "firstByteMs", "responseMs", "byt const stages = ["repository", "credential_resolution", "upstream", "cache_lookup", "cache_fill", "worker_wait", "anonymize", "streamer_headers"]; export function logRecord(level: string, module: string, args: unknown[]) { const context = logContext.getStore() || {}; - const details = Object.assign({}, ...args.filter(a => a && typeof a === "object" && !Array.isArray(a))); - const first = typeof args[0] === "string" ? args[0] : ""; - const event = events[first] || (module.includes("webhook") ? "webhook_log" : "application_log"); + const details: Record = {}; + const keys = [...numbers, "requestId", "jobId", "code", "message", "outcome", "method", "route", "queue", + "stages", "reason", "phase", "event", "deliveryId", "hostname", "service"]; + // Do not traverse arbitrary objects or invoke error getters while logging. + for (const arg of args.slice(0, 8)) if (arg && typeof arg === "object") { + for (const key of keys) { + try { + const property = Object.getOwnPropertyDescriptor(arg, key); + if (property && "value" in property) details[key] = property.value; + } catch { /* A hostile Proxy must not make logging throw. */ } + } + } + const first = typeof args[0] === "string" && args[0].length < 256 ? args[0] : ""; + const event = (Object.hasOwn(events, first) ? events[first] : undefined) || (module.includes("webhook") ? "webhook_log" : "application_log"); const record: Record = { timestamp: new Date().toISOString(), severity: level, - service: context.service || (["api", "streamer", "worker"].includes(details.service) ? details.service : undefined) || (process.env.SERVICE_NAME?.toLowerCase() === "streamer" ? "streamer" : "api"), + service: context.service || (allowed(["api", "streamer", "worker"], details.service) ? details.service : undefined) || (process.env.SERVICE_NAME?.toLowerCase() === "streamer" ? "streamer" : "api"), instance, release: /^[a-f0-9]{7,40}$/.test(process.env.APP_RELEASE || "") ? process.env.APP_RELEASE : "unknown", event, message: event, module: LOG_MODULES.includes(module) ? module : "application" }; for (const key of ["requestId", "jobId"] as const) { @@ -38,21 +50,22 @@ export function logRecord(level: string, module: string, args: unknown[]) { } if (Number.isSafeInteger(context.attempt)) record.attempt = context.attempt; for (const key of numbers) if (typeof details[key] === "number" && Number.isFinite(details[key])) record[key] = details[key]; - const code = [details.code, details.message].find(value => errorCodes.has(value)); + const code = [details.code, details.message].find(value => typeof value === "string" && errorCodes.has(value)); if (code) record.errorCode = code; - if (["protocol", "credentials", "port", "host"].includes(details.reason)) record.reason = details.reason; - if (["parse", "find_grant_owners", "reconcile_grant", "persist_installation_block", "reconcile_installation"].includes(details.phase)) record.phase = details.phase; - if (["github_app_authorization", "installation", "installation_repositories"].includes(details.event)) record.webhookEvent = details.event; + if (allowed(["protocol", "credentials", "port", "host"], details.reason)) record.reason = details.reason; + if (allowed(["parse", "find_grant_owners", "reconcile_grant", "persist_installation_block", "reconcile_installation"], details.phase)) record.phase = details.phase; + if (allowed(["github_app_authorization", "installation", "installation_repositories"], details.event)) record.webhookEvent = details.event; if (validTraceId(details.deliveryId)) record.deliveryId = details.deliveryId; - if (["github.com", "api.github.com", "codeload.github.com", "raw.githubusercontent.com", "media.githubusercontent.com"].includes(details.hostname)) record.upstreamHost = details.hostname; + if (allowed(["github.com", "api.github.com", "codeload.github.com", "raw.githubusercontent.com", "media.githubusercontent.com"], details.hostname)) record.upstreamHost = details.hostname; else if (details.hostname) record.upstreamHost = "other"; - if (["completed", "interrupted"].includes(details.outcome)) record.outcome = details.outcome; - if (["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "OTHER"].includes(details.method)) record.method = details.method; + if (allowed(["completed", "interrupted"], details.outcome)) record.outcome = details.outcome; + if (allowed(["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "OTHER"], details.method)) record.method = details.method; // Route labels are generated by request-monitoring, never from URLs in arbitrary errors. if (module === "requests" && typeof details.route === "string") record.route = details.route; - if (["download", "remove", "cache"].includes(details.queue)) record.queue = details.queue; + if (allowed(["download", "remove", "cache"], details.queue)) record.queue = details.queue; if (details.stages && typeof details.stages === "object") for (const key of stages) { - const value = details.stages[key]; + let value: unknown; + try { value = Object.getOwnPropertyDescriptor(details.stages, key)?.value; } catch { continue; } if (typeof value === "number" && Number.isFinite(value)) record[`stage_${key}_ms`] = value; } return record; diff --git a/src/core/logger.ts b/src/core/logger.ts index 32bd902..3da76aa 100644 --- a/src/core/logger.ts +++ b/src/core/logger.ts @@ -253,7 +253,6 @@ function persistError(entry: { function emit(level: Level, module: string, args: unknown[]) { if (LEVEL_ORDER[level] < threshold) return; - args = args.map(arg => redactSecrets(arg instanceof Error ? serializeError(arg) : arg)); const ts = new Date().toISOString(); const record = logRecord(level, module, args); // stdout is a pipe in Docker. Respect backpressure without awaiting it or @@ -272,8 +271,8 @@ function emit(level: Level, module: string, args: unknown[]) { ts, level, module, - message: String(record.event), - raw: [{ ...record, code: record.errorCode || record.event }], + message: String(record.errorCode || record.event), + raw: [{ ...record, message: record.errorCode || record.event, code: record.errorCode || record.event }], }); } } @@ -301,10 +300,12 @@ type ErrorLike = { url?: string | (() => string | undefined); }; -export function serializeError(err: unknown): Record { +export function serializeError(err: unknown, seen = new WeakSet(), depth = 0): Record { if (err == null) return { value: err }; if (typeof err !== "object") return { value: String(err) }; + if (seen.has(err) || depth >= 4) return { message: "error_details_truncated" }; + seen.add(err); const e = err as ErrorLike; const out: Record = {}; if (e.name) out.name = e.name; @@ -339,7 +340,7 @@ export function serializeError(err: unknown): Record { /* ignore */ } } - if (e.cause) out.cause = serializeError(e.cause); + if (e.cause) out.cause = serializeError(e.cause, seen, depth + 1); // Only include the stack when there's nothing else useful — avoids dumping // a stack for handled HTTP errors but keeps debuggability for plain Errors. diff --git a/test/repository-reconnect.test.js b/test/repository-reconnect.test.js index 794306a..231ab51 100644 --- a/test/repository-reconnect.test.js +++ b/test/repository-reconnect.test.js @@ -107,7 +107,8 @@ describe("reconnecting a recreated repository", () => { expect(saved.update.$set.statusDate).to.be.instanceOf(Date); expect(saved.update.$set.source.commitDate).to.deep.equal(new Date("2026-09-29T00:00:00Z")); expect(removals).to.deep.equal([{ accessRevision: "old", replacement: true }]); - expect(jobs[0][1]).to.deep.equal({ repoId: "recreated" }); + expect(jobs[0][1]).to.include({ repoId: "recreated" }); + expect(jobs[0][1].logJobId).to.match(/^[a-f0-9-]{36}$/); }); it("rebuilds when only the repository identity changed", async () => { await handler("/:repoId/", "post")(form({ source: { branch: "main", commit: "deadbeef" } }), { json: () => {} }); diff --git a/test/structured-logging.test.js b/test/structured-logging.test.js index ae457e7..7d59cc9 100644 --- a/test/structured-logging.test.js +++ b/test/structured-logging.test.js @@ -7,7 +7,8 @@ const { logRecord } = require('../src/core/log-record'); const { logContext, jobTrace } = require('../src/core/log-context'); const { loggedJob } = require('../src/queue/log-job'); -describe('structured logging privacy and context', () => { +describe('structured logging privacy and context', function () { + this.timeout(15000); // Child processes compile TypeScript before exercising pipe failures. it('keeps the production summary compatible with JSON and legacy records', () => { const { logSummary, observeLine } = require('../scripts/check-production-logs'); const summary = logSummary(); @@ -48,6 +49,15 @@ describe('structured logging privacy and context', () => { catch (error) { expect(error.message).to.equal('fixture failure'); return; } throw new Error('job failure was swallowed'); }); + it('ignores cyclic objects and hostile getters without losing safe fields', () => { + const value = {status: 502}; value.self = value; + Object.defineProperty(value, 'message', {get() { throw new Error('getter invoked'); }}); + value.method = {toString() { throw new Error('toString invoked'); }}; + const record = logRecord('error', 'route', ['__proto__', value]); + expect(record).to.include({event: 'application_log', status: 502}); + const failure = new Error('fixture'); failure.cause = failure; + expect(require('../src/core/logger').serializeError(failure).cause.message).to.equal('error_details_truncated'); + }); it('rejects injected correlation IDs and non-finite measurements', () => { const row = logRecord('warn', 'fixture', ['request', { requestId: 'bad\nsecret', jobId: 'private', ms: Infinity, status: NaN }]); expect(row).not.to.have.any.keys('requestId', 'jobId', 'ms', 'status'); @@ -77,12 +87,14 @@ describe('structured logging privacy and context', () => { it('preserves admin error codes and expected-error ring suppression', () => { const child = spawnSync(process.execPath, ['-r', 'ts-node/register/transpile-only', '-e', ` const redis = require('redis'); - let pushes = 0; const fields = []; - const tx = {lPush() {pushes++; return tx;}, lTrim() {return tx;}, + let pushes = 0, last; const fields = []; + const tx = {lPush(key, value) {pushes++; last = JSON.parse(value); return tx;}, lTrim() {return tx;}, hIncrBy(key, field) { fields.push(field); return tx; }, expire() {return tx;}, exec() {return Promise.resolve([]);} }; redis.createClient = () => ({isReady:true, isOpen:true, on() {}, connect() {return Promise.resolve();}, multi() {return tx;}}); require('./src/core/logger').createLogger('route').warn('repository_not_ready', {message:'repository_not_ready', httpStatus:409}); if (pushes !== 0 || !fields.includes('cb:warn:repository_not_ready')) process.exitCode = 1; + require('./src/core/logger').createLogger('route').warn('invalid_request', {message:'invalid_request', httpStatus:400}); + if (pushes !== 1 || last.raw[0].message !== 'invalid_request' || last.message !== 'invalid_request') process.exitCode = 1; `], {env: {...process.env, LOG_REDIS_MIRROR: 'on'}, encoding: 'utf8', timeout: 10000}); expect(child.status, child.stderr).to.equal(0); });