mirror of
https://github.com/tdurieux/anonymous_github.git
synced 2026-11-17 17:44:09 +01:00
fix: make buffer losses observable and preserve admin error codes
This commit is contained in:
1 parent
7e2a6e9d75
commit
f084f5cbfc
8 files changed
+90
-31
No files matched your search
@@ -102,7 +102,10 @@ for this stack and alert at 8 GiB used or under 8 GiB free. Keep application dat
|
||||
on a different filesystem for protection from a logging flood. At the free-space
|
||||
reserve, ingest stops and Vector queues sanitized records.
|
||||
|
||||
Vector's persistent disk buffer is 256 MiB, with `drop_newest`. Its files and
|
||||
Vector's persistent disk buffer is 256 MiB, followed by a 1,000-event memory
|
||||
overflow buffer with `drop_newest`. The memory stage makes full-buffer losses
|
||||
observable in Vector 0.61.0; its disk-only drop path omits fullness-loss counters.
|
||||
The overflow stage is volatile and can deliver records out of order. Disk files and
|
||||
checkpoints survive collector restarts. Allocation and metadata add overhead;
|
||||
budget 512 MiB on disk. Outages cause retries with one in-flight request. Once
|
||||
the buffer fills, new records are lost; applications do not wait. Vector syncs
|
||||
@@ -195,9 +198,23 @@ privacy, correlation, query syntax, rotation, restart, backend outage and recove
|
||||
retention rejection and fixture storage size. It never reads production logs or
|
||||
uses the application `.env`. The optional `--stress` writes 500,000 synthetic records while the backend is
|
||||
stopped, checks the buffer bound and collector liveness, and waits for the real
|
||||
CollectorDrops alert to fire. Allow 1 GiB temporary disk and several minutes.
|
||||
CollectorDrops alert to fire. Allow 1 GiB temporary disk and up to 15 minutes with the 0.5 CPU collector limit.
|
||||
These fixtures do not establish sustained production throughput.
|
||||
|
||||
Local results on 2026-11-14: application build, TypeScript and lint passed. The full
|
||||
suite had 1,656 passing, 441 skipped and five failures also reproduced on base
|
||||
`b456ab8`: one missing `cache_miss` translation, three activity-date mocks missing
|
||||
`hint()`, and one existing Vue split-preview expectation. A focused logging and
|
||||
job-reconnection run passed 34 tests. Skipped database tests were not executed.
|
||||
The synthetic 10,000-record batch contained 2,070,000 raw JSON bytes; the entire
|
||||
VictoriaLogs fixture store occupied about 357–358 KB of data and indexes,
|
||||
including earlier records. This is a fixture measurement, not a production ratio.
|
||||
The backend-outage stress run submitted 500,000 records, observed 136,532,570
|
||||
buffered bytes without collector restart/OOM, and verified that `CollectorDrops`
|
||||
reached firing after its sustained two-minute threshold. The overflow stage
|
||||
reported its losses. A separate real query verified terminal worker drop counts
|
||||
in the dashboard.
|
||||
|
||||
Official references checked 2026-11-14:
|
||||
|
||||
- [VictoriaLogs retention and resource flags](https://docs.victoriametrics.com/victorialogs/)
|
||||
@@ -206,3 +223,5 @@ Official references checked 2026-11-14:
|
||||
- [VictoriaLogs metrics](https://docs.victoriametrics.com/victorialogs/metrics/)
|
||||
- [vmalert LogsQL integration](https://docs.victoriametrics.com/victorialogs/vmalert/)
|
||||
- [Grafana plugin provisioning](https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/)
|
||||
|
||||
- [Pinned Vector buffer sender accounting](https://github.com/vectordotdev/vector/blob/v0.61.0/lib/vector-buffers/src/topology/channel/sender.rs)
|
||||
@@ -284,7 +284,7 @@
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "event:process_heartbeat | stats by (service, instance) max(rss) as rss, max(memoryRatio) as memory_ratio, max(droppedLogs) as dropped_logs",
|
||||
"expr": "(event:process_heartbeat OR event:job_completed OR event:job_failed) | stats by (service, instance) max(rss) as rss, max(memoryRatio) as memory_ratio, max(droppedLogs) as dropped_logs",
|
||||
"queryType": "instant",
|
||||
"editorMode": "code",
|
||||
"maxLines": 200
|
||||
|
||||
@@ -39,10 +39,15 @@ sinks:
|
||||
_stream_fields: service,severity
|
||||
batch: {max_bytes: 1048576, timeout_secs: 1}
|
||||
request: {timeout_secs: 5, concurrency: 1}
|
||||
# Disk fullness drops lack accounting in Vector 0.61.0. Overflow into a
|
||||
# bounded memory stage whose drop_newest losses have observable counters.
|
||||
buffer:
|
||||
type: disk
|
||||
max_size: 268435488
|
||||
when_full: drop_newest
|
||||
- type: disk
|
||||
max_size: 268435488
|
||||
when_full: overflow
|
||||
- type: memory
|
||||
max_events: 1000
|
||||
when_full: drop_newest
|
||||
metrics:
|
||||
type: prometheus_exporter
|
||||
inputs: [internal]
|
||||
|
||||
+11
-3
@@ -216,10 +216,15 @@ with tempfile.TemporaryDirectory(prefix=PROJECT) as temp:
|
||||
output.write(line * 1000)
|
||||
def metric_value(name):
|
||||
lines = get(vector, '/metrics').splitlines()
|
||||
return sum(float(line.split()[1]) for line in lines if line.startswith(name + '{') and ('buffer_id="logs"' in line if name.startswith('vector_buffer_') else 'component_id="logs"' in line))
|
||||
wait_for(lambda: metric_value('vector_buffer_discarded_events_total') > 0 or metric_value('vector_component_discarded_events_total') > 0, 240)
|
||||
return sum(float(line.split()[1]) for line in lines if line.startswith(name + '{') and (name.startswith('vector_buffer_') or 'component_id="logs"' in line))
|
||||
wait_for(lambda: metric_value('vector_buffer_discarded_events_total') > 0 or metric_value('vector_component_discarded_events_total') > 0, 600)
|
||||
buffered = metric_value('vector_buffer_size_bytes')
|
||||
assert buffered <= 268435488 + 1048576, buffered
|
||||
# Memory-stage metrics inherit buffer_type=disk from the pipeline;
|
||||
# select its stage explicitly instead of that misleading label.
|
||||
memory_events = sum(float(line.split()[1]) for line in get(vector, '/metrics').splitlines()
|
||||
if line.startswith('vector_buffer_size_events{') and 'buffer_id="logs"' in line and 'stage="1"' in line)
|
||||
assert memory_events <= 1000, memory_events
|
||||
assert buffered <= 268435488 + 8 * 1048576, buffered
|
||||
inspection = json.loads(run(['docker', 'inspect', compose('ps', '-q', 'vector')]))[0]
|
||||
assert inspection['RestartCount'] == 0 and not inspection['State']['OOMKilled']
|
||||
print(f'PASS full-spool drop-newest stress: 500000 events; buffered={int(buffered)} bytes; collector stayed alive')
|
||||
@@ -229,6 +234,9 @@ with tempfile.TemporaryDirectory(prefix=PROJECT) as temp:
|
||||
'ALERTS{alertname="CollectorDrops",alertstate="firing"}'}))['data']['result'], 210)
|
||||
print('PASS sustained CollectorDrops alert fires on full-spool loss')
|
||||
except Exception:
|
||||
if '--stress' in sys.argv:
|
||||
print('\n'.join(line for line in get(url('vector', 9598), '/metrics').splitlines()
|
||||
if line.startswith(('vector_buffer_', 'vector_component_discarded')) and not '_bucket' in line))
|
||||
# Only synthetic fixture logs exist in this project. No production reads.
|
||||
print(compose('logs', '--tail', '30', 'vector', 'victorialogs', 'vmalert', 'grafana'))
|
||||
raise
|
||||
|
||||
+26
-13
@@ -5,6 +5,7 @@ import { logContext, validTraceId } from "./log-context";
|
||||
// No arbitrary strings: upstream errors, paths and even innocuous-looking messages
|
||||
// can contain a private repository name. Add diagnostics here deliberately.
|
||||
const instance = randomUUID();
|
||||
const allowed = (values: readonly string[], value: unknown): value is string => typeof value === "string" && values.includes(value);
|
||||
const events: Record<string, string> = {
|
||||
...LOG_EVENTS,
|
||||
request: "request_completed", "slow request": "request_slow", "request interrupted": "request_cancelled",
|
||||
@@ -25,11 +26,22 @@ const numbers = ["status", "httpStatus", "ms", "firstByteMs", "responseMs", "byt
|
||||
const stages = ["repository", "credential_resolution", "upstream", "cache_lookup", "cache_fill", "worker_wait", "anonymize", "streamer_headers"];
|
||||
export function logRecord(level: string, module: string, args: unknown[]) {
|
||||
const context = logContext.getStore() || {};
|
||||
const details = Object.assign({}, ...args.filter(a => a && typeof a === "object" && !Array.isArray(a)));
|
||||
const first = typeof args[0] === "string" ? args[0] : "";
|
||||
const event = events[first] || (module.includes("webhook") ? "webhook_log" : "application_log");
|
||||
const details: Record<string, unknown> = {};
|
||||
const keys = [...numbers, "requestId", "jobId", "code", "message", "outcome", "method", "route", "queue",
|
||||
"stages", "reason", "phase", "event", "deliveryId", "hostname", "service"];
|
||||
// Do not traverse arbitrary objects or invoke error getters while logging.
|
||||
for (const arg of args.slice(0, 8)) if (arg && typeof arg === "object") {
|
||||
for (const key of keys) {
|
||||
try {
|
||||
const property = Object.getOwnPropertyDescriptor(arg, key);
|
||||
if (property && "value" in property) details[key] = property.value;
|
||||
} catch { /* A hostile Proxy must not make logging throw. */ }
|
||||
}
|
||||
}
|
||||
const first = typeof args[0] === "string" && args[0].length < 256 ? args[0] : "";
|
||||
const event = (Object.hasOwn(events, first) ? events[first] : undefined) || (module.includes("webhook") ? "webhook_log" : "application_log");
|
||||
const record: Record<string, unknown> = { timestamp: new Date().toISOString(), severity: level,
|
||||
service: context.service || (["api", "streamer", "worker"].includes(details.service) ? details.service : undefined) || (process.env.SERVICE_NAME?.toLowerCase() === "streamer" ? "streamer" : "api"),
|
||||
service: context.service || (allowed(["api", "streamer", "worker"], details.service) ? details.service : undefined) || (process.env.SERVICE_NAME?.toLowerCase() === "streamer" ? "streamer" : "api"),
|
||||
instance, release: /^[a-f0-9]{7,40}$/.test(process.env.APP_RELEASE || "") ? process.env.APP_RELEASE : "unknown",
|
||||
event, message: event, module: LOG_MODULES.includes(module) ? module : "application" };
|
||||
for (const key of ["requestId", "jobId"] as const) {
|
||||
@@ -38,21 +50,22 @@ export function logRecord(level: string, module: string, args: unknown[]) {
|
||||
}
|
||||
if (Number.isSafeInteger(context.attempt)) record.attempt = context.attempt;
|
||||
for (const key of numbers) if (typeof details[key] === "number" && Number.isFinite(details[key])) record[key] = details[key];
|
||||
const code = [details.code, details.message].find(value => errorCodes.has(value));
|
||||
const code = [details.code, details.message].find(value => typeof value === "string" && errorCodes.has(value));
|
||||
if (code) record.errorCode = code;
|
||||
if (["protocol", "credentials", "port", "host"].includes(details.reason)) record.reason = details.reason;
|
||||
if (["parse", "find_grant_owners", "reconcile_grant", "persist_installation_block", "reconcile_installation"].includes(details.phase)) record.phase = details.phase;
|
||||
if (["github_app_authorization", "installation", "installation_repositories"].includes(details.event)) record.webhookEvent = details.event;
|
||||
if (allowed(["protocol", "credentials", "port", "host"], details.reason)) record.reason = details.reason;
|
||||
if (allowed(["parse", "find_grant_owners", "reconcile_grant", "persist_installation_block", "reconcile_installation"], details.phase)) record.phase = details.phase;
|
||||
if (allowed(["github_app_authorization", "installation", "installation_repositories"], details.event)) record.webhookEvent = details.event;
|
||||
if (validTraceId(details.deliveryId)) record.deliveryId = details.deliveryId;
|
||||
if (["github.com", "api.github.com", "codeload.github.com", "raw.githubusercontent.com", "media.githubusercontent.com"].includes(details.hostname)) record.upstreamHost = details.hostname;
|
||||
if (allowed(["github.com", "api.github.com", "codeload.github.com", "raw.githubusercontent.com", "media.githubusercontent.com"], details.hostname)) record.upstreamHost = details.hostname;
|
||||
else if (details.hostname) record.upstreamHost = "other";
|
||||
if (["completed", "interrupted"].includes(details.outcome)) record.outcome = details.outcome;
|
||||
if (["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "OTHER"].includes(details.method)) record.method = details.method;
|
||||
if (allowed(["completed", "interrupted"], details.outcome)) record.outcome = details.outcome;
|
||||
if (allowed(["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "OTHER"], details.method)) record.method = details.method;
|
||||
// Route labels are generated by request-monitoring, never from URLs in arbitrary errors.
|
||||
if (module === "requests" && typeof details.route === "string") record.route = details.route;
|
||||
if (["download", "remove", "cache"].includes(details.queue)) record.queue = details.queue;
|
||||
if (allowed(["download", "remove", "cache"], details.queue)) record.queue = details.queue;
|
||||
if (details.stages && typeof details.stages === "object") for (const key of stages) {
|
||||
const value = details.stages[key];
|
||||
let value: unknown;
|
||||
try { value = Object.getOwnPropertyDescriptor(details.stages, key)?.value; } catch { continue; }
|
||||
if (typeof value === "number" && Number.isFinite(value)) record[`stage_${key}_ms`] = value;
|
||||
}
|
||||
return record;
|
||||
|
||||
+6
-5
@@ -253,7 +253,6 @@ function persistError(entry: {
|
||||
|
||||
function emit(level: Level, module: string, args: unknown[]) {
|
||||
if (LEVEL_ORDER[level] < threshold) return;
|
||||
args = args.map(arg => redactSecrets(arg instanceof Error ? serializeError(arg) : arg));
|
||||
const ts = new Date().toISOString();
|
||||
const record = logRecord(level, module, args);
|
||||
// stdout is a pipe in Docker. Respect backpressure without awaiting it or
|
||||
@@ -272,8 +271,8 @@ function emit(level: Level, module: string, args: unknown[]) {
|
||||
ts,
|
||||
level,
|
||||
module,
|
||||
message: String(record.event),
|
||||
raw: [{ ...record, code: record.errorCode || record.event }],
|
||||
message: String(record.errorCode || record.event),
|
||||
raw: [{ ...record, message: record.errorCode || record.event, code: record.errorCode || record.event }],
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -301,10 +300,12 @@ type ErrorLike = {
|
||||
url?: string | (() => string | undefined);
|
||||
};
|
||||
|
||||
export function serializeError(err: unknown): Record<string, unknown> {
|
||||
export function serializeError(err: unknown, seen = new WeakSet<object>(), depth = 0): Record<string, unknown> {
|
||||
if (err == null) return { value: err };
|
||||
if (typeof err !== "object") return { value: String(err) };
|
||||
|
||||
if (seen.has(err) || depth >= 4) return { message: "error_details_truncated" };
|
||||
seen.add(err);
|
||||
const e = err as ErrorLike;
|
||||
const out: Record<string, unknown> = {};
|
||||
if (e.name) out.name = e.name;
|
||||
@@ -339,7 +340,7 @@ export function serializeError(err: unknown): Record<string, unknown> {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
if (e.cause) out.cause = serializeError(e.cause);
|
||||
if (e.cause) out.cause = serializeError(e.cause, seen, depth + 1);
|
||||
|
||||
// Only include the stack when there's nothing else useful — avoids dumping
|
||||
// a stack for handled HTTP errors but keeps debuggability for plain Errors.
|
||||
|
||||
@@ -107,7 +107,8 @@ describe("reconnecting a recreated repository", () => {
|
||||
expect(saved.update.$set.statusDate).to.be.instanceOf(Date);
|
||||
expect(saved.update.$set.source.commitDate).to.deep.equal(new Date("2026-09-29T00:00:00Z"));
|
||||
expect(removals).to.deep.equal([{ accessRevision: "old", replacement: true }]);
|
||||
expect(jobs[0][1]).to.deep.equal({ repoId: "recreated" });
|
||||
expect(jobs[0][1]).to.include({ repoId: "recreated" });
|
||||
expect(jobs[0][1].logJobId).to.match(/^[a-f0-9-]{36}$/);
|
||||
});
|
||||
it("rebuilds when only the repository identity changed", async () => {
|
||||
await handler("/:repoId/", "post")(form({ source: { branch: "main", commit: "deadbeef" } }), { json: () => {} });
|
||||
|
||||
@@ -7,7 +7,8 @@ const { logRecord } = require('../src/core/log-record');
|
||||
const { logContext, jobTrace } = require('../src/core/log-context');
|
||||
const { loggedJob } = require('../src/queue/log-job');
|
||||
|
||||
describe('structured logging privacy and context', () => {
|
||||
describe('structured logging privacy and context', function () {
|
||||
this.timeout(15000); // Child processes compile TypeScript before exercising pipe failures.
|
||||
it('keeps the production summary compatible with JSON and legacy records', () => {
|
||||
const { logSummary, observeLine } = require('../scripts/check-production-logs');
|
||||
const summary = logSummary();
|
||||
@@ -48,6 +49,15 @@ describe('structured logging privacy and context', () => {
|
||||
catch (error) { expect(error.message).to.equal('fixture failure'); return; }
|
||||
throw new Error('job failure was swallowed');
|
||||
});
|
||||
it('ignores cyclic objects and hostile getters without losing safe fields', () => {
|
||||
const value = {status: 502}; value.self = value;
|
||||
Object.defineProperty(value, 'message', {get() { throw new Error('getter invoked'); }});
|
||||
value.method = {toString() { throw new Error('toString invoked'); }};
|
||||
const record = logRecord('error', 'route', ['__proto__', value]);
|
||||
expect(record).to.include({event: 'application_log', status: 502});
|
||||
const failure = new Error('fixture'); failure.cause = failure;
|
||||
expect(require('../src/core/logger').serializeError(failure).cause.message).to.equal('error_details_truncated');
|
||||
});
|
||||
it('rejects injected correlation IDs and non-finite measurements', () => {
|
||||
const row = logRecord('warn', 'fixture', ['request', { requestId: 'bad\nsecret', jobId: 'private', ms: Infinity, status: NaN }]);
|
||||
expect(row).not.to.have.any.keys('requestId', 'jobId', 'ms', 'status');
|
||||
@@ -77,12 +87,14 @@ describe('structured logging privacy and context', () => {
|
||||
it('preserves admin error codes and expected-error ring suppression', () => {
|
||||
const child = spawnSync(process.execPath, ['-r', 'ts-node/register/transpile-only', '-e', `
|
||||
const redis = require('redis');
|
||||
let pushes = 0; const fields = [];
|
||||
const tx = {lPush() {pushes++; return tx;}, lTrim() {return tx;},
|
||||
let pushes = 0, last; const fields = [];
|
||||
const tx = {lPush(key, value) {pushes++; last = JSON.parse(value); return tx;}, lTrim() {return tx;},
|
||||
hIncrBy(key, field) { fields.push(field); return tx; }, expire() {return tx;}, exec() {return Promise.resolve([]);} };
|
||||
redis.createClient = () => ({isReady:true, isOpen:true, on() {}, connect() {return Promise.resolve();}, multi() {return tx;}});
|
||||
require('./src/core/logger').createLogger('route').warn('repository_not_ready', {message:'repository_not_ready', httpStatus:409});
|
||||
if (pushes !== 0 || !fields.includes('cb:warn:repository_not_ready')) process.exitCode = 1;
|
||||
require('./src/core/logger').createLogger('route').warn('invalid_request', {message:'invalid_request', httpStatus:400});
|
||||
if (pushes !== 1 || last.raw[0].message !== 'invalid_request' || last.message !== 'invalid_request') process.exitCode = 1;
|
||||
`], {env: {...process.env, LOG_REDIS_MIRROR: 'on'}, encoding: 'utf8', timeout: 10000});
|
||||
expect(child.status, child.stderr).to.equal(0);
|
||||
});
|
||||
|
||||
Reference in new issue
Block a user