mirror of
https://github.com/tdurieux/anonymous_github.git
synced 2026-11-18 01:54:23 +01:00
Merge pull request #956 from tdurieux/t3/oss-logging-infrastructure
feat: add private structured logging and investigation dashboards
This commit is contained in:
40 files changed
+1835
-75
No files matched your search
@@ -125,7 +125,8 @@ describe("removal recovery", function () {
|
||||
AnonymizedRepositoryModel.find = () => ({ lean: async () => [] });
|
||||
await queueModule.recoverStuckRemoving();
|
||||
expect(repo.status).to.equal("removing");
|
||||
expect(added[1]).to.deep.equal({ repoId: repo.repoId });
|
||||
expect(added[1]).to.deep.include({ repoId: repo.repoId });
|
||||
expect(added[1].logJobId).to.match(/^[a-f0-9-]{36}$/);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -164,7 +165,8 @@ describe("removal recovery", function () {
|
||||
let added;
|
||||
queueModule.removeQueue.add = async (...args) => { added = args; };
|
||||
await queueModule.recoverStuckRemoving();
|
||||
expect(added[1]).to.deep.equal({ repoId: repo.repoId });
|
||||
expect(added[1]).to.deep.include({ repoId: repo.repoId });
|
||||
expect(added[1].logJobId).to.match(/^[a-f0-9-]{36}$/);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -392,7 +394,8 @@ describe("removal workers", function () {
|
||||
expect(await addRemovalJob("repo-1", queue)).to.equal(true);
|
||||
expect(removed).to.equal(true);
|
||||
expect(added[0]).to.equal("repo-1");
|
||||
expect(added[1]).to.deep.equal({ repoId: "repo-1" });
|
||||
expect(added[1]).to.deep.include({ repoId: "repo-1" });
|
||||
expect(added[1].logJobId).to.match(/^[a-f0-9-]{36}$/);
|
||||
expect(added[2].jobId).to.equal("repo-repo-1");
|
||||
expect(added[2].attempts).to.equal(3);
|
||||
expect(added[2].removeOnFail).to.deep.equal({ count: 1000 });
|
||||
|
||||
@@ -19,15 +19,17 @@ describe("credential and webhook failure diagnostics", () => {
|
||||
afterEach(() => restores.splice(0).reverse().forEach(restore => restore()));
|
||||
it("logs generic upstream 401 as warning while retaining server errors", () => {
|
||||
const warnings = [], errors = [];
|
||||
stub(console, "warn", (...args) => warnings.push(args));
|
||||
stub(console, "error", (...args) => errors.push(args));
|
||||
stub(process.stdout, "write", line => {
|
||||
const record = JSON.parse(line);
|
||||
(record.severity === "warn" ? warnings : errors).push(line); return true;
|
||||
});
|
||||
handleError(Object.assign(new Error("denied"), { status: 401 }), response());
|
||||
handleError(Object.assign(new Error("failed"), { response: { statusCode: 503 } }), response());
|
||||
expect(warnings).to.have.length(1); expect(errors).to.have.length(1);
|
||||
expect(errors.flat().join(" ")).to.include('"httpStatus":503');
|
||||
});
|
||||
it("logs got HTTP401 as warning with its actual status", () => {
|
||||
const warnings = []; stub(console, "warn", (...args) => warnings.push(args));
|
||||
const warnings = []; stub(process.stdout, "write", line => { warnings.push(line); return true; });
|
||||
const failure = Object.assign(new Error("Unauthorized"), { response: { statusCode: 401 } });
|
||||
Object.setPrototypeOf(failure, require("got").HTTPError.prototype);
|
||||
const res = response(); handleError(failure, res);
|
||||
@@ -51,16 +53,16 @@ describe("credential and webhook failure diagnostics", () => {
|
||||
it("logs signed webhook persistence failures without secrets and still returns 503", async () => {
|
||||
stub(config, "GITHUB_APP_ENABLED", true); stub(config, "GITHUB_APP_ID", "123");
|
||||
stub(config, "GITHUB_APP_WEBHOOK_SECRET", "test-secret");
|
||||
const messages = []; stub(console, "error", (...args) => messages.push(args));
|
||||
const messages = []; stub(process.stdout, "write", line => { messages.push(line); return true; });
|
||||
stub(Installation, "updateOne", async () => { throw new Error("secret-token https://private/?token=secret"); });
|
||||
const body = Buffer.from(JSON.stringify({ action: "added", installation: { id: 4, app_id: 123 }, private: "private-body" }));
|
||||
const req = { body, headers: { "x-github-event": "installation_repositories", "x-github-delivery": "abcd-1234",
|
||||
const req = { body, headers: { "x-github-event": "installation_repositories", "x-github-delivery": "abcdef12-1234-1234-1234-abcdef123456",
|
||||
"x-hub-signature-256": "sha256=" + createHmac("sha256", "test-secret").update(body).digest("hex") } };
|
||||
const handler = githubAppWebhook.stack.find(layer => layer.route).route.stack.at(-1).handle;
|
||||
const res = response(); await handler(req, res);
|
||||
expect(res.code).to.equal(503); expect(res.body.error).to.equal("webhook_processing_failed");
|
||||
const text = JSON.stringify(messages);
|
||||
expect(text).to.include("persist_installation_block").and.include("abcd-1234");
|
||||
expect(text).to.include("persist_installation_block").and.include("abcdef12-1234-1234-1234-abcdef123456");
|
||||
expect(text).not.to.include("secret-token"); expect(text).not.to.include("private-body"); expect(text).not.to.include("https://");
|
||||
expect(text).not.to.include(req.headers["x-hub-signature-256"]);
|
||||
});
|
||||
|
||||
@@ -27,7 +27,7 @@ describe("download progress publication fences", function () {
|
||||
|
||||
it("keeps a captured publication fence valid when a queue progress tick runs", async function () {
|
||||
const { repo, stored } = setup(); let tick;
|
||||
stub(global, "setInterval", callback => { tick = callback; return 1; });
|
||||
stub(global, "setInterval", callback => { tick = callback; return { unref() {} }; });
|
||||
stub(global, "clearInterval", () => {});
|
||||
stub(db, "connect", async () => {});
|
||||
stub(db, "getRepository", async () => repo);
|
||||
|
||||
@@ -52,7 +52,7 @@ describe("folder file endpoint disclosure", function () {
|
||||
});
|
||||
it("blocks a real HTTP redirect to a GitHub tree page before fetching its HTML", async () => {
|
||||
const warnings = [];
|
||||
stub(console, "warn", (...args) => warnings.push(args.join(" ")));
|
||||
stub(process.stdout, "write", line => { warnings.push(line); return true; });
|
||||
const server = await listen(http.createServer((_req, res) => { res.writeHead(302, { location: "https://github.com/example/private-repo/tree/main/folder?token=secret-query" }); res.end(); }));
|
||||
const original = got.stream;
|
||||
stub(got, "stream", (_url, options) => original(`http://127.0.0.1:${server.address().port}/raw`, options));
|
||||
|
||||
@@ -107,7 +107,8 @@ describe("reconnecting a recreated repository", () => {
|
||||
expect(saved.update.$set.statusDate).to.be.instanceOf(Date);
|
||||
expect(saved.update.$set.source.commitDate).to.deep.equal(new Date("2026-09-29T00:00:00Z"));
|
||||
expect(removals).to.deep.equal([{ accessRevision: "old", replacement: true }]);
|
||||
expect(jobs[0][1]).to.deep.equal({ repoId: "recreated" });
|
||||
expect(jobs[0][1]).to.include({ repoId: "recreated" });
|
||||
expect(jobs[0][1].logJobId).to.match(/^[a-f0-9-]{36}$/);
|
||||
});
|
||||
it("rebuilds when only the repository identity changed", async () => {
|
||||
await handler("/:repoId/", "post")(form({ source: { branch: "main", commit: "deadbeef" } }), { json: () => {} });
|
||||
|
||||
@@ -84,7 +84,8 @@ describe("repository refresh and restoration", () => {
|
||||
expect(response.status).to.equal("preparing");
|
||||
expect(repo.model.source.commit).to.equal(commit);
|
||||
expect(repo.repoId).to.equal("restore-me");
|
||||
expect(added[1]).to.deep.equal({ repoId: "restore-me" });
|
||||
expect(added[1]).to.deep.include({ repoId: "restore-me" });
|
||||
expect(added[1].logJobId).to.match(/^[a-f0-9-]{36}$/);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -9,17 +9,16 @@ require("../src/core/storage");
|
||||
const FileSystem = require("../src/core/storage/FileSystem").default;
|
||||
|
||||
describe("cache write error origins", function () {
|
||||
let directory, previousFolder, messages, previousWarn, previousError;
|
||||
let directory, previousFolder, messages, previousWrite;
|
||||
beforeEach(function () {
|
||||
directory = fs.mkdtempSync(path.join(os.tmpdir(), "cache-errors-"));
|
||||
previousFolder = config.FOLDER; config.FOLDER = directory;
|
||||
messages = [];
|
||||
previousWarn = console.warn; previousError = console.error;
|
||||
console.warn = (...args) => messages.push(["warn", args.join(" ")]);
|
||||
console.error = (...args) => messages.push(["error", args.join(" ")]);
|
||||
previousWrite = process.stdout.write;
|
||||
process.stdout.write = line => { const record = JSON.parse(line); messages.push([record.severity, record.event]); return true; };
|
||||
});
|
||||
afterEach(function () {
|
||||
console.warn = previousWarn; console.error = previousError;
|
||||
process.stdout.write = previousWrite;
|
||||
config.FOLDER = previousFolder;
|
||||
fs.rmSync(directory, { recursive: true, force: true });
|
||||
});
|
||||
@@ -34,8 +33,8 @@ describe("cache write error origins", function () {
|
||||
let caught;
|
||||
try { await storage.write("repo", "file.txt", input); } catch (error) { caught = error; }
|
||||
expect(caught).to.equal(failure);
|
||||
expect(messages.some(([level, text]) => level === "warn" && text.includes("source stream failed"))).to.equal(true);
|
||||
expect(messages.some(([level, text]) => level === "error" && text.includes("write failed"))).to.equal(false);
|
||||
expect(messages.some(([level, text]) => level === "warn" && text.includes("source_stream_failed"))).to.equal(true);
|
||||
expect(messages.some(([level, text]) => level === "error" && text.includes("write_failed"))).to.equal(false);
|
||||
expect(fs.readFileSync(path.join(directory, "repo/original/file.txt"), "utf8")).to.equal("previous");
|
||||
expect(fs.readdirSync(path.join(directory, "repo/original"))).to.deep.equal(["file.txt"]);
|
||||
});
|
||||
@@ -45,7 +44,7 @@ describe("cache write error origins", function () {
|
||||
let caught;
|
||||
try { await storage.write("repo", "file.txt", "new content"); } catch (error) { caught = error; }
|
||||
expect(caught).to.be.instanceOf(Error);
|
||||
expect(messages.some(([level, text]) => level === "error" && text.includes("write failed"))).to.equal(true);
|
||||
expect(messages.some(([level, text]) => level === "error" && text.includes("write_failed"))).to.equal(true);
|
||||
expect(messages.some(([level]) => level === "warn")).to.equal(false);
|
||||
expect(fs.readdirSync(path.join(directory, "repo/original"))).to.deep.equal(["file.txt"]);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
const { setTimeout } = require("node:timers");
|
||||
const { expect } = require('chai');
|
||||
const { randomUUID } = require('node:crypto');
|
||||
const { spawn, spawnSync } = require('node:child_process');
|
||||
require('ts-node/register/transpile-only');
|
||||
const { logRecord } = require('../src/core/log-record');
|
||||
const { logContext, jobTrace } = require('../src/core/log-context');
|
||||
const { loggedJob } = require('../src/queue/log-job');
|
||||
|
||||
describe('structured logging privacy and context', function () {
|
||||
this.timeout(15000); // Child processes compile TypeScript before exercising pipe failures.
|
||||
it('keeps the production summary compatible with JSON and legacy records', () => {
|
||||
const { logSummary, observeLine } = require('../scripts/check-production-logs');
|
||||
const summary = logSummary();
|
||||
observeLine(summary, JSON.stringify(logRecord('info', 'requests', ['request', {ms: 1, status: 503, outcome: 'completed'}])));
|
||||
observeLine(summary, JSON.stringify(logRecord('error', 'process', ['process fatal error'])));
|
||||
observeLine(summary, '2026-01-01T00:00:00Z WARN [requests] request {"ms":2,"status":429,"outcome":"completed"}');
|
||||
expect(summary).to.include({completed: 2, completed5xx: 1, fatalErrors: 1, errors: 1, warnings: 1, rateLimited: 1});
|
||||
});
|
||||
it('omits arbitrary text, private identities, request bodies and nested secrets', () => {
|
||||
const secret = 'private-owner/private-repository?state=secret';
|
||||
const record = logRecord('error', 'upstream', [secret, {
|
||||
message: secret, url: secret, headers: { authorization: secret }, repoId: secret,
|
||||
requestBody: secret, status: 502, code: 'ETIMEDOUT', stages: { upstream: 42, [secret]: 1 },
|
||||
}]);
|
||||
expect(JSON.stringify(record)).not.to.include(secret);
|
||||
expect(record).to.include({ status: 502, errorCode: 'ETIMEDOUT', stage_upstream_ms: 42 });
|
||||
expect(record).to.have.keys('timestamp', 'severity', 'service', 'instance', 'release', 'event', 'message', 'module', 'status', 'errorCode', 'stage_upstream_ms');
|
||||
});
|
||||
it('keeps concurrent request contexts separate and propagates an opaque job trace', async () => {
|
||||
const ids = [randomUUID(), randomUUID()];
|
||||
const rows = await Promise.all(ids.map((requestId, i) => logContext.run({ requestId }, async () => {
|
||||
await new Promise(resolve => setTimeout(resolve, 10 - i));
|
||||
const trace = jobTrace();
|
||||
expect(trace.requestId).to.equal(requestId);
|
||||
expect(trace.logJobId).not.to.equal(requestId);
|
||||
return logRecord('info', 'fixture', ['request']);
|
||||
})));
|
||||
expect(rows.map(row => row.requestId)).to.deep.equal(ids);
|
||||
expect(logRecord('info', 'fixture', ['request'])).not.to.have.property('requestId');
|
||||
});
|
||||
it('sets sandbox job context and preserves success/failure semantics', async () => {
|
||||
const requestId = randomUUID(), jobId = randomUUID();
|
||||
const processor = loggedJob('download', async () => {
|
||||
expect(logContext.getStore()).to.deep.equal({ service: 'worker', requestId, jobId, attempt: 3 });
|
||||
throw new Error('fixture failure');
|
||||
});
|
||||
try { await processor({ data: { repoId: 'private-id', requestId, logJobId: jobId }, attemptsMade: 2 }); }
|
||||
catch (error) { expect(error.message).to.equal('fixture failure'); return; }
|
||||
throw new Error('job failure was swallowed');
|
||||
});
|
||||
it('ignores cyclic objects and hostile getters without losing safe fields', () => {
|
||||
const value = {status: 502}; value.self = value;
|
||||
Object.defineProperty(value, 'message', {get() { throw new Error('getter invoked'); }});
|
||||
value.method = {toString() { throw new Error('toString invoked'); }};
|
||||
const record = logRecord('error', 'route', ['__proto__', value]);
|
||||
expect(record).to.include({event: 'application_log', status: 502});
|
||||
const failure = new Error('fixture'); failure.cause = failure;
|
||||
expect(require('../src/core/logger').serializeError(failure).cause.message).to.equal('error_details_truncated');
|
||||
});
|
||||
it('rejects injected correlation IDs and non-finite measurements', () => {
|
||||
const row = logRecord('warn', 'fixture', ['request', { requestId: 'bad\nsecret', jobId: 'private', ms: Infinity, status: NaN }]);
|
||||
expect(row).not.to.have.any.keys('requestId', 'jobId', 'ms', 'status');
|
||||
});
|
||||
it('survives a reader closing stdout', async function () {
|
||||
this.timeout(10000);
|
||||
const child = spawn(process.execPath, ['-r', 'ts-node/register/transpile-only', '-e', `
|
||||
const { createLogger, getDroppedOutput } = require('./src/core/logger');
|
||||
const logger = createLogger('process');
|
||||
process.on('message', () => {
|
||||
logger.info('process heartbeat');
|
||||
setTimeout(() => { logger.info('process heartbeat'); process.send(getDroppedOutput()); process.disconnect(); }, 50);
|
||||
});
|
||||
process.send('ready');
|
||||
`], { env: { ...process.env, LOG_REDIS_MIRROR: 'off' }, stdio: ['ignore', 'pipe', 'pipe', 'ipc'] });
|
||||
let drops, fallback = "";
|
||||
child.stderr.on("data", chunk => { fallback += chunk; });
|
||||
child.on('message', message => {
|
||||
if (message === 'ready') { child.stdout.destroy(); child.send('write'); }
|
||||
else drops = message;
|
||||
});
|
||||
const code = await new Promise((resolve, reject) => { child.on('exit', resolve); child.on('error', reject); });
|
||||
expect(code).to.equal(0);
|
||||
expect(drops).to.be.greaterThan(0);
|
||||
expect(fallback).to.include('"event":"process_heartbeat"');
|
||||
});
|
||||
it('preserves admin error codes and expected-error ring suppression', () => {
|
||||
const child = spawnSync(process.execPath, ['-r', 'ts-node/register/transpile-only', '-e', `
|
||||
const redis = require('redis');
|
||||
let pushes = 0, last; const fields = [];
|
||||
const tx = {lPush(key, value) {pushes++; last = JSON.parse(value); return tx;}, lTrim() {return tx;},
|
||||
hIncrBy(key, field) { fields.push(field); return tx; }, expire() {return tx;}, exec() {return Promise.resolve([]);} };
|
||||
redis.createClient = () => ({isReady:true, isOpen:true, on() {}, connect() {return Promise.resolve();}, multi() {return tx;}});
|
||||
require('./src/core/logger').createLogger('route').warn('repository_not_ready', {message:'repository_not_ready', httpStatus:409});
|
||||
if (pushes !== 0 || !fields.includes('cb:warn:repository_not_ready')) process.exitCode = 1;
|
||||
require('./src/core/logger').createLogger('route').warn('invalid_request', {message:'invalid_request', httpStatus:400});
|
||||
if (pushes !== 1 || last.raw[0].message !== 'invalid_request' || last.message !== 'invalid_request') process.exitCode = 1;
|
||||
`], {env: {...process.env, LOG_REDIS_MIRROR: 'on'}, encoding: 'utf8', timeout: 10000});
|
||||
expect(child.status, child.stderr).to.equal(0);
|
||||
});
|
||||
it('writes JSON without Redis and bounds a blocked stdout pipe', () => {
|
||||
const child = spawnSync(process.execPath, ['-r', 'ts-node/register/transpile-only', '-e', `
|
||||
const { createLogger, getDroppedOutput } = require('./src/core/logger');
|
||||
const logger = createLogger('fixture');
|
||||
logger.info('request', {status: 200, requestBody: 'secret-fixture'});
|
||||
Object.defineProperty(process.stdout, 'writableLength', {value: 300000});
|
||||
for (let i = 0; i < 10000; i++) logger.warn('request', {status: 429});
|
||||
if (getDroppedOutput() !== 10000) process.exitCode = 1;
|
||||
`], { cwd: process.cwd(), env: { ...process.env, LOG_REDIS_MIRROR: 'off' }, encoding: 'utf8', timeout: 10000 });
|
||||
expect(child.status, child.stderr).to.equal(0);
|
||||
const lines = child.stdout.trim().split('\n');
|
||||
expect(lines).to.have.length(1);
|
||||
expect(JSON.parse(lines[0])).to.include({ event: 'request_completed', status: 200 });
|
||||
expect(child.stdout).not.to.include('secret-fixture');
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user