mirror of
https://github.com/tdurieux/anonymous_github.git
synced 2026-11-18 01:54:23 +01:00
fix: explain interrupted requests and account for overlapping latency
This commit is contained in:
1 parent
1e617d2df0
commit
d2e3954d21
18 files changed
+394
-33
No files matched your search
@@ -0,0 +1,28 @@
|
||||
const {expect}=require('chai');
|
||||
require('ts-node/register/transpile-only');
|
||||
const mongoose=require('mongoose');
|
||||
const {installDatabaseTiming}=require('../src/core/database-timing');
|
||||
const {monitorRequests,RequestMetrics}=require('../src/core/request-monitoring');
|
||||
const express=require('express');
|
||||
const http=require('node:http');
|
||||
describe('database execution timing',()=>{
|
||||
it('preserves query receiver, result and failure while measuring execution once',async()=>{
|
||||
const Model=mongoose.model("TimingFixture",new mongoose.Schema({}));
|
||||
const logs=[];const app=express();let server;
|
||||
try{
|
||||
installDatabaseTiming();const wrapped=mongoose.Query.prototype.exec;installDatabaseTiming();expect(mongoose.Query.prototype.exec).equal(wrapped);
|
||||
app.use(monitorRequests('api',new RequestMetrics(),{info:(_,row)=>logs.push(row),warn:(_,row)=>logs.push(row)}));
|
||||
app.get('/',async(_req,res)=>{
|
||||
const query=Model.find();query._find=async function(){expect(this).equal(query);await new Promise(r=>setTimeout(r,20));return ['ok'];};
|
||||
expect(await query.exec()).deep.equal(['ok']);
|
||||
const failed=Model.find();const error=new Error('PRIVATE query');failed._find=async()=>{throw error;};
|
||||
try{await failed.exec();throw Error('expected failure');}catch(e){expect(e).equal(error);}
|
||||
res.end('ok');
|
||||
});
|
||||
server=await new Promise(resolve=>{const s=app.listen(0,'127.0.0.1',()=>resolve(s));});
|
||||
await new Promise((resolve,reject)=>http.get({host:'127.0.0.1',port:server.address().port},res=>{res.resume();res.on('end',resolve);}).on('error',reject));
|
||||
expect(logs[0].stageCounts.database).equal(2);expect(logs[0].stages.database).greaterThan(15);expect(logs[0].stageOpen.database).equal(0);
|
||||
expect(JSON.stringify(logs)).not.include('PRIVATE');
|
||||
}finally{mongoose.deleteModel("TimingFixture");if(server)await new Promise(resolve=>server.close(resolve));}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,68 @@
|
||||
const { expect } = require('chai');
|
||||
require('ts-node/register/transpile-only');
|
||||
const { RequestTiming, requestRoute } = require('../src/core/request-monitoring');
|
||||
const { recordResponseFailure, recordUpstreamFailure, responseEvidence, expectedResponseBytes } = require('../src/core/request-evidence');
|
||||
const { logRecord } = require('../src/core/log-record');
|
||||
const { safeRedisLog } = require('../src/server/admin-log-investigation');
|
||||
|
||||
describe('request causal evidence and timing accounting', () => {
|
||||
it('integrates nested, parallel, repeated and unfinished spans without invented totals', () => {
|
||||
let now = 0;
|
||||
const timing = new RequestTiming(0, () => now);
|
||||
now = 10; const credential = timing.start('credential_resolution');
|
||||
now = 20; const db = timing.start('database');
|
||||
now = 30; const upstream = timing.start('upstream');
|
||||
now = 40; db(); db();
|
||||
now = 50; upstream(); credential();
|
||||
now = 60; timing.start('upstream');
|
||||
now = 70; const snapshot = timing.finish();
|
||||
expect(snapshot).deep.equal({stages:{credential_resolution:40,database:20,upstream:30},
|
||||
stageCounts:{credential_resolution:1,database:1,upstream:2},stageOpen:{credential_resolution:0,database:0,upstream:1},
|
||||
measuredMs:50,unattributedMs:20,overlapMs:40});
|
||||
now = 100; upstream(); timing.start('database')();
|
||||
expect(timing.finish()).deep.equal(snapshot);
|
||||
});
|
||||
it('bounds storage across repeated spans', () => {
|
||||
let now = 0; const timing = new RequestTiming(0, () => now++);
|
||||
for(let i=0;i<100000;i++) timing.start('upstream')();
|
||||
const result=timing.finish();
|
||||
expect(result.stageCounts.upstream).equal(100000);
|
||||
expect(JSON.stringify(timing).length).lessThan(500);
|
||||
});
|
||||
it('keeps disconnect origins unknown and preserves the first causal evidence', () => {
|
||||
expect(responseEvidence({})).deep.equal({cancellationOrigin:'unknown',cancellationEvidence:'response_close'});
|
||||
expect(responseEvidence({},true).cancellationOrigin).equal('unknown');
|
||||
const res={}; recordUpstreamFailure(res,Object.assign(new Error('PRIVATE'),{code:'ETIMEDOUT'}));
|
||||
recordResponseFailure(res,'api','local_abort');
|
||||
expect(responseEvidence(res)).deep.equal({cancellationOrigin:'timeout',cancellationEvidence:'timeout'});
|
||||
const upstream={};recordUpstreamFailure(upstream,new Error('PRIVATE'),'streamer');
|
||||
expect(responseEvidence(upstream)).deep.equal({cancellationOrigin:'streamer',cancellationEvidence:'upstream_error'});
|
||||
const local={};recordResponseFailure(local,'api','local_abort');expect(responseEvidence(local).cancellationOrigin).equal('api');
|
||||
});
|
||||
it('only compares body bytes with the output content length', () => {
|
||||
expect(expectedResponseBytes('GET',200,'123')).equal(123);
|
||||
expect(expectedResponseBytes('HEAD',200,'123')).equal(0);
|
||||
expect(expectedResponseBytes('GET',304,'123')).equal(0);
|
||||
for(const value of [undefined,-1,'1.2','9e5','9007199254740992',{},['123']]) expect(expectedResponseBytes('GET',200,value)).equal(undefined);
|
||||
});
|
||||
it('preserves bounded evidence and timings through the admin projection', () => {
|
||||
const row=logRecord('warn','requests',['request interrupted',{outcome:'interrupted',bytes:12,expectedBytes:50,
|
||||
cancellationOrigin:'unknown',cancellationEvidence:'response_close',endpointKind:'public',measuredMs:20,unattributedMs:10,overlapMs:5,
|
||||
stages:{database:3,upstream:17,PRIVATE:123},stageCounts:{upstream:2},stageOpen:{upstream:1}}]);
|
||||
const safe=safeRedisLog({ts:row.timestamp,level:'warn',module:'requests',raw:[row]}).raw[0];
|
||||
expect(safe).include({bytes:12,expectedBytes:50,cancellationOrigin:'unknown',cancellationEvidence:'response_close',
|
||||
endpointKind:'public',measuredMs:20,unattributedMs:10,overlapMs:5,stage_database_ms:3,stage_upstream_ms:17,stage_upstream_count:2,stage_upstream_open:1});
|
||||
expect(JSON.stringify(safe)).not.include('PRIVATE');
|
||||
const bad=logRecord('warn','requests',['request interrupted',{cancellationOrigin:'PRIVATE',cancellationEvidence:'PRIVATE',endpointKind:'PRIVATE'}]);
|
||||
expect(JSON.stringify(bad)).not.include('PRIVATE');
|
||||
});
|
||||
it('respects source handler precedence and canonical anonymous route IDs', () => {
|
||||
for(const action of ['readme','branches','tree','file']) expect(requestRoute('/api/repo/private/files/'+action,'api')).equal('/api/repo/:owner/:repo/'+action);
|
||||
expect(requestRoute('/api/repo/demo/file/readme','api')).equal('/api/repo/:repoId/file/:path');
|
||||
for(const action of ['coauthors','extend','refresh']) {
|
||||
expect(requestRoute('/api/repo/private/'+action,'api','GET')).equal('/api/repo/:owner/:repo');
|
||||
expect(requestRoute('/api/repo/demo/'+action,'api','POST')).equal('/api/repo/:repoId/'+action);
|
||||
}
|
||||
expect(logRecord('info','requests',['request',{route:'/api/repo/:id/files'}]).route).equal('/api/repo/:repoId/files');
|
||||
});
|
||||
});
|
||||
@@ -174,6 +174,7 @@ describe("bounded request monitoring", function () {
|
||||
await ready; client.destroy(); await terminal;
|
||||
expect(logs).to.have.length(1); expect(logs[0][0]).to.equal("request interrupted");
|
||||
expect(logs[0][1].outcome).to.equal("interrupted");
|
||||
expect(logs[0][1].cancellationOrigin).to.equal("unknown");
|
||||
expect(logs[0][1]).include({requestPath:"/api/repo/demo-123/zip",repoId:"demo-123"});
|
||||
expect(JSON.stringify(logs)).not.include("SECRET");
|
||||
const row = metrics.drain().find(row => row.data.metric === "request").data;
|
||||
@@ -186,6 +187,50 @@ describe("bounded request monitoring", function () {
|
||||
});
|
||||
}
|
||||
|
||||
for (const code of ["ETIMEDOUT", "ECONNRESET"]) {
|
||||
it(`records causal ${code} before a truncated 200 closes`, async function () {
|
||||
const { streamResponse } = require("../src/core/response-stream");
|
||||
app.get("/", (_req, res) => {
|
||||
const source = new Readable({ read() {} });
|
||||
res.setHeader("Content-Length", "100");
|
||||
void streamResponse(source, res, () => res.destroy());
|
||||
source.push("partial");
|
||||
setTimeout(() => source.destroy(Object.assign(new Error("PRIVATE"), {code})), 20);
|
||||
});
|
||||
await listen();
|
||||
await fetch(server, "/").catch(() => undefined);
|
||||
await delay(20);
|
||||
const detail = logs[0][1];
|
||||
expect(detail).include({status:200,outcome:"interrupted",bytes:7,expectedBytes:100,
|
||||
cancellationOrigin:code === "ETIMEDOUT" ? "timeout" : "streamer"});
|
||||
expect(JSON.stringify(logs)).not.include("PRIVATE");
|
||||
});
|
||||
}
|
||||
|
||||
it("correlates repeated streamer hops without accepting public caller IDs", async function () {
|
||||
const upstream = express(); const hops=[];
|
||||
upstream.use(monitorRequests("streamer",new RequestMetrics(),{info:(_,row)=>hops.push(row),warn:(_,row)=>hops.push(row)}));
|
||||
upstream.post("/api",(_req,res)=>res.end("ok"));
|
||||
const streamer = await new Promise(resolve => {const s=upstream.listen(0,"127.0.0.1",()=>resolve(s));});
|
||||
try {
|
||||
app.get("/", async (_req,res)=>{
|
||||
await fetch(streamer,"/api",{method:"POST",headers:requestHeaders()});
|
||||
await fetch(streamer,"/api",{method:"POST",headers:requestHeaders()});
|
||||
res.end("ok");
|
||||
});
|
||||
await listen();
|
||||
const forged="11111111-1111-1111-1111-111111111111";
|
||||
await fetch(server,"/",{headers:{"x-request-id":forged,"x-parent-hop-id":forged}});
|
||||
const parent=logs[0][1];
|
||||
expect(parent.requestId).not.equal(forged);
|
||||
expect(parent.parentHopId).equal(undefined);
|
||||
expect(hops).length(2);
|
||||
expect(hops[0].hopId).not.equal(hops[1].hopId);
|
||||
for(const hop of hops) expect(hop).include({requestId:parent.requestId,parentHopId:parent.hopId,endpointKind:"internal",outcome:"completed"});
|
||||
expect(parent).not.have.property("cancellationOrigin");
|
||||
} finally {streamer.closeAllConnections();await new Promise(resolve=>streamer.close(resolve));}
|
||||
});
|
||||
|
||||
it("keeps overlapping request IDs and asynchronous stages isolated", async function () {
|
||||
let doneA, doneB;
|
||||
app.get("/a", (_req, res) => { doneA = startStage("upstream"); setTimeout(() => { doneA(); doneA(); res.json(requestHeaders()); }, 30); });
|
||||
|
||||
Reference in new issue
Block a user