fix: explain interrupted requests and account for overlapping latency

This commit is contained in:
Thomas Durieux committed 2026-11-15 13:31:16 +00:00
1 parent 1e617d2df0
commit d2e3954d21
18 files changed
+394 -33

No files matched your search

+28
View File
@@ -0,0 +1,28 @@
const {expect}=require('chai');
require('ts-node/register/transpile-only');
const mongoose=require('mongoose');
const {installDatabaseTiming}=require('../src/core/database-timing');
const {monitorRequests,RequestMetrics}=require('../src/core/request-monitoring');
const express=require('express');
const http=require('node:http');
describe('database execution timing',()=>{
it('preserves query receiver, result and failure while measuring execution once',async()=>{
const Model=mongoose.model("TimingFixture",new mongoose.Schema({}));
const logs=[];const app=express();let server;
try{
installDatabaseTiming();const wrapped=mongoose.Query.prototype.exec;installDatabaseTiming();expect(mongoose.Query.prototype.exec).equal(wrapped);
app.use(monitorRequests('api',new RequestMetrics(),{info:(_,row)=>logs.push(row),warn:(_,row)=>logs.push(row)}));
app.get('/',async(_req,res)=>{
const query=Model.find();query._find=async function(){expect(this).equal(query);await new Promise(r=>setTimeout(r,20));return ['ok'];};
expect(await query.exec()).deep.equal(['ok']);
const failed=Model.find();const error=new Error('PRIVATE query');failed._find=async()=>{throw error;};
try{await failed.exec();throw Error('expected failure');}catch(e){expect(e).equal(error);}
res.end('ok');
});
server=await new Promise(resolve=>{const s=app.listen(0,'127.0.0.1',()=>resolve(s));});
await new Promise((resolve,reject)=>http.get({host:'127.0.0.1',port:server.address().port},res=>{res.resume();res.on('end',resolve);}).on('error',reject));
expect(logs[0].stageCounts.database).equal(2);expect(logs[0].stages.database).greaterThan(15);expect(logs[0].stageOpen.database).equal(0);
expect(JSON.stringify(logs)).not.include('PRIVATE');
}finally{mongoose.deleteModel("TimingFixture");if(server)await new Promise(resolve=>server.close(resolve));}
});
});
+68
View File
@@ -0,0 +1,68 @@
const { expect } = require('chai');
require('ts-node/register/transpile-only');
const { RequestTiming, requestRoute } = require('../src/core/request-monitoring');
const { recordResponseFailure, recordUpstreamFailure, responseEvidence, expectedResponseBytes } = require('../src/core/request-evidence');
const { logRecord } = require('../src/core/log-record');
const { safeRedisLog } = require('../src/server/admin-log-investigation');
describe('request causal evidence and timing accounting', () => {
it('integrates nested, parallel, repeated and unfinished spans without invented totals', () => {
let now = 0;
const timing = new RequestTiming(0, () => now);
now = 10; const credential = timing.start('credential_resolution');
now = 20; const db = timing.start('database');
now = 30; const upstream = timing.start('upstream');
now = 40; db(); db();
now = 50; upstream(); credential();
now = 60; timing.start('upstream');
now = 70; const snapshot = timing.finish();
expect(snapshot).deep.equal({stages:{credential_resolution:40,database:20,upstream:30},
stageCounts:{credential_resolution:1,database:1,upstream:2},stageOpen:{credential_resolution:0,database:0,upstream:1},
measuredMs:50,unattributedMs:20,overlapMs:40});
now = 100; upstream(); timing.start('database')();
expect(timing.finish()).deep.equal(snapshot);
});
it('bounds storage across repeated spans', () => {
let now = 0; const timing = new RequestTiming(0, () => now++);
for(let i=0;i<100000;i++) timing.start('upstream')();
const result=timing.finish();
expect(result.stageCounts.upstream).equal(100000);
expect(JSON.stringify(timing).length).lessThan(500);
});
it('keeps disconnect origins unknown and preserves the first causal evidence', () => {
expect(responseEvidence({})).deep.equal({cancellationOrigin:'unknown',cancellationEvidence:'response_close'});
expect(responseEvidence({},true).cancellationOrigin).equal('unknown');
const res={}; recordUpstreamFailure(res,Object.assign(new Error('PRIVATE'),{code:'ETIMEDOUT'}));
recordResponseFailure(res,'api','local_abort');
expect(responseEvidence(res)).deep.equal({cancellationOrigin:'timeout',cancellationEvidence:'timeout'});
const upstream={};recordUpstreamFailure(upstream,new Error('PRIVATE'),'streamer');
expect(responseEvidence(upstream)).deep.equal({cancellationOrigin:'streamer',cancellationEvidence:'upstream_error'});
const local={};recordResponseFailure(local,'api','local_abort');expect(responseEvidence(local).cancellationOrigin).equal('api');
});
it('only compares body bytes with the output content length', () => {
expect(expectedResponseBytes('GET',200,'123')).equal(123);
expect(expectedResponseBytes('HEAD',200,'123')).equal(0);
expect(expectedResponseBytes('GET',304,'123')).equal(0);
for(const value of [undefined,-1,'1.2','9e5','9007199254740992',{},['123']]) expect(expectedResponseBytes('GET',200,value)).equal(undefined);
});
it('preserves bounded evidence and timings through the admin projection', () => {
const row=logRecord('warn','requests',['request interrupted',{outcome:'interrupted',bytes:12,expectedBytes:50,
cancellationOrigin:'unknown',cancellationEvidence:'response_close',endpointKind:'public',measuredMs:20,unattributedMs:10,overlapMs:5,
stages:{database:3,upstream:17,PRIVATE:123},stageCounts:{upstream:2},stageOpen:{upstream:1}}]);
const safe=safeRedisLog({ts:row.timestamp,level:'warn',module:'requests',raw:[row]}).raw[0];
expect(safe).include({bytes:12,expectedBytes:50,cancellationOrigin:'unknown',cancellationEvidence:'response_close',
endpointKind:'public',measuredMs:20,unattributedMs:10,overlapMs:5,stage_database_ms:3,stage_upstream_ms:17,stage_upstream_count:2,stage_upstream_open:1});
expect(JSON.stringify(safe)).not.include('PRIVATE');
const bad=logRecord('warn','requests',['request interrupted',{cancellationOrigin:'PRIVATE',cancellationEvidence:'PRIVATE',endpointKind:'PRIVATE'}]);
expect(JSON.stringify(bad)).not.include('PRIVATE');
});
it('respects source handler precedence and canonical anonymous route IDs', () => {
for(const action of ['readme','branches','tree','file']) expect(requestRoute('/api/repo/private/files/'+action,'api')).equal('/api/repo/:owner/:repo/'+action);
expect(requestRoute('/api/repo/demo/file/readme','api')).equal('/api/repo/:repoId/file/:path');
for(const action of ['coauthors','extend','refresh']) {
expect(requestRoute('/api/repo/private/'+action,'api','GET')).equal('/api/repo/:owner/:repo');
expect(requestRoute('/api/repo/demo/'+action,'api','POST')).equal('/api/repo/:repoId/'+action);
}
expect(logRecord('info','requests',['request',{route:'/api/repo/:id/files'}]).route).equal('/api/repo/:repoId/files');
});
});
+45
View File
@@ -174,6 +174,7 @@ describe("bounded request monitoring", function () {
await ready; client.destroy(); await terminal;
expect(logs).to.have.length(1); expect(logs[0][0]).to.equal("request interrupted");
expect(logs[0][1].outcome).to.equal("interrupted");
expect(logs[0][1].cancellationOrigin).to.equal("unknown");
expect(logs[0][1]).include({requestPath:"/api/repo/demo-123/zip",repoId:"demo-123"});
expect(JSON.stringify(logs)).not.include("SECRET");
const row = metrics.drain().find(row => row.data.metric === "request").data;
@@ -186,6 +187,50 @@ describe("bounded request monitoring", function () {
});
}
for (const code of ["ETIMEDOUT", "ECONNRESET"]) {
it(`records causal ${code} before a truncated 200 closes`, async function () {
const { streamResponse } = require("../src/core/response-stream");
app.get("/", (_req, res) => {
const source = new Readable({ read() {} });
res.setHeader("Content-Length", "100");
void streamResponse(source, res, () => res.destroy());
source.push("partial");
setTimeout(() => source.destroy(Object.assign(new Error("PRIVATE"), {code})), 20);
});
await listen();
await fetch(server, "/").catch(() => undefined);
await delay(20);
const detail = logs[0][1];
expect(detail).include({status:200,outcome:"interrupted",bytes:7,expectedBytes:100,
cancellationOrigin:code === "ETIMEDOUT" ? "timeout" : "streamer"});
expect(JSON.stringify(logs)).not.include("PRIVATE");
});
}
it("correlates repeated streamer hops without accepting public caller IDs", async function () {
const upstream = express(); const hops=[];
upstream.use(monitorRequests("streamer",new RequestMetrics(),{info:(_,row)=>hops.push(row),warn:(_,row)=>hops.push(row)}));
upstream.post("/api",(_req,res)=>res.end("ok"));
const streamer = await new Promise(resolve => {const s=upstream.listen(0,"127.0.0.1",()=>resolve(s));});
try {
app.get("/", async (_req,res)=>{
await fetch(streamer,"/api",{method:"POST",headers:requestHeaders()});
await fetch(streamer,"/api",{method:"POST",headers:requestHeaders()});
res.end("ok");
});
await listen();
const forged="11111111-1111-1111-1111-111111111111";
await fetch(server,"/",{headers:{"x-request-id":forged,"x-parent-hop-id":forged}});
const parent=logs[0][1];
expect(parent.requestId).not.equal(forged);
expect(parent.parentHopId).equal(undefined);
expect(hops).length(2);
expect(hops[0].hopId).not.equal(hops[1].hopId);
for(const hop of hops) expect(hop).include({requestId:parent.requestId,parentHopId:parent.hopId,endpointKind:"internal",outcome:"completed"});
expect(parent).not.have.property("cancellationOrigin");
} finally {streamer.closeAllConnections();await new Promise(resolve=>streamer.close(resolve));}
});
it("keeps overlapping request IDs and asynchronous stages isolated", async function () {
let doneA, doneB;
app.get("/a", (_req, res) => { doneA = startStage("upstream"); setTimeout(() => { doneA(); doneA(); res.json(requestHeaders()); }, 30); });