W4 T12: size the finding-floor assessment cap from measured latency (25 runs, p95 27.4s -> 90s cap)

This commit is contained in:
garrytan committed 2026-10-03 20:38:29 +00:00
1 parent f33763b40a
commit 4605627409
2 files changed
+13 -6

No files matched your search

+8 -2
View File
@@ -206,7 +206,13 @@ function deterministicPlanFloorFinding(input: PlanFloorReview): PlanFloorAssessm
});
}
/** Same warmup CLI, one turn and 30s cap as the replaced waiting-state judge.
/** Per-call cap on the warmup-model assessment. Measured 2026-10-03 with
* Claude Code 2.1.284 and claude-haiku-4-5 on the committed captured inputs:
* 25 calls (10 sequential, 15 at 3-way concurrency), p50 14.7s, p95 27.4s,
* max 31.3s, so the old 30s cap cut the tail. 90s is about 3x p95. */
export const PLAN_FLOOR_ASSESSMENT_CAP_MS = 90_000;
/** Same warmup CLI and one turn as the replaced waiting-state judge.
* The original case deadline bounds each call; complete input is never truncated. */
export function judgePlanFloorReview(input: PlanFloorReview, opts: {
binary: string; model: string; deadlineAt: number;
@@ -225,7 +231,7 @@ export function judgePlanFloorReview(input: PlanFloorReview, opts: {
try {
const result = (opts.invoke ?? spawnSync)(opts.binary,
['-p', '--model', opts.model, '--max-turns', '1'],
{ input: prompt, stdio: ['pipe', 'pipe', 'pipe'], timeout: Math.min(30_000, remaining), encoding: 'utf8' });
{ input: prompt, stdio: ['pipe', 'pipe', 'pipe'], timeout: Math.min(PLAN_FLOOR_ASSESSMENT_CAP_MS, remaining), encoding: 'utf8' });
Object.assign(diagnostic, { rawOutput: String(result.stdout ?? ''), stderr: String(result.stderr ?? ''), status: result.status });
if (result.error || result.status !== 0 || Date.now() >= opts.deadlineAt)
throw Error(`Floor assessment did not complete: ${result.error?.message ?? `exit ${result.status}`} ${String(result.stderr ?? '').slice(-3000)}`.trim());
+5 -4
View File
@@ -1,5 +1,5 @@
import {expect,test,spyOn} from 'bun:test';
import {buildPlanFloorReviewPrompt,validatePlanFloorAssessment,resolvePlanFloorCitations,judgePlanFloorReview,pickPlanFloorMode,pickPlanFloorProductType,type PlanFloorReview} from './helpers/plan-floor-review';
import {buildPlanFloorReviewPrompt,validatePlanFloorAssessment,resolvePlanFloorCitations,judgePlanFloorReview,pickPlanFloorMode,pickPlanFloorProductType,PLAN_FLOOR_ASSESSMENT_CAP_MS,type PlanFloorReview} from './helpers/plan-floor-review';
import {FORCING_FLOOR_CEO, FORCING_FLOOR_DEVEX} from './fixtures/forcing-finding-seeds';
import capturedQuotes from './fixtures/plan-floor-quote-70b.json';
import productTypes from './fixtures/plan-floor-product-type-70b.json';
@@ -57,14 +57,15 @@ test('large complete input is preserved; over-limit input is rejected without in
expect(()=>judgePlanFloorReview(input,{binary:'fake',model:'warmup',deadlineAt:Date.now()+30_000,invoke:(()=>{calls++;throw Error('must not execute');}) as any})).toThrow();
expect(calls).toBe(0);
});
test('replacement judge retains the original CLI model, one turn, 30s cap and absolute case deadline',()=>{
for(const remaining of [5000,60_000]){
test('replacement judge retains the original CLI model, one turn, measured cap and absolute case deadline',()=>{
expect(PLAN_FLOOR_ASSESSMENT_CAP_MS).toBeGreaterThan(30_000);
for(const remaining of [5000,60_000,PLAN_FLOOR_ASSESSMENT_CAP_MS+60_000]){
const deadlineAt=Date.now()+remaining;let calls=0;
const actual=judgePlanFloorReview(review(),{binary:'/fake/claude',model:'unchanged-warmup',deadlineAt,invoke:((file,args,opts)=>{
calls++;expect(file).toBe('/fake/claude');expect(args).toEqual(['-p','--model','unchanged-warmup','--max-turns','1']);
expect(opts.stdio).toEqual(['pipe','pipe','pipe']);expect(opts.encoding).toBe('utf8');
expect(opts.input).toBe(buildPlanFloorReviewPrompt(review()));
expect(opts.timeout).toBeGreaterThan(0);expect(opts.timeout).toBeLessThanOrEqual(Math.min(30_000,remaining));
expect(opts.timeout).toBeGreaterThan(0);expect(opts.timeout).toBeLessThanOrEqual(Math.min(PLAN_FLOOR_ASSESSMENT_CAP_MS,remaining));
return {status:0,stdout:JSON.stringify(citationFinding()),stderr:''};
}) as any});expect(calls).toBe(1);expect(actual.kind).toBe('finding');
}