mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-06 00:44:07 +01:00
W4 T12: size the finding-floor assessment cap from measured latency (25 runs, p95 27.4s -> 90s cap)
This commit is contained in:
1 parent
f33763b40a
commit
4605627409
2 files changed
+13
-6
No files matched your search
@@ -206,7 +206,13 @@ function deterministicPlanFloorFinding(input: PlanFloorReview): PlanFloorAssessm
|
||||
});
|
||||
}
|
||||
|
||||
/** Same warmup CLI, one turn and 30s cap as the replaced waiting-state judge.
|
||||
/** Per-call cap on the warmup-model assessment. Measured 2026-10-03 with
|
||||
* Claude Code 2.1.284 and claude-haiku-4-5 on the committed captured inputs:
|
||||
* 25 calls (10 sequential, 15 at 3-way concurrency), p50 14.7s, p95 27.4s,
|
||||
* max 31.3s, so the old 30s cap cut the tail. 90s is about 3x p95. */
|
||||
export const PLAN_FLOOR_ASSESSMENT_CAP_MS = 90_000;
|
||||
|
||||
/** Same warmup CLI and one turn as the replaced waiting-state judge.
|
||||
* The original case deadline bounds each call; complete input is never truncated. */
|
||||
export function judgePlanFloorReview(input: PlanFloorReview, opts: {
|
||||
binary: string; model: string; deadlineAt: number;
|
||||
@@ -225,7 +231,7 @@ export function judgePlanFloorReview(input: PlanFloorReview, opts: {
|
||||
try {
|
||||
const result = (opts.invoke ?? spawnSync)(opts.binary,
|
||||
['-p', '--model', opts.model, '--max-turns', '1'],
|
||||
{ input: prompt, stdio: ['pipe', 'pipe', 'pipe'], timeout: Math.min(30_000, remaining), encoding: 'utf8' });
|
||||
{ input: prompt, stdio: ['pipe', 'pipe', 'pipe'], timeout: Math.min(PLAN_FLOOR_ASSESSMENT_CAP_MS, remaining), encoding: 'utf8' });
|
||||
Object.assign(diagnostic, { rawOutput: String(result.stdout ?? ''), stderr: String(result.stderr ?? ''), status: result.status });
|
||||
if (result.error || result.status !== 0 || Date.now() >= opts.deadlineAt)
|
||||
throw Error(`Floor assessment did not complete: ${result.error?.message ?? `exit ${result.status}`} ${String(result.stderr ?? '').slice(-3000)}`.trim());
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {expect,test,spyOn} from 'bun:test';
|
||||
import {buildPlanFloorReviewPrompt,validatePlanFloorAssessment,resolvePlanFloorCitations,judgePlanFloorReview,pickPlanFloorMode,pickPlanFloorProductType,type PlanFloorReview} from './helpers/plan-floor-review';
|
||||
import {buildPlanFloorReviewPrompt,validatePlanFloorAssessment,resolvePlanFloorCitations,judgePlanFloorReview,pickPlanFloorMode,pickPlanFloorProductType,PLAN_FLOOR_ASSESSMENT_CAP_MS,type PlanFloorReview} from './helpers/plan-floor-review';
|
||||
import {FORCING_FLOOR_CEO, FORCING_FLOOR_DEVEX} from './fixtures/forcing-finding-seeds';
|
||||
import capturedQuotes from './fixtures/plan-floor-quote-70b.json';
|
||||
import productTypes from './fixtures/plan-floor-product-type-70b.json';
|
||||
@@ -57,14 +57,15 @@ test('large complete input is preserved; over-limit input is rejected without in
|
||||
expect(()=>judgePlanFloorReview(input,{binary:'fake',model:'warmup',deadlineAt:Date.now()+30_000,invoke:(()=>{calls++;throw Error('must not execute');}) as any})).toThrow();
|
||||
expect(calls).toBe(0);
|
||||
});
|
||||
test('replacement judge retains the original CLI model, one turn, 30s cap and absolute case deadline',()=>{
|
||||
for(const remaining of [5000,60_000]){
|
||||
test('replacement judge retains the original CLI model, one turn, measured cap and absolute case deadline',()=>{
|
||||
expect(PLAN_FLOOR_ASSESSMENT_CAP_MS).toBeGreaterThan(30_000);
|
||||
for(const remaining of [5000,60_000,PLAN_FLOOR_ASSESSMENT_CAP_MS+60_000]){
|
||||
const deadlineAt=Date.now()+remaining;let calls=0;
|
||||
const actual=judgePlanFloorReview(review(),{binary:'/fake/claude',model:'unchanged-warmup',deadlineAt,invoke:((file,args,opts)=>{
|
||||
calls++;expect(file).toBe('/fake/claude');expect(args).toEqual(['-p','--model','unchanged-warmup','--max-turns','1']);
|
||||
expect(opts.stdio).toEqual(['pipe','pipe','pipe']);expect(opts.encoding).toBe('utf8');
|
||||
expect(opts.input).toBe(buildPlanFloorReviewPrompt(review()));
|
||||
expect(opts.timeout).toBeGreaterThan(0);expect(opts.timeout).toBeLessThanOrEqual(Math.min(30_000,remaining));
|
||||
expect(opts.timeout).toBeGreaterThan(0);expect(opts.timeout).toBeLessThanOrEqual(Math.min(PLAN_FLOOR_ASSESSMENT_CAP_MS,remaining));
|
||||
return {status:0,stdout:JSON.stringify(citationFinding()),stderr:''};
|
||||
}) as any});expect(calls).toBe(1);expect(actual.kind).toBe('finding');
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user