diff --git a/CHANGELOG.md b/CHANGELOG.md index a6847d87a..de23edfbe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ Source: the Sept 28 weekly census (run 36385945043) and the two proof censuses o | Weekly periodic census wall clock | 2h 45m | 15 min for 32 of 33 machines (proof run 2) | | Longest planned slice | 160 min (one test, twice) | ~10 min | | Automatic retries on paid evals | up to 2 per file | 0 | -| Product-code type errors | 105 (no check) | 0, required in `free-tests` | +| Product-code type errors | 103 on v1.91.8.0 (no check) | 0, required in `free-tests` | | `lib/cso` longest source line | 2,159 chars | 785 (a string literal) | The biggest change is honesty. With 238 live cases, a retry used to hide a failing test; now every trial is recorded, `bun run eval:pass-rates` shows each case's pass rate with a confidence range, and a case that slides gets flagged by its history instead of passing on a lucky rerun. diff --git a/test/ceo-hold-posture-review.test.ts b/test/ceo-hold-posture-review.test.ts index 262b7c08b..ccc814e15 100644 --- a/test/ceo-hold-posture-review.test.ts +++ b/test/ceo-hold-posture-review.test.ts @@ -3,7 +3,7 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; import captured from './fixtures/ceo-hold-proof-fb10.json'; import { buildCeoHoldPostureReview, evaluateCeoHoldPostureReview, type CeoHoldPostureReviewInput } from './helpers/ceo-hold-posture-review'; -import { hasNativePostAnswerCeoPosture } from './helpers/ceo-mode-option'; +import { hasNativePostAnswerCeoPosture, holdDeferKeepIndex } from './helpers/ceo-mode-option'; import type { PlanReviewDecisionInput, PlanReviewDecisionJudgment } from './helpers/plan-review-decisions'; import type { NativePublicToolEvent } from './helpers/plan-count-transcript'; import { nativePlanCallFingerprint } from './helpers/claude-pty-runner'; @@ -243,7 +243,7 @@ async function registered(scenario:'accept'|'uncertain'|'missing source'|'missin navigateToModeAskUserQuestion:async()=>({modeIndex:3,visibleAtMode:'captured mode',question:{nativeCall:mode(f)}}), planCountQuestionInput:(_v:string,q:any)=>q.nativeCall.toolUseId===modeId?'3':'1',selectPtyNumberedOption:async()=>{throw Error('unexpected legacy key');}, hasNativePostAnswerCeoPosture:scenario==='lexical pass'||scenario==='expansion'?()=>true:hasNativePostAnswerCeoPosture, - ceoModeSubmissionInput:()=>null,ceoExpansionPacingReady:()=>false,ceoExpansionPacingChoice:()=>null, + ceoModeSubmissionInput:()=>null,ceoExpansionPacingReady:()=>false,ceoExpansionPacingChoice:()=>null,holdDeferKeepIndex, nextCeoPostureContinuation:(_a:any,_b:any,_c:any,_d:any,_e:any,continued:boolean)=>continued?null:'question', capturePlanCountQuestion:()=>({nativeCall:pending}),isPlanReadyVisible:()=>false,isNumberedOptionListVisible:()=>false, buildCeoHoldPostureReview,evaluateCeoHoldPostureReview:async(review:PlanReviewDecisionInput)=>{ diff --git a/test/ceo-mode-routing-fixture.test.ts b/test/ceo-mode-routing-fixture.test.ts index aa5309849..c90d9d375 100644 --- a/test/ceo-mode-routing-fixture.test.ts +++ b/test/ceo-mode-routing-fixture.test.ts @@ -52,6 +52,7 @@ mock.module(path.join(root,'test/helpers/ceo-mode-option.ts'),()=>({ ceoExpansionPacingChoice:()=>{current.pacingChoices++;return scenario.startsWith('pacing')&&(!current.pacingSent||scenario==='pacing-repeated')?{call:{questions:[]},index:scenario==='pacing-unsupported'?0:1}:null;}, ceoExpansionPacingReady:()=>{current.pacingChecks++;return (scenario==='pacing'||scenario==='pacing-repeated')&¤t.pacingChecks>=3;}, ceoModeSubmissionInput:()=>scenario==='mode-submit'&&!current.submitted?'\\r':null, + holdDeferKeepIndex:()=>null, nextCeoModeNavigation:(_visible,target)=>{ if(scenario==='navigation')throw new Error('fixture navigation failed'); current.mode=target; return {kind:'mode',index:target==='HOLD SCOPE'?2:1,question}; diff --git a/test/ceo-split-collection.test.ts b/test/ceo-split-collection.test.ts index 17a0da978..2dd9a39c6 100644 --- a/test/ceo-split-collection.test.ts +++ b/test/ceo-split-collection.test.ts @@ -176,7 +176,7 @@ const state = JSON.parse(fs.readFileSync(${JSON.stringify(inputPath)}, 'utf8')); const facts = { runs: 0, evaluators: 0, judges: 0, directory: '', candidateCalls: 0, suppliedCalls: 0 }; const save = () => fs.writeFileSync(${JSON.stringify(factsPath)}, JSON.stringify(facts)); mock.module(${JSON.stringify(path.join(ROOT, 'test/helpers/e2e-gate.ts'))}, () => ({ - describeE2ETier: tier => { expect(tier).toBe('periodic'); return describe; }, + describeE2ETier: tier => { expect(tier).toBe('marathon'); return describe; }, })); mock.module(${JSON.stringify(path.join(ROOT, 'test/helpers/plan-review-decisions.ts'))}, () => ({ evaluatePlanReviewDecisions: async input => { diff --git a/test/ceo-split-question-policy.test.ts b/test/ceo-split-question-policy.test.ts index cf5cd96c2..184754c79 100644 --- a/test/ceo-split-question-policy.test.ts +++ b/test/ceo-split-question-policy.test.ts @@ -300,7 +300,7 @@ mock.module(${JSON.stringify(path.join(ROOT, 'test/helpers/plan-review-decisions }, })); mock.module(${JSON.stringify(path.join(ROOT, 'test/helpers/e2e-gate.ts'))}, () => ({ - describeE2ETier: tier => { expect(tier).toBe('periodic'); return describe; }, + describeE2ETier: tier => { expect(tier).toBe('marathon'); return describe; }, })); const boundary = () => false; mock.module(${JSON.stringify(path.join(ROOT, 'test/helpers/claude-pty-runner.ts'))}, () => ({ diff --git a/test/eng-finding-retry-budget.test.ts b/test/eng-finding-retry-budget.test.ts index b8aa70b78..823f05d92 100644 --- a/test/eng-finding-retry-budget.test.ts +++ b/test/eng-finding-retry-budget.test.ts @@ -168,9 +168,14 @@ test('explicit allocation keeps its timer across load scheduling', () => { }); test('single-slice manifest retains all registered files with one allocation', () => { - const m = buildRunManifest({ tier: 'periodic', sliceCount: 1, evalsAll: true, env: { EVALS_ALL: '1' } }); - expect(m.entries.filter(e => e.status === 'planned').every(e => e.slice === 1)).toBe(true); - for (const budget of FINDING_RETRY_BUDGETS) expect(m.entries.find(e => e.file === budget.file)?.budget).toEqual(resolvePaidShardBudget([budget.file])); + // A registered file runs in exactly one scheduled lane: periodic, or the marathon lane for full flows. + const manifests = (['periodic', 'marathon'] as const).map(tier => buildRunManifest({ tier, sliceCount: 1, evalsAll: true, env: { EVALS_ALL: '1' } })); + for (const m of manifests) expect(m.entries.filter(e => e.status === 'planned').every(e => e.slice === 1)).toBe(true); + for (const budget of FINDING_RETRY_BUDGETS) { + const entries = manifests.flatMap(m => m.entries.filter(e => e.file === budget.file && e.status === 'planned')); + expect(entries, budget.file).toHaveLength(1); + expect(entries[0]!.budget).toEqual(resolvePaidShardBudget([budget.file])); + } }); test('current detach supervision covers the live-census floor', () => {