diff --git a/CHANGELOG.md b/CHANGELOG.md index b4652fd20..d1c33189e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,8 @@ When /ship's measure loop prints needs-classify, read the listed reds and record #### Fixed - `ship-measure measure` no longer refuses a case that shares its test file with other cases (for example `ship-exploratory-late-input`): the no-cost selection check before the first paid trial now accepts the per-case trial shard such a case lists. +- `ship-measure measure` runs cases that keep native evidence (functional QA, docs faults) outside CI: each trial gets its own `EVALS_RUN_ID`, where before every trial stopped at "requires EVALS_RUN_ID" without starting a session. +- The `plan-mode-no-op` gate case no longer times out on large pull requests: its CEO review reads a fixed small plan instead of the branch diff. #### For contributors - The bar is one pure module, `scripts/lib/measure-bar.ts`, pinned by `test/ship-measure-bar.test.ts`; the sweep is `scripts/ship-measure-sweep.ts`, pinned by `test/ship-measure-sweep.test.ts` and `test/eval-sweep-workflow.test.ts`. `eval:pass-rates` history loading is now `loadPassRateHistory()` in `scripts/eval-flake-rank.ts`, shared with the sweep. diff --git a/scripts/ship-measure.ts b/scripts/ship-measure.ts index 6519ef36e..187d232f9 100644 --- a/scripts/ship-measure.ts +++ b/scripts/ship-measure.ts @@ -540,9 +540,11 @@ export function caseSelectionPreflight(caseId: string, rootDir = ROOT): { ok: bo * and never publish: with the judge input cache enabled, a trial could reuse a * stored pass, and its own passes would flow into the gate's reuse. */ -export function diagnosticBaseEnv(base: NodeJS.ProcessEnv, evalDir: string): NodeJS.ProcessEnv { +export function diagnosticBaseEnv(base: NodeJS.ProcessEnv, evalDir: string, trialId: string): NodeJS.ProcessEnv { const { EVALS_CACHE_DIR: _cacheDir, EVALS_CACHE_RUNTIME_ID: _cacheRuntime, ...inherited } = base; - return { ...inherited, GSTACK_EVAL_DIR: evalDir, GSTACK_SHIP_MEASURE_LABEL: 'diagnostic', EVALS_JOBS: '1' }; + // Cases that retain native evidence (functional QA, docs faults) require EVALS_RUN_ID, which only CI sets; + // each trial gets its own so parallel trials never share an evidence directory. + return { ...inherited, GSTACK_EVAL_DIR: evalDir, GSTACK_SHIP_MEASURE_LABEL: 'diagnostic', EVALS_JOBS: '1', EVALS_RUN_ID: `${base.EVALS_RUN_ID || 'local'}-measure-${trialId}` }; } /** True when some junit.xml under evalDir holds an executed, passing testcase and no failed or errored one. */ @@ -636,7 +638,7 @@ export function judgeFile(id: string, rootDir = ROOT): string { * with the judge selected alone and passes only on its own passing record. */ export function gstackRunner(rootDir = ROOT): TrialRunner { - return async ({ caseId, dir }) => { + return async ({ caseId, round, trial, dir }) => { const evalDir = path.join(dir, 'eval'); const isJudge = !Object.hasOwn(E2E_TIERS, caseId); if (isJudge && !Object.hasOwn(LLM_JUDGE_TOUCHFILES, caseId)) throw new Error(`${caseId} is neither an E2E case nor a standalone judge`); @@ -644,7 +646,7 @@ export function gstackRunner(rootDir = ROOT): TrialRunner { ? [process.execPath, 'test', path.join(rootDir, judgeFile(caseId, rootDir))] : [process.execPath, 'run', path.join(rootDir, 'scripts/test-paid-shards.ts'), '--tier', E2E_TIERS[caseId]!, '--case', caseId, '--trials', '1']; const env = { - ...diagnosticBaseEnv(process.env, evalDir), + ...diagnosticBaseEnv(process.env, evalDir, `${caseSlug(caseId)}-${round}-t${trial}-${process.pid}`), ...(isJudge ? { EVALS: '1', EVALS_TIER: 'gate', EVALS_ALL: '1', ...paidSelectionEnv('full', { e2e: [], judges: [caseId] }, 'ship-measure judge') } : {}), }; const { code, output } = await runTrialProcess(argv, rootDir, env, dir); diff --git a/test/ship-measure.test.ts b/test/ship-measure.test.ts index b8f6cf6d4..b4d2a12a6 100644 --- a/test/ship-measure.test.ts +++ b/test/ship-measure.test.ts @@ -341,11 +341,13 @@ describe('diagnostic trials prove they ran and never touch the judge cache', () expect(junitExecuted(dirWith(junit('')))).toBe(true); expect(junitExecuted(dirWith(junit('')))).toBe(true); }); - test('the judge input cache is never inherited by a diagnostic trial', () => { - const env = diagnosticBaseEnv({ EVALS_CACHE_DIR: '/c', EVALS_CACHE_RUNTIME_ID: 'img', KEEP: '1' }, '/e'); + test('the judge input cache is never inherited by a diagnostic trial; each trial gets its own run id', () => { + const env = diagnosticBaseEnv({ EVALS_CACHE_DIR: '/c', EVALS_CACHE_RUNTIME_ID: 'img', KEEP: '1' }, '/e', 'case-baseline-t3'); expect(env).not.toHaveProperty('EVALS_CACHE_DIR'); expect(env).not.toHaveProperty('EVALS_CACHE_RUNTIME_ID'); - expect(env).toMatchObject({ KEEP: '1', GSTACK_EVAL_DIR: '/e', GSTACK_SHIP_MEASURE_LABEL: 'diagnostic' }); + expect(env).toMatchObject({ KEEP: '1', GSTACK_EVAL_DIR: '/e', GSTACK_SHIP_MEASURE_LABEL: 'diagnostic', EVALS_RUN_ID: 'local-measure-case-baseline-t3' }); + // Every trial has its own run id (evidence-retaining cases refuse to start without one); CI's id is kept as the prefix. + expect(diagnosticBaseEnv({ EVALS_RUN_ID: 'ci-9-1-eval-sweep' }, '/e', 'x-t1').EVALS_RUN_ID).toBe('ci-9-1-eval-sweep-measure-x-t1'); }); });