fix(ship-measure): each diagnostic trial gets its own EVALS_RUN_ID

Evidence-retaining cases (functional QA, docs faults) refuse to start
without EVALS_RUN_ID, which only CI sets, so measuring them locally ran
0/10 sessions. qa-functional-cli-report, red once in this PR's gate,
measured 10/10 once the trials started ($11.13).

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

CHANGELOG.md

@@ -43,6 +43,8 @@ When /ship's measure loop prints needs-classify, read the listed reds and record
 
 #### Fixed
 - `ship-measure measure` no longer refuses a case that shares its test file with other cases (for example `ship-exploratory-late-input`): the no-cost selection check before the first paid trial now accepts the per-case trial shard such a case lists.
+- `ship-measure measure` runs cases that keep native evidence (functional QA, docs faults) outside CI: each trial gets its own `EVALS_RUN_ID`, where before every trial stopped at "requires EVALS_RUN_ID" without starting a session.
+- The `plan-mode-no-op` gate case no longer times out on large pull requests: its CEO review reads a fixed small plan instead of the branch diff.
 
 #### For contributors
 - The bar is one pure module, `scripts/lib/measure-bar.ts`, pinned by `test/ship-measure-bar.test.ts`; the sweep is `scripts/ship-measure-sweep.ts`, pinned by `test/ship-measure-sweep.test.ts` and `test/eval-sweep-workflow.test.ts`. `eval:pass-rates` history loading is now `loadPassRateHistory()` in `scripts/eval-flake-rank.ts`, shared with the sweep.

scripts/ship-measure.ts

@@ -540,9 +540,11 @@ export function caseSelectionPreflight(caseId: string, rootDir = ROOT): { ok: bo
  * and never publish: with the judge input cache enabled, a trial could reuse a
  * stored pass, and its own passes would flow into the gate's reuse.
  */
-export function diagnosticBaseEnv(base: NodeJS.ProcessEnv, evalDir: string): NodeJS.ProcessEnv {
+export function diagnosticBaseEnv(base: NodeJS.ProcessEnv, evalDir: string, trialId: string): NodeJS.ProcessEnv {
   const { EVALS_CACHE_DIR: _cacheDir, EVALS_CACHE_RUNTIME_ID: _cacheRuntime, ...inherited } = base;
-  return { ...inherited, GSTACK_EVAL_DIR: evalDir, GSTACK_SHIP_MEASURE_LABEL: 'diagnostic', EVALS_JOBS: '1' };
+  // Cases that retain native evidence (functional QA, docs faults) require EVALS_RUN_ID, which only CI sets;
+  // each trial gets its own so parallel trials never share an evidence directory.
+  return { ...inherited, GSTACK_EVAL_DIR: evalDir, GSTACK_SHIP_MEASURE_LABEL: 'diagnostic', EVALS_JOBS: '1', EVALS_RUN_ID: `${base.EVALS_RUN_ID || 'local'}-measure-${trialId}` };
 }
 
 /** True when some junit.xml under evalDir holds an executed, passing testcase and no failed or errored one. */
@@ -636,7 +638,7 @@ export function judgeFile(id: string, rootDir = ROOT): string {
  * with the judge selected alone and passes only on its own passing record.
  */
 export function gstackRunner(rootDir = ROOT): TrialRunner {
-  return async ({ caseId, dir }) => {
+  return async ({ caseId, round, trial, dir }) => {
     const evalDir = path.join(dir, 'eval');
     const isJudge = !Object.hasOwn(E2E_TIERS, caseId);
     if (isJudge && !Object.hasOwn(LLM_JUDGE_TOUCHFILES, caseId)) throw new Error(`${caseId} is neither an E2E case nor a standalone judge`);
@@ -644,7 +646,7 @@ export function gstackRunner(rootDir = ROOT): TrialRunner {
       ? [process.execPath, 'test', path.join(rootDir, judgeFile(caseId, rootDir))]
       : [process.execPath, 'run', path.join(rootDir, 'scripts/test-paid-shards.ts'), '--tier', E2E_TIERS[caseId]!, '--case', caseId, '--trials', '1'];
     const env = {
-      ...diagnosticBaseEnv(process.env, evalDir),
+      ...diagnosticBaseEnv(process.env, evalDir, `${caseSlug(caseId)}-${round}-t${trial}-${process.pid}`),
       ...(isJudge ? { EVALS: '1', EVALS_TIER: 'gate', EVALS_ALL: '1', ...paidSelectionEnv('full', { e2e: [], judges: [caseId] }, 'ship-measure judge') } : {}),
     };
     const { code, output } = await runTrialProcess(argv, rootDir, env, dir);

test/ship-measure.test.ts

@@ -341,11 +341,13 @@ describe('diagnostic trials prove they ran and never touch the judge cache', ()
     expect(junitExecuted(dirWith(junit('<testcase name="a" time="1"/>')))).toBe(true);
     expect(junitExecuted(dirWith(junit('<testcase name="a" time="1"></testcase><testcase name="b"><skipped/></testcase>')))).toBe(true);
   });
-  test('the judge input cache is never inherited by a diagnostic trial', () => {
-    const env = diagnosticBaseEnv({ EVALS_CACHE_DIR: '/c', EVALS_CACHE_RUNTIME_ID: 'img', KEEP: '1' }, '/e');
+  test('the judge input cache is never inherited by a diagnostic trial; each trial gets its own run id', () => {
+    const env = diagnosticBaseEnv({ EVALS_CACHE_DIR: '/c', EVALS_CACHE_RUNTIME_ID: 'img', KEEP: '1' }, '/e', 'case-baseline-t3');
     expect(env).not.toHaveProperty('EVALS_CACHE_DIR');
     expect(env).not.toHaveProperty('EVALS_CACHE_RUNTIME_ID');
-    expect(env).toMatchObject({ KEEP: '1', GSTACK_EVAL_DIR: '/e', GSTACK_SHIP_MEASURE_LABEL: 'diagnostic' });
+    expect(env).toMatchObject({ KEEP: '1', GSTACK_EVAL_DIR: '/e', GSTACK_SHIP_MEASURE_LABEL: 'diagnostic', EVALS_RUN_ID: 'local-measure-case-baseline-t3' });
+    // Every trial has its own run id (evidence-retaining cases refuse to start without one); CI's id is kept as the prefix.
+    expect(diagnosticBaseEnv({ EVALS_RUN_ID: 'ci-9-1-eval-sweep' }, '/e', 'x-t1').EVALS_RUN_ID).toBe('ci-9-1-eval-sweep-measure-x-t1');
   });
 });