diff --git a/CHANGELOG.md b/CHANGELOG.md
index b4652fd20..d1c33189e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -43,6 +43,8 @@ When /ship's measure loop prints needs-classify, read the listed reds and record
#### Fixed
- `ship-measure measure` no longer refuses a case that shares its test file with other cases (for example `ship-exploratory-late-input`): the no-cost selection check before the first paid trial now accepts the per-case trial shard such a case lists.
+- `ship-measure measure` runs cases that keep native evidence (functional QA, docs faults) outside CI: each trial gets its own `EVALS_RUN_ID`, where before every trial stopped at "requires EVALS_RUN_ID" without starting a session.
+- The `plan-mode-no-op` gate case no longer times out on large pull requests: its CEO review reads a fixed small plan instead of the branch diff.
#### For contributors
- The bar is one pure module, `scripts/lib/measure-bar.ts`, pinned by `test/ship-measure-bar.test.ts`; the sweep is `scripts/ship-measure-sweep.ts`, pinned by `test/ship-measure-sweep.test.ts` and `test/eval-sweep-workflow.test.ts`. `eval:pass-rates` history loading is now `loadPassRateHistory()` in `scripts/eval-flake-rank.ts`, shared with the sweep.
diff --git a/scripts/ship-measure.ts b/scripts/ship-measure.ts
index 6519ef36e..187d232f9 100644
--- a/scripts/ship-measure.ts
+++ b/scripts/ship-measure.ts
@@ -540,9 +540,11 @@ export function caseSelectionPreflight(caseId: string, rootDir = ROOT): { ok: bo
* and never publish: with the judge input cache enabled, a trial could reuse a
* stored pass, and its own passes would flow into the gate's reuse.
*/
-export function diagnosticBaseEnv(base: NodeJS.ProcessEnv, evalDir: string): NodeJS.ProcessEnv {
+export function diagnosticBaseEnv(base: NodeJS.ProcessEnv, evalDir: string, trialId: string): NodeJS.ProcessEnv {
const { EVALS_CACHE_DIR: _cacheDir, EVALS_CACHE_RUNTIME_ID: _cacheRuntime, ...inherited } = base;
- return { ...inherited, GSTACK_EVAL_DIR: evalDir, GSTACK_SHIP_MEASURE_LABEL: 'diagnostic', EVALS_JOBS: '1' };
+ // Cases that retain native evidence (functional QA, docs faults) require EVALS_RUN_ID, which only CI sets;
+ // each trial gets its own so parallel trials never share an evidence directory.
+ return { ...inherited, GSTACK_EVAL_DIR: evalDir, GSTACK_SHIP_MEASURE_LABEL: 'diagnostic', EVALS_JOBS: '1', EVALS_RUN_ID: `${base.EVALS_RUN_ID || 'local'}-measure-${trialId}` };
}
/** True when some junit.xml under evalDir holds an executed, passing testcase and no failed or errored one. */
@@ -636,7 +638,7 @@ export function judgeFile(id: string, rootDir = ROOT): string {
* with the judge selected alone and passes only on its own passing record.
*/
export function gstackRunner(rootDir = ROOT): TrialRunner {
- return async ({ caseId, dir }) => {
+ return async ({ caseId, round, trial, dir }) => {
const evalDir = path.join(dir, 'eval');
const isJudge = !Object.hasOwn(E2E_TIERS, caseId);
if (isJudge && !Object.hasOwn(LLM_JUDGE_TOUCHFILES, caseId)) throw new Error(`${caseId} is neither an E2E case nor a standalone judge`);
@@ -644,7 +646,7 @@ export function gstackRunner(rootDir = ROOT): TrialRunner {
? [process.execPath, 'test', path.join(rootDir, judgeFile(caseId, rootDir))]
: [process.execPath, 'run', path.join(rootDir, 'scripts/test-paid-shards.ts'), '--tier', E2E_TIERS[caseId]!, '--case', caseId, '--trials', '1'];
const env = {
- ...diagnosticBaseEnv(process.env, evalDir),
+ ...diagnosticBaseEnv(process.env, evalDir, `${caseSlug(caseId)}-${round}-t${trial}-${process.pid}`),
...(isJudge ? { EVALS: '1', EVALS_TIER: 'gate', EVALS_ALL: '1', ...paidSelectionEnv('full', { e2e: [], judges: [caseId] }, 'ship-measure judge') } : {}),
};
const { code, output } = await runTrialProcess(argv, rootDir, env, dir);
diff --git a/test/ship-measure.test.ts b/test/ship-measure.test.ts
index b8f6cf6d4..b4d2a12a6 100644
--- a/test/ship-measure.test.ts
+++ b/test/ship-measure.test.ts
@@ -341,11 +341,13 @@ describe('diagnostic trials prove they ran and never touch the judge cache', ()
expect(junitExecuted(dirWith(junit('')))).toBe(true);
expect(junitExecuted(dirWith(junit('')))).toBe(true);
});
- test('the judge input cache is never inherited by a diagnostic trial', () => {
- const env = diagnosticBaseEnv({ EVALS_CACHE_DIR: '/c', EVALS_CACHE_RUNTIME_ID: 'img', KEEP: '1' }, '/e');
+ test('the judge input cache is never inherited by a diagnostic trial; each trial gets its own run id', () => {
+ const env = diagnosticBaseEnv({ EVALS_CACHE_DIR: '/c', EVALS_CACHE_RUNTIME_ID: 'img', KEEP: '1' }, '/e', 'case-baseline-t3');
expect(env).not.toHaveProperty('EVALS_CACHE_DIR');
expect(env).not.toHaveProperty('EVALS_CACHE_RUNTIME_ID');
- expect(env).toMatchObject({ KEEP: '1', GSTACK_EVAL_DIR: '/e', GSTACK_SHIP_MEASURE_LABEL: 'diagnostic' });
+ expect(env).toMatchObject({ KEEP: '1', GSTACK_EVAL_DIR: '/e', GSTACK_SHIP_MEASURE_LABEL: 'diagnostic', EVALS_RUN_ID: 'local-measure-case-baseline-t3' });
+ // Every trial has its own run id (evidence-retaining cases refuse to start without one); CI's id is kept as the prefix.
+ expect(diagnosticBaseEnv({ EVALS_RUN_ID: 'ci-9-1-eval-sweep' }, '/e', 'x-t1').EVALS_RUN_ID).toBe('ci-9-1-eval-sweep-measure-x-t1');
});
});