Merge remote-tracking branch 'origin/main' into capy/merge-train-workflow-side

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

# Conflicts:
#	CHANGELOG.md
#	scripts/harness-version.json

scripts/census-arms.ts

@@ -0,0 +1,81 @@
+#!/usr/bin/env bun
+/**
+ * Gate census arms for evals-periodic.yml (GSTA-49 TM2). Each arm is one host
+ * and model running the weekly gate census plan:
+ *   - Monday cron (0 6 * * 1): claude-opus (the capture model, lib/eval-model.ts)
+ *     and codex-sol (gpt-6.1-sol on the Codex host, adapter cases only);
+ *   - monthly cron (0 6 1 * *): claude-fable and codex-astra, census jobs only;
+ *   - dispatch: `census_arms` runs only the census with the named arms; without
+ *     it a dispatch keeps the periodic lane plus the Monday arms when the census
+ *     runs (main, or include_gate_census).
+ * Judges stay in the Claude lanes: every census plan skips them.
+ *
+ * CLI (GitHub step outputs and env lines):
+ *   --select                 env SCHEDULE, CENSUS_ARMS, INCLUDE_GATE_CENSUS -> arms=<json>, periodic_lane=<bool>
+ *   --env <arm>              KEY=VALUE lines selecting the arm's host and model
+ *   --matrix <dir> <arms>    gate_matrix=<json> from <dir>/<arm>/manifest.json
+ */
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import type { EvalHost } from '../test/helpers/e2e-hosts';
+
+export interface CensusArm { host: EvalHost; model?: string }
+
+export const CENSUS_ARMS: Readonly<Record<string, CensusArm>> = Object.freeze({
+  // No model: the Claude runners' capture default (claude-opus-5-5 since TM1).
+  'claude-opus': { host: 'claude' },
+  'codex-sol': { host: 'codex', model: 'gpt-6.1-sol' },
+  'claude-fable': { host: 'claude', model: 'claude-fable-5-1' },
+  'codex-astra': { host: 'codex', model: 'gpt-6-astra' },
+});
+
+export const WEEKLY_CENSUS_SCHEDULE = '0 6 * * 1';
+export const MONTHLY_CENSUS_SCHEDULE = '0 6 1 * *';
+export const WEEKLY_CENSUS_ARMS: readonly string[] = ['claude-opus', 'codex-sol'];
+export const MONTHLY_CENSUS_ARMS: readonly string[] = ['claude-fable', 'codex-astra'];
+
+/** Which census arms run, and whether the periodic lane runs beside them. */
+export function selectCensus(opts: { schedule?: string; censusArms?: string; includeGateCensus: boolean }): { arms: string[]; periodicLane: boolean } {
+  if (opts.schedule === MONTHLY_CENSUS_SCHEDULE) return { arms: [...MONTHLY_CENSUS_ARMS], periodicLane: false };
+  const named = (opts.censusArms ?? '').split(/[\s,]+/).filter(Boolean);
+  const unknown = named.filter(arm => !Object.hasOwn(CENSUS_ARMS, arm));
+  if (unknown.length) throw new Error(`unknown census arm(s): ${unknown.join(', ')}; known: ${Object.keys(CENSUS_ARMS).join(', ')}`);
+  if (named.length) return { arms: [...new Set(named)], periodicLane: false };
+  return { arms: opts.includeGateCensus ? [...WEEKLY_CENSUS_ARMS] : [], periodicLane: true };
+}
+
+/** The env one arm's planner and executor run with. */
+export function censusArmEnv(arm: string): Record<string, string> {
+  const spec = CENSUS_ARMS[arm];
+  if (!spec) throw new Error(`unknown census arm: ${arm}`);
+  if (!spec.model) return { EVALS_HOST: spec.host };
+  return { EVALS_HOST: spec.host, [spec.host === 'codex' ? 'GSTACK_CODEX_MODEL' : 'EVALS_MODEL']: spec.model };
+}
+
+/** One executor per arm and planned slice, with that slice's planned CI ceiling. */
+export function censusMatrix(arms: readonly string[], planDir: string): Array<{ arm: string; slice: number; timeout: number }> {
+  return arms.flatMap(arm => {
+    const manifest = JSON.parse(fs.readFileSync(path.join(planDir, arm, 'manifest.json'), 'utf8')) as
+      { sliceCount: number; plan?: { ciTimeoutMinutes: number; sliceCiTimeoutMinutes?: number[] } };
+    if (!manifest.plan) throw new Error(`${arm}: census manifest has no budget plan`);
+    return Array.from({ length: manifest.sliceCount }, (_, i) => ({
+      arm, slice: i + 1, timeout: manifest.plan!.sliceCiTimeoutMinutes?.[i] ?? manifest.plan!.ciTimeoutMinutes,
+    }));
+  });
+}
+
+if (import.meta.main) {
+  const [mode, ...args] = process.argv.slice(2);
+  if (mode === '--select') {
+    const { arms, periodicLane } = selectCensus({ schedule: process.env.SCHEDULE, censusArms: process.env.CENSUS_ARMS,
+      includeGateCensus: process.env.INCLUDE_GATE_CENSUS === 'true' });
+    console.log(`arms=${JSON.stringify(arms)}\nperiodic_lane=${periodicLane}`);
+  } else if (mode === '--env' && args.length === 1) {
+    for (const [key, value] of Object.entries(censusArmEnv(args[0]!))) console.log(`${key}=${value}`);
+  } else if (mode === '--matrix' && args.length === 2) {
+    console.log(`gate_matrix=${JSON.stringify(censusMatrix(JSON.parse(args[1]!) as string[], args[0]!))}`);
+  } else {
+    console.error('usage: census-arms.ts --select | --env <arm> | --matrix <plan dir> <arms json>');
+    process.exit(2);
+  }
+}

scripts/e2e-shard-reuse.ts

@@ -83,6 +83,7 @@ export function e2eReuseLaneProblem(env: NodeJS.ProcessEnv, profileMode: string
   if (!/^(?:sha256:)?[a-f0-9]{64}$/.test(env.EVALS_CACHE_RUNTIME_ID ?? '')) return 'No immutable runtime identity';
   if (env.EVALS_TIER !== 'gate' || env.EVALS_FRESH === '1') return 'Fresh validation requested';
   if (['release', 'periodic', 'marathon'].includes(env.EVALS_CACHE_PURPOSE ?? '')) return 'Scheduled and release lanes execute fresh';
+  if ((env.EVALS_HOST ?? '').trim() && env.EVALS_HOST!.trim() !== 'claude') return 'Non-Claude hosts execute fresh';
   if (env.NODE_OPTIONS || env.BUN_OPTIONS) return 'Preload options change execution outside the consumed source';
   if (env.ANTHROPIC_BASE_URL && env.ANTHROPIC_BASE_URL !== 'https://api.anthropic.com') return 'Custom model endpoint';
   return null;

scripts/harness-version.json

@@ -3,6 +3,7 @@
   "patterns": [
     "test/helpers/agent-sdk-runner.ts",
     "test/helpers/codex-session-runner.ts",
+    "test/helpers/codex-host-runner.ts",
     "test/helpers/claude-pty-runner.ts",
     "test/helpers/pty/**",
     "test/helpers/pty-screen.ts",
@@ -10,22 +11,23 @@
     "test/helpers/llm-judge.ts"
   ],
   "files": {
-    "scripts/lib/paid-cases.ts": {"blob":"9bacd52446192a7ecf3dfaae901ea443cf9c9ae2","since":6},
-    "scripts/lib/paid-plan.ts": {"blob":"45721ce128a112666a0caed6c13c66637b74ca54","since":6},
-    "scripts/lib/paid-report.ts": {"blob":"b66c41237611ccecaaa7bd6a4d7b7aac8dc548cd","since":6},
+    "scripts/lib/paid-cases.ts": {"blob":"f862cf5241580c7593b9ee62eacf5ef046248d26","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
+    "scripts/lib/paid-plan.ts": {"blob":"95e1798e7e2495ace350d2624e59f83412f341bc","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
+    "scripts/lib/paid-report.ts": {"blob":"f92c6bb723b158e328aa040fa5de6a0ee6d3916d","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
     "scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":6},
     "scripts/lib/shard-engine.ts": {"blob":"3cd0b1aaa84018b00bf9ea0ce48093f3c8492f66","since":6},
-    "scripts/test-paid-shards.ts": {"blob":"8ddee5f11ed7856876135520dd8025ab989e2db9","since":6},
+    "scripts/test-paid-shards.ts": {"blob":"19c9b2376d09090aee1a8acacfe721d66f55b0fc","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
     "scripts/test-pr-profile.ts": {"blob":"d334195b0d0393622e2c380dc3e2feebe099b6e6","since":6,"nonBehavioral":"classify the new evals-main-backstop.yml workflow as free-lane only: it dispatches evals.yml on main and runs inside no case, so no case's selection on a PR, input or verdict changes"},
     "scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":6},
     "test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":6},
     "test/helpers/auq-substance-panel.ts": {"blob":"4881260ef793668e4963a062db6d06a5e1f7a40f","since":6},
     "test/helpers/claude-pty-runner.ts": {"blob":"20cc5894532873aa6145fc6617fdeec8cef94a2d","since":6},
-    "test/helpers/codex-session-runner.ts": {"blob":"4eea35931a35a0afb006363402e3d4191e379737","since":6},
+    "test/helpers/codex-host-runner.ts": {"blob":"bff225017e235f0be6ece29f04971ac04b95b544","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
+    "test/helpers/codex-session-runner.ts": {"blob":"c040ebbfa815b8cce19ef80c6133bd5a5474149a","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
     "test/helpers/e2e-gate.ts": {"blob":"efb7e197dd5f66c3ad0e34074b14e640754f7a45","since":6},
-    "test/helpers/e2e-helpers.ts": {"blob":"2d4a2a53063cfc1f003db366d2c861ab1e379e6e","since":6},
+    "test/helpers/e2e-helpers.ts": {"blob":"6ee6937e6c3ca081c77131b5a755278c7b714834","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
     "test/helpers/eval-budgets.ts": {"blob":"667d0e09b20fe5d7366f57b77b5fb9b04baa37ef","since":6},
-    "test/helpers/eval-store.ts": {"blob":"609c6d099108ff2696eb014b47aa318a618dd2af","since":6},
+    "test/helpers/eval-store.ts": {"blob":"ed8721f13d39f4ef62e0a1d814efbe9352da781a","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
     "test/helpers/hermetic-env.ts": {"blob":"cc072895461dc52ea317d1beff72121342420d30","since":6},
     "test/helpers/llm-judge.ts": {"blob":"d0bc6bbbe9f478603d11a1ece95e66c812ad34e6","since":6},
     "test/helpers/paid-test-set.ts": {"blob":"ff9bfe43c43ec2bdd11114f7b8ec7c5736116f75","since":6},
@@ -44,8 +46,8 @@
     "test/helpers/pty/screen.ts": {"blob":"ba7a80bb68b1959296ab2a827cfacdaa46f4ba2d","since":6},
     "test/helpers/pty/session.ts": {"blob":"58533cea9d81cc87bb3b499bdfe0db8e3a8709c5","since":6},
     "test/helpers/session-drain-policy.ts": {"blob":"6211239dd0485e6495a700b824a7e0e61d1bcd97","since":6},
-    "test/helpers/session-ledger.ts": {"blob":"6e3c9e837e46f98addba2d12eaebff802da320bb","since":6},
-    "test/helpers/session-runner.ts": {"blob":"d40eb29e8b8fdc0675b511693e4a90823b5963fc","since":6},
+    "test/helpers/session-ledger.ts": {"blob":"425323284a368f5e77937e03884db4dea4e6134e","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
+    "test/helpers/session-runner.ts": {"blob":"cc65a66adafa9c053e3da3c6f17b5ac86c0eade7","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
     "test/helpers/skill-fixture.ts": {"blob":"b188064594eb4201a54a80ba1bc6f28e31d7f49e","since":6},
     "test/helpers/test-selection.ts": {"blob":"51655f89f66be7a5ec487e2abfe4744300ebd5f6","since":6},
     "test/helpers/touchfiles.ts": {"blob":"15f369056d41b9dc42dcd86f81f60c878074704c","since":6}

scripts/lib/paid-cases.ts

@@ -9,6 +9,7 @@ import type { EvalCaseKind, PanelShape } from '../../test/helpers/eval-store';
 import { E2E_KINDS } from '../../test/helpers/touchfiles-data';
 import { E2E_TOUCHFILES, E2E_TIERS } from '../../test/helpers/touchfiles';
 import { type PaidTier, ROOT } from './paid-types';
+import { type EvalHost, hostSkipReason, resolveEvalHost } from '../../test/helpers/e2e-hosts';
 
 /**
  * The E2E ids a paid file registers: the touchfile registrations that list the
@@ -226,12 +227,38 @@ export function partitionCaseExclusions(keys: string[]): { runnable: string[]; e
   return { runnable, excluded };
 }
 
+/**
+ * Host exclusions (EVALS_HOST, test/helpers/e2e-hosts.ts): on a non-Claude
+ * host a case key whose case is Claude-only, or a file key none of whose
+ * `tier` cases runs on the host, is excluded with its reason. A file that
+ * keeps one host case stays runnable; its Claude-only cases skip at module
+ * load and the report deselects them. The Claude host excludes nothing.
+ */
+export function partitionHostExclusions(keys: string[], tier: PaidTier, host: EvalHost, rootDir = ROOT): { runnable: string[]; excluded: Array<{ file: string; reason: string }> } {
+  if (host === 'claude') return { runnable: keys, excluded: [] };
+  const excluded: Array<{ file: string; reason: string }> = [];
+  const runnable = keys.filter(key => {
+    const caseId = shardCaseId(key);
+    const ids = caseId !== null ? [caseId]
+      : fileCaseRegistration(shardFile(key), fs.readFileSync(path.join(rootDir, shardFile(key)), 'utf8')).registered.filter(id => E2E_TIERS[id] === tier);
+    if (ids.some(id => hostSkipReason(id, host) === null)) return true;
+    const reason = ids.length === 1 ? hostSkipReason(ids[0]!, host)!
+      : ids.length === 0 ? `Claude-only on the ${host} host: no Codex host metadata (no ${tier} case registered)`
+      : `Claude-only on the ${host} host: no case runs on it (${ids.join(', ')})`;
+    excluded.push({ file: key, reason: `skipped: ${reason}` });
+    return false;
+  });
+  return { runnable, excluded };
+}
+
 /**
  * Codex access in the CI eval image, scoped per paid shard.
  *
  * The image installs the pinned Codex CLI off PATH (GSTACK_CI_CODEX_BIN_DIR)
  * and the eval workflows log it in under a non-default home
- * (GSTACK_CI_CODEX_HOME). A gate shard never sees either. A non-gate shard
+ * (GSTACK_CI_CODEX_HOME). On the Codex host (EVALS_HOST=codex) Codex is the
+ * agent, so every shard gets the CLI on PATH and the home as CODEX_HOME.
+ * Otherwise a gate shard never sees either. A non-gate shard
  * whose every file exercises Codex gets the CLI on PATH and the home as
  * CODEX_HOME. A file that mixes one Codex case with other cases keeps only
  * the two variables, and that case opts in on its own session; hermetic
@@ -266,8 +293,10 @@ export const CODEX_CI_CASES: Readonly<Record<string, readonly string[]>> = {
 export type CodexShardAccess = 'none' | 'case-opt-in' | 'path';
 
 /** What one shard, given the test files behind its keys, may see of the CI Codex install in `tier`. */
-export function codexShardAccess(files: readonly string[], tier: string | undefined): CodexShardAccess {
-  if (tier === 'gate' || files.length === 0) return 'none';
+export function codexShardAccess(files: readonly string[], tier: string | undefined, host: EvalHost = 'claude'): CodexShardAccess {
+  if (files.length === 0) return 'none';
+  if (host === 'codex') return 'path';
+  if (tier === 'gate') return 'none';
   if (files.every(file => CODEX_CI_FILES.includes(file))) return 'path';
   return files.every(file => file in CODEX_CI_CASES) ? 'case-opt-in' : 'none';
 }
@@ -277,7 +306,7 @@ export function scopeCodexAccess(env: NodeJS.ProcessEnv, files: readonly string[
   const binDir = env[CODEX_CI_ENV.binDir];
   if (!binDir) return;
   const home = env[CODEX_CI_ENV.home];
-  const access = codexShardAccess(files, env.EVALS_TIER);
+  const access = codexShardAccess(files, env.EVALS_TIER, resolveEvalHost(env));
   delete env.CODEX_HOME;
   if (access === 'case-opt-in') return;
   delete env[CODEX_CI_ENV.binDir];

scripts/lib/paid-plan.ts

@@ -10,7 +10,8 @@ import { evalEntryOutcome, type EvalCaseKind } from '../../test/helpers/eval-sto
 import { E2E_KINDS } from '../../test/helpers/touchfiles-data';
 import { prProfileCaseAllowed, prProfileFileMap, type PrProfileSelection } from '../test-pr-profile';
 import { E2E_TOUCHFILES, E2E_TIERS, LLM_JUDGE_TOUCHFILES } from '../../test/helpers/touchfiles';
-import { CASE_KEY_SEPARATOR, CASE_SHARDED_FILES, type CaseTrialPlan, caseTrialPlan, codexShardAccess, expandCaseShards, expandTrialShards, isIsolatedCase, partitionCaseExclusions, sameTrialPlan, shardCaseId, shardFile, shardTrial } from './paid-cases';
+import { CASE_KEY_SEPARATOR, CASE_SHARDED_FILES, type CaseTrialPlan, caseTrialPlan, codexShardAccess, expandCaseShards, expandTrialShards, isIsolatedCase, partitionCaseExclusions, partitionHostExclusions, sameTrialPlan, shardCaseId, shardFile, shardTrial } from './paid-cases';
+import { EVAL_HOSTS, type EvalHost, resolveEvalHost } from '../../test/helpers/e2e-hosts';
 import { DEFAULT_JOBS, OVERLAY_MAX_ACTIVE_SHARDS, PAID_TIERS, SLICE_UPLOAD_RESERVE_MS, type PaidCaseSelection, type PaidProfile, type PaidShardBudget, type PaidTier, ROOT, type ShardOutcome, type ShardStatus, type ShardTrialRecord, isOverlayTestFile } from './paid-types';
 import { collectPaidTestFiles, computePaidCaseSelection, expectedPrCaseCount, isAllSkippedPass, paidShardWallUpperBoundMs, partitionShardsByDiffSelection, planPaidShards, prProfileFileSelected, resolvePaidShardBudget, resolvePaidShardTimeoutMs, sameBudget, selectPaidTestFiles, shardSlug, validatedProfile } from './paid-select';
 
@@ -58,6 +59,8 @@ export interface PaidRunManifest {
   evalsAll: boolean;
   sliceCount: number;
   selectionReason: string;
+  /** The agent host every session runs on (EVALS_HOST); absent means claude. */
+  host?: EvalHost;
   /** Legacy v1 manifests omit these; new plans bind case-level execution. */
   profile?: PaidProfile;
   selection?: PaidCaseSelection;
@@ -314,9 +317,11 @@ export function buildRunManifest(opts: {
   const quarantine = opts.quarantine ?? CASE_QUARANTINE;
   const notLive = [...Object.keys(kinds).filter(id => kinds[id] === 'behavior'), ...Object.keys(quarantine)].filter(id => !Object.hasOwn(E2E_TIERS, id));
   if (notLive.length) throw new Error(`Only live E2E cases can be behavior or quarantined (judges sample their panel inside the case): ${notLive.join(', ')}`);
-  const caseKeys = partitionCaseExclusions(expandCaseShards(selected, opts.tier, rootDir));
+  const host = resolveEvalHost(env);
+  const ciKeys = partitionCaseExclusions(expandCaseShards(selected, opts.tier, rootDir));
+  const caseKeys = partitionHostExclusions(ciKeys.runnable, opts.tier, host, rootDir);
   const excluded = [...tierSelection.excluded, ...(opts.skipJudges ? tierSelection.selected.filter(judge)
-    .map(file => ({ file, reason: 'skipped: LLM judges run in the periodic census and PR gate lanes' })) : []), ...caseKeys.excluded];
+    .map(file => ({ file, reason: 'skipped: LLM judges run in the periodic census and PR gate lanes' })) : []), ...ciKeys.excluded, ...caseKeys.excluded];
   const expansion = expandTrialShards(caseKeys.runnable, opts.tier, rootDir, { kinds, quarantine });
   const excludeOf = (key: string) => expansion.excludeCases[normalizeRelativePath(key)] ?? [];
   const shards = planPaidShards(expansion.keys, { maxFilesPerShard: 1 });
@@ -346,7 +351,7 @@ export function buildRunManifest(opts: {
     entries.sort((a, b) => (a.file < b.file ? -1 : 1));
     return parseRunManifest(JSON.stringify({
       version: 1, tier: opts.tier, evalsAll: opts.evalsAll, sliceCount: plan.slices.length,
-      selectionReason: cases.reason, profile, selection: cases.selection,
+      selectionReason: cases.reason, ...(host !== 'claude' ? { host } : {}), profile, selection: cases.selection,
       ...(cases.coverage ? { prCoverage: cases.coverage } : {}),
       plan: { sliceBudgetMs: opts.sliceBudgetMs!, jobs: opts.jobs!, estimatedSliceMs: plan.estimatedSliceMs, ciTimeoutMinutes: plan.ciTimeoutMinutes,
         sliceCiTimeoutMinutes: plan.sliceCiTimeoutMinutes, codexSlices: codexSlicesOf(opts.tier, entries) },
@@ -466,6 +471,7 @@ export function buildRunManifest(opts: {
     evalsAll: opts.evalsAll,
     sliceCount,
     selectionReason: cases.reason,
+    ...(host !== 'claude' ? { host } : {}),
     profile,
     selection: cases.selection,
     ...(cases.coverage ? { prCoverage: cases.coverage } : {}),
@@ -479,6 +485,7 @@ export function parseRunManifest(raw: string): PaidRunManifest {
   if (parsed.version !== 1) throw new Error(`unsupported manifest version: ${(parsed as { version?: unknown }).version}`);
   if (!PAID_TIERS.includes(parsed.tier)) throw new Error(`manifest tier invalid: ${parsed.tier}`);
   if (parsed.profile !== undefined && parsed.profile !== 'pr' && parsed.profile !== 'full') throw new Error('manifest profile invalid');
+  if (parsed.host !== undefined && (parsed.host === 'claude' || !EVAL_HOSTS.includes(parsed.host))) throw new Error(`manifest host invalid: ${parsed.host}`);
   if (parsed.selection !== undefined) {
     for (const [key, inventory] of [['e2e', E2E_TOUCHFILES], ['judges', LLM_JUDGE_TOUCHFILES]] as const) {
       const ids = parsed.selection?.[key];

scripts/lib/paid-report.ts

@@ -17,6 +17,7 @@ import { E2E_KINDS } from '../../test/helpers/touchfiles-data';
 import { manualReviewProblem } from '../../test/helpers/cookie-workflow-manual-review';
 import { writeNegativeReceipt, writePanelReceipt } from '../e2e-shard-reuse';
 import { E2E_TOUCHFILES, E2E_TIERS, LLM_JUDGE_TOUCHFILES } from '../../test/helpers/touchfiles';
+import { type EvalHost, hostSkipReason } from '../../test/helpers/e2e-hosts';
 import { CASE_TEST_NAMES, type CaseTrialPlan, caseTestNamePattern, caseTrialPlan, fileCaseRegistration, shardCaseId, shardFile, trialShardKey } from './paid-cases';
 import { type ManifestEntry, PAID_TEST_DURATIONS_FILE, type PaidRunManifest, type SliceResult, collectorOutcomeCounts, formatProfileCoverage, loadPaidTestDurations, mergePaidTestDurations, parseRunManifest, trialPanelKey, verifySliceResults, writePaidTestDurations } from './paid-plan';
 import { DEFAULT_JOBS, type PaidCaseSelection, type PaidTier, ROOT, type ShardTrialRecord } from './paid-types';
@@ -170,6 +171,8 @@ export interface JUnitShardContext {
   selection?: PaidCaseSelection;
   /** Ids this file shard leaves to their isolated trial shards. */
   excludeCases?: readonly string[];
+  /** The run's agent host (manifest.host); absent means claude. */
+  host?: EvalHost;
 }
 
 export const DESELECTION_REASONS = {
@@ -205,6 +208,8 @@ export function attributeJUnitCase(tc: JUnitCase, ctx: JUnitShardContext,
   const tier = tiers[id];
   if (tier !== undefined && tier !== ctx.tier) return { kind: 'deselected', reason: `${tier}-tier case (this lane runs ${ctx.tier})` };
   if (ctx.excludeCases?.includes(id)) return { kind: 'deselected', reason: DESELECTION_REASONS.isolated };
+  const hostReason = hostSkipReason(id, ctx.host ?? 'claude');
+  if (hostReason) return { kind: 'deselected', reason: hostReason };
   const pool = tier !== undefined ? ctx.selection?.e2e : Object.hasOwn(LLM_JUDGE_TOUCHFILES, id) ? ctx.selection?.judges : null;
   if (pool && !pool.includes(id)) return { kind: 'deselected', reason: DESELECTION_REASONS.selection };
   return { kind: 'case', id };
@@ -241,7 +246,7 @@ export const CENSUS_RED_GUIDE = 'docs/evals/census-red.md';
 
 export function sliceArtifactName(root: string, slice: number, attempt: number): string {
   const name = path.basename(root);
-  return /^(?:paid-slice|gate-census)-\d+-a\d+$/.test(name) ? name : `slice ${slice} attempt ${attempt}`;
+  return /^(?:paid-slice|gate-census)-\d+-a\d+(?:-[a-z0-9-]+)?$/.test(name) ? name : `slice ${slice} attempt ${attempt}`;
 }
 
 export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact[], recordsByShard: Map<string, any[]>,
@@ -254,7 +259,7 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact
       if (outcome.trial || outcome.files.length !== 1) continue;
       let xml = '';
       try { xml = fs.readFileSync(path.join(root, 'shards', shardSlug([key]), 'junit.xml'), 'utf8'); } catch { continue; }
-      const ctx: JUnitShardContext = { key, tier: manifest.tier, selection: manifest.selection, excludeCases: entries.get(key)?.excludeCases };
+      const ctx: JUnitShardContext = { key, tier: manifest.tier, selection: manifest.selection, excludeCases: entries.get(key)?.excludeCases, host: manifest.host };
       const byCase = new Map<string, JUnitCase[]>();
       const reasons = new Map<string, number>();
       for (const tc of parseJUnitCases(xml)) {
@@ -530,7 +535,7 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo
   const results = artifacts.filter(a => (a.result.attempt ?? 1) === primary).map(a => a.result);
   const verdict = verifySliceResults(manifest, results);
   const planned = manifest.entries.filter((e) => e.status === 'planned').length;
-  const lane = `${manifest.tier}/${manifest.profile ?? 'full'}${manifest.evalsAll ? ' census' : ''}`;
+  const lane = `${manifest.tier}/${manifest.profile ?? 'full'}${manifest.evalsAll ? ' census' : ''}${manifest.host ? ` on ${manifest.host}` : ''}`;
   console.log(`[test:paid] report: ${results.length}/${manifest.sliceCount} slices, ${planned} planned shards, tier=${manifest.tier}, attempt ${primary}${attempts.length > 1 ? ` (later attempts ${attempts.slice(1).join(', ')} reported, never replacing it)` : ''}`);
   for (const line of formatProfileCoverage(manifest)) console.log(line);
   for (const result of [...results].sort((a, b) => a.sliceIndex - b.sliceIndex)) {

scripts/test-paid-shards.ts

@@ -73,6 +73,7 @@ import { readSessionLedger, type SessionLedgerRow } from '../test/helpers/sessio
 import { preflightAnthropicApi } from '../test/helpers/anthropic-preflight';
 import { e2eReuseLaneProblem, prepareE2EShardReuse, selectPlanReceipts } from './e2e-shard-reuse';
 import { E2E_TOUCHFILES, E2E_TIERS } from '../test/helpers/touchfiles';
+import { resolveEvalHost } from '../test/helpers/e2e-hosts';
 import { scopeCodexAccess, shardFile, shardCaseId, shardTrial, trialShardKey, type CaseTrialPlan, caseTrialPlan, excludedCasesNamePattern, caseTestNamePattern, expandCaseShards, expandTrialShards, fileCaseRegistration, partitionCaseExclusions } from './lib/paid-cases';
 import { retriesForFiles, trialPanelKey, sliceExecutionOrder, buildRunManifest, parseRunManifest, type SliceResult, sliceExitCode, guardTrialRecords, formatSlicePlan, formatCapacityPreflight, sliceDeadlineMs } from './lib/paid-plan';
 import { caseSelection, runCaseDiagnosis, formatPanelLine, runPaidReport } from './lib/paid-report';
@@ -909,6 +910,9 @@ async function main(): Promise<number> {
     if (manifest.tier !== options.tier) {
       throw new Error(`manifest tier ${manifest.tier} != requested tier ${options.tier} — refusing a cross-tier run`);
     }
+    if ((manifest.host ?? 'claude') !== resolveEvalHost()) {
+      throw new Error(`manifest host ${manifest.host ?? 'claude'} != EVALS_HOST ${resolveEvalHost()} — refusing a cross-host run`);
+    }
     const profile = manifest.profile ?? 'full';
     if (options.profileExplicit && options.profile !== profile) throw new Error(`manifest profile ${profile} != requested profile ${options.profile}`);
     if (options.sliceIndex > manifest.sliceCount) {