fix: a cost-only record never sets the trial model

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed 2026-10-29 04:45:05 +00:00
1 parent fa5629423d
commit b889070fb9
3 files changed
+7 -2

No files matched your search

+1 -1
View File
@@ -13,7 +13,7 @@
"files": {
"scripts/lib/paid-cases.ts": {"blob":"f862cf5241580c7593b9ee62eacf5ef046248d26","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
"scripts/lib/paid-plan.ts": {"blob":"e8ce3cb98474dded38c5128a631d283b0ca35b63","since":6,"nonBehavioral":"F1 rule-16 flaky rerun: report aggregation, rerun planning and the gate slice exit code; no case's execution, trial outcome, trial record or panel verdict changes, so no pass-rate series resets"},
"scripts/lib/paid-report.ts": {"blob":"6adbde198309328bae17c3b17d199963e19a4c7e","since":6,"nonBehavioral":"the paid report headline adds session cost no collector record carries; cost fields only, no verdict or failure class changes"},
"scripts/lib/paid-report.ts": {"blob":"a4ef799c233c65e9209a4b654360476fdbf51802","since":6,"nonBehavioral":"the trial model comes from the named records only, as before cost_case_id; cost fields only, no verdict, failure class or series key changes"},
"scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":6},
"scripts/lib/shard-engine.ts": {"blob":"dc714d7d897aed185c0ec3d2c92ad22d954039e5","since":6,"nonBehavioral":"createShardSandbox drops the Capy launch markers (GSTACK_SESSION_KIND, GSTACK_CAPY, GSTACK_EPHEMERAL, GSTACK_INSTALL_VERDICT) from shard environments; CI never sets them, so no case's input or verdict changes there, and on a Capy machine fixtures now see the session kind they set themselves instead of the installer's"},
"scripts/test-paid-shards.ts": {"blob":"5439d0a8f6e1569fa7c924af20d67c30c81d5d30","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"},
+1 -1
View File
@@ -334,7 +334,7 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact
...(failureClass ? { failure_class: failureClass } : {}), ...(failedRecord?.exit_reason ? { exit_reason: String(failedRecord.exit_reason) } : {}),
...(error && failed ? { error } : {}), ...diagnosis, ...trialSessions(sessions), duration_ms: Math.min(cases.reduce((sum, tc) => sum + tc.timeMs, 0), outcome.elapsedMs),
...cost,
...(typeof mine[0]?.model === 'string' ? { model: mine[0].model } : {}), ...(cliVersion ? { cli_version: cliVersion } : {}),
...(typeof named[0]?.model === 'string' ? { model: named[0].model } : {}), ...(cliVersion ? { cli_version: cliVersion } : {}),
quarantined: false, execution: outcome.reused ? 'reused' : 'executed', source: 'junit' } as TrialOutcomeRecord);
}
census.shardSkipReasons.set(key, reasons);
+5
View File
@@ -102,4 +102,9 @@ describe('junitCensus cost binding', () => {
.toEqual({ failureClass: unbound.failureClass, error: unbound.error, line: unbound.line?.replace(/ {2}\[slice.*$/, '') });
expect(bound.failureClass).toBe('assertion');
});
test('a cost-only record never sets the trial\'s model, which keys its pass-rate series', () => {
const history = census(labelled.map(r => ({ ...r, model: 'claude-opus-5-5' }))).history;
expect(history.map(h => [h.case, h.model])).toEqual([['browse-basic', undefined], ['browse-snapshot', undefined]]);
});
});