From 55bd791f32259d9a004026411dd586c970a41f20 Mon Sep 17 00:00:00 2001 From: garrytan Date: Thu, 29 Oct 2026 04:36:10 +0000 Subject: [PATCH 1/8] Price PTY sessions from their transcripts: pin a session id, sum the session's and its subagents' usage into the ledger's streamed_cost_usd Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- scripts/harness-version.json | 6 +- .../claude-pty-runner.launch.unit.test.ts | 13 ++++- test/helpers/pty-transcript-cost.unit.test.ts | 58 +++++++++++++++++++ test/helpers/pty/launch.ts | 15 ++++- test/helpers/pty/session.ts | 3 +- test/helpers/pty/transcript-cost.ts | 42 ++++++++++++++ test/helpers/session-ledger.ts | 2 +- 7 files changed, 130 insertions(+), 9 deletions(-) create mode 100644 test/helpers/pty-transcript-cost.unit.test.ts create mode 100644 test/helpers/pty/transcript-cost.ts diff --git a/scripts/harness-version.json b/scripts/harness-version.json index e225bbbad..695848771 100644 --- a/scripts/harness-version.json +++ b/scripts/harness-version.json @@ -38,15 +38,15 @@ "test/helpers/pty/classify.ts": {"blob":"3f2daa12ff6484f96b8d28d910a50231c4bb3ae8","since":6}, "test/helpers/pty/fake-session.ts": {"blob":"554e064ef259b8444280eddfe9b0db93c66a8202","since":6}, "test/helpers/pty/judge.ts": {"blob":"81c5bb4c48ab928aca2f186a1ade9df7169d7fbe","since":6}, - "test/helpers/pty/launch.ts": {"blob":"e4413de380b799659170e7da248d51afda3b0e3f","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"}, + "test/helpers/pty/launch.ts": {"blob":"d2e51418ce284815a26e174d777ca8b47ffd7e68","since":6,"nonBehavioral":"PTY sessions price their ledger row from the session transcript; the launcher pins a random --session-id (Claude otherwise picks its own random one) when the caller passes none and is not resuming; no prompt, answer, timeout or verdict changes"}, "test/helpers/pty/plan-native.ts": {"blob":"ddba9cb68c7c0da935cff7c1cea258dee2dff740","since":6}, "test/helpers/pty/runners/counting.ts": {"blob":"8c5c05e54abf961259d34d9c3d77db67138d7794","since":6}, "test/helpers/pty/runners/floor.ts": {"blob":"54e2145832bb98913644d10ff303792d67f9362e","since":6,"nonBehavioral":"Removes the plan-design 0D focus picker from the shared harness; floor.ts gains an optional pickSetupChoice hook that no other caller passes, so no other case's input or verdict can change. plan-design-finding-floor, the only case whose seed raises that menu, passes the picker from its owned fixture test/fixtures/design-floor-focus-picker.ts, so its owned bytes start its own new series."}, "test/helpers/pty/runners/observation.ts": {"blob":"1c2b75fbb984708725abaa8939125f7cab969f9f","since":6}, "test/helpers/pty/screen.ts": {"blob":"f3743152fc9bb7a8e8c70002f75bf45914f260d5","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"}, - "test/helpers/pty/session.ts": {"blob":"54fe04b61d953e4e55e2fc251532c04a49fab66f","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"}, + "test/helpers/pty/session.ts": {"blob":"8fd1749a62f24dc37ae186c8a21c2165f7f7645e","since":6,"nonBehavioral":"PTY sessions price their ledger row from the session transcript; the launcher pins a random --session-id (Claude otherwise picks its own random one) when the caller passes none and is not resuming; no prompt, answer, timeout or verdict changes"}, "test/helpers/session-drain-policy.ts": {"blob":"6211239dd0485e6495a700b824a7e0e61d1bcd97","since":6}, - "test/helpers/session-ledger.ts": {"blob":"5d8b796ccd422f4e1e2ab395bf6ce5a2e4c0e167","since":6,"nonBehavioral":"Cost attribution only (GSTA-150 PR 2): a display-label record carries cost_case_id from runSkillTest's testName and the report binds its cost by it while the failed record and failure class still come from the named records (test/paid-report-cost.test.ts pins both); an aborted Agent SDK session logs its streamed usage as streamed_cost_usd, used only when a case has no record; a listed model-free case with no record or session is a known $0. No case's execution, outcome, failure class, rerun eligibility or panel verdict changes, so no pass-rate series resets"}, + "test/helpers/session-ledger.ts": {"blob":"c99baaaa0a77a5a72384493fd8a85a4253ecb05d","since":6,"nonBehavioral":"PTY sessions price their ledger row from the session transcript; the launcher pins a random --session-id (Claude otherwise picks its own random one) when the caller passes none and is not resuming; no prompt, answer, timeout or verdict changes"}, "test/helpers/session-runner.ts": {"blob":"c73b2596d9eb5377530a6bf1d284e0b6ea1556a0","since":6,"nonBehavioral":"Cost attribution only (GSTA-150 PR 2): a display-label record carries cost_case_id from runSkillTest's testName and the report binds its cost by it while the failed record and failure class still come from the named records (test/paid-report-cost.test.ts pins both); an aborted Agent SDK session logs its streamed usage as streamed_cost_usd, used only when a case has no record; a listed model-free case with no record or session is a known $0. No case's execution, outcome, failure class, rerun eligibility or panel verdict changes, so no pass-rate series resets"}, "test/helpers/skill-fixture.ts": {"blob":"8a654dae6b0a396d839b02179119ae3e3bb13f69","since":6,"nonBehavioral":"extractSkillBody and extractSkillSections follow the shared-preamble hoist (pointer plus sections/preamble.md); every fixture they build is byte-identical to the pre-hoist output (compared for 8 body and 4 section extractions), so no case's input or verdict changes; the /ship review then reworded only comments and two error messages, so no fixture byte changes"}, "test/helpers/skill-startup.ts": {"blob":"a8d21e3817c3f35a1d06af325c92fde7d420e377","since":6,"nonBehavioral":"readSkillEffective also appends a variant's added ### subsections (office-hours' Open-ended questions); readSkillStartup, readSharedSections and readSkillEffective return bytes identical to the pinned helper for every tier 2+ render without added subsections (735 across all hosts) and for office-hours' SKILL.md before this PR, so office-hours' fixture input changes only because its prompt changed in this PR, not because the helper did"}, diff --git a/test/helpers/claude-pty-runner.launch.unit.test.ts b/test/helpers/claude-pty-runner.launch.unit.test.ts index 0f6123afe..5913435b6 100644 --- a/test/helpers/claude-pty-runner.launch.unit.test.ts +++ b/test/helpers/claude-pty-runner.launch.unit.test.ts @@ -39,7 +39,7 @@ describe('launchClaudePty model pin', () => { expect(opts.model).toBe('claude-sonnet-4-6'); }); - test.skipIf(process.platform === 'win32')('spawn args pin --model from the fallback chain, before extraArgs, with hermetic MCP gating', () => { + test.skipIf(process.platform === 'win32')('spawn args pin --model from the fallback chain, before extraArgs, with hermetic MCP gating and a pinned session id', () => { const dir = mkdtempSync(join(tmpdir(), 'pty-model-pin-')); try { const fake = join(dir, 'fake-claude'); @@ -56,7 +56,9 @@ const argv = async (opts, evalsModel) => { }; process.stdout.write(JSON.stringify({ capture: resolveEvalModel('capture'), fallback: await argv({}, undefined), env: await argv({}, 'env-model'), explicit: await argv({ model: 'opts-model' }, 'env-model'), - extra: await argv({ extraArgs: ['--model', 'override'] }, undefined) })); + extra: await argv({ extraArgs: ['--model', 'override'] }, undefined), + pinned: await argv({ extraArgs: ['--session-id', '11111111-2222-4333-8444-555555555555'] }, undefined), + resumed: await argv({ extraArgs: ['--resume', 'prior'] }, undefined) })); `); const run = (hermetic: string) => { const result = spawnSync(process.execPath, [worker], { cwd: join(import.meta.dir, '../..'), encoding: 'utf8', timeout: 30_000, @@ -73,6 +75,13 @@ process.stdout.write(JSON.stringify({ capture: resolveEvalModel('capture'), expect(hermetic.extra.slice(-2)).toEqual(['--model', 'override']); expect(hermetic.fallback).toContain('--strict-mcp-config'); expect(run('0').fallback).not.toContain('--strict-mcp-config'); + const sessionIds = (args: string[]) => args.flatMap((arg, i) => arg === '--session-id' ? [args[i + 1]] : []); + expect(sessionIds(hermetic.fallback)).toHaveLength(1); + expect(sessionIds(hermetic.fallback)[0]).toMatch(/^[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}$/); + expect(sessionIds(hermetic.extra)).toHaveLength(1); + expect(sessionIds(hermetic.pinned)).toEqual(['11111111-2222-4333-8444-555555555555']); + expect(sessionIds(hermetic.resumed)).toEqual([]); + expect(sessionIds(run('0').fallback)).toEqual([]); } finally { rmSync(dir, { recursive: true, force: true }); } }, 60_000); }); diff --git a/test/helpers/pty-transcript-cost.unit.test.ts b/test/helpers/pty-transcript-cost.unit.test.ts new file mode 100644 index 000000000..c02e5db07 --- /dev/null +++ b/test/helpers/pty-transcript-cost.unit.test.ts @@ -0,0 +1,58 @@ +import { describe, test, expect } from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { pricedTranscript, ptyTranscriptCostUsd } from './pty/transcript-cost'; + +const SID = '11111111-2222-4333-8444-555555555555'; +const assistant = (id: string, model: string, usage: object, extra: object = {}) => + JSON.stringify({ type: 'assistant', sessionId: SID, message: { id, model, role: 'assistant', usage }, ...extra }); + +function fixture(write: (project: string) => void): string { + const config = fs.mkdtempSync(path.join(os.tmpdir(), 'pty-cost-')); + const project = path.join(config, 'projects', '-fixture-cwd'); + fs.mkdirSync(project, { recursive: true }); + write(project); + return config; +} + +describe('ptyTranscriptCostUsd', () => { + test('prices the session, its inline sidechains and subagent files, each message id once at its largest output', () => { + const config = fixture(project => { + fs.writeFileSync(path.join(project, `${SID}.jsonl`), [ + JSON.stringify({ type: 'user', sessionId: SID, message: { role: 'user', content: 'hi' } }), + assistant('msg_a', 'claude-opus-5-5', { input_tokens: 1000, output_tokens: 10 }), + assistant('msg_a', 'claude-opus-5-5', { input_tokens: 1000, output_tokens: 500 }), + assistant('msg_side', 'claude-opus-5-5', { input_tokens: 0, output_tokens: 1000 }, { isSidechain: true }), + assistant('msg_syn', '', { input_tokens: 0, output_tokens: 0 }), + '{"partial":', + ].join('\n')); + fs.mkdirSync(path.join(project, SID, 'subagents'), { recursive: true }); + fs.writeFileSync(path.join(project, SID, 'subagents', 'agent-x.jsonl'), + assistant('msg_sub', 'claude-opus-5-5', { input_tokens: 0, cache_read_input_tokens: 10000, output_tokens: 0 }) + '\n'); + fs.writeFileSync(path.join(project, '99999999-2222-4333-8444-555555555555.jsonl'), + assistant('msg_foreign', 'claude-opus-5-5', { input_tokens: 1_000_000, output_tokens: 0 }) + '\n'); + }); + try { + // opus-5-5: $4 in, $20 out, $0.20 cached per MTok. + const expected = (1000 * 4 + 500 * 20 + 1000 * 20 + 10000 * 0.2) / 1_000_000; + expect(ptyTranscriptCostUsd(config, SID)).toBeCloseTo(expected, 6); + expect(pricedTranscript({ hermeticConfigDir: config, sessionId: SID }).streamed_cost_usd).toBeCloseTo(expected, 6); + } finally { fs.rmSync(config, { recursive: true, force: true }); } + }); + + test('stays cost-unknown without a transcript, a session id, usage or a priced model', () => { + const unpriced = fixture(project => fs.writeFileSync(path.join(project, `${SID}.jsonl`), + assistant('msg_a', 'claude-opus-5-5', { output_tokens: 1 }) + '\n' + assistant('msg_b', 'some-unpriced-model', { output_tokens: 1 }) + '\n')); + const empty = fixture(project => fs.writeFileSync(path.join(project, `${SID}.jsonl`), JSON.stringify({ type: 'user', sessionId: SID }) + '\n')); + try { + expect(ptyTranscriptCostUsd(unpriced, SID)).toBeUndefined(); + expect(ptyTranscriptCostUsd(empty, SID)).toBeUndefined(); + expect(ptyTranscriptCostUsd(empty, '99999999-2222-4333-8444-555555555555')).toBeUndefined(); + expect(ptyTranscriptCostUsd(unpriced, undefined)).toBeUndefined(); + expect(ptyTranscriptCostUsd(null, SID)).toBeUndefined(); + expect(ptyTranscriptCostUsd(path.join(empty, 'missing'), SID)).toBeUndefined(); + expect(pricedTranscript(undefined)).toEqual({}); + } finally { for (const dir of [unpriced, empty]) fs.rmSync(dir, { recursive: true, force: true }); } + }); +}); diff --git a/test/helpers/pty/launch.ts b/test/helpers/pty/launch.ts index e4413de38..d2e51418c 100644 --- a/test/helpers/pty/launch.ts +++ b/test/helpers/pty/launch.ts @@ -2,6 +2,7 @@ * Spawn `claude` in a real PTY (launchClaudePty) and the session handle. Moved from claude-pty-runner.ts. * Import through test/helpers/claude-pty-runner.ts from tests; pty/ modules import siblings directly. */ +import { randomUUID } from 'node:crypto'; import * as path from 'node:path'; import { resolveEvalModel } from '../../../lib/eval-model'; import { hermeticChildEnv, hermeticSkillsConfigDir, isHermeticEnabled } from '../hermetic-env'; @@ -15,6 +16,7 @@ import { trustDialogInput } from '../pty-trust-dialog'; import { resolveClaudeBinary } from './binary'; import { createPtyScreen, screenPanelEnd, stripAnsi } from './screen'; import { appendSessionLedger, sessionKey } from '../session-ledger'; +import { pricedTranscript } from './transcript-cost'; export interface ClaudePtyOptions { /** Register the repo's shipped skills in the child's user scope via @@ -129,6 +131,8 @@ export interface ClaudePtySession { * the dir name ends in `/.claude` by contract). */ hermeticConfigDir: string | null; + /** The Claude session id the launcher pinned (or the caller's --session-id): its transcript prices the session ledger row. */ + sessionId?: string; /** Owned HOME/.gstack created by the seeded launcher; absent for caller overrides. */ hermeticSkillStateRoot?: string; /** Owned pre-tool identity record, removed by close(). */ @@ -249,7 +253,8 @@ export async function launchClaudePty( recorded = true; const elapsed = performance.now() - startedMono; appendSessionLedger({ key: sessionKey('pty', undefined), runner: 'pty', started_at: startedAt, budget_ms: timeoutMs, - elapsed_ms: elapsed, end: elapsed >= timeoutMs ? 'session_timeout' : 'completed', ...screenPanelEnd(handle.visibleText()), billed: false }); + elapsed_ms: elapsed, end: elapsed >= timeoutMs ? 'session_timeout' : 'completed', ...screenPanelEnd(handle.visibleText()), billed: false, + ...pricedTranscript(handle) }); } return close(); } }); @@ -263,6 +268,7 @@ interface LaunchSetup { hermeticSkillStateRoot: string | undefined; autoplanArtifactStateRoot: string | undefined; autoplanEngTestPlanStateRoot: string | undefined; + sessionId: string | undefined; } function prepareLaunch(opts: ClaudePtyOptions): LaunchSetup { @@ -283,6 +289,10 @@ function prepareLaunch(opts: ClaudePtyOptions): LaunchSetup { // Before opts.extraArgs so a test could theoretically supply --mcp-config. const hermetic = isHermeticEnabled(); if (hermetic) args.push('--strict-mcp-config'); + const extra = opts.extraArgs ?? []; + const resumes = extra.some(arg => ['--resume', '-r', '--continue', '-c'].includes(arg)); + let sessionId = extra.includes('--session-id') ? extra[extra.indexOf('--session-id') + 1] : undefined; + if (hermetic && !sessionId && !resumes) args.push('--session-id', sessionId = randomUUID()); if (opts.extraArgs) args.push(...opts.extraArgs); // Hermetic by default (test/helpers/hermetic-env.ts): operator session @@ -319,7 +329,7 @@ function prepareLaunch(opts: ClaudePtyOptions): LaunchSetup { const autoplanEngTestPlanStateRoot = opts.approveAutoplanArtifactEdits && opts.autoplanArtifactState !== undefined ? hermeticSkillStateRoot : undefined; - return { args, childEnv, hermetic, hermeticSkillStateRoot, autoplanArtifactStateRoot, autoplanEngTestPlanStateRoot }; + return { args, childEnv, hermetic, hermeticSkillStateRoot, autoplanArtifactStateRoot, autoplanEngTestPlanStateRoot, sessionId }; } interface LaunchRecorders { @@ -568,6 +578,7 @@ function sessionHandle(pty: PtyProcess, launch: LaunchSetup, wallTimer: ReturnTy exited: () => pty.exited, exitCode: () => pty.exitCode, hermeticConfigDir: launch.hermetic ? launch.childEnv.CLAUDE_CONFIG_DIR ?? null : null, + sessionId: launch.sessionId, hermeticSkillStateRoot: launch.hermeticSkillStateRoot, pendingPlanReadyFile: pendingExit?.file, pendingQuestionFile: pendingQuestion?.file, diff --git a/test/helpers/pty/session.ts b/test/helpers/pty/session.ts index 54fe04b61..8fd1749a6 100644 --- a/test/helpers/pty/session.ts +++ b/test/helpers/pty/session.ts @@ -10,6 +10,7 @@ import { launchClaudePty, type ClaudePtyOptions, type ClaudePtySession } from './launch'; import { screenPanelEnd } from './screen'; import { appendSessionLedger, sessionKey, type SessionEnd } from '../session-ledger'; +import { pricedTranscript } from './transcript-cost'; /** * The runners' launch seam: how a session starts, and the clock the runner @@ -97,7 +98,7 @@ export async function runPtySession(plan: PtySessionPlan): Promise { } finally { appendSessionLedger({ key: sessionKey('pty', undefined), runner: 'pty', started_at: startedAt, budget_ms: plan.budgetMs ?? plan.launch.timeoutMs ?? 240_000, elapsed_ms: plan.driver.monotonic() - started, - end, ...safePanelEnd(session), billed: false }); + end, ...safePanelEnd(session), billed: false, ...pricedTranscript(session) }); try { try { plan.beforeClose?.(session, failure); } catch (error) { if (!failure) { failure = { error }; throw error; } } diff --git a/test/helpers/pty/transcript-cost.ts b/test/helpers/pty/transcript-cost.ts new file mode 100644 index 000000000..064d520b8 --- /dev/null +++ b/test/helpers/pty/transcript-cost.ts @@ -0,0 +1,42 @@ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { streamedUsageCostUsd } from '../../../lib/pricing'; + +/** The session ledger's cost field for a PTY session: present only when its transcript prices. */ +export function pricedTranscript(session: { hermeticConfigDir?: string | null; sessionId?: string } | undefined): { streamed_cost_usd?: number } { + const cost = ptyTranscriptCostUsd(session?.hermeticConfigDir, session?.sessionId); + return cost === undefined ? {} : { streamed_cost_usd: cost }; +} + +/** + * Price one PTY session from the transcripts Claude Code wrote under its owned config dir: + * `projects//.jsonl` (inline sidechains included) and `projects///subagents/*.jsonl`, + * rows deduplicated by message id. Undefined (cost-unknown, never $0) without a config dir, session id, transcript or + * usage, or when a message's model has no price row. + */ +export function ptyTranscriptCostUsd(configDir: string | null | undefined, sessionId: string | undefined): number | undefined { + if (!configDir || !sessionId) return undefined; + const projects = path.join(configDir, 'projects'); + let dirs: fs.Dirent[]; + try { + dirs = fs.readdirSync(projects, { withFileTypes: true }); + } catch { + return undefined; + } + const files: string[] = []; + for (const dir of dirs) { + if (!dir.isDirectory()) continue; + const main = path.join(projects, dir.name, `${sessionId}.jsonl`); + if (fs.existsSync(main)) files.push(main); + const subagents = path.join(projects, dir.name, sessionId, 'subagents'); + if (fs.existsSync(subagents)) files.push(...fs.readdirSync(subagents).filter(f => f.endsWith('.jsonl')).map(f => path.join(subagents, f))); + } + const rows = files.flatMap(file => fs.readFileSync(file, 'utf8').split('\n').flatMap(line => { + try { + return [JSON.parse(line)]; + } catch { + return []; + } + })); + return streamedUsageCostUsd(rows); +} diff --git a/test/helpers/session-ledger.ts b/test/helpers/session-ledger.ts index 5d8b796cc..c99baaaa0 100644 --- a/test/helpers/session-ledger.ts +++ b/test/helpers/session-ledger.ts @@ -52,7 +52,7 @@ export interface SessionLedgerRow { liveness?: LivenessSummary; /** Whether the runner captured this session's billing (a terminal result with total_cost_usd). */ billed?: boolean; - /** A session that ended without a terminal result (aborted, thrown): its assistant messages' streamed usage at lib/pricing.ts rates, absent when unpriced. */ + /** A session without a terminal result (an aborted Agent SDK session, a PTY session): its assistant messages' usage at lib/pricing.ts rates, from the stream or the PTY transcript; absent when unpriced. */ streamed_cost_usd?: number; /** Codex sessions: the model, its billed cost and token classes. */ model?: string; From 876cf1a8d3ec2732befff3792575ca6f74b49d4b Mon Sep 17 00:00:00 2001 From: garrytan Date: Thu, 29 Oct 2026 04:43:28 +0000 Subject: [PATCH 2/8] fix: transcript pricing never throws from a PTY ledger writer Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- test/helpers/pty-transcript-cost.unit.test.ts | 4 +- test/helpers/pty/transcript-cost.ts | 39 +++++++++---------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/test/helpers/pty-transcript-cost.unit.test.ts b/test/helpers/pty-transcript-cost.unit.test.ts index c02e5db07..d05d3591e 100644 --- a/test/helpers/pty-transcript-cost.unit.test.ts +++ b/test/helpers/pty-transcript-cost.unit.test.ts @@ -45,6 +45,7 @@ describe('ptyTranscriptCostUsd', () => { const unpriced = fixture(project => fs.writeFileSync(path.join(project, `${SID}.jsonl`), assistant('msg_a', 'claude-opus-5-5', { output_tokens: 1 }) + '\n' + assistant('msg_b', 'some-unpriced-model', { output_tokens: 1 }) + '\n')); const empty = fixture(project => fs.writeFileSync(path.join(project, `${SID}.jsonl`), JSON.stringify({ type: 'user', sessionId: SID }) + '\n')); + const unreadable = fixture(project => fs.mkdirSync(path.join(project, `${SID}.jsonl`))); try { expect(ptyTranscriptCostUsd(unpriced, SID)).toBeUndefined(); expect(ptyTranscriptCostUsd(empty, SID)).toBeUndefined(); @@ -53,6 +54,7 @@ describe('ptyTranscriptCostUsd', () => { expect(ptyTranscriptCostUsd(null, SID)).toBeUndefined(); expect(ptyTranscriptCostUsd(path.join(empty, 'missing'), SID)).toBeUndefined(); expect(pricedTranscript(undefined)).toEqual({}); - } finally { for (const dir of [unpriced, empty]) fs.rmSync(dir, { recursive: true, force: true }); } + expect(ptyTranscriptCostUsd(unreadable, SID)).toBeUndefined(); + } finally { for (const dir of [unpriced, empty, unreadable]) fs.rmSync(dir, { recursive: true, force: true }); } }); }); diff --git a/test/helpers/pty/transcript-cost.ts b/test/helpers/pty/transcript-cost.ts index 064d520b8..c54a70bed 100644 --- a/test/helpers/pty/transcript-cost.ts +++ b/test/helpers/pty/transcript-cost.ts @@ -12,31 +12,30 @@ export function pricedTranscript(session: { hermeticConfigDir?: string | null; s * Price one PTY session from the transcripts Claude Code wrote under its owned config dir: * `projects//.jsonl` (inline sidechains included) and `projects///subagents/*.jsonl`, * rows deduplicated by message id. Undefined (cost-unknown, never $0) without a config dir, session id, transcript or - * usage, or when a message's model has no price row. + * usage, when a message's model has no price row, or when any read fails: it runs in ledger writers before close and + * cleanup, so it never throws. */ export function ptyTranscriptCostUsd(configDir: string | null | undefined, sessionId: string | undefined): number | undefined { if (!configDir || !sessionId) return undefined; - const projects = path.join(configDir, 'projects'); - let dirs: fs.Dirent[]; try { - dirs = fs.readdirSync(projects, { withFileTypes: true }); + const projects = path.join(configDir, 'projects'); + const files: string[] = []; + for (const dir of fs.readdirSync(projects, { withFileTypes: true })) { + if (!dir.isDirectory()) continue; + const main = path.join(projects, dir.name, `${sessionId}.jsonl`); + if (fs.existsSync(main)) files.push(main); + const subagents = path.join(projects, dir.name, sessionId, 'subagents'); + if (fs.existsSync(subagents)) files.push(...fs.readdirSync(subagents).filter(f => f.endsWith('.jsonl')).map(f => path.join(subagents, f))); + } + const rows = files.flatMap(file => fs.readFileSync(file, 'utf8').split('\n').flatMap(line => { + try { + return [JSON.parse(line)]; + } catch { + return []; + } + })); + return streamedUsageCostUsd(rows); } catch { return undefined; } - const files: string[] = []; - for (const dir of dirs) { - if (!dir.isDirectory()) continue; - const main = path.join(projects, dir.name, `${sessionId}.jsonl`); - if (fs.existsSync(main)) files.push(main); - const subagents = path.join(projects, dir.name, sessionId, 'subagents'); - if (fs.existsSync(subagents)) files.push(...fs.readdirSync(subagents).filter(f => f.endsWith('.jsonl')).map(f => path.join(subagents, f))); - } - const rows = files.flatMap(file => fs.readFileSync(file, 'utf8').split('\n').flatMap(line => { - try { - return [JSON.parse(line)]; - } catch { - return []; - } - })); - return streamedUsageCostUsd(rows); } From 40b1fbb898c1596b1e58d8f1dbe337090df59a0a Mon Sep 17 00:00:00 2001 From: garrytan Date: Thu, 29 Oct 2026 04:43:59 +0000 Subject: [PATCH 3/8] chore: pin transcript-cost.ts as non-behavioral at HARNESS_VERSION 6 Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- scripts/harness-version.json | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/harness-version.json b/scripts/harness-version.json index 695848771..4a087d7c2 100644 --- a/scripts/harness-version.json +++ b/scripts/harness-version.json @@ -45,6 +45,7 @@ "test/helpers/pty/runners/observation.ts": {"blob":"1c2b75fbb984708725abaa8939125f7cab969f9f","since":6}, "test/helpers/pty/screen.ts": {"blob":"f3743152fc9bb7a8e8c70002f75bf45914f260d5","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"}, "test/helpers/pty/session.ts": {"blob":"8fd1749a62f24dc37ae186c8a21c2165f7f7645e","since":6,"nonBehavioral":"PTY sessions price their ledger row from the session transcript; the launcher pins a random --session-id (Claude otherwise picks its own random one) when the caller passes none and is not resuming; no prompt, answer, timeout or verdict changes"}, + "test/helpers/pty/transcript-cost.ts": {"blob":"c54a70bed59c60f1249ba8cb13e83ecd7e6d7471","since":6,"nonBehavioral":"transcript-cost.ts prices a PTY session's ledger row from its transcript and never throws; cost fields only, no prompt, answer, timeout or verdict changes"}, "test/helpers/session-drain-policy.ts": {"blob":"6211239dd0485e6495a700b824a7e0e61d1bcd97","since":6}, "test/helpers/session-ledger.ts": {"blob":"c99baaaa0a77a5a72384493fd8a85a4253ecb05d","since":6,"nonBehavioral":"PTY sessions price their ledger row from the session transcript; the launcher pins a random --session-id (Claude otherwise picks its own random one) when the caller passes none and is not resuming; no prompt, answer, timeout or verdict changes"}, "test/helpers/session-runner.ts": {"blob":"c73b2596d9eb5377530a6bf1d284e0b6ea1556a0","since":6,"nonBehavioral":"Cost attribution only (GSTA-150 PR 2): a display-label record carries cost_case_id from runSkillTest's testName and the report binds its cost by it while the failed record and failure class still come from the named records (test/paid-report-cost.test.ts pins both); an aborted Agent SDK session logs its streamed usage as streamed_cost_usd, used only when a case has no record; a listed model-free case with no record or session is a known $0. No case's execution, outcome, failure class, rerun eligibility or panel verdict changes, so no pass-rate series resets"}, From c140df6d432eecbb68190cc69806dd39252e6bcf Mon Sep 17 00:00:00 2001 From: garrytan Date: Thu, 29 Oct 2026 04:45:46 +0000 Subject: [PATCH 4/8] fix: recognise --session-id= and --resume= forms Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- scripts/harness-version.json | 2 +- test/helpers/claude-pty-runner.launch.unit.test.ts | 6 +++++- test/helpers/pty/launch.ts | 7 ++++--- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/scripts/harness-version.json b/scripts/harness-version.json index 4a087d7c2..5dd942307 100644 --- a/scripts/harness-version.json +++ b/scripts/harness-version.json @@ -38,7 +38,7 @@ "test/helpers/pty/classify.ts": {"blob":"3f2daa12ff6484f96b8d28d910a50231c4bb3ae8","since":6}, "test/helpers/pty/fake-session.ts": {"blob":"554e064ef259b8444280eddfe9b0db93c66a8202","since":6}, "test/helpers/pty/judge.ts": {"blob":"81c5bb4c48ab928aca2f186a1ade9df7169d7fbe","since":6}, - "test/helpers/pty/launch.ts": {"blob":"d2e51418ce284815a26e174d777ca8b47ffd7e68","since":6,"nonBehavioral":"PTY sessions price their ledger row from the session transcript; the launcher pins a random --session-id (Claude otherwise picks its own random one) when the caller passes none and is not resuming; no prompt, answer, timeout or verdict changes"}, + "test/helpers/pty/launch.ts": {"blob":"11bbba2ed8425edd91a6214c05ecb32b33a00c41","since":6,"nonBehavioral":"the launcher also recognises --session-id= and --resume= forms before pinning a session id; no prompt, answer, timeout or verdict changes"}, "test/helpers/pty/plan-native.ts": {"blob":"ddba9cb68c7c0da935cff7c1cea258dee2dff740","since":6}, "test/helpers/pty/runners/counting.ts": {"blob":"8c5c05e54abf961259d34d9c3d77db67138d7794","since":6}, "test/helpers/pty/runners/floor.ts": {"blob":"54e2145832bb98913644d10ff303792d67f9362e","since":6,"nonBehavioral":"Removes the plan-design 0D focus picker from the shared harness; floor.ts gains an optional pickSetupChoice hook that no other caller passes, so no other case's input or verdict can change. plan-design-finding-floor, the only case whose seed raises that menu, passes the picker from its owned fixture test/fixtures/design-floor-focus-picker.ts, so its owned bytes start its own new series."}, diff --git a/test/helpers/claude-pty-runner.launch.unit.test.ts b/test/helpers/claude-pty-runner.launch.unit.test.ts index 5913435b6..413836faf 100644 --- a/test/helpers/claude-pty-runner.launch.unit.test.ts +++ b/test/helpers/claude-pty-runner.launch.unit.test.ts @@ -58,7 +58,9 @@ process.stdout.write(JSON.stringify({ capture: resolveEvalModel('capture'), fallback: await argv({}, undefined), env: await argv({}, 'env-model'), explicit: await argv({ model: 'opts-model' }, 'env-model'), extra: await argv({ extraArgs: ['--model', 'override'] }, undefined), pinned: await argv({ extraArgs: ['--session-id', '11111111-2222-4333-8444-555555555555'] }, undefined), - resumed: await argv({ extraArgs: ['--resume', 'prior'] }, undefined) })); + resumed: await argv({ extraArgs: ['--resume', 'prior'] }, undefined), + pinnedEq: await argv({ extraArgs: ['--session-id=11111111-2222-4333-8444-555555555555'] }, undefined), + resumedEq: await argv({ extraArgs: ['--resume=prior'] }, undefined) })); `); const run = (hermetic: string) => { const result = spawnSync(process.execPath, [worker], { cwd: join(import.meta.dir, '../..'), encoding: 'utf8', timeout: 30_000, @@ -81,6 +83,8 @@ process.stdout.write(JSON.stringify({ capture: resolveEvalModel('capture'), expect(sessionIds(hermetic.extra)).toHaveLength(1); expect(sessionIds(hermetic.pinned)).toEqual(['11111111-2222-4333-8444-555555555555']); expect(sessionIds(hermetic.resumed)).toEqual([]); + expect(hermetic.pinnedEq.filter((arg: string) => arg.startsWith('--session-id'))).toEqual(['--session-id=11111111-2222-4333-8444-555555555555']); + expect(sessionIds(hermetic.resumedEq)).toEqual([]); expect(sessionIds(run('0').fallback)).toEqual([]); } finally { rmSync(dir, { recursive: true, force: true }); } }, 60_000); diff --git a/test/helpers/pty/launch.ts b/test/helpers/pty/launch.ts index d2e51418c..11bbba2ed 100644 --- a/test/helpers/pty/launch.ts +++ b/test/helpers/pty/launch.ts @@ -290,9 +290,10 @@ function prepareLaunch(opts: ClaudePtyOptions): LaunchSetup { const hermetic = isHermeticEnabled(); if (hermetic) args.push('--strict-mcp-config'); const extra = opts.extraArgs ?? []; - const resumes = extra.some(arg => ['--resume', '-r', '--continue', '-c'].includes(arg)); - let sessionId = extra.includes('--session-id') ? extra[extra.indexOf('--session-id') + 1] : undefined; - if (hermetic && !sessionId && !resumes) args.push('--session-id', sessionId = randomUUID()); + const resumes = extra.some(arg => /^(--resume|--continue|-r|-c)(=|$)/.test(arg)); + const pinned = extra.findIndex(arg => /^--session-id(=|$)/.test(arg)); + let sessionId = pinned < 0 ? undefined : extra[pinned].startsWith('--session-id=') ? extra[pinned].slice('--session-id='.length) : extra[pinned + 1]; + if (hermetic && pinned < 0 && !resumes) args.push('--session-id', sessionId = randomUUID()); if (opts.extraArgs) args.push(...opts.extraArgs); // Hermetic by default (test/helpers/hermetic-env.ts): operator session From eb0be6ec1dd19f6219d4c719240546af2467fa23 Mon Sep 17 00:00:00 2001 From: garrytan Date: Thu, 29 Oct 2026 04:50:49 +0000 Subject: [PATCH 5/8] docs: CHANGELOG for PTY transcript pricing (Unreleased) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 31ded5130..54a819cf2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,16 +2,17 @@ ## [Unreleased] -**The train gate's paid report now prices the trials it used to call "cost unknown".** 32 of 139 gate trials had no cost on every gate since Oct 26, so the report's total understated a gate by about $7.50 (GSTA-150 audit). Three of the four causes are fixed here; PTY sessions follow in the next PR. +**The train gate's paid report now prices the trials it used to call "cost unknown".** 32 of 139 gate trials had no cost on every gate since Oct 26, so the report's total understated a gate by about $7.50 (GSTA-150 audit). All four causes are fixed. ### Fixed - **Label-named records in multi-case shards** (`test/helpers/e2e-helpers.ts`, `scripts/lib/paid-report.ts`): a `claude -p` record named by its display label ("browse basic commands") carries `cost_case_id` from `runSkillTest`'s registered test name, and the report binds its cost to that case. The failed record, failure class and model still come from the records named by case id, so no verdict, class or pass-rate series changes (22 trials: bws, session-intelligence, workflow, skillify, review-attribution, review, gate-integrity, diagram). - **Aborted Agent SDK sessions** (`test/helpers/agent-sdk-runner.ts`, `lib/pricing.ts`): a session that ends without a result event logs its streamed usage as `streamed_cost_usd` in the session ledger, priced once per message id at its largest output, subagent turns included, with 5-minute cache writes at 1.25x and 1-hour writes at 2x input. An unpriced model leaves the cost unknown (auq-format-gate). - **Model-free cases**: `timeline-event-flow` runs no model, so with no record and no session it is a known $0. A test checks every listed case's body starts no model session. +- **PTY sessions** (`test/helpers/pty/launch.ts`, `test/helpers/pty/session.ts`, `test/helpers/pty/transcript-cost.ts`): the hermetic launcher pins a random `--session-id` when the caller passes none and is not resuming, and each PTY ledger row prices that session's transcript (`projects/*/.jsonl` plus its `subagents/` files) as `streamed_cost_usd`. No transcript, no usage or an unpriced model leaves the cost unknown, never $0, and a read error never reaches the test. This covers 8 cases and 14 sessions per gate: plan-mode-no-op, plan-design-with-ui-scope, the plan-ceo and plan-devex finding-floor and plan-mode cases, autoplan-guard-pty and office-hours-auto-mode. Checked live, a `claude -p` session with one subagent priced $0.090376 from its transcripts, equal to its `total_cost_usd`. - **The headline total** adds session cost that no collector record carries, so newly priced trials reach the gate's `cost $X` line exactly once. ### Tests -- `test/paid-report-cost.test.ts`: pricing arithmetic, streamed dedupe, `caseCost`, `recordE2E`'s `cost_case_id`, a two-case shard whose label-named records bind by cost without changing the failed record, class or model. `test/paid-report-census.test.ts`: a record-less trial's streamed cost reaches the trial and the verdict total once. +- `test/paid-report-cost.test.ts`: pricing arithmetic, streamed dedupe, `caseCost`, `recordE2E`'s `cost_case_id`, a two-case shard whose label-named records bind by cost without changing the failed record, class or model. `test/paid-report-census.test.ts`: a record-less trial's streamed cost reaches the trial and the verdict total once. `test/helpers/pty-transcript-cost.unit.test.ts`: a session, its inline sidechains and subagent files priced once per message id; a foreign session, missing or unreadable transcript, or unpriced model stays unknown. `test/helpers/claude-pty-runner.launch.unit.test.ts`: one pinned session id, a caller's id kept in either form, none when resuming or non-hermetic. ## [1.91.304.0] - 2026-10-29 From 2c2c18901c17505caa07bffdf2946a76faa006ca Mon Sep 17 00:00:00 2001 From: garrytan Date: Thu, 29 Oct 2026 04:59:35 +0000 Subject: [PATCH 6/8] docs: PTY sessions priced from their transcripts Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- docs/TESTING_INTERNALS.md | 30 ++++++++++++++++++++++-------- docs/evals/census-red.md | 2 +- docs/troubleshooting.md | 12 +++++++----- 3 files changed, 30 insertions(+), 14 deletions(-) diff --git a/docs/TESTING_INTERNALS.md b/docs/TESTING_INTERNALS.md index eb6592644..e82046594 100644 --- a/docs/TESTING_INTERNALS.md +++ b/docs/TESTING_INTERNALS.md @@ -648,9 +648,10 @@ no price row). A rule case's trial in the JUnit census (`caseCost` in carries: each ledger session without a terminal result (`billed: false`) adds its `streamed_cost_usd`, and a trial with no record adds each billed session's own `cost_usd` (Codex). The trial is cost-unknown when any record says -`cost_known: false`, when any of those sessions has no priced cost (every PTY -session, a `claude -p` session that ended without a result, an aborted Agent -SDK session with no usage or an unpriced model), or when it has neither records +`cost_known: false`, when any of those sessions has no priced cost (a PTY +session whose transcript does not price, a `claude -p` session that ended +without a result, an aborted Agent SDK session with no usage or an unpriced +model), or when it has neither records nor sessions. A case in `MODEL_FREE_CASES` (`timeline-event-flow`, which runs no model) with neither is a known $0. A panel trial (`behavior`, `judge`) from an isolated trial shard keeps the older rule (`trialCostKnown` in @@ -694,8 +695,19 @@ by `streamedUsageCostUsd` (`lib/pricing.ts`), each message id once at its largest output, subagent turns included, `` messages skipped, 5-minute cache writes at 1.25x and 1-hour writes at 2x the input rate. The field is absent when no message carries usage or any message's model has no -price row, so an unpriced session stays cost-unknown, never $0. The ledger -rides in each slice artifact beside the shard results. +price row, so an unpriced session stays cost-unknown, never $0. A PTY session +records `streamed_cost_usd` from its transcript the same way: the hermetic +launcher (`prepareLaunch` in `test/helpers/pty/launch.ts`) pins +`--session-id ` when the caller passes none and is not resuming +(`--resume`, `-r`, `--continue`, `-c`), and both PTY ledger writers price +`/projects/*/.jsonl` plus `/subagents/*.jsonl` +(`test/helpers/pty/transcript-cost.ts`). Pricing keys on the session id, not +the whole directory, because concurrent PTY tests in one process share one +hermetic config dir. A non-hermetic or resumed launch without a caller +`--session-id`, a missing transcript, no usage, an unpriced model or a read +error leaves the field absent (cost-unknown), and a pricing failure never +fails the test. The row keeps `billed: false`. The ledger rides in each slice +artifact beside the shard results. **Quarantine** (`CASE_QUARANTINE`, same file). An entry needs: a per-trial rate below 95% over at least 10 post-policy trials of the case's current input @@ -1207,9 +1219,11 @@ observation never removes a command and a missing or empty artifact never erases one; removing a command is a reviewed edit. While the map is in shadow mode it changes no selection, so a PR that changes only the map selects no paid case. -Dollar figures are lower bounds. Known cost excludes PTY trials, whose cost is -unknown: about 39% of observed case runs in the committed map. The shadow block -and `test:health` both say so. +Dollar figures are lower bounds. Known cost excludes trials whose cost is +unknown: about 39% of observed case runs in the committed map, mostly PTY +trials recorded before PTY sessions were priced from their transcripts +([Failure causes and details](#failure-causes)). The shadow block and +`test:health` both say so. - **Shadow block:** `prCoverage.shadow` in the paid-plan manifest lists the gate cases the map would select, would drop and would add, a provenance line per diff --git a/docs/evals/census-red.md b/docs/evals/census-red.md index cc8d7b273..e3bb69991 100644 --- a/docs/evals/census-red.md +++ b/docs/evals/census-red.md @@ -93,7 +93,7 @@ What incomplete data means: | headroom `censored` | a sample timed out, so its real duration is at least the budget | | cause `unrecorded` | the trial was written before `failure_cause` existed | | cause `unknown` | the trial failed with no evidence for any other cause | -| `cost unknown` | a session's cost was not captured (a PTY session, an unbilled Codex session, a `claude -p` session without a result, an aborted Agent SDK session with no priced streamed usage), or no record or session can be attributed to the trial (a multi-case shard's display-label record without `cost_case_id`); only the known sum is shown | +| `cost unknown` | a session's cost was not captured (a PTY session without a priced transcript, an unbilled Codex session, a `claude -p` session without a result, an aborted Agent SDK session with no priced streamed usage), or no record or session can be attributed to the trial (a multi-case shard's display-label record without `cost_case_id`); only the known sum is shown | | `history unavailable (...)` | the GitHub history could not be read; `--gate` fails closed | ## 4. Replay and repair (free) diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 37ed4fb07..f0c74c96e 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -2263,11 +2263,13 @@ case with `bun run eval:pass-rates --headroom --case `. ### `cost unknown (no billing captured for any of trial(s))` / `cost $ known + of trial(s) cost unknown` **Meaning.** Some trials' cost was not captured, so only the known sum is -shown. A trial is cost-unknown when it ran a PTY session (PTY sessions record -no billing), a Codex session that completed no turn or whose model has no price -row, a `claude -p` session that ended without a result, or an aborted Agent -SDK session whose streamed usage could not be priced (no usage, or a model with -no price row). An aborted Agent SDK session with priced usage counts its +shown. A trial is cost-unknown when it ran a PTY session whose transcript +could not be priced (a non-hermetic or resumed launch without a caller +`--session-id`, no transcript or usage, or a model with no price row), a +Codex session that completed no turn or whose model has no price row, a +`claude -p` session that ended without a result, or an aborted Agent SDK +session whose streamed usage could not be priced (no usage, or a model with no +price row). A PTY or aborted Agent SDK session that does price counts its `streamed_cost_usd` instead. A trial is also unknown when no record and no session can be attributed to it, for example a multi-case shard's display-label record without `cost_case_id` (a one-case shard's records are From 267c55acbceea2d7d5c0fbd82764d32125ef234b Mon Sep 17 00:00:00 2001 From: garrytan Date: Thu, 29 Oct 2026 05:10:10 +0000 Subject: [PATCH 7/8] fix: price PTY transcripts without adding a launch argument; native AUQ capture opts out Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CHANGELOG.md | 4 +- docs/TESTING_INTERNALS.md | 23 +++--- docs/troubleshooting.md | 5 +- scripts/harness-version.json | 4 +- test/helpers/auq-native-capture.ts | 2 +- .../claude-pty-runner.launch.unit.test.ts | 57 +++++++++++---- test/helpers/pty-transcript-cost.unit.test.ts | 60 --------------- test/helpers/pty/launch.ts | 33 ++++++--- test/helpers/pty/transcript-cost.ts | 37 +++++++++- test/paid-report-cost.test.ts | 73 +++++++++++++++++++ 10 files changed, 191 insertions(+), 107 deletions(-) delete mode 100644 test/helpers/pty-transcript-cost.unit.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 54a819cf2..f788f8dfe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,11 +8,11 @@ - **Label-named records in multi-case shards** (`test/helpers/e2e-helpers.ts`, `scripts/lib/paid-report.ts`): a `claude -p` record named by its display label ("browse basic commands") carries `cost_case_id` from `runSkillTest`'s registered test name, and the report binds its cost to that case. The failed record, failure class and model still come from the records named by case id, so no verdict, class or pass-rate series changes (22 trials: bws, session-intelligence, workflow, skillify, review-attribution, review, gate-integrity, diagram). - **Aborted Agent SDK sessions** (`test/helpers/agent-sdk-runner.ts`, `lib/pricing.ts`): a session that ends without a result event logs its streamed usage as `streamed_cost_usd` in the session ledger, priced once per message id at its largest output, subagent turns included, with 5-minute cache writes at 1.25x and 1-hour writes at 2x input. An unpriced model leaves the cost unknown (auq-format-gate). - **Model-free cases**: `timeline-event-flow` runs no model, so with no record and no session it is a known $0. A test checks every listed case's body starts no model session. -- **PTY sessions** (`test/helpers/pty/launch.ts`, `test/helpers/pty/session.ts`, `test/helpers/pty/transcript-cost.ts`): the hermetic launcher pins a random `--session-id` when the caller passes none and is not resuming, and each PTY ledger row prices that session's transcript (`projects/*/.jsonl` plus its `subagents/` files) as `streamed_cost_usd`. No transcript, no usage or an unpriced model leaves the cost unknown, never $0, and a read error never reaches the test. This covers 8 cases and 14 sessions per gate: plan-mode-no-op, plan-design-with-ui-scope, the plan-ceo and plan-devex finding-floor and plan-mode cases, autoplan-guard-pty and office-hours-auto-mode. Checked live, a `claude -p` session with one subagent priced $0.090376 from its transcripts, equal to its `total_cost_usd`. +- **PTY sessions** (`test/helpers/pty/launch.ts`, `test/helpers/pty/session.ts`, `test/helpers/pty/transcript-cost.ts`): each PTY ledger row prices the session's transcript (`projects/*/.jsonl` plus its `subagents/` files) as `streamed_cost_usd`. The session is the caller's `--session-id`, else the one transcript that appeared during the launch for its cwd, so the launcher adds no argument and concurrent tests sharing a config dir never price each other. No transcript, two new transcripts for one cwd, no usage or an unpriced model leaves the cost unknown, never $0, and a read error never reaches the test. The native AskUserQuestion capture, which never reads model transcripts, opts out (`priceTranscript: false`). This covers 8 cases and 14 sessions per gate: plan-mode-no-op, plan-design-with-ui-scope, the plan-ceo and plan-devex finding-floor and plan-mode cases, autoplan-guard-pty and office-hours-auto-mode. Checked live, a `claude -p` session with one subagent priced $0.090376 from its transcripts, equal to its `total_cost_usd`. - **The headline total** adds session cost that no collector record carries, so newly priced trials reach the gate's `cost $X` line exactly once. ### Tests -- `test/paid-report-cost.test.ts`: pricing arithmetic, streamed dedupe, `caseCost`, `recordE2E`'s `cost_case_id`, a two-case shard whose label-named records bind by cost without changing the failed record, class or model. `test/paid-report-census.test.ts`: a record-less trial's streamed cost reaches the trial and the verdict total once. `test/helpers/pty-transcript-cost.unit.test.ts`: a session, its inline sidechains and subagent files priced once per message id; a foreign session, missing or unreadable transcript, or unpriced model stays unknown. `test/helpers/claude-pty-runner.launch.unit.test.ts`: one pinned session id, a caller's id kept in either form, none when resuming or non-hermetic. +- `test/paid-report-cost.test.ts`: pricing arithmetic, streamed dedupe, `caseCost`, `recordE2E`'s `cost_case_id`, a two-case shard whose label-named records bind by cost without changing the failed record, class or model. `test/paid-report-census.test.ts`: a record-less trial's streamed cost reaches the trial and the verdict total once. It also prices a PTY session, its inline sidechains and subagent files once per message id, leaves a foreign session, a missing or unreadable transcript or an unpriced model unknown, and claims only the one new transcript for a launch's cwd. `test/helpers/claude-pty-runner.launch.unit.test.ts`: a real launch prices the transcript it created without adding an argument, and an opted-out launch prices nothing. ## [1.91.304.0] - 2026-10-29 diff --git a/docs/TESTING_INTERNALS.md b/docs/TESTING_INTERNALS.md index e82046594..2d3d6888f 100644 --- a/docs/TESTING_INTERNALS.md +++ b/docs/TESTING_INTERNALS.md @@ -696,17 +696,18 @@ largest output, subagent turns included, `` messages skipped, 5-minute cache writes at 1.25x and 1-hour writes at 2x the input rate. The field is absent when no message carries usage or any message's model has no price row, so an unpriced session stays cost-unknown, never $0. A PTY session -records `streamed_cost_usd` from its transcript the same way: the hermetic -launcher (`prepareLaunch` in `test/helpers/pty/launch.ts`) pins -`--session-id ` when the caller passes none and is not resuming -(`--resume`, `-r`, `--continue`, `-c`), and both PTY ledger writers price -`/projects/*/.jsonl` plus `/subagents/*.jsonl` -(`test/helpers/pty/transcript-cost.ts`). Pricing keys on the session id, not -the whole directory, because concurrent PTY tests in one process share one -hermetic config dir. A non-hermetic or resumed launch without a caller -`--session-id`, a missing transcript, no usage, an unpriced model or a read -error leaves the field absent (cost-unknown), and a pricing failure never -fails the test. The row keeps `billed: false`. The ledger rides in each slice +records `streamed_cost_usd` from its transcript the same way. Both PTY ledger +writers price `/projects/*/.jsonl` plus +`/subagents/*.jsonl` (`test/helpers/pty/transcript-cost.ts`), where the +id is the caller's `--session-id`, else the one transcript that appeared +during this launch for its cwd. The launcher adds no argument. Pricing never +takes the whole directory, because concurrent PTY tests in one process share +one hermetic config dir; two new transcripts for one cwd are ambiguous and stay +unknown. A non-hermetic or resumed launch, a caller that opts out +(`priceTranscript: false`, the native AskUserQuestion capture, which never +reads model transcripts), a missing transcript, no usage, an unpriced model or +a read error leaves the field absent (cost-unknown), and a pricing failure +never fails the test. The row keeps `billed: false`. The ledger rides in each slice artifact beside the shard results. **Quarantine** (`CASE_QUARANTINE`, same file). An entry needs: a per-trial rate diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index f0c74c96e..a0de5f624 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -2264,8 +2264,9 @@ case with `bun run eval:pass-rates --headroom --case `. **Meaning.** Some trials' cost was not captured, so only the known sum is shown. A trial is cost-unknown when it ran a PTY session whose transcript -could not be priced (a non-hermetic or resumed launch without a caller -`--session-id`, no transcript or usage, or a model with no price row), a +could not be priced (a non-hermetic or resumed launch, the native +AskUserQuestion capture, two new transcripts for one cwd, no transcript or +usage, or a model with no price row), a Codex session that completed no turn or whose model has no price row, a `claude -p` session that ended without a result, or an aborted Agent SDK session whose streamed usage could not be priced (no usage, or a model with no diff --git a/scripts/harness-version.json b/scripts/harness-version.json index 5dd942307..13e76e098 100644 --- a/scripts/harness-version.json +++ b/scripts/harness-version.json @@ -38,14 +38,14 @@ "test/helpers/pty/classify.ts": {"blob":"3f2daa12ff6484f96b8d28d910a50231c4bb3ae8","since":6}, "test/helpers/pty/fake-session.ts": {"blob":"554e064ef259b8444280eddfe9b0db93c66a8202","since":6}, "test/helpers/pty/judge.ts": {"blob":"81c5bb4c48ab928aca2f186a1ade9df7169d7fbe","since":6}, - "test/helpers/pty/launch.ts": {"blob":"11bbba2ed8425edd91a6214c05ecb32b33a00c41","since":6,"nonBehavioral":"the launcher also recognises --session-id= and --resume= forms before pinning a session id; no prompt, answer, timeout or verdict changes"}, + "test/helpers/pty/launch.ts": {"blob":"4f0731ef39bc3a6a7c5c592ce98690a24cb1ed7f","since":6,"nonBehavioral":"PTY transcript pricing finds the session by the caller's --session-id or the one new transcript for the launch cwd; the launcher adds no argument; native AUQ capture opts out; ledger cost fields only, no prompt, argv, answer, timeout or verdict changes"}, "test/helpers/pty/plan-native.ts": {"blob":"ddba9cb68c7c0da935cff7c1cea258dee2dff740","since":6}, "test/helpers/pty/runners/counting.ts": {"blob":"8c5c05e54abf961259d34d9c3d77db67138d7794","since":6}, "test/helpers/pty/runners/floor.ts": {"blob":"54e2145832bb98913644d10ff303792d67f9362e","since":6,"nonBehavioral":"Removes the plan-design 0D focus picker from the shared harness; floor.ts gains an optional pickSetupChoice hook that no other caller passes, so no other case's input or verdict can change. plan-design-finding-floor, the only case whose seed raises that menu, passes the picker from its owned fixture test/fixtures/design-floor-focus-picker.ts, so its owned bytes start its own new series."}, "test/helpers/pty/runners/observation.ts": {"blob":"1c2b75fbb984708725abaa8939125f7cab969f9f","since":6}, "test/helpers/pty/screen.ts": {"blob":"f3743152fc9bb7a8e8c70002f75bf45914f260d5","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"}, "test/helpers/pty/session.ts": {"blob":"8fd1749a62f24dc37ae186c8a21c2165f7f7645e","since":6,"nonBehavioral":"PTY sessions price their ledger row from the session transcript; the launcher pins a random --session-id (Claude otherwise picks its own random one) when the caller passes none and is not resuming; no prompt, answer, timeout or verdict changes"}, - "test/helpers/pty/transcript-cost.ts": {"blob":"c54a70bed59c60f1249ba8cb13e83ecd7e6d7471","since":6,"nonBehavioral":"transcript-cost.ts prices a PTY session's ledger row from its transcript and never throws; cost fields only, no prompt, answer, timeout or verdict changes"}, + "test/helpers/pty/transcript-cost.ts": {"blob":"08f7ccbdbd9d71a5dc86edd2f8fced423a78ad9a","since":6,"nonBehavioral":"PTY transcript pricing finds the session by the caller's --session-id or the one new transcript for the launch cwd; the launcher adds no argument; native AUQ capture opts out; ledger cost fields only, no prompt, argv, answer, timeout or verdict changes"}, "test/helpers/session-drain-policy.ts": {"blob":"6211239dd0485e6495a700b824a7e0e61d1bcd97","since":6}, "test/helpers/session-ledger.ts": {"blob":"c99baaaa0a77a5a72384493fd8a85a4253ecb05d","since":6,"nonBehavioral":"PTY sessions price their ledger row from the session transcript; the launcher pins a random --session-id (Claude otherwise picks its own random one) when the caller passes none and is not resuming; no prompt, answer, timeout or verdict changes"}, "test/helpers/session-runner.ts": {"blob":"c73b2596d9eb5377530a6bf1d284e0b6ea1556a0","since":6,"nonBehavioral":"Cost attribution only (GSTA-150 PR 2): a display-label record carries cost_case_id from runSkillTest's testName and the report binds its cost by it while the failed record and failure class still come from the named records (test/paid-report-cost.test.ts pins both); an aborted Agent SDK session logs its streamed usage as streamed_cost_usd, used only when a case has no record; a listed model-free case with no record or session is a known $0. No case's execution, outcome, failure class, rerun eligibility or panel verdict changes, so no pass-rate series resets"}, diff --git a/test/helpers/auq-native-capture.ts b/test/helpers/auq-native-capture.ts index ba70bb8c4..27b1cee56 100644 --- a/test/helpers/auq-native-capture.ts +++ b/test/helpers/auq-native-capture.ts @@ -313,7 +313,7 @@ Skip any system-audit / environment-setup / codebase-exploration steps. At the f session = await launchClaudePty({ // Explicit tool approvals suffice. Bypass mode introduces a separate // safety-consent screen in a fresh CLI config; never accept that screen. - cwd, model, seedSkills: false, permissionMode: 'default', + cwd, model, seedSkills: false, permissionMode: 'default', priceTranscript: false, observeScreen: true, observeSetupQuestions: true, timeoutMs: Math.max(1, deadline - Date.now()), // Explicit availability as well as approval: no Bash/Agent/Skill or MCP. diff --git a/test/helpers/claude-pty-runner.launch.unit.test.ts b/test/helpers/claude-pty-runner.launch.unit.test.ts index 413836faf..5abbb8e05 100644 --- a/test/helpers/claude-pty-runner.launch.unit.test.ts +++ b/test/helpers/claude-pty-runner.launch.unit.test.ts @@ -39,7 +39,7 @@ describe('launchClaudePty model pin', () => { expect(opts.model).toBe('claude-sonnet-4-6'); }); - test.skipIf(process.platform === 'win32')('spawn args pin --model from the fallback chain, before extraArgs, with hermetic MCP gating and a pinned session id', () => { + test.skipIf(process.platform === 'win32')('spawn args pin --model from the fallback chain, before extraArgs, with hermetic MCP gating', () => { const dir = mkdtempSync(join(tmpdir(), 'pty-model-pin-')); try { const fake = join(dir, 'fake-claude'); @@ -56,11 +56,7 @@ const argv = async (opts, evalsModel) => { }; process.stdout.write(JSON.stringify({ capture: resolveEvalModel('capture'), fallback: await argv({}, undefined), env: await argv({}, 'env-model'), explicit: await argv({ model: 'opts-model' }, 'env-model'), - extra: await argv({ extraArgs: ['--model', 'override'] }, undefined), - pinned: await argv({ extraArgs: ['--session-id', '11111111-2222-4333-8444-555555555555'] }, undefined), - resumed: await argv({ extraArgs: ['--resume', 'prior'] }, undefined), - pinnedEq: await argv({ extraArgs: ['--session-id=11111111-2222-4333-8444-555555555555'] }, undefined), - resumedEq: await argv({ extraArgs: ['--resume=prior'] }, undefined) })); + extra: await argv({ extraArgs: ['--model', 'override'] }, undefined) })); `); const run = (hermetic: string) => { const result = spawnSync(process.execPath, [worker], { cwd: join(import.meta.dir, '../..'), encoding: 'utf8', timeout: 30_000, @@ -77,15 +73,46 @@ process.stdout.write(JSON.stringify({ capture: resolveEvalModel('capture'), expect(hermetic.extra.slice(-2)).toEqual(['--model', 'override']); expect(hermetic.fallback).toContain('--strict-mcp-config'); expect(run('0').fallback).not.toContain('--strict-mcp-config'); - const sessionIds = (args: string[]) => args.flatMap((arg, i) => arg === '--session-id' ? [args[i + 1]] : []); - expect(sessionIds(hermetic.fallback)).toHaveLength(1); - expect(sessionIds(hermetic.fallback)[0]).toMatch(/^[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}$/); - expect(sessionIds(hermetic.extra)).toHaveLength(1); - expect(sessionIds(hermetic.pinned)).toEqual(['11111111-2222-4333-8444-555555555555']); - expect(sessionIds(hermetic.resumed)).toEqual([]); - expect(hermetic.pinnedEq.filter((arg: string) => arg.startsWith('--session-id'))).toEqual(['--session-id=11111111-2222-4333-8444-555555555555']); - expect(sessionIds(hermetic.resumedEq)).toEqual([]); - expect(sessionIds(run('0').fallback)).toEqual([]); + } finally { rmSync(dir, { recursive: true, force: true }); } + }, 60_000); +}); + +describe('launchClaudePty transcript pricing (GSTA-150 PR 3)', () => { + test.skipIf(process.platform === 'win32')('prices the transcript this launch created for its cwd, without adding a launch argument; opting out prices nothing', () => { + const dir = mkdtempSync(join(tmpdir(), 'pty-transcript-cost-')); + try { + const fake = join(dir, 'fake-claude'); + writeFileSync(fake, `#!${process.execPath} +const fs = require('node:fs'), path = require('node:path'), crypto = require('node:crypto'); +const project = path.join(process.env.CLAUDE_CONFIG_DIR, 'projects', 'fake-project'); +fs.mkdirSync(project, { recursive: true }); +fs.writeFileSync(path.join(project, crypto.randomUUID() + '.jsonl'), JSON.stringify({ type: 'assistant', cwd: process.cwd(), + message: { id: 'msg_' + crypto.randomUUID(), model: 'claude-opus-5-5', usage: { input_tokens: 0, output_tokens: 1000 } } }) + '\\n'); +process.stdout.write('ARGV=' + JSON.stringify(process.argv.slice(2)) + '\\n'); +setInterval(() => {}, 1000); +`, { mode: 0o755 }); + const runner = pathToFileURL(join(import.meta.dir, 'claude-pty-runner.ts')).href; + const worker = join(dir, 'worker.ts'); + writeFileSync(worker, `import { launchClaudePty } from ${JSON.stringify(runner)}; +import * as fs from 'node:fs'; +const run = async (opts) => { + const cwd = fs.mkdtempSync(${JSON.stringify(join(dir, 'cwd-'))}); + const session = await launchClaudePty({ cwd, timeoutMs: 5000, ...opts }); + try { + await session.waitFor(/ARGV=\\[.*\\]/, { timeoutMs: 4000, pollMs: 20 }); + return { argv: JSON.parse(/ARGV=(\\[.*\\])/.exec(session.visibleText())[1]), cost: session.transcriptCostUsd?.() ?? null }; + } finally { await session.close(); } +}; +const [priced, optedOut] = await Promise.all([run({}), run({ priceTranscript: false })]); +process.stdout.write(JSON.stringify({ priced, optedOut })); +`); + const result = spawnSync(process.execPath, [worker], { cwd: join(import.meta.dir, '../..'), encoding: 'utf8', timeout: 30_000, + env: { ...process.env, BROWSE_TERMINAL_BINARY: fake, EVALS_HERMETIC: '1' } }); + expect(result.status, result.stderr).toBe(0); + const { priced, optedOut } = JSON.parse(result.stdout.trim().split('\n').at(-1)!); + expect(priced.cost).toBeCloseTo(0.02, 6); + expect(optedOut.cost).toBeNull(); + expect(priced.argv).not.toContain('--session-id'); } finally { rmSync(dir, { recursive: true, force: true }); } }, 60_000); }); diff --git a/test/helpers/pty-transcript-cost.unit.test.ts b/test/helpers/pty-transcript-cost.unit.test.ts deleted file mode 100644 index d05d3591e..000000000 --- a/test/helpers/pty-transcript-cost.unit.test.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { describe, test, expect } from 'bun:test'; -import * as fs from 'node:fs'; -import * as os from 'node:os'; -import * as path from 'node:path'; -import { pricedTranscript, ptyTranscriptCostUsd } from './pty/transcript-cost'; - -const SID = '11111111-2222-4333-8444-555555555555'; -const assistant = (id: string, model: string, usage: object, extra: object = {}) => - JSON.stringify({ type: 'assistant', sessionId: SID, message: { id, model, role: 'assistant', usage }, ...extra }); - -function fixture(write: (project: string) => void): string { - const config = fs.mkdtempSync(path.join(os.tmpdir(), 'pty-cost-')); - const project = path.join(config, 'projects', '-fixture-cwd'); - fs.mkdirSync(project, { recursive: true }); - write(project); - return config; -} - -describe('ptyTranscriptCostUsd', () => { - test('prices the session, its inline sidechains and subagent files, each message id once at its largest output', () => { - const config = fixture(project => { - fs.writeFileSync(path.join(project, `${SID}.jsonl`), [ - JSON.stringify({ type: 'user', sessionId: SID, message: { role: 'user', content: 'hi' } }), - assistant('msg_a', 'claude-opus-5-5', { input_tokens: 1000, output_tokens: 10 }), - assistant('msg_a', 'claude-opus-5-5', { input_tokens: 1000, output_tokens: 500 }), - assistant('msg_side', 'claude-opus-5-5', { input_tokens: 0, output_tokens: 1000 }, { isSidechain: true }), - assistant('msg_syn', '', { input_tokens: 0, output_tokens: 0 }), - '{"partial":', - ].join('\n')); - fs.mkdirSync(path.join(project, SID, 'subagents'), { recursive: true }); - fs.writeFileSync(path.join(project, SID, 'subagents', 'agent-x.jsonl'), - assistant('msg_sub', 'claude-opus-5-5', { input_tokens: 0, cache_read_input_tokens: 10000, output_tokens: 0 }) + '\n'); - fs.writeFileSync(path.join(project, '99999999-2222-4333-8444-555555555555.jsonl'), - assistant('msg_foreign', 'claude-opus-5-5', { input_tokens: 1_000_000, output_tokens: 0 }) + '\n'); - }); - try { - // opus-5-5: $4 in, $20 out, $0.20 cached per MTok. - const expected = (1000 * 4 + 500 * 20 + 1000 * 20 + 10000 * 0.2) / 1_000_000; - expect(ptyTranscriptCostUsd(config, SID)).toBeCloseTo(expected, 6); - expect(pricedTranscript({ hermeticConfigDir: config, sessionId: SID }).streamed_cost_usd).toBeCloseTo(expected, 6); - } finally { fs.rmSync(config, { recursive: true, force: true }); } - }); - - test('stays cost-unknown without a transcript, a session id, usage or a priced model', () => { - const unpriced = fixture(project => fs.writeFileSync(path.join(project, `${SID}.jsonl`), - assistant('msg_a', 'claude-opus-5-5', { output_tokens: 1 }) + '\n' + assistant('msg_b', 'some-unpriced-model', { output_tokens: 1 }) + '\n')); - const empty = fixture(project => fs.writeFileSync(path.join(project, `${SID}.jsonl`), JSON.stringify({ type: 'user', sessionId: SID }) + '\n')); - const unreadable = fixture(project => fs.mkdirSync(path.join(project, `${SID}.jsonl`))); - try { - expect(ptyTranscriptCostUsd(unpriced, SID)).toBeUndefined(); - expect(ptyTranscriptCostUsd(empty, SID)).toBeUndefined(); - expect(ptyTranscriptCostUsd(empty, '99999999-2222-4333-8444-555555555555')).toBeUndefined(); - expect(ptyTranscriptCostUsd(unpriced, undefined)).toBeUndefined(); - expect(ptyTranscriptCostUsd(null, SID)).toBeUndefined(); - expect(ptyTranscriptCostUsd(path.join(empty, 'missing'), SID)).toBeUndefined(); - expect(pricedTranscript(undefined)).toEqual({}); - expect(ptyTranscriptCostUsd(unreadable, SID)).toBeUndefined(); - } finally { for (const dir of [unpriced, empty, unreadable]) fs.rmSync(dir, { recursive: true, force: true }); } - }); -}); diff --git a/test/helpers/pty/launch.ts b/test/helpers/pty/launch.ts index 11bbba2ed..4f0731ef3 100644 --- a/test/helpers/pty/launch.ts +++ b/test/helpers/pty/launch.ts @@ -2,7 +2,6 @@ * Spawn `claude` in a real PTY (launchClaudePty) and the session handle. Moved from claude-pty-runner.ts. * Import through test/helpers/claude-pty-runner.ts from tests; pty/ modules import siblings directly. */ -import { randomUUID } from 'node:crypto'; import * as path from 'node:path'; import { resolveEvalModel } from '../../../lib/eval-model'; import { hermeticChildEnv, hermeticSkillsConfigDir, isHermeticEnabled } from '../hermetic-env'; @@ -16,9 +15,12 @@ import { trustDialogInput } from '../pty-trust-dialog'; import { resolveClaudeBinary } from './binary'; import { createPtyScreen, screenPanelEnd, stripAnsi } from './screen'; import { appendSessionLedger, sessionKey } from '../session-ledger'; -import { pricedTranscript } from './transcript-cost'; +import { newTranscriptSessionId, pricedTranscript, ptyTranscriptCostUsd, transcriptFiles } from './transcript-cost'; export interface ClaudePtyOptions { + /** Price this session's ledger row from the transcript Claude writes under the hermetic config dir (default true). + * false for a caller that must never read model transcripts, such as the native AskUserQuestion capture. */ + priceTranscript?: boolean; /** Register the repo's shipped skills in the child's user scope via * hermeticSkillsConfigDir(). Required by any test that types a /skill * slash command; without it hermetic claude rejects the command as @@ -131,8 +133,9 @@ export interface ClaudePtySession { * the dir name ends in `/.claude` by contract). */ hermeticConfigDir: string | null; - /** The Claude session id the launcher pinned (or the caller's --session-id): its transcript prices the session ledger row. */ - sessionId?: string; + /** This session's cost from its transcript (the caller's --session-id, else the one transcript this launch created + * for its cwd); undefined when unpriced, ambiguous or opted out. Never throws. */ + transcriptCostUsd?(): number | undefined; /** Owned HOME/.gstack created by the seeded launcher; absent for caller overrides. */ hermeticSkillStateRoot?: string; /** Owned pre-tool identity record, removed by close(). */ @@ -268,7 +271,8 @@ interface LaunchSetup { hermeticSkillStateRoot: string | undefined; autoplanArtifactStateRoot: string | undefined; autoplanEngTestPlanStateRoot: string | undefined; - sessionId: string | undefined; + /** What prices the transcript: the caller's session id, else the transcripts present before spawn. */ + transcript: { sessionId: string | undefined; before: Set; cwd: string } | undefined; } function prepareLaunch(opts: ClaudePtyOptions): LaunchSetup { @@ -289,11 +293,6 @@ function prepareLaunch(opts: ClaudePtyOptions): LaunchSetup { // Before opts.extraArgs so a test could theoretically supply --mcp-config. const hermetic = isHermeticEnabled(); if (hermetic) args.push('--strict-mcp-config'); - const extra = opts.extraArgs ?? []; - const resumes = extra.some(arg => /^(--resume|--continue|-r|-c)(=|$)/.test(arg)); - const pinned = extra.findIndex(arg => /^--session-id(=|$)/.test(arg)); - let sessionId = pinned < 0 ? undefined : extra[pinned].startsWith('--session-id=') ? extra[pinned].slice('--session-id='.length) : extra[pinned + 1]; - if (hermetic && pinned < 0 && !resumes) args.push('--session-id', sessionId = randomUUID()); if (opts.extraArgs) args.push(...opts.extraArgs); // Hermetic by default (test/helpers/hermetic-env.ts): operator session @@ -330,7 +329,13 @@ function prepareLaunch(opts: ClaudePtyOptions): LaunchSetup { const autoplanEngTestPlanStateRoot = opts.approveAutoplanArtifactEdits && opts.autoplanArtifactState !== undefined ? hermeticSkillStateRoot : undefined; - return { args, childEnv, hermetic, hermeticSkillStateRoot, autoplanArtifactStateRoot, autoplanEngTestPlanStateRoot, sessionId }; + const extra = opts.extraArgs ?? []; + const pinned = extra.findIndex(arg => /^--session-id(=|$)/.test(arg)); + const sessionId = pinned < 0 ? undefined : extra[pinned].startsWith('--session-id=') ? extra[pinned].slice('--session-id='.length) : extra[pinned + 1]; + const resumes = extra.some(arg => /^(--resume|--continue|-r|-c)(=|$)/.test(arg)); + const transcript = hermetic && childEnv.CLAUDE_CONFIG_DIR && opts.priceTranscript !== false && !resumes + ? { sessionId, before: transcriptFiles(childEnv.CLAUDE_CONFIG_DIR), cwd: opts.cwd ?? process.cwd() } : undefined; + return { args, childEnv, hermetic, hermeticSkillStateRoot, autoplanArtifactStateRoot, autoplanEngTestPlanStateRoot, transcript }; } interface LaunchRecorders { @@ -579,7 +584,11 @@ function sessionHandle(pty: PtyProcess, launch: LaunchSetup, wallTimer: ReturnTy exited: () => pty.exited, exitCode: () => pty.exitCode, hermeticConfigDir: launch.hermetic ? launch.childEnv.CLAUDE_CONFIG_DIR ?? null : null, - sessionId: launch.sessionId, + transcriptCostUsd: () => { + const { transcript, childEnv } = launch; + return transcript && ptyTranscriptCostUsd(childEnv.CLAUDE_CONFIG_DIR, + transcript.sessionId ?? newTranscriptSessionId(childEnv.CLAUDE_CONFIG_DIR, transcript.before, transcript.cwd)); + }, hermeticSkillStateRoot: launch.hermeticSkillStateRoot, pendingPlanReadyFile: pendingExit?.file, pendingQuestionFile: pendingQuestion?.file, diff --git a/test/helpers/pty/transcript-cost.ts b/test/helpers/pty/transcript-cost.ts index c54a70bed..08f7ccbdb 100644 --- a/test/helpers/pty/transcript-cost.ts +++ b/test/helpers/pty/transcript-cost.ts @@ -3,11 +3,44 @@ import * as path from 'node:path'; import { streamedUsageCostUsd } from '../../../lib/pricing'; /** The session ledger's cost field for a PTY session: present only when its transcript prices. */ -export function pricedTranscript(session: { hermeticConfigDir?: string | null; sessionId?: string } | undefined): { streamed_cost_usd?: number } { - const cost = ptyTranscriptCostUsd(session?.hermeticConfigDir, session?.sessionId); +export function pricedTranscript(session: { transcriptCostUsd?(): number | undefined } | undefined): { streamed_cost_usd?: number } { + const cost = session?.transcriptCostUsd?.(); return cost === undefined ? {} : { streamed_cost_usd: cost }; } +/** Top-level session transcripts (`projects//.jsonl`) under a config dir; empty when unreadable. */ +export function transcriptFiles(configDir: string | undefined): Set { + if (!configDir) return new Set(); + try { + const projects = path.join(configDir, 'projects'); + return new Set(fs.readdirSync(projects, { withFileTypes: true }).filter(dir => dir.isDirectory()) + .flatMap(dir => fs.readdirSync(path.join(projects, dir.name)).filter(f => f.endsWith('.jsonl')).map(f => path.join(projects, dir.name, f)))); + } catch { + return new Set(); + } +} + +/** + * The session id of the one transcript that appeared since `before` and was written for `cwd`. Concurrent PTY tests in + * one process share a hermetic config dir, so a launch claims only a new transcript for its own cwd; none or several + * (two sessions in one cwd) is undefined, which leaves the cost unknown. + */ +export function newTranscriptSessionId(configDir: string | undefined, before: ReadonlySet, cwd: string): string | undefined { + try { + const cwds = new Set([cwd, fs.realpathSync(cwd)]); + const mine = [...transcriptFiles(configDir)].filter(file => !before.has(file) && fs.readFileSync(file, 'utf8').split('\n').some(line => { + try { + return cwds.has(JSON.parse(line)?.cwd); + } catch { + return false; + } + })); + return mine.length === 1 ? path.basename(mine[0], '.jsonl') : undefined; + } catch { + return undefined; + } +} + /** * Price one PTY session from the transcripts Claude Code wrote under its owned config dir: * `projects//.jsonl` (inline sidechains included) and `projects///subagents/*.jsonl`, diff --git a/test/paid-report-cost.test.ts b/test/paid-report-cost.test.ts index ac5e08eff..fcfc2dd30 100644 --- a/test/paid-report-cost.test.ts +++ b/test/paid-report-cost.test.ts @@ -12,6 +12,7 @@ import { anthropicUsageCostUsd, streamedUsageCostUsd } from '../lib/pricing'; import { E2E_TOUCHFILES } from './helpers/touchfiles-data'; import type { SessionLedgerRow } from './helpers/session-ledger'; import { recordE2E } from './helpers/e2e-helpers'; +import { newTranscriptSessionId, pricedTranscript, ptyTranscriptCostUsd, transcriptFiles } from './helpers/pty/transcript-cost'; const ROOT = path.resolve(import.meta.dir, '..'); const session = (over: Partial): SessionLedgerRow => ({ key: 'k', runner: 'agent-sdk', started_at: 't', elapsed_ms: 1, end: 'aborted', ...over }); @@ -128,3 +129,75 @@ describe('junitCensus cost binding', () => { expect(history.map(h => [h.case, h.model])).toEqual([['browse-basic', undefined], ['browse-snapshot', undefined]]); }); }); + +describe('PTY transcript pricing (GSTA-150 PR 3)', () => { + const SID = '11111111-2222-4333-8444-555555555555'; + const OTHER = '99999999-2222-4333-8444-555555555555'; + const assistant = (sid: string, id: string, model: string, usage: object, extra: object = {}) => + JSON.stringify({ type: 'assistant', sessionId: sid, cwd: '/fixture/cwd', message: { id, model, role: 'assistant', usage }, ...extra }); + const fixture = (write: (project: string) => void) => { + const config = fs.mkdtempSync(path.join(os.tmpdir(), 'pty-cost-')); + const project = path.join(config, 'projects', '-fixture-cwd'); + fs.mkdirSync(project, { recursive: true }); + write(project); + return config; + }; + + test('prices the session, its inline sidechains and subagent files, each message id once at its largest output', () => { + const config = fixture(project => { + fs.writeFileSync(path.join(project, `${SID}.jsonl`), [ + JSON.stringify({ type: 'user', sessionId: SID, message: { role: 'user', content: 'hi' } }), + assistant(SID, 'msg_a', 'claude-opus-5-5', { input_tokens: 1000, output_tokens: 10 }), + assistant(SID, 'msg_a', 'claude-opus-5-5', { input_tokens: 1000, output_tokens: 500 }), + assistant(SID, 'msg_side', 'claude-opus-5-5', { input_tokens: 0, output_tokens: 1000 }, { isSidechain: true }), + assistant(SID, 'msg_syn', '', { input_tokens: 0, output_tokens: 0 }), + '{"partial":', + ].join('\n')); + fs.mkdirSync(path.join(project, SID, 'subagents'), { recursive: true }); + fs.writeFileSync(path.join(project, SID, 'subagents', 'agent-x.jsonl'), + assistant(SID, 'msg_sub', 'claude-opus-5-5', { input_tokens: 0, cache_read_input_tokens: 10000, output_tokens: 0 }) + '\n'); + fs.writeFileSync(path.join(project, `${OTHER}.jsonl`), assistant(OTHER, 'msg_foreign', 'claude-opus-5-5', { input_tokens: 1_000_000 }) + '\n'); + }); + try { + // opus-5-5: $4 in, $20 out, $0.20 cached per MTok. + const expected = (1000 * 4 + 500 * 20 + 1000 * 20 + 10000 * 0.2) / 1_000_000; + expect(ptyTranscriptCostUsd(config, SID)).toBeCloseTo(expected, 6); + expect(pricedTranscript({ transcriptCostUsd: () => ptyTranscriptCostUsd(config, SID) }).streamed_cost_usd).toBeCloseTo(expected, 6); + } finally { fs.rmSync(config, { recursive: true, force: true }); } + }); + + test('stays cost-unknown without a transcript, a session id, usage, a priced model or a readable file, and never throws', () => { + const unpriced = fixture(project => fs.writeFileSync(path.join(project, `${SID}.jsonl`), + assistant(SID, 'msg_a', 'claude-opus-5-5', { output_tokens: 1 }) + '\n' + assistant(SID, 'msg_b', 'some-unpriced-model', { output_tokens: 1 }) + '\n')); + const empty = fixture(project => fs.writeFileSync(path.join(project, `${SID}.jsonl`), JSON.stringify({ type: 'user', sessionId: SID }) + '\n')); + const unreadable = fixture(project => fs.mkdirSync(path.join(project, `${SID}.jsonl`))); + try { + expect(ptyTranscriptCostUsd(unpriced, SID)).toBeUndefined(); + expect(ptyTranscriptCostUsd(empty, SID)).toBeUndefined(); + expect(ptyTranscriptCostUsd(empty, OTHER)).toBeUndefined(); + expect(ptyTranscriptCostUsd(unpriced, undefined)).toBeUndefined(); + expect(ptyTranscriptCostUsd(null, SID)).toBeUndefined(); + expect(ptyTranscriptCostUsd(path.join(empty, 'missing'), SID)).toBeUndefined(); + expect(ptyTranscriptCostUsd(unreadable, SID)).toBeUndefined(); + expect(newTranscriptSessionId(unreadable, new Set(), '/fixture/cwd')).toBeUndefined(); + expect(pricedTranscript(undefined)).toEqual({}); + expect(pricedTranscript({})).toEqual({}); + } finally { for (const dir of [unpriced, empty, unreadable]) fs.rmSync(dir, { recursive: true, force: true }); } + }); + + test('a launch claims only the one transcript it created for its own cwd', () => { + const cwd = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'pty-cost-cwd-'))); + const line = (sid: string, at: string) => JSON.stringify({ type: 'user', sessionId: sid, cwd: at }) + '\n'; + const config = fixture(project => fs.writeFileSync(path.join(project, `${OTHER}.jsonl`), line(OTHER, cwd))); + try { + const before = transcriptFiles(config); + const project = path.join(config, 'projects', '-fixture-cwd'); + fs.writeFileSync(path.join(project, '22222222-2222-4333-8444-555555555555.jsonl'), line('22222222-2222-4333-8444-555555555555', '/a/concurrent/test')); + expect(newTranscriptSessionId(config, before, cwd)).toBeUndefined(); + fs.writeFileSync(path.join(project, `${SID}.jsonl`), line(SID, cwd)); + expect(newTranscriptSessionId(config, before, cwd)).toBe(SID); + fs.writeFileSync(path.join(project, '33333333-2222-4333-8444-555555555555.jsonl'), line('33333333-2222-4333-8444-555555555555', cwd)); + expect(newTranscriptSessionId(config, before, cwd)).toBeUndefined(); + } finally { fs.rmSync(config, { recursive: true, force: true }); fs.rmSync(cwd, { recursive: true, force: true }); } + }); +}); From eccaea13200fb7eeed15f5501f38a37340fc3f02 Mon Sep 17 00:00:00 2001 From: garrytan Date: Thu, 29 Oct 2026 05:13:22 +0000 Subject: [PATCH 8/8] docs: harness notes and CHANGELOG match the transcript claim Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CHANGELOG.md | 2 +- scripts/harness-version.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f788f8dfe..d19d8dbb3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ - **The headline total** adds session cost that no collector record carries, so newly priced trials reach the gate's `cost $X` line exactly once. ### Tests -- `test/paid-report-cost.test.ts`: pricing arithmetic, streamed dedupe, `caseCost`, `recordE2E`'s `cost_case_id`, a two-case shard whose label-named records bind by cost without changing the failed record, class or model. `test/paid-report-census.test.ts`: a record-less trial's streamed cost reaches the trial and the verdict total once. It also prices a PTY session, its inline sidechains and subagent files once per message id, leaves a foreign session, a missing or unreadable transcript or an unpriced model unknown, and claims only the one new transcript for a launch's cwd. `test/helpers/claude-pty-runner.launch.unit.test.ts`: a real launch prices the transcript it created without adding an argument, and an opted-out launch prices nothing. +- `test/paid-report-cost.test.ts`: pricing arithmetic, streamed dedupe, `caseCost`, `recordE2E`'s `cost_case_id`, a two-case shard whose label-named records bind by cost without changing the failed record, class or model. `test/paid-report-census.test.ts`: a record-less trial's streamed cost reaches the trial and the verdict total once. `test/paid-report-cost.test.ts` also prices a PTY session, its inline sidechains and subagent files once per message id, leaves a foreign session, a missing or unreadable transcript or an unpriced model unknown, and claims only the one new transcript for a launch's cwd. `test/helpers/claude-pty-runner.launch.unit.test.ts`: a real launch prices the transcript it created without adding an argument, and an opted-out launch prices nothing. ## [1.91.304.0] - 2026-10-29 diff --git a/scripts/harness-version.json b/scripts/harness-version.json index 13e76e098..823b6f5ad 100644 --- a/scripts/harness-version.json +++ b/scripts/harness-version.json @@ -44,10 +44,10 @@ "test/helpers/pty/runners/floor.ts": {"blob":"54e2145832bb98913644d10ff303792d67f9362e","since":6,"nonBehavioral":"Removes the plan-design 0D focus picker from the shared harness; floor.ts gains an optional pickSetupChoice hook that no other caller passes, so no other case's input or verdict can change. plan-design-finding-floor, the only case whose seed raises that menu, passes the picker from its owned fixture test/fixtures/design-floor-focus-picker.ts, so its owned bytes start its own new series."}, "test/helpers/pty/runners/observation.ts": {"blob":"1c2b75fbb984708725abaa8939125f7cab969f9f","since":6}, "test/helpers/pty/screen.ts": {"blob":"f3743152fc9bb7a8e8c70002f75bf45914f260d5","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"}, - "test/helpers/pty/session.ts": {"blob":"8fd1749a62f24dc37ae186c8a21c2165f7f7645e","since":6,"nonBehavioral":"PTY sessions price their ledger row from the session transcript; the launcher pins a random --session-id (Claude otherwise picks its own random one) when the caller passes none and is not resuming; no prompt, answer, timeout or verdict changes"}, + "test/helpers/pty/session.ts": {"blob":"8fd1749a62f24dc37ae186c8a21c2165f7f7645e","since":6,"nonBehavioral":"PTY sessions price their ledger row (streamed_cost_usd) from the session transcript found by the caller's --session-id or the one new transcript for the launch cwd; ledger cost fields only, no prompt, argv, answer, timeout or verdict changes"}, "test/helpers/pty/transcript-cost.ts": {"blob":"08f7ccbdbd9d71a5dc86edd2f8fced423a78ad9a","since":6,"nonBehavioral":"PTY transcript pricing finds the session by the caller's --session-id or the one new transcript for the launch cwd; the launcher adds no argument; native AUQ capture opts out; ledger cost fields only, no prompt, argv, answer, timeout or verdict changes"}, "test/helpers/session-drain-policy.ts": {"blob":"6211239dd0485e6495a700b824a7e0e61d1bcd97","since":6}, - "test/helpers/session-ledger.ts": {"blob":"c99baaaa0a77a5a72384493fd8a85a4253ecb05d","since":6,"nonBehavioral":"PTY sessions price their ledger row from the session transcript; the launcher pins a random --session-id (Claude otherwise picks its own random one) when the caller passes none and is not resuming; no prompt, answer, timeout or verdict changes"}, + "test/helpers/session-ledger.ts": {"blob":"c99baaaa0a77a5a72384493fd8a85a4253ecb05d","since":6,"nonBehavioral":"PTY sessions price their ledger row (streamed_cost_usd) from the session transcript found by the caller's --session-id or the one new transcript for the launch cwd; ledger cost fields only, no prompt, argv, answer, timeout or verdict changes"}, "test/helpers/session-runner.ts": {"blob":"c73b2596d9eb5377530a6bf1d284e0b6ea1556a0","since":6,"nonBehavioral":"Cost attribution only (GSTA-150 PR 2): a display-label record carries cost_case_id from runSkillTest's testName and the report binds its cost by it while the failed record and failure class still come from the named records (test/paid-report-cost.test.ts pins both); an aborted Agent SDK session logs its streamed usage as streamed_cost_usd, used only when a case has no record; a listed model-free case with no record or session is a known $0. No case's execution, outcome, failure class, rerun eligibility or panel verdict changes, so no pass-rate series resets"}, "test/helpers/skill-fixture.ts": {"blob":"8a654dae6b0a396d839b02179119ae3e3bb13f69","since":6,"nonBehavioral":"extractSkillBody and extractSkillSections follow the shared-preamble hoist (pointer plus sections/preamble.md); every fixture they build is byte-identical to the pre-hoist output (compared for 8 body and 4 section extractions), so no case's input or verdict changes; the /ship review then reworded only comments and two error messages, so no fixture byte changes"}, "test/helpers/skill-startup.ts": {"blob":"a8d21e3817c3f35a1d06af325c92fde7d420e377","since":6,"nonBehavioral":"readSkillEffective also appends a variant's added ### subsections (office-hours' Open-ended questions); readSkillStartup, readSharedSections and readSkillEffective return bytes identical to the pinned helper for every tier 2+ render without added subsections (735 across all hosts) and for office-hours' SKILL.md before this PR, so office-hours' fixture input changes only because its prompt changed in this PR, not because the helper did"},