diff --git a/scripts/harness-version.json b/scripts/harness-version.json index e225bbbad..695848771 100644 --- a/scripts/harness-version.json +++ b/scripts/harness-version.json @@ -38,15 +38,15 @@ "test/helpers/pty/classify.ts": {"blob":"3f2daa12ff6484f96b8d28d910a50231c4bb3ae8","since":6}, "test/helpers/pty/fake-session.ts": {"blob":"554e064ef259b8444280eddfe9b0db93c66a8202","since":6}, "test/helpers/pty/judge.ts": {"blob":"81c5bb4c48ab928aca2f186a1ade9df7169d7fbe","since":6}, - "test/helpers/pty/launch.ts": {"blob":"e4413de380b799659170e7da248d51afda3b0e3f","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"}, + "test/helpers/pty/launch.ts": {"blob":"d2e51418ce284815a26e174d777ca8b47ffd7e68","since":6,"nonBehavioral":"PTY sessions price their ledger row from the session transcript; the launcher pins a random --session-id (Claude otherwise picks its own random one) when the caller passes none and is not resuming; no prompt, answer, timeout or verdict changes"}, "test/helpers/pty/plan-native.ts": {"blob":"ddba9cb68c7c0da935cff7c1cea258dee2dff740","since":6}, "test/helpers/pty/runners/counting.ts": {"blob":"8c5c05e54abf961259d34d9c3d77db67138d7794","since":6}, "test/helpers/pty/runners/floor.ts": {"blob":"54e2145832bb98913644d10ff303792d67f9362e","since":6,"nonBehavioral":"Removes the plan-design 0D focus picker from the shared harness; floor.ts gains an optional pickSetupChoice hook that no other caller passes, so no other case's input or verdict can change. plan-design-finding-floor, the only case whose seed raises that menu, passes the picker from its owned fixture test/fixtures/design-floor-focus-picker.ts, so its owned bytes start its own new series."}, "test/helpers/pty/runners/observation.ts": {"blob":"1c2b75fbb984708725abaa8939125f7cab969f9f","since":6}, "test/helpers/pty/screen.ts": {"blob":"f3743152fc9bb7a8e8c70002f75bf45914f260d5","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"}, - "test/helpers/pty/session.ts": {"blob":"54fe04b61d953e4e55e2fc251532c04a49fab66f","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"}, + "test/helpers/pty/session.ts": {"blob":"8fd1749a62f24dc37ae186c8a21c2165f7f7645e","since":6,"nonBehavioral":"PTY sessions price their ledger row from the session transcript; the launcher pins a random --session-id (Claude otherwise picks its own random one) when the caller passes none and is not resuming; no prompt, answer, timeout or verdict changes"}, "test/helpers/session-drain-policy.ts": {"blob":"6211239dd0485e6495a700b824a7e0e61d1bcd97","since":6}, - "test/helpers/session-ledger.ts": {"blob":"5d8b796ccd422f4e1e2ab395bf6ce5a2e4c0e167","since":6,"nonBehavioral":"Cost attribution only (GSTA-150 PR 2): a display-label record carries cost_case_id from runSkillTest's testName and the report binds its cost by it while the failed record and failure class still come from the named records (test/paid-report-cost.test.ts pins both); an aborted Agent SDK session logs its streamed usage as streamed_cost_usd, used only when a case has no record; a listed model-free case with no record or session is a known $0. No case's execution, outcome, failure class, rerun eligibility or panel verdict changes, so no pass-rate series resets"}, + "test/helpers/session-ledger.ts": {"blob":"c99baaaa0a77a5a72384493fd8a85a4253ecb05d","since":6,"nonBehavioral":"PTY sessions price their ledger row from the session transcript; the launcher pins a random --session-id (Claude otherwise picks its own random one) when the caller passes none and is not resuming; no prompt, answer, timeout or verdict changes"}, "test/helpers/session-runner.ts": {"blob":"c73b2596d9eb5377530a6bf1d284e0b6ea1556a0","since":6,"nonBehavioral":"Cost attribution only (GSTA-150 PR 2): a display-label record carries cost_case_id from runSkillTest's testName and the report binds its cost by it while the failed record and failure class still come from the named records (test/paid-report-cost.test.ts pins both); an aborted Agent SDK session logs its streamed usage as streamed_cost_usd, used only when a case has no record; a listed model-free case with no record or session is a known $0. No case's execution, outcome, failure class, rerun eligibility or panel verdict changes, so no pass-rate series resets"}, "test/helpers/skill-fixture.ts": {"blob":"8a654dae6b0a396d839b02179119ae3e3bb13f69","since":6,"nonBehavioral":"extractSkillBody and extractSkillSections follow the shared-preamble hoist (pointer plus sections/preamble.md); every fixture they build is byte-identical to the pre-hoist output (compared for 8 body and 4 section extractions), so no case's input or verdict changes; the /ship review then reworded only comments and two error messages, so no fixture byte changes"}, "test/helpers/skill-startup.ts": {"blob":"a8d21e3817c3f35a1d06af325c92fde7d420e377","since":6,"nonBehavioral":"readSkillEffective also appends a variant's added ### subsections (office-hours' Open-ended questions); readSkillStartup, readSharedSections and readSkillEffective return bytes identical to the pinned helper for every tier 2+ render without added subsections (735 across all hosts) and for office-hours' SKILL.md before this PR, so office-hours' fixture input changes only because its prompt changed in this PR, not because the helper did"}, diff --git a/test/helpers/claude-pty-runner.launch.unit.test.ts b/test/helpers/claude-pty-runner.launch.unit.test.ts index 0f6123afe..5913435b6 100644 --- a/test/helpers/claude-pty-runner.launch.unit.test.ts +++ b/test/helpers/claude-pty-runner.launch.unit.test.ts @@ -39,7 +39,7 @@ describe('launchClaudePty model pin', () => { expect(opts.model).toBe('claude-sonnet-4-6'); }); - test.skipIf(process.platform === 'win32')('spawn args pin --model from the fallback chain, before extraArgs, with hermetic MCP gating', () => { + test.skipIf(process.platform === 'win32')('spawn args pin --model from the fallback chain, before extraArgs, with hermetic MCP gating and a pinned session id', () => { const dir = mkdtempSync(join(tmpdir(), 'pty-model-pin-')); try { const fake = join(dir, 'fake-claude'); @@ -56,7 +56,9 @@ const argv = async (opts, evalsModel) => { }; process.stdout.write(JSON.stringify({ capture: resolveEvalModel('capture'), fallback: await argv({}, undefined), env: await argv({}, 'env-model'), explicit: await argv({ model: 'opts-model' }, 'env-model'), - extra: await argv({ extraArgs: ['--model', 'override'] }, undefined) })); + extra: await argv({ extraArgs: ['--model', 'override'] }, undefined), + pinned: await argv({ extraArgs: ['--session-id', '11111111-2222-4333-8444-555555555555'] }, undefined), + resumed: await argv({ extraArgs: ['--resume', 'prior'] }, undefined) })); `); const run = (hermetic: string) => { const result = spawnSync(process.execPath, [worker], { cwd: join(import.meta.dir, '../..'), encoding: 'utf8', timeout: 30_000, @@ -73,6 +75,13 @@ process.stdout.write(JSON.stringify({ capture: resolveEvalModel('capture'), expect(hermetic.extra.slice(-2)).toEqual(['--model', 'override']); expect(hermetic.fallback).toContain('--strict-mcp-config'); expect(run('0').fallback).not.toContain('--strict-mcp-config'); + const sessionIds = (args: string[]) => args.flatMap((arg, i) => arg === '--session-id' ? [args[i + 1]] : []); + expect(sessionIds(hermetic.fallback)).toHaveLength(1); + expect(sessionIds(hermetic.fallback)[0]).toMatch(/^[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}$/); + expect(sessionIds(hermetic.extra)).toHaveLength(1); + expect(sessionIds(hermetic.pinned)).toEqual(['11111111-2222-4333-8444-555555555555']); + expect(sessionIds(hermetic.resumed)).toEqual([]); + expect(sessionIds(run('0').fallback)).toEqual([]); } finally { rmSync(dir, { recursive: true, force: true }); } }, 60_000); }); diff --git a/test/helpers/pty-transcript-cost.unit.test.ts b/test/helpers/pty-transcript-cost.unit.test.ts new file mode 100644 index 000000000..c02e5db07 --- /dev/null +++ b/test/helpers/pty-transcript-cost.unit.test.ts @@ -0,0 +1,58 @@ +import { describe, test, expect } from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { pricedTranscript, ptyTranscriptCostUsd } from './pty/transcript-cost'; + +const SID = '11111111-2222-4333-8444-555555555555'; +const assistant = (id: string, model: string, usage: object, extra: object = {}) => + JSON.stringify({ type: 'assistant', sessionId: SID, message: { id, model, role: 'assistant', usage }, ...extra }); + +function fixture(write: (project: string) => void): string { + const config = fs.mkdtempSync(path.join(os.tmpdir(), 'pty-cost-')); + const project = path.join(config, 'projects', '-fixture-cwd'); + fs.mkdirSync(project, { recursive: true }); + write(project); + return config; +} + +describe('ptyTranscriptCostUsd', () => { + test('prices the session, its inline sidechains and subagent files, each message id once at its largest output', () => { + const config = fixture(project => { + fs.writeFileSync(path.join(project, `${SID}.jsonl`), [ + JSON.stringify({ type: 'user', sessionId: SID, message: { role: 'user', content: 'hi' } }), + assistant('msg_a', 'claude-opus-5-5', { input_tokens: 1000, output_tokens: 10 }), + assistant('msg_a', 'claude-opus-5-5', { input_tokens: 1000, output_tokens: 500 }), + assistant('msg_side', 'claude-opus-5-5', { input_tokens: 0, output_tokens: 1000 }, { isSidechain: true }), + assistant('msg_syn', '', { input_tokens: 0, output_tokens: 0 }), + '{"partial":', + ].join('\n')); + fs.mkdirSync(path.join(project, SID, 'subagents'), { recursive: true }); + fs.writeFileSync(path.join(project, SID, 'subagents', 'agent-x.jsonl'), + assistant('msg_sub', 'claude-opus-5-5', { input_tokens: 0, cache_read_input_tokens: 10000, output_tokens: 0 }) + '\n'); + fs.writeFileSync(path.join(project, '99999999-2222-4333-8444-555555555555.jsonl'), + assistant('msg_foreign', 'claude-opus-5-5', { input_tokens: 1_000_000, output_tokens: 0 }) + '\n'); + }); + try { + // opus-5-5: $4 in, $20 out, $0.20 cached per MTok. + const expected = (1000 * 4 + 500 * 20 + 1000 * 20 + 10000 * 0.2) / 1_000_000; + expect(ptyTranscriptCostUsd(config, SID)).toBeCloseTo(expected, 6); + expect(pricedTranscript({ hermeticConfigDir: config, sessionId: SID }).streamed_cost_usd).toBeCloseTo(expected, 6); + } finally { fs.rmSync(config, { recursive: true, force: true }); } + }); + + test('stays cost-unknown without a transcript, a session id, usage or a priced model', () => { + const unpriced = fixture(project => fs.writeFileSync(path.join(project, `${SID}.jsonl`), + assistant('msg_a', 'claude-opus-5-5', { output_tokens: 1 }) + '\n' + assistant('msg_b', 'some-unpriced-model', { output_tokens: 1 }) + '\n')); + const empty = fixture(project => fs.writeFileSync(path.join(project, `${SID}.jsonl`), JSON.stringify({ type: 'user', sessionId: SID }) + '\n')); + try { + expect(ptyTranscriptCostUsd(unpriced, SID)).toBeUndefined(); + expect(ptyTranscriptCostUsd(empty, SID)).toBeUndefined(); + expect(ptyTranscriptCostUsd(empty, '99999999-2222-4333-8444-555555555555')).toBeUndefined(); + expect(ptyTranscriptCostUsd(unpriced, undefined)).toBeUndefined(); + expect(ptyTranscriptCostUsd(null, SID)).toBeUndefined(); + expect(ptyTranscriptCostUsd(path.join(empty, 'missing'), SID)).toBeUndefined(); + expect(pricedTranscript(undefined)).toEqual({}); + } finally { for (const dir of [unpriced, empty]) fs.rmSync(dir, { recursive: true, force: true }); } + }); +}); diff --git a/test/helpers/pty/launch.ts b/test/helpers/pty/launch.ts index e4413de38..d2e51418c 100644 --- a/test/helpers/pty/launch.ts +++ b/test/helpers/pty/launch.ts @@ -2,6 +2,7 @@ * Spawn `claude` in a real PTY (launchClaudePty) and the session handle. Moved from claude-pty-runner.ts. * Import through test/helpers/claude-pty-runner.ts from tests; pty/ modules import siblings directly. */ +import { randomUUID } from 'node:crypto'; import * as path from 'node:path'; import { resolveEvalModel } from '../../../lib/eval-model'; import { hermeticChildEnv, hermeticSkillsConfigDir, isHermeticEnabled } from '../hermetic-env'; @@ -15,6 +16,7 @@ import { trustDialogInput } from '../pty-trust-dialog'; import { resolveClaudeBinary } from './binary'; import { createPtyScreen, screenPanelEnd, stripAnsi } from './screen'; import { appendSessionLedger, sessionKey } from '../session-ledger'; +import { pricedTranscript } from './transcript-cost'; export interface ClaudePtyOptions { /** Register the repo's shipped skills in the child's user scope via @@ -129,6 +131,8 @@ export interface ClaudePtySession { * the dir name ends in `/.claude` by contract). */ hermeticConfigDir: string | null; + /** The Claude session id the launcher pinned (or the caller's --session-id): its transcript prices the session ledger row. */ + sessionId?: string; /** Owned HOME/.gstack created by the seeded launcher; absent for caller overrides. */ hermeticSkillStateRoot?: string; /** Owned pre-tool identity record, removed by close(). */ @@ -249,7 +253,8 @@ export async function launchClaudePty( recorded = true; const elapsed = performance.now() - startedMono; appendSessionLedger({ key: sessionKey('pty', undefined), runner: 'pty', started_at: startedAt, budget_ms: timeoutMs, - elapsed_ms: elapsed, end: elapsed >= timeoutMs ? 'session_timeout' : 'completed', ...screenPanelEnd(handle.visibleText()), billed: false }); + elapsed_ms: elapsed, end: elapsed >= timeoutMs ? 'session_timeout' : 'completed', ...screenPanelEnd(handle.visibleText()), billed: false, + ...pricedTranscript(handle) }); } return close(); } }); @@ -263,6 +268,7 @@ interface LaunchSetup { hermeticSkillStateRoot: string | undefined; autoplanArtifactStateRoot: string | undefined; autoplanEngTestPlanStateRoot: string | undefined; + sessionId: string | undefined; } function prepareLaunch(opts: ClaudePtyOptions): LaunchSetup { @@ -283,6 +289,10 @@ function prepareLaunch(opts: ClaudePtyOptions): LaunchSetup { // Before opts.extraArgs so a test could theoretically supply --mcp-config. const hermetic = isHermeticEnabled(); if (hermetic) args.push('--strict-mcp-config'); + const extra = opts.extraArgs ?? []; + const resumes = extra.some(arg => ['--resume', '-r', '--continue', '-c'].includes(arg)); + let sessionId = extra.includes('--session-id') ? extra[extra.indexOf('--session-id') + 1] : undefined; + if (hermetic && !sessionId && !resumes) args.push('--session-id', sessionId = randomUUID()); if (opts.extraArgs) args.push(...opts.extraArgs); // Hermetic by default (test/helpers/hermetic-env.ts): operator session @@ -319,7 +329,7 @@ function prepareLaunch(opts: ClaudePtyOptions): LaunchSetup { const autoplanEngTestPlanStateRoot = opts.approveAutoplanArtifactEdits && opts.autoplanArtifactState !== undefined ? hermeticSkillStateRoot : undefined; - return { args, childEnv, hermetic, hermeticSkillStateRoot, autoplanArtifactStateRoot, autoplanEngTestPlanStateRoot }; + return { args, childEnv, hermetic, hermeticSkillStateRoot, autoplanArtifactStateRoot, autoplanEngTestPlanStateRoot, sessionId }; } interface LaunchRecorders { @@ -568,6 +578,7 @@ function sessionHandle(pty: PtyProcess, launch: LaunchSetup, wallTimer: ReturnTy exited: () => pty.exited, exitCode: () => pty.exitCode, hermeticConfigDir: launch.hermetic ? launch.childEnv.CLAUDE_CONFIG_DIR ?? null : null, + sessionId: launch.sessionId, hermeticSkillStateRoot: launch.hermeticSkillStateRoot, pendingPlanReadyFile: pendingExit?.file, pendingQuestionFile: pendingQuestion?.file, diff --git a/test/helpers/pty/session.ts b/test/helpers/pty/session.ts index 54fe04b61..8fd1749a6 100644 --- a/test/helpers/pty/session.ts +++ b/test/helpers/pty/session.ts @@ -10,6 +10,7 @@ import { launchClaudePty, type ClaudePtyOptions, type ClaudePtySession } from './launch'; import { screenPanelEnd } from './screen'; import { appendSessionLedger, sessionKey, type SessionEnd } from '../session-ledger'; +import { pricedTranscript } from './transcript-cost'; /** * The runners' launch seam: how a session starts, and the clock the runner @@ -97,7 +98,7 @@ export async function runPtySession(plan: PtySessionPlan): Promise { } finally { appendSessionLedger({ key: sessionKey('pty', undefined), runner: 'pty', started_at: startedAt, budget_ms: plan.budgetMs ?? plan.launch.timeoutMs ?? 240_000, elapsed_ms: plan.driver.monotonic() - started, - end, ...safePanelEnd(session), billed: false }); + end, ...safePanelEnd(session), billed: false, ...pricedTranscript(session) }); try { try { plan.beforeClose?.(session, failure); } catch (error) { if (!failure) { failure = { error }; throw error; } } diff --git a/test/helpers/pty/transcript-cost.ts b/test/helpers/pty/transcript-cost.ts new file mode 100644 index 000000000..064d520b8 --- /dev/null +++ b/test/helpers/pty/transcript-cost.ts @@ -0,0 +1,42 @@ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { streamedUsageCostUsd } from '../../../lib/pricing'; + +/** The session ledger's cost field for a PTY session: present only when its transcript prices. */ +export function pricedTranscript(session: { hermeticConfigDir?: string | null; sessionId?: string } | undefined): { streamed_cost_usd?: number } { + const cost = ptyTranscriptCostUsd(session?.hermeticConfigDir, session?.sessionId); + return cost === undefined ? {} : { streamed_cost_usd: cost }; +} + +/** + * Price one PTY session from the transcripts Claude Code wrote under its owned config dir: + * `projects//.jsonl` (inline sidechains included) and `projects///subagents/*.jsonl`, + * rows deduplicated by message id. Undefined (cost-unknown, never $0) without a config dir, session id, transcript or + * usage, or when a message's model has no price row. + */ +export function ptyTranscriptCostUsd(configDir: string | null | undefined, sessionId: string | undefined): number | undefined { + if (!configDir || !sessionId) return undefined; + const projects = path.join(configDir, 'projects'); + let dirs: fs.Dirent[]; + try { + dirs = fs.readdirSync(projects, { withFileTypes: true }); + } catch { + return undefined; + } + const files: string[] = []; + for (const dir of dirs) { + if (!dir.isDirectory()) continue; + const main = path.join(projects, dir.name, `${sessionId}.jsonl`); + if (fs.existsSync(main)) files.push(main); + const subagents = path.join(projects, dir.name, sessionId, 'subagents'); + if (fs.existsSync(subagents)) files.push(...fs.readdirSync(subagents).filter(f => f.endsWith('.jsonl')).map(f => path.join(subagents, f))); + } + const rows = files.flatMap(file => fs.readFileSync(file, 'utf8').split('\n').flatMap(line => { + try { + return [JSON.parse(line)]; + } catch { + return []; + } + })); + return streamedUsageCostUsd(rows); +} diff --git a/test/helpers/session-ledger.ts b/test/helpers/session-ledger.ts index 5d8b796cc..c99baaaa0 100644 --- a/test/helpers/session-ledger.ts +++ b/test/helpers/session-ledger.ts @@ -52,7 +52,7 @@ export interface SessionLedgerRow { liveness?: LivenessSummary; /** Whether the runner captured this session's billing (a terminal result with total_cost_usd). */ billed?: boolean; - /** A session that ended without a terminal result (aborted, thrown): its assistant messages' streamed usage at lib/pricing.ts rates, absent when unpriced. */ + /** A session without a terminal result (an aborted Agent SDK session, a PTY session): its assistant messages' usage at lib/pricing.ts rates, from the stream or the PTY transcript; absent when unpriced. */ streamed_cost_usd?: number; /** Codex sessions: the model, its billed cost and token classes. */ model?: string;