mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-04 07:47:27 +01:00
fix: price PTY transcripts without adding a launch argument; native AUQ capture opts out
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
2c2c18901c
commit
267c55acbc
10 files changed
+191
-107
No files matched your search
+2
-2
@@ -8,11 +8,11 @@
|
||||
- **Label-named records in multi-case shards** (`test/helpers/e2e-helpers.ts`, `scripts/lib/paid-report.ts`): a `claude -p` record named by its display label ("browse basic commands") carries `cost_case_id` from `runSkillTest`'s registered test name, and the report binds its cost to that case. The failed record, failure class and model still come from the records named by case id, so no verdict, class or pass-rate series changes (22 trials: bws, session-intelligence, workflow, skillify, review-attribution, review, gate-integrity, diagram).
|
||||
- **Aborted Agent SDK sessions** (`test/helpers/agent-sdk-runner.ts`, `lib/pricing.ts`): a session that ends without a result event logs its streamed usage as `streamed_cost_usd` in the session ledger, priced once per message id at its largest output, subagent turns included, with 5-minute cache writes at 1.25x and 1-hour writes at 2x input. An unpriced model leaves the cost unknown (auq-format-gate).
|
||||
- **Model-free cases**: `timeline-event-flow` runs no model, so with no record and no session it is a known $0. A test checks every listed case's body starts no model session.
|
||||
- **PTY sessions** (`test/helpers/pty/launch.ts`, `test/helpers/pty/session.ts`, `test/helpers/pty/transcript-cost.ts`): the hermetic launcher pins a random `--session-id` when the caller passes none and is not resuming, and each PTY ledger row prices that session's transcript (`projects/*/<id>.jsonl` plus its `subagents/` files) as `streamed_cost_usd`. No transcript, no usage or an unpriced model leaves the cost unknown, never $0, and a read error never reaches the test. This covers 8 cases and 14 sessions per gate: plan-mode-no-op, plan-design-with-ui-scope, the plan-ceo and plan-devex finding-floor and plan-mode cases, autoplan-guard-pty and office-hours-auto-mode. Checked live, a `claude -p` session with one subagent priced $0.090376 from its transcripts, equal to its `total_cost_usd`.
|
||||
- **PTY sessions** (`test/helpers/pty/launch.ts`, `test/helpers/pty/session.ts`, `test/helpers/pty/transcript-cost.ts`): each PTY ledger row prices the session's transcript (`projects/*/<id>.jsonl` plus its `subagents/` files) as `streamed_cost_usd`. The session is the caller's `--session-id`, else the one transcript that appeared during the launch for its cwd, so the launcher adds no argument and concurrent tests sharing a config dir never price each other. No transcript, two new transcripts for one cwd, no usage or an unpriced model leaves the cost unknown, never $0, and a read error never reaches the test. The native AskUserQuestion capture, which never reads model transcripts, opts out (`priceTranscript: false`). This covers 8 cases and 14 sessions per gate: plan-mode-no-op, plan-design-with-ui-scope, the plan-ceo and plan-devex finding-floor and plan-mode cases, autoplan-guard-pty and office-hours-auto-mode. Checked live, a `claude -p` session with one subagent priced $0.090376 from its transcripts, equal to its `total_cost_usd`.
|
||||
- **The headline total** adds session cost that no collector record carries, so newly priced trials reach the gate's `cost $X` line exactly once.
|
||||
|
||||
### Tests
|
||||
- `test/paid-report-cost.test.ts`: pricing arithmetic, streamed dedupe, `caseCost`, `recordE2E`'s `cost_case_id`, a two-case shard whose label-named records bind by cost without changing the failed record, class or model. `test/paid-report-census.test.ts`: a record-less trial's streamed cost reaches the trial and the verdict total once. `test/helpers/pty-transcript-cost.unit.test.ts`: a session, its inline sidechains and subagent files priced once per message id; a foreign session, missing or unreadable transcript, or unpriced model stays unknown. `test/helpers/claude-pty-runner.launch.unit.test.ts`: one pinned session id, a caller's id kept in either form, none when resuming or non-hermetic.
|
||||
- `test/paid-report-cost.test.ts`: pricing arithmetic, streamed dedupe, `caseCost`, `recordE2E`'s `cost_case_id`, a two-case shard whose label-named records bind by cost without changing the failed record, class or model. `test/paid-report-census.test.ts`: a record-less trial's streamed cost reaches the trial and the verdict total once. It also prices a PTY session, its inline sidechains and subagent files once per message id, leaves a foreign session, a missing or unreadable transcript or an unpriced model unknown, and claims only the one new transcript for a launch's cwd. `test/helpers/claude-pty-runner.launch.unit.test.ts`: a real launch prices the transcript it created without adding an argument, and an opted-out launch prices nothing.
|
||||
|
||||
## [1.91.304.0] - 2026-10-29
|
||||
|
||||
|
||||
+12
-11
@@ -696,17 +696,18 @@ largest output, subagent turns included, `<synthetic>` messages skipped,
|
||||
5-minute cache writes at 1.25x and 1-hour writes at 2x the input rate. The
|
||||
field is absent when no message carries usage or any message's model has no
|
||||
price row, so an unpriced session stays cost-unknown, never $0. A PTY session
|
||||
records `streamed_cost_usd` from its transcript the same way: the hermetic
|
||||
launcher (`prepareLaunch` in `test/helpers/pty/launch.ts`) pins
|
||||
`--session-id <uuid>` when the caller passes none and is not resuming
|
||||
(`--resume`, `-r`, `--continue`, `-c`), and both PTY ledger writers price
|
||||
`<CLAUDE_CONFIG_DIR>/projects/*/<id>.jsonl` plus `<id>/subagents/*.jsonl`
|
||||
(`test/helpers/pty/transcript-cost.ts`). Pricing keys on the session id, not
|
||||
the whole directory, because concurrent PTY tests in one process share one
|
||||
hermetic config dir. A non-hermetic or resumed launch without a caller
|
||||
`--session-id`, a missing transcript, no usage, an unpriced model or a read
|
||||
error leaves the field absent (cost-unknown), and a pricing failure never
|
||||
fails the test. The row keeps `billed: false`. The ledger rides in each slice
|
||||
records `streamed_cost_usd` from its transcript the same way. Both PTY ledger
|
||||
writers price `<CLAUDE_CONFIG_DIR>/projects/*/<id>.jsonl` plus
|
||||
`<id>/subagents/*.jsonl` (`test/helpers/pty/transcript-cost.ts`), where the
|
||||
id is the caller's `--session-id`, else the one transcript that appeared
|
||||
during this launch for its cwd. The launcher adds no argument. Pricing never
|
||||
takes the whole directory, because concurrent PTY tests in one process share
|
||||
one hermetic config dir; two new transcripts for one cwd are ambiguous and stay
|
||||
unknown. A non-hermetic or resumed launch, a caller that opts out
|
||||
(`priceTranscript: false`, the native AskUserQuestion capture, which never
|
||||
reads model transcripts), a missing transcript, no usage, an unpriced model or
|
||||
a read error leaves the field absent (cost-unknown), and a pricing failure
|
||||
never fails the test. The row keeps `billed: false`. The ledger rides in each slice
|
||||
artifact beside the shard results.
|
||||
|
||||
**Quarantine** (`CASE_QUARANTINE`, same file). An entry needs: a per-trial rate
|
||||
|
||||
@@ -2264,8 +2264,9 @@ case with `bun run eval:pass-rates --headroom --case <id>`.
|
||||
|
||||
**Meaning.** Some trials' cost was not captured, so only the known sum is
|
||||
shown. A trial is cost-unknown when it ran a PTY session whose transcript
|
||||
could not be priced (a non-hermetic or resumed launch without a caller
|
||||
`--session-id`, no transcript or usage, or a model with no price row), a
|
||||
could not be priced (a non-hermetic or resumed launch, the native
|
||||
AskUserQuestion capture, two new transcripts for one cwd, no transcript or
|
||||
usage, or a model with no price row), a
|
||||
Codex session that completed no turn or whose model has no price row, a
|
||||
`claude -p` session that ended without a result, or an aborted Agent SDK
|
||||
session whose streamed usage could not be priced (no usage, or a model with no
|
||||
|
||||
@@ -38,14 +38,14 @@
|
||||
"test/helpers/pty/classify.ts": {"blob":"3f2daa12ff6484f96b8d28d910a50231c4bb3ae8","since":6},
|
||||
"test/helpers/pty/fake-session.ts": {"blob":"554e064ef259b8444280eddfe9b0db93c66a8202","since":6},
|
||||
"test/helpers/pty/judge.ts": {"blob":"81c5bb4c48ab928aca2f186a1ade9df7169d7fbe","since":6},
|
||||
"test/helpers/pty/launch.ts": {"blob":"11bbba2ed8425edd91a6214c05ecb32b33a00c41","since":6,"nonBehavioral":"the launcher also recognises --session-id= and --resume= forms before pinning a session id; no prompt, answer, timeout or verdict changes"},
|
||||
"test/helpers/pty/launch.ts": {"blob":"4f0731ef39bc3a6a7c5c592ce98690a24cb1ed7f","since":6,"nonBehavioral":"PTY transcript pricing finds the session by the caller's --session-id or the one new transcript for the launch cwd; the launcher adds no argument; native AUQ capture opts out; ledger cost fields only, no prompt, argv, answer, timeout or verdict changes"},
|
||||
"test/helpers/pty/plan-native.ts": {"blob":"ddba9cb68c7c0da935cff7c1cea258dee2dff740","since":6},
|
||||
"test/helpers/pty/runners/counting.ts": {"blob":"8c5c05e54abf961259d34d9c3d77db67138d7794","since":6},
|
||||
"test/helpers/pty/runners/floor.ts": {"blob":"54e2145832bb98913644d10ff303792d67f9362e","since":6,"nonBehavioral":"Removes the plan-design 0D focus picker from the shared harness; floor.ts gains an optional pickSetupChoice hook that no other caller passes, so no other case's input or verdict can change. plan-design-finding-floor, the only case whose seed raises that menu, passes the picker from its owned fixture test/fixtures/design-floor-focus-picker.ts, so its owned bytes start its own new series."},
|
||||
"test/helpers/pty/runners/observation.ts": {"blob":"1c2b75fbb984708725abaa8939125f7cab969f9f","since":6},
|
||||
"test/helpers/pty/screen.ts": {"blob":"f3743152fc9bb7a8e8c70002f75bf45914f260d5","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"},
|
||||
"test/helpers/pty/session.ts": {"blob":"8fd1749a62f24dc37ae186c8a21c2165f7f7645e","since":6,"nonBehavioral":"PTY sessions price their ledger row from the session transcript; the launcher pins a random --session-id (Claude otherwise picks its own random one) when the caller passes none and is not resuming; no prompt, answer, timeout or verdict changes"},
|
||||
"test/helpers/pty/transcript-cost.ts": {"blob":"c54a70bed59c60f1249ba8cb13e83ecd7e6d7471","since":6,"nonBehavioral":"transcript-cost.ts prices a PTY session's ledger row from its transcript and never throws; cost fields only, no prompt, answer, timeout or verdict changes"},
|
||||
"test/helpers/pty/transcript-cost.ts": {"blob":"08f7ccbdbd9d71a5dc86edd2f8fced423a78ad9a","since":6,"nonBehavioral":"PTY transcript pricing finds the session by the caller's --session-id or the one new transcript for the launch cwd; the launcher adds no argument; native AUQ capture opts out; ledger cost fields only, no prompt, argv, answer, timeout or verdict changes"},
|
||||
"test/helpers/session-drain-policy.ts": {"blob":"6211239dd0485e6495a700b824a7e0e61d1bcd97","since":6},
|
||||
"test/helpers/session-ledger.ts": {"blob":"c99baaaa0a77a5a72384493fd8a85a4253ecb05d","since":6,"nonBehavioral":"PTY sessions price their ledger row from the session transcript; the launcher pins a random --session-id (Claude otherwise picks its own random one) when the caller passes none and is not resuming; no prompt, answer, timeout or verdict changes"},
|
||||
"test/helpers/session-runner.ts": {"blob":"c73b2596d9eb5377530a6bf1d284e0b6ea1556a0","since":6,"nonBehavioral":"Cost attribution only (GSTA-150 PR 2): a display-label record carries cost_case_id from runSkillTest's testName and the report binds its cost by it while the failed record and failure class still come from the named records (test/paid-report-cost.test.ts pins both); an aborted Agent SDK session logs its streamed usage as streamed_cost_usd, used only when a case has no record; a listed model-free case with no record or session is a known $0. No case's execution, outcome, failure class, rerun eligibility or panel verdict changes, so no pass-rate series resets"},
|
||||
|
||||
@@ -313,7 +313,7 @@ Skip any system-audit / environment-setup / codebase-exploration steps. At the f
|
||||
session = await launchClaudePty({
|
||||
// Explicit tool approvals suffice. Bypass mode introduces a separate
|
||||
// safety-consent screen in a fresh CLI config; never accept that screen.
|
||||
cwd, model, seedSkills: false, permissionMode: 'default',
|
||||
cwd, model, seedSkills: false, permissionMode: 'default', priceTranscript: false,
|
||||
observeScreen: true, observeSetupQuestions: true,
|
||||
timeoutMs: Math.max(1, deadline - Date.now()),
|
||||
// Explicit availability as well as approval: no Bash/Agent/Skill or MCP.
|
||||
|
||||
@@ -39,7 +39,7 @@ describe('launchClaudePty model pin', () => {
|
||||
expect(opts.model).toBe('claude-sonnet-4-6');
|
||||
});
|
||||
|
||||
test.skipIf(process.platform === 'win32')('spawn args pin --model from the fallback chain, before extraArgs, with hermetic MCP gating and a pinned session id', () => {
|
||||
test.skipIf(process.platform === 'win32')('spawn args pin --model from the fallback chain, before extraArgs, with hermetic MCP gating', () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'pty-model-pin-'));
|
||||
try {
|
||||
const fake = join(dir, 'fake-claude');
|
||||
@@ -56,11 +56,7 @@ const argv = async (opts, evalsModel) => {
|
||||
};
|
||||
process.stdout.write(JSON.stringify({ capture: resolveEvalModel('capture'),
|
||||
fallback: await argv({}, undefined), env: await argv({}, 'env-model'), explicit: await argv({ model: 'opts-model' }, 'env-model'),
|
||||
extra: await argv({ extraArgs: ['--model', 'override'] }, undefined),
|
||||
pinned: await argv({ extraArgs: ['--session-id', '11111111-2222-4333-8444-555555555555'] }, undefined),
|
||||
resumed: await argv({ extraArgs: ['--resume', 'prior'] }, undefined),
|
||||
pinnedEq: await argv({ extraArgs: ['--session-id=11111111-2222-4333-8444-555555555555'] }, undefined),
|
||||
resumedEq: await argv({ extraArgs: ['--resume=prior'] }, undefined) }));
|
||||
extra: await argv({ extraArgs: ['--model', 'override'] }, undefined) }));
|
||||
`);
|
||||
const run = (hermetic: string) => {
|
||||
const result = spawnSync(process.execPath, [worker], { cwd: join(import.meta.dir, '../..'), encoding: 'utf8', timeout: 30_000,
|
||||
@@ -77,15 +73,46 @@ process.stdout.write(JSON.stringify({ capture: resolveEvalModel('capture'),
|
||||
expect(hermetic.extra.slice(-2)).toEqual(['--model', 'override']);
|
||||
expect(hermetic.fallback).toContain('--strict-mcp-config');
|
||||
expect(run('0').fallback).not.toContain('--strict-mcp-config');
|
||||
const sessionIds = (args: string[]) => args.flatMap((arg, i) => arg === '--session-id' ? [args[i + 1]] : []);
|
||||
expect(sessionIds(hermetic.fallback)).toHaveLength(1);
|
||||
expect(sessionIds(hermetic.fallback)[0]).toMatch(/^[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}$/);
|
||||
expect(sessionIds(hermetic.extra)).toHaveLength(1);
|
||||
expect(sessionIds(hermetic.pinned)).toEqual(['11111111-2222-4333-8444-555555555555']);
|
||||
expect(sessionIds(hermetic.resumed)).toEqual([]);
|
||||
expect(hermetic.pinnedEq.filter((arg: string) => arg.startsWith('--session-id'))).toEqual(['--session-id=11111111-2222-4333-8444-555555555555']);
|
||||
expect(sessionIds(hermetic.resumedEq)).toEqual([]);
|
||||
expect(sessionIds(run('0').fallback)).toEqual([]);
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
}, 60_000);
|
||||
});
|
||||
|
||||
describe('launchClaudePty transcript pricing (GSTA-150 PR 3)', () => {
|
||||
test.skipIf(process.platform === 'win32')('prices the transcript this launch created for its cwd, without adding a launch argument; opting out prices nothing', () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'pty-transcript-cost-'));
|
||||
try {
|
||||
const fake = join(dir, 'fake-claude');
|
||||
writeFileSync(fake, `#!${process.execPath}
|
||||
const fs = require('node:fs'), path = require('node:path'), crypto = require('node:crypto');
|
||||
const project = path.join(process.env.CLAUDE_CONFIG_DIR, 'projects', 'fake-project');
|
||||
fs.mkdirSync(project, { recursive: true });
|
||||
fs.writeFileSync(path.join(project, crypto.randomUUID() + '.jsonl'), JSON.stringify({ type: 'assistant', cwd: process.cwd(),
|
||||
message: { id: 'msg_' + crypto.randomUUID(), model: 'claude-opus-5-5', usage: { input_tokens: 0, output_tokens: 1000 } } }) + '\\n');
|
||||
process.stdout.write('ARGV=' + JSON.stringify(process.argv.slice(2)) + '\\n');
|
||||
setInterval(() => {}, 1000);
|
||||
`, { mode: 0o755 });
|
||||
const runner = pathToFileURL(join(import.meta.dir, 'claude-pty-runner.ts')).href;
|
||||
const worker = join(dir, 'worker.ts');
|
||||
writeFileSync(worker, `import { launchClaudePty } from ${JSON.stringify(runner)};
|
||||
import * as fs from 'node:fs';
|
||||
const run = async (opts) => {
|
||||
const cwd = fs.mkdtempSync(${JSON.stringify(join(dir, 'cwd-'))});
|
||||
const session = await launchClaudePty({ cwd, timeoutMs: 5000, ...opts });
|
||||
try {
|
||||
await session.waitFor(/ARGV=\\[.*\\]/, { timeoutMs: 4000, pollMs: 20 });
|
||||
return { argv: JSON.parse(/ARGV=(\\[.*\\])/.exec(session.visibleText())[1]), cost: session.transcriptCostUsd?.() ?? null };
|
||||
} finally { await session.close(); }
|
||||
};
|
||||
const [priced, optedOut] = await Promise.all([run({}), run({ priceTranscript: false })]);
|
||||
process.stdout.write(JSON.stringify({ priced, optedOut }));
|
||||
`);
|
||||
const result = spawnSync(process.execPath, [worker], { cwd: join(import.meta.dir, '../..'), encoding: 'utf8', timeout: 30_000,
|
||||
env: { ...process.env, BROWSE_TERMINAL_BINARY: fake, EVALS_HERMETIC: '1' } });
|
||||
expect(result.status, result.stderr).toBe(0);
|
||||
const { priced, optedOut } = JSON.parse(result.stdout.trim().split('\n').at(-1)!);
|
||||
expect(priced.cost).toBeCloseTo(0.02, 6);
|
||||
expect(optedOut.cost).toBeNull();
|
||||
expect(priced.argv).not.toContain('--session-id');
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
}, 60_000);
|
||||
});
|
||||
@@ -1,60 +0,0 @@
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { pricedTranscript, ptyTranscriptCostUsd } from './pty/transcript-cost';
|
||||
|
||||
const SID = '11111111-2222-4333-8444-555555555555';
|
||||
const assistant = (id: string, model: string, usage: object, extra: object = {}) =>
|
||||
JSON.stringify({ type: 'assistant', sessionId: SID, message: { id, model, role: 'assistant', usage }, ...extra });
|
||||
|
||||
function fixture(write: (project: string) => void): string {
|
||||
const config = fs.mkdtempSync(path.join(os.tmpdir(), 'pty-cost-'));
|
||||
const project = path.join(config, 'projects', '-fixture-cwd');
|
||||
fs.mkdirSync(project, { recursive: true });
|
||||
write(project);
|
||||
return config;
|
||||
}
|
||||
|
||||
describe('ptyTranscriptCostUsd', () => {
|
||||
test('prices the session, its inline sidechains and subagent files, each message id once at its largest output', () => {
|
||||
const config = fixture(project => {
|
||||
fs.writeFileSync(path.join(project, `${SID}.jsonl`), [
|
||||
JSON.stringify({ type: 'user', sessionId: SID, message: { role: 'user', content: 'hi' } }),
|
||||
assistant('msg_a', 'claude-opus-5-5', { input_tokens: 1000, output_tokens: 10 }),
|
||||
assistant('msg_a', 'claude-opus-5-5', { input_tokens: 1000, output_tokens: 500 }),
|
||||
assistant('msg_side', 'claude-opus-5-5', { input_tokens: 0, output_tokens: 1000 }, { isSidechain: true }),
|
||||
assistant('msg_syn', '<synthetic>', { input_tokens: 0, output_tokens: 0 }),
|
||||
'{"partial":',
|
||||
].join('\n'));
|
||||
fs.mkdirSync(path.join(project, SID, 'subagents'), { recursive: true });
|
||||
fs.writeFileSync(path.join(project, SID, 'subagents', 'agent-x.jsonl'),
|
||||
assistant('msg_sub', 'claude-opus-5-5', { input_tokens: 0, cache_read_input_tokens: 10000, output_tokens: 0 }) + '\n');
|
||||
fs.writeFileSync(path.join(project, '99999999-2222-4333-8444-555555555555.jsonl'),
|
||||
assistant('msg_foreign', 'claude-opus-5-5', { input_tokens: 1_000_000, output_tokens: 0 }) + '\n');
|
||||
});
|
||||
try {
|
||||
// opus-5-5: $4 in, $20 out, $0.20 cached per MTok.
|
||||
const expected = (1000 * 4 + 500 * 20 + 1000 * 20 + 10000 * 0.2) / 1_000_000;
|
||||
expect(ptyTranscriptCostUsd(config, SID)).toBeCloseTo(expected, 6);
|
||||
expect(pricedTranscript({ hermeticConfigDir: config, sessionId: SID }).streamed_cost_usd).toBeCloseTo(expected, 6);
|
||||
} finally { fs.rmSync(config, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('stays cost-unknown without a transcript, a session id, usage or a priced model', () => {
|
||||
const unpriced = fixture(project => fs.writeFileSync(path.join(project, `${SID}.jsonl`),
|
||||
assistant('msg_a', 'claude-opus-5-5', { output_tokens: 1 }) + '\n' + assistant('msg_b', 'some-unpriced-model', { output_tokens: 1 }) + '\n'));
|
||||
const empty = fixture(project => fs.writeFileSync(path.join(project, `${SID}.jsonl`), JSON.stringify({ type: 'user', sessionId: SID }) + '\n'));
|
||||
const unreadable = fixture(project => fs.mkdirSync(path.join(project, `${SID}.jsonl`)));
|
||||
try {
|
||||
expect(ptyTranscriptCostUsd(unpriced, SID)).toBeUndefined();
|
||||
expect(ptyTranscriptCostUsd(empty, SID)).toBeUndefined();
|
||||
expect(ptyTranscriptCostUsd(empty, '99999999-2222-4333-8444-555555555555')).toBeUndefined();
|
||||
expect(ptyTranscriptCostUsd(unpriced, undefined)).toBeUndefined();
|
||||
expect(ptyTranscriptCostUsd(null, SID)).toBeUndefined();
|
||||
expect(ptyTranscriptCostUsd(path.join(empty, 'missing'), SID)).toBeUndefined();
|
||||
expect(pricedTranscript(undefined)).toEqual({});
|
||||
expect(ptyTranscriptCostUsd(unreadable, SID)).toBeUndefined();
|
||||
} finally { for (const dir of [unpriced, empty, unreadable]) fs.rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
});
|
||||
+21
-12
@@ -2,7 +2,6 @@
|
||||
* Spawn `claude` in a real PTY (launchClaudePty) and the session handle. Moved from claude-pty-runner.ts.
|
||||
* Import through test/helpers/claude-pty-runner.ts from tests; pty/ modules import siblings directly.
|
||||
*/
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import * as path from 'node:path';
|
||||
import { resolveEvalModel } from '../../../lib/eval-model';
|
||||
import { hermeticChildEnv, hermeticSkillsConfigDir, isHermeticEnabled } from '../hermetic-env';
|
||||
@@ -16,9 +15,12 @@ import { trustDialogInput } from '../pty-trust-dialog';
|
||||
import { resolveClaudeBinary } from './binary';
|
||||
import { createPtyScreen, screenPanelEnd, stripAnsi } from './screen';
|
||||
import { appendSessionLedger, sessionKey } from '../session-ledger';
|
||||
import { pricedTranscript } from './transcript-cost';
|
||||
import { newTranscriptSessionId, pricedTranscript, ptyTranscriptCostUsd, transcriptFiles } from './transcript-cost';
|
||||
|
||||
export interface ClaudePtyOptions {
|
||||
/** Price this session's ledger row from the transcript Claude writes under the hermetic config dir (default true).
|
||||
* false for a caller that must never read model transcripts, such as the native AskUserQuestion capture. */
|
||||
priceTranscript?: boolean;
|
||||
/** Register the repo's shipped skills in the child's user scope via
|
||||
* hermeticSkillsConfigDir(). Required by any test that types a /skill
|
||||
* slash command; without it hermetic claude rejects the command as
|
||||
@@ -131,8 +133,9 @@ export interface ClaudePtySession {
|
||||
* the dir name ends in `/.claude` by contract).
|
||||
*/
|
||||
hermeticConfigDir: string | null;
|
||||
/** The Claude session id the launcher pinned (or the caller's --session-id): its transcript prices the session ledger row. */
|
||||
sessionId?: string;
|
||||
/** This session's cost from its transcript (the caller's --session-id, else the one transcript this launch created
|
||||
* for its cwd); undefined when unpriced, ambiguous or opted out. Never throws. */
|
||||
transcriptCostUsd?(): number | undefined;
|
||||
/** Owned HOME/.gstack created by the seeded launcher; absent for caller overrides. */
|
||||
hermeticSkillStateRoot?: string;
|
||||
/** Owned pre-tool identity record, removed by close(). */
|
||||
@@ -268,7 +271,8 @@ interface LaunchSetup {
|
||||
hermeticSkillStateRoot: string | undefined;
|
||||
autoplanArtifactStateRoot: string | undefined;
|
||||
autoplanEngTestPlanStateRoot: string | undefined;
|
||||
sessionId: string | undefined;
|
||||
/** What prices the transcript: the caller's session id, else the transcripts present before spawn. */
|
||||
transcript: { sessionId: string | undefined; before: Set<string>; cwd: string } | undefined;
|
||||
}
|
||||
|
||||
function prepareLaunch(opts: ClaudePtyOptions): LaunchSetup {
|
||||
@@ -289,11 +293,6 @@ function prepareLaunch(opts: ClaudePtyOptions): LaunchSetup {
|
||||
// Before opts.extraArgs so a test could theoretically supply --mcp-config.
|
||||
const hermetic = isHermeticEnabled();
|
||||
if (hermetic) args.push('--strict-mcp-config');
|
||||
const extra = opts.extraArgs ?? [];
|
||||
const resumes = extra.some(arg => /^(--resume|--continue|-r|-c)(=|$)/.test(arg));
|
||||
const pinned = extra.findIndex(arg => /^--session-id(=|$)/.test(arg));
|
||||
let sessionId = pinned < 0 ? undefined : extra[pinned].startsWith('--session-id=') ? extra[pinned].slice('--session-id='.length) : extra[pinned + 1];
|
||||
if (hermetic && pinned < 0 && !resumes) args.push('--session-id', sessionId = randomUUID());
|
||||
if (opts.extraArgs) args.push(...opts.extraArgs);
|
||||
|
||||
// Hermetic by default (test/helpers/hermetic-env.ts): operator session
|
||||
@@ -330,7 +329,13 @@ function prepareLaunch(opts: ClaudePtyOptions): LaunchSetup {
|
||||
|
||||
const autoplanEngTestPlanStateRoot = opts.approveAutoplanArtifactEdits && opts.autoplanArtifactState !== undefined
|
||||
? hermeticSkillStateRoot : undefined;
|
||||
return { args, childEnv, hermetic, hermeticSkillStateRoot, autoplanArtifactStateRoot, autoplanEngTestPlanStateRoot, sessionId };
|
||||
const extra = opts.extraArgs ?? [];
|
||||
const pinned = extra.findIndex(arg => /^--session-id(=|$)/.test(arg));
|
||||
const sessionId = pinned < 0 ? undefined : extra[pinned].startsWith('--session-id=') ? extra[pinned].slice('--session-id='.length) : extra[pinned + 1];
|
||||
const resumes = extra.some(arg => /^(--resume|--continue|-r|-c)(=|$)/.test(arg));
|
||||
const transcript = hermetic && childEnv.CLAUDE_CONFIG_DIR && opts.priceTranscript !== false && !resumes
|
||||
? { sessionId, before: transcriptFiles(childEnv.CLAUDE_CONFIG_DIR), cwd: opts.cwd ?? process.cwd() } : undefined;
|
||||
return { args, childEnv, hermetic, hermeticSkillStateRoot, autoplanArtifactStateRoot, autoplanEngTestPlanStateRoot, transcript };
|
||||
}
|
||||
|
||||
interface LaunchRecorders {
|
||||
@@ -579,7 +584,11 @@ function sessionHandle(pty: PtyProcess, launch: LaunchSetup, wallTimer: ReturnTy
|
||||
exited: () => pty.exited,
|
||||
exitCode: () => pty.exitCode,
|
||||
hermeticConfigDir: launch.hermetic ? launch.childEnv.CLAUDE_CONFIG_DIR ?? null : null,
|
||||
sessionId: launch.sessionId,
|
||||
transcriptCostUsd: () => {
|
||||
const { transcript, childEnv } = launch;
|
||||
return transcript && ptyTranscriptCostUsd(childEnv.CLAUDE_CONFIG_DIR,
|
||||
transcript.sessionId ?? newTranscriptSessionId(childEnv.CLAUDE_CONFIG_DIR, transcript.before, transcript.cwd));
|
||||
},
|
||||
hermeticSkillStateRoot: launch.hermeticSkillStateRoot,
|
||||
pendingPlanReadyFile: pendingExit?.file,
|
||||
pendingQuestionFile: pendingQuestion?.file,
|
||||
|
||||
@@ -3,11 +3,44 @@ import * as path from 'node:path';
|
||||
import { streamedUsageCostUsd } from '../../../lib/pricing';
|
||||
|
||||
/** The session ledger's cost field for a PTY session: present only when its transcript prices. */
|
||||
export function pricedTranscript(session: { hermeticConfigDir?: string | null; sessionId?: string } | undefined): { streamed_cost_usd?: number } {
|
||||
const cost = ptyTranscriptCostUsd(session?.hermeticConfigDir, session?.sessionId);
|
||||
export function pricedTranscript(session: { transcriptCostUsd?(): number | undefined } | undefined): { streamed_cost_usd?: number } {
|
||||
const cost = session?.transcriptCostUsd?.();
|
||||
return cost === undefined ? {} : { streamed_cost_usd: cost };
|
||||
}
|
||||
|
||||
/** Top-level session transcripts (`projects/<dir>/<id>.jsonl`) under a config dir; empty when unreadable. */
|
||||
export function transcriptFiles(configDir: string | undefined): Set<string> {
|
||||
if (!configDir) return new Set();
|
||||
try {
|
||||
const projects = path.join(configDir, 'projects');
|
||||
return new Set(fs.readdirSync(projects, { withFileTypes: true }).filter(dir => dir.isDirectory())
|
||||
.flatMap(dir => fs.readdirSync(path.join(projects, dir.name)).filter(f => f.endsWith('.jsonl')).map(f => path.join(projects, dir.name, f))));
|
||||
} catch {
|
||||
return new Set();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The session id of the one transcript that appeared since `before` and was written for `cwd`. Concurrent PTY tests in
|
||||
* one process share a hermetic config dir, so a launch claims only a new transcript for its own cwd; none or several
|
||||
* (two sessions in one cwd) is undefined, which leaves the cost unknown.
|
||||
*/
|
||||
export function newTranscriptSessionId(configDir: string | undefined, before: ReadonlySet<string>, cwd: string): string | undefined {
|
||||
try {
|
||||
const cwds = new Set([cwd, fs.realpathSync(cwd)]);
|
||||
const mine = [...transcriptFiles(configDir)].filter(file => !before.has(file) && fs.readFileSync(file, 'utf8').split('\n').some(line => {
|
||||
try {
|
||||
return cwds.has(JSON.parse(line)?.cwd);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}));
|
||||
return mine.length === 1 ? path.basename(mine[0], '.jsonl') : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Price one PTY session from the transcripts Claude Code wrote under its owned config dir:
|
||||
* `projects/<dir>/<sessionId>.jsonl` (inline sidechains included) and `projects/<dir>/<sessionId>/subagents/*.jsonl`,
|
||||
|
||||
@@ -12,6 +12,7 @@ import { anthropicUsageCostUsd, streamedUsageCostUsd } from '../lib/pricing';
|
||||
import { E2E_TOUCHFILES } from './helpers/touchfiles-data';
|
||||
import type { SessionLedgerRow } from './helpers/session-ledger';
|
||||
import { recordE2E } from './helpers/e2e-helpers';
|
||||
import { newTranscriptSessionId, pricedTranscript, ptyTranscriptCostUsd, transcriptFiles } from './helpers/pty/transcript-cost';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const session = (over: Partial<SessionLedgerRow>): SessionLedgerRow => ({ key: 'k', runner: 'agent-sdk', started_at: 't', elapsed_ms: 1, end: 'aborted', ...over });
|
||||
@@ -128,3 +129,75 @@ describe('junitCensus cost binding', () => {
|
||||
expect(history.map(h => [h.case, h.model])).toEqual([['browse-basic', undefined], ['browse-snapshot', undefined]]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('PTY transcript pricing (GSTA-150 PR 3)', () => {
|
||||
const SID = '11111111-2222-4333-8444-555555555555';
|
||||
const OTHER = '99999999-2222-4333-8444-555555555555';
|
||||
const assistant = (sid: string, id: string, model: string, usage: object, extra: object = {}) =>
|
||||
JSON.stringify({ type: 'assistant', sessionId: sid, cwd: '/fixture/cwd', message: { id, model, role: 'assistant', usage }, ...extra });
|
||||
const fixture = (write: (project: string) => void) => {
|
||||
const config = fs.mkdtempSync(path.join(os.tmpdir(), 'pty-cost-'));
|
||||
const project = path.join(config, 'projects', '-fixture-cwd');
|
||||
fs.mkdirSync(project, { recursive: true });
|
||||
write(project);
|
||||
return config;
|
||||
};
|
||||
|
||||
test('prices the session, its inline sidechains and subagent files, each message id once at its largest output', () => {
|
||||
const config = fixture(project => {
|
||||
fs.writeFileSync(path.join(project, `${SID}.jsonl`), [
|
||||
JSON.stringify({ type: 'user', sessionId: SID, message: { role: 'user', content: 'hi' } }),
|
||||
assistant(SID, 'msg_a', 'claude-opus-5-5', { input_tokens: 1000, output_tokens: 10 }),
|
||||
assistant(SID, 'msg_a', 'claude-opus-5-5', { input_tokens: 1000, output_tokens: 500 }),
|
||||
assistant(SID, 'msg_side', 'claude-opus-5-5', { input_tokens: 0, output_tokens: 1000 }, { isSidechain: true }),
|
||||
assistant(SID, 'msg_syn', '<synthetic>', { input_tokens: 0, output_tokens: 0 }),
|
||||
'{"partial":',
|
||||
].join('\n'));
|
||||
fs.mkdirSync(path.join(project, SID, 'subagents'), { recursive: true });
|
||||
fs.writeFileSync(path.join(project, SID, 'subagents', 'agent-x.jsonl'),
|
||||
assistant(SID, 'msg_sub', 'claude-opus-5-5', { input_tokens: 0, cache_read_input_tokens: 10000, output_tokens: 0 }) + '\n');
|
||||
fs.writeFileSync(path.join(project, `${OTHER}.jsonl`), assistant(OTHER, 'msg_foreign', 'claude-opus-5-5', { input_tokens: 1_000_000 }) + '\n');
|
||||
});
|
||||
try {
|
||||
// opus-5-5: $4 in, $20 out, $0.20 cached per MTok.
|
||||
const expected = (1000 * 4 + 500 * 20 + 1000 * 20 + 10000 * 0.2) / 1_000_000;
|
||||
expect(ptyTranscriptCostUsd(config, SID)).toBeCloseTo(expected, 6);
|
||||
expect(pricedTranscript({ transcriptCostUsd: () => ptyTranscriptCostUsd(config, SID) }).streamed_cost_usd).toBeCloseTo(expected, 6);
|
||||
} finally { fs.rmSync(config, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('stays cost-unknown without a transcript, a session id, usage, a priced model or a readable file, and never throws', () => {
|
||||
const unpriced = fixture(project => fs.writeFileSync(path.join(project, `${SID}.jsonl`),
|
||||
assistant(SID, 'msg_a', 'claude-opus-5-5', { output_tokens: 1 }) + '\n' + assistant(SID, 'msg_b', 'some-unpriced-model', { output_tokens: 1 }) + '\n'));
|
||||
const empty = fixture(project => fs.writeFileSync(path.join(project, `${SID}.jsonl`), JSON.stringify({ type: 'user', sessionId: SID }) + '\n'));
|
||||
const unreadable = fixture(project => fs.mkdirSync(path.join(project, `${SID}.jsonl`)));
|
||||
try {
|
||||
expect(ptyTranscriptCostUsd(unpriced, SID)).toBeUndefined();
|
||||
expect(ptyTranscriptCostUsd(empty, SID)).toBeUndefined();
|
||||
expect(ptyTranscriptCostUsd(empty, OTHER)).toBeUndefined();
|
||||
expect(ptyTranscriptCostUsd(unpriced, undefined)).toBeUndefined();
|
||||
expect(ptyTranscriptCostUsd(null, SID)).toBeUndefined();
|
||||
expect(ptyTranscriptCostUsd(path.join(empty, 'missing'), SID)).toBeUndefined();
|
||||
expect(ptyTranscriptCostUsd(unreadable, SID)).toBeUndefined();
|
||||
expect(newTranscriptSessionId(unreadable, new Set(), '/fixture/cwd')).toBeUndefined();
|
||||
expect(pricedTranscript(undefined)).toEqual({});
|
||||
expect(pricedTranscript({})).toEqual({});
|
||||
} finally { for (const dir of [unpriced, empty, unreadable]) fs.rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('a launch claims only the one transcript it created for its own cwd', () => {
|
||||
const cwd = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'pty-cost-cwd-')));
|
||||
const line = (sid: string, at: string) => JSON.stringify({ type: 'user', sessionId: sid, cwd: at }) + '\n';
|
||||
const config = fixture(project => fs.writeFileSync(path.join(project, `${OTHER}.jsonl`), line(OTHER, cwd)));
|
||||
try {
|
||||
const before = transcriptFiles(config);
|
||||
const project = path.join(config, 'projects', '-fixture-cwd');
|
||||
fs.writeFileSync(path.join(project, '22222222-2222-4333-8444-555555555555.jsonl'), line('22222222-2222-4333-8444-555555555555', '/a/concurrent/test'));
|
||||
expect(newTranscriptSessionId(config, before, cwd)).toBeUndefined();
|
||||
fs.writeFileSync(path.join(project, `${SID}.jsonl`), line(SID, cwd));
|
||||
expect(newTranscriptSessionId(config, before, cwd)).toBe(SID);
|
||||
fs.writeFileSync(path.join(project, '33333333-2222-4333-8444-555555555555.jsonl'), line('33333333-2222-4333-8444-555555555555', cwd));
|
||||
expect(newTranscriptSessionId(config, before, cwd)).toBeUndefined();
|
||||
} finally { fs.rmSync(config, { recursive: true, force: true }); fs.rmSync(cwd, { recursive: true, force: true }); }
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user