From 58715ba660adc4ac846ee90173e48f11dd41cf9f Mon Sep 17 00:00:00 2001 From: garrytan Date: Mon, 26 Oct 2026 03:03:05 +0000 Subject: [PATCH] fix(context-bill): a bill with any estimated item under --exact is labelled mixed and keeps its error band, in --json and text Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- lib/context-bill.ts | 21 ++++++++++++++++++--- test/context-bill.test.ts | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/lib/context-bill.ts b/lib/context-bill.ts index 568e0d079..2c9584fb0 100644 --- a/lib/context-bill.ts +++ b/lib/context-bill.ts @@ -568,10 +568,12 @@ export interface Bill { export function buildBill( root: string, - { tokensOf = estimateTokensOf, tokenSource, calibration }: { + { tokensOf = estimateTokensOf, tokenSource, calibration, missedKeys }: { tokensOf?: TokensOf; tokenSource?: string; calibration?: Calibration; + /** Keys a measured tokensOf had to estimate; filled while the bill is priced. */ + missedKeys?: ReadonlySet; } = {}, ): Bill { const resolved = path.resolve(root); @@ -581,12 +583,17 @@ export function buildBill( ); const total = skills.reduce((n, s) => n + s.totalMdBytes, 0); const totalTokens = skills.reduce((n, s) => n + s.totalMdTokens, 0); + const estimated = tokenSource ? (missedKeys?.size ?? 0) : 0; return { root: resolved, // Named so a reader never has to guess whether a figure was measured. - tokenSource: tokenSource ?? "estimate: calibrated bytes/token per content class", + tokenSource: !tokenSource + ? "estimate: calibrated bytes/token per content class" + : estimated + ? `mixed: ${tokenSource}, with ${estimated} item(s) estimated offline` + : tokenSource, tokenEstimate: TOKEN_DIVISORS, - tokenEstimateErrorPct: tokenSource ? 0 : TOKEN_ESTIMATE_ERROR_PCT, + tokenEstimateErrorPct: tokenSource && !estimated ? 0 : TOKEN_ESTIMATE_ERROR_PCT, // Present only under --exact: how far the offline estimate was off, per file. ...(calibration ? { calibration } : {}), skills, @@ -769,6 +776,12 @@ export function tokenDisclaimer(bill: Pick `${k} /${v}`).join(", "); return ( `Token counts are ESTIMATES: bytes divided per content class (${per}), calibrated against ` + @@ -1211,6 +1224,7 @@ export async function contextBillMain(argv: string[], options: MainOptions = {}) const exactFor = async (tree: string, set: ExactSendSet | undefined): Promise<{ tokensOf?: TokensOf; tokenSource?: string; + missedKeys?: ReadonlySet; calibration?: Calibration; onDone?: () => void; }> => { @@ -1260,6 +1274,7 @@ export async function contextBillMain(argv: string[], options: MainOptions = {}) return { tokensOf: measured.tokensOf, tokenSource: measured.tokenSource, + missedKeys: measured.missedKeys, calibration: calibrationTable(measured.counts, tree, measured.realOf), onDone: () => { if (measured.missedKeys.size) { diff --git a/test/context-bill.test.ts b/test/context-bill.test.ts index 522113229..714638ce3 100644 --- a/test/context-bill.test.ts +++ b/test/context-bill.test.ts @@ -1292,6 +1292,39 @@ describe("--exact (opt-in measurement; offline here via an injected fetch)", () expect(bill.skills[0].totalMdBytes).toBe(skillMd.length + "CHECKOUT-FLOW".length + "PRIVATE-DIARY".length); }); + it("labels a bill with any estimated item as mixed, in --json and in text", async () => { + if (process.platform === "win32") return; + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "context-bill-mixed-")); + const outside = path.join(tmp, "outside", "ext"); + fs.mkdirSync(outside, { recursive: true }); + fs.writeFileSync(path.join(outside, "SKILL.md"), "---\nname: ext\ndescription: d\n---\n" + "e".repeat(6_000)); + const skills = path.join(tmp, "skills"); + fs.mkdirSync(path.join(skills, "qa"), { recursive: true }); + fs.writeFileSync(path.join(skills, "qa", "SKILL.md"), "---\nname: qa\ndescription: d\n---\n# qa\n"); + fs.symlinkSync(outside, path.join(skills, "ext")); + const run = async (args: string[]) => { + const stdout = capture(); + const code = await contextBillMain([skills, "--exact", ...args], { + stdout: stdout.stream, + stderr: capture().stream, + apiKey: "sk-test", + fetchImpl: fakeFetch(), + egressHome, + }); + return { code, out: stdout.text() }; + }; + const json = await run(["--json"]); + const text = await run([]); + fs.rmSync(tmp, { recursive: true, force: true }); + expect(json.code).toBe(0); + const bill = JSON.parse(json.out); + expect(bill.tokenSource).toStartWith("mixed: count_tokens"); + expect(bill.tokenSource).toContain("estimated offline"); + expect(bill.tokenEstimateErrorPct).toBeGreaterThan(0); + expect(text.out).toContain("Token counts are MIXED"); + expect(text.out).not.toContain("Token counts measured with"); + }); + async function exactRun(skills: string) { const calls: { body: string }[] = []; const stderr = capture();