From de5d385e772e5378e8175ea6d17ed3fef587792c Mon Sep 17 00:00:00 2001 From: garrytan Date: Mon, 26 Oct 2026 15:17:49 +0000 Subject: [PATCH] test: pin a failed display-labelled trial's fields and the $0.00 cost line; say so in the CHANGELOG The widened record filter also feeds failedRecord, so a failed display-labelled trial in a case shard now takes failure_class, exit_reason and error from its record (train-gate classes a red infra record INFRA instead of VERDICT). The CHANGELOG entry said only the cost fields changed; it now says both. A train priced at zero prints cost=$0.00. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CHANGELOG.md | 7 ++++--- test/paid-report-census.test.ts | 20 ++++++++++++++++++++ test/ship-queue-train-cost.test.ts | 9 +++++++++ 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2352a057d..a89dc2984 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,13 +8,14 @@ - **`.gstack/ship-policy.json`**: `train.max_size` 5 → 3 and `train.fill_hours` 3 → 1. `fillDue` already dispatches at either bound, whichever comes first; a fourth READY PR is `HELD:` for the next train. docs/ship-policy.md says so in the fill paragraph and its example block. ### Fixed -- **`scripts/lib/paid-report.ts`**: when a shard ran exactly one case (every case shard since W5a), its result records belong to that case even when they are named by a display label, so the trial's `cost_usd` is the record's and `cost_known` is no longer false. A multi-case shard whose records carry display labels keeps those trials cost-unknown rather than guessing which case paid. The train record's per-case `cost_usd` (scripts/train-gate.ts) and the slice report's headline now agree for case shards; the headline was never wrong. +- **`scripts/lib/paid-report.ts`**: when a shard ran exactly one case (every case shard since W5a), its result records belong to that case even when they are named by a display label, so the trial's `cost_usd` is the record's and `cost_known` is no longer false; a failed trial in such a shard also takes its `failure_class`, `exit_reason` and `error` from that record (train-gate classes a red `infra` record INFRA instead of VERDICT) where it was a bare `assertion` with the JUnit message. No case's pass or fail changes. A multi-case shard whose records carry display labels keeps those trials cost-unknown rather than guessing which case paid. The train record's per-case `cost_usd` (scripts/train-gate.ts) and the slice report's headline now agree for case shards; the headline was never wrong. - **`lib/ship-queue-train-verdict.ts`**: the `ATTEMPT:` line prints `cost=$ known + of case(s) cost unknown` when any panel has no cost, and the bare `cost=$` only when every panel is priced. ### Tests - `test/gstack-ship-policy.test.ts` pins the live fill rule (3 PRs or one hour) and exercises `fillDue` with the live block at 59 and 60 minutes and at 3 PRs. -- `test/paid-report-census.test.ts`: a display-labelled record in the periodic fixture's `qa-fix-loop` case shard gives the trial `cost_usd: 1.23` with no `cost_known: false`, `verdict.costUsd` 1.23 and the headline `cost $1.23 known + 6 of 7 trial(s) cost unknown` (red before the fix: `cost_usd: 0, cost_known: false`); the two-case `skill-e2e-cso` shard keeps its display-labelled trial unknown while the id-named one is priced. -- `test/ship-queue-train-cost.test.ts`: a green train whose record has two unpriced panels prints `cost=$0.70 known + 2 of 3 case(s) cost unknown`; a fully priced record prints `cost=$1.50` alone (red before the fix: the bare `cost=$0.70`). +- `test/paid-report-census.test.ts`: a display-labelled record in the periodic fixture's `qa-fix-loop` case shard gives the trial `cost_usd: 1.23` with no `cost_known: false`, `verdict.costUsd` 1.23 and the headline `cost $1.23 known + 6 of 7 trial(s) cost unknown` (red before the fix: `cost_usd: 0, cost_known: false`); a failed display-labelled record gives the trial `failure_class: 'timeout'`, its exit reason, error and cost (red before the fix: `assertion` with the JUnit message); the two-case `skill-e2e-cso` shard keeps its display-labelled trial unknown while the id-named one is priced. +- `test/ship-queue-train-cost.test.ts`: a green train whose record has two unpriced panels prints `cost=$0.70 known + 2 of 3 case(s) cost unknown`; a fully priced record prints `cost=$1.50` alone (red before the fix: the bare `cost=$0.70`); every panel priced at zero prints `cost=$0.00`. +- `scripts/lib/windows-curation.ts` excludes `test/ship-queue-train-cost.test.ts` from the Windows subset like `test/ship-queue-train.test.ts`: both drive `bin/gstack-ship-queue` against the `#!/bin/sh` gh shim, which windows-latest cannot run. ### Gate integrity - No threshold, rubric, trial count, case kind or judge prompt changed. Cost attribution changes what a trial record reports, never whether a case passed. diff --git a/test/paid-report-census.test.ts b/test/paid-report-census.test.ts index 32292dd54..ff9dbf168 100644 --- a/test/paid-report-census.test.ts +++ b/test/paid-report-census.test.ts @@ -122,6 +122,26 @@ describe('a case shard\'s result records are its one case\'s cost (2026-10-26 tr expect(r.history.find(record => record.case === 'cso-infra-scope').cost_known).toBeUndefined(); expect(collector(r.dir).verdict.costUsd).toBe(0.75); }); + + // Value: protects=a failed case shard's trial reads its failure class, exit reason, error and cost from the display-labelled record the shard + // wrote, not a generic 'assertion' with the JUnit message (the same attribution the cost fix made; train-gate's INFRA/VERDICT class follows failure_class); + // fails_when=failedRecord is looked up through the old name/case_id-only filter again so the record's class and error are dropped; + // existing_gap=both attribution rows above use passing records, so the failedRecord branch of the widened filter is unexercised; seam=none + test('a failed display-labelled record in a case shard supplies the trial\'s failure class, exit reason and error, not a bare assertion', () => { + const r = report('periodic', dir => { + const qa = junit(dir, 'skill-e2e-qa-workflow--qa-fix-loop'); + fs.writeFileSync(qa, fs.readFileSync(qa, 'utf8').replace('failures="0"', 'failures="1"') + .replace(/(]*?) \/>/, '$1>')); + results(dir, 'skill-e2e-qa-workflow--qa-fix-loop', [{ name: '/qa fix loop', suite: 'QA skill E2E', tier: 'e2e', passed: false, duration_ms: 1000, cost_usd: 2.5, + failure_class: 'timeout', exit_reason: 'timeout', timeout_at_turn: 7, error: 'no terminal outcome within 1800000ms\nsecond line' }]); + }); + expect(r.status, r.out).toBe(1); + const trial = r.history.find(record => record.case === 'qa-fix-loop'); + expect(trial).toMatchObject({ outcome: 'failed', failure_class: 'timeout', exit_reason: 'timeout', error: 'no terminal outcome within 1800000ms', cost_usd: 2.5 }); + expect(trial.cost_known).toBeUndefined(); + expect(r.out).toContain('✗ qa-fix-loop rule FAIL timeout at turn 7 — no terminal outcome within 1800000ms'); + expect(trialRecordProblemsV1(trial)).toEqual([]); + }); }); describe('record builders attach ledger sessions and failure causes (plan 0.1/0.2)', () => { diff --git a/test/ship-queue-train-cost.test.ts b/test/ship-queue-train-cost.test.ts index 1499c47d4..d9ccd2c66 100644 --- a/test/ship-queue-train-cost.test.ts +++ b/test/ship-queue-train-cost.test.ts @@ -37,4 +37,13 @@ describe('train verdict cost line', () => { expect(line(queue(all, ['train', 'verdict', '--train', priced]).out, 'ATTEMPT:')).toBe('ATTEMPT: a1 gate run 101 GREEN cost=$1.50 (rule 2/2, judge 1/1)'); }, 120_000); + // Value: protects=a train whose every case is priced at $0.00 (reused or free trials) still reports its cost as known; + // fails_when=the null guard on cost_usd becomes a falsy check, so a zero sum prints nothing and reads like an unpriced run; + // existing_gap=the two rows above price every panel above zero or leave some unpriced, never a known zero; seam=none + test('a train priced at zero prints cost=$0.00 rather than no cost at all', () => { + const fx = make(); + const free = { ...GREEN_GATE, record: { ...GREEN_GATE.record, panels: [PASS('c1', 'rule', 0), PASS('c2', 'rule', 0), PASS('j1', 'judge', 0)] } }; + const nonce = greenTrain(fx, free); + expect(line(queue(fx, ['train', 'verdict', '--train', nonce]).out, 'ATTEMPT:')).toBe('ATTEMPT: a1 gate run 101 GREEN cost=$0.00 (rule 2/2, judge 1/1)'); + }, 60_000); });