From d5e6a7d5f7f62cdbf77ff46daabd27a826e34c37 Mon Sep 17 00:00:00 2001 From: garrytan Date: Mon, 26 Oct 2026 14:54:16 +0000 Subject: [PATCH] fix(ship-queue): fill trains at 3 READY PRs or one hour, and price every case shard's trial The 5-PR / 3-hour fill left READY PRs waiting most of a day; .gstack/ship-policy.json now fills at max_size 3 or fill_hours 1, whichever comes first (fillDue already dispatches at either bound). train verdict printed cost=$24.41 for a train whose slice report said cost $32.92 known + 54 of 136 trial(s) cost unknown: E2E records named by a display label ("/benchmark workflow") were never attributed to their case id, so the per-case sum was short. A case shard's records now belong to its one case; a multi-case shard with display-labelled records keeps those trials unknown. The ATTEMPT: line names how many cases have no cost instead of printing a partial sum as the total. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- .gstack/ship-policy.json | 2 +- CHANGELOG.md | 19 ++++++++++++++ docs/ship-policy.md | 10 +++++--- lib/ship-queue-train-verdict.ts | 7 +++++- scripts/lib/paid-report.ts | 4 ++- test/gstack-ship-policy.test.ts | 13 ++++++++++ test/paid-report-census.test.ts | 27 ++++++++++++++++++++ test/ship-queue-train-cost.test.ts | 40 ++++++++++++++++++++++++++++++ 8 files changed, 115 insertions(+), 7 deletions(-) create mode 100644 test/ship-queue-train-cost.test.ts diff --git a/.gstack/ship-policy.json b/.gstack/ship-policy.json index cd1e67330..f5b84ce2c 100644 --- a/.gstack/ship-policy.json +++ b/.gstack/ship-policy.json @@ -12,6 +12,6 @@ "stamp_sensitive_gates": ["tests"], "pr_count": { "default_pr_count": 1, "split_requires_owner": true }, "maintainers": ["garrytan"], - "train": { "gate_workflow": "evals.yml", "free_workflow": "free-tests.yml", "full_gate_check": "gstack/full-gate", "record_artifact": "gstack-train-record", "max_size": 5, "fill_hours": 3, "screen_selection": "gstack-paid", "trial_reservation_usd": 1.5 }, + "train": { "gate_workflow": "evals.yml", "free_workflow": "free-tests.yml", "full_gate_check": "gstack/full-gate", "record_artifact": "gstack-train-record", "max_size": 3, "fill_hours": 1, "screen_selection": "gstack-paid", "trial_reservation_usd": 1.5 }, "min_gstack_version": "1.91.146.0" } diff --git a/CHANGELOG.md b/CHANGELOG.md index bcc901d87..2352a057d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,24 @@ # Changelog +## [Unreleased] + +**Merge trains fill at 3 READY PRs or one hour after the first READY, and a train verdict's cost line no longer hides the trials it could not price.** On 2026-10-26 the 5-PR / 3-hour fill left READY PRs waiting most of a day, and `train verdict` printed `cost=$24.41` for a train whose slice report said `cost $32.92 known + 54 of 136 trial(s) cost unknown`. The slice report was right: 54 E2E trials had no cost attributed because their result records are named by a display label ("/benchmark workflow") while the case is named by its id (`benchmark-workflow`), so the per-case sum the train record carries was short by $8.51. + +### Changed +- **`.gstack/ship-policy.json`**: `train.max_size` 5 → 3 and `train.fill_hours` 3 → 1. `fillDue` already dispatches at either bound, whichever comes first; a fourth READY PR is `HELD:` for the next train. docs/ship-policy.md says so in the fill paragraph and its example block. + +### Fixed +- **`scripts/lib/paid-report.ts`**: when a shard ran exactly one case (every case shard since W5a), its result records belong to that case even when they are named by a display label, so the trial's `cost_usd` is the record's and `cost_known` is no longer false. A multi-case shard whose records carry display labels keeps those trials cost-unknown rather than guessing which case paid. The train record's per-case `cost_usd` (scripts/train-gate.ts) and the slice report's headline now agree for case shards; the headline was never wrong. +- **`lib/ship-queue-train-verdict.ts`**: the `ATTEMPT:` line prints `cost=$ known + of case(s) cost unknown` when any panel has no cost, and the bare `cost=$` only when every panel is priced. + +### Tests +- `test/gstack-ship-policy.test.ts` pins the live fill rule (3 PRs or one hour) and exercises `fillDue` with the live block at 59 and 60 minutes and at 3 PRs. +- `test/paid-report-census.test.ts`: a display-labelled record in the periodic fixture's `qa-fix-loop` case shard gives the trial `cost_usd: 1.23` with no `cost_known: false`, `verdict.costUsd` 1.23 and the headline `cost $1.23 known + 6 of 7 trial(s) cost unknown` (red before the fix: `cost_usd: 0, cost_known: false`); the two-case `skill-e2e-cso` shard keeps its display-labelled trial unknown while the id-named one is priced. +- `test/ship-queue-train-cost.test.ts`: a green train whose record has two unpriced panels prints `cost=$0.70 known + 2 of 3 case(s) cost unknown`; a fully priced record prints `cost=$1.50` alone (red before the fix: the bare `cost=$0.70`). + +### Gate integrity +- No threshold, rubric, trial count, case kind or judge prompt changed. Cost attribution changes what a trial record reports, never whether a case passed. + ## [1.91.248.0] - 2026-10-26 **`$B` in direct mode no longer lets Chromium pick a proxy from the environment (#3440 item 8).** `BROWSE_PROXY_URL=direct` (and an unparseable `HTTPS_PROXY`) only stopped gstack from configuring a proxy. The daemon inherits the whole environment and the launch passed no proxy switch, so Chromium read `HTTPS_PROXY`, `HTTP_PROXY` or `ALL_PROXY` itself and every `goto` still went through that proxy while `$B` reported a direct connection. That made the BROWSER.md precedence list false and any certificate check on a proxied machine unreliable. Reproduced on Linux with Playwright 1.62.1: a logging CONNECT proxy in `HTTPS_PROXY` received `CONNECT :443` with `BROWSE_PROXY_URL=direct` set. diff --git a/docs/ship-policy.md b/docs/ship-policy.md index b336b20eb..5a12da56c 100644 --- a/docs/ship-policy.md +++ b/docs/ship-policy.md @@ -317,7 +317,7 @@ warning on stderr). A block for gstack itself would read: "train": { "gate_workflow": "evals.yml", "free_workflow": "free-tests.yml", "full_gate_check": "gstack/full-gate", "record_artifact": "gstack-train-record", - "max_size": 5, "fill_hours": 3, "screen_selection": "gstack-paid", + "max_size": 3, "fill_hours": 1, "screen_selection": "gstack-paid", "trial_reservation_usd": 1.5 }, "min_gstack_version": "" @@ -437,9 +437,11 @@ tree=` per prefix and one `SLOT:` per PR. `--dry-run` adds the writes `train.json`, pushes `train/`, dispatches `free_workflow` there and waits (`--free-timeout `, default 2700) for a green `workflow_dispatch` run on that exact SHA, then dispatches the full gate -through eval-bg's nonce. A train under `max_size` waits until its oldest READY -has waited `fill_hours` (`QUEUE_TRAIN_WAITING`); `--urgent` dispatches a -one-PR train at once. The same pins on the same base return the live train +through eval-bg's nonce. A train fills at `max_size` READY PRs or when its +oldest READY has waited `fill_hours`, whichever comes first; until then +`QUEUE_TRAIN_WAITING` names the time, and `--urgent` dispatches a one-PR +train at once. gstack's own block fills at 3 PRs or one hour (it was 5 PRs or +three hours until 2026-10-26, which left READY PRs waiting most of a day). The same pins on the same base return the live train instead of a second one; `train build --train ` resumes a train from its record. Refusals, all before any spend: `QUEUE_TRAIN_UNTRUSTED` (a fork head), `QUEUE_TRAIN_MOVED` (a head moved from its pin), diff --git a/lib/ship-queue-train-verdict.ts b/lib/ship-queue-train-verdict.ts index 9d27bafae..3aa226431 100644 --- a/lib/ship-queue-train-verdict.ts +++ b/lib/ship-queue-train-verdict.ts @@ -82,7 +82,12 @@ export function classifyGate(raw: unknown, expect: { nonce: string; head: string } const gateAttempts = (t: TrainRecord) => attemptsOf(t).filter(a => a.purpose !== 'bisect'); -const attemptLine = (a: AttemptRecord) => `ATTEMPT: ${a.id} ${a.purpose} run ${a.run?.id ?? '-'} ${a.outcome ?? a.state}${a.red_cases.length ? ` red=${a.red_cases.join(',')}` : ''}${a.cost_usd !== null ? ` cost=$${a.cost_usd.toFixed(2)}` : ''}${a.detail ? ` (${a.detail})` : ''}`; +const costText = (a: AttemptRecord) => { + if (a.cost_usd === null) return ''; + const unknown = a.panels.filter(p => p.cost_usd === null).length; + return ` cost=$${a.cost_usd.toFixed(2)}${unknown ? ` known + ${unknown} of ${a.panels.length} case(s) cost unknown` : ''}`; +}; +const attemptLine = (a: AttemptRecord) => `ATTEMPT: ${a.id} ${a.purpose} run ${a.run?.id ?? '-'} ${a.outcome ?? a.state}${a.red_cases.length ? ` red=${a.red_cases.join(',')}` : ''}${costText(a)}${a.detail ? ` (${a.detail})` : ''}`; function casePrice(env: TrainEnv, red: AttemptRecord, id: string): number { return red.panels.find(p => p.case === id)?.cost_usd ?? env.train.trial_reservation_usd; diff --git a/scripts/lib/paid-report.ts b/scripts/lib/paid-report.ts index 19a87cb02..224dc5e8b 100644 --- a/scripts/lib/paid-report.ts +++ b/scripts/lib/paid-report.ts @@ -275,7 +275,9 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact const kind = (E2E_KINDS[id] ?? 'rule') as EvalCaseKind; const failed = cases.find(tc => tc.outcome === 'failed'); const caseOutcome: TrialOutcome = failed ? 'failed' : cases.some(tc => tc.outcome === 'passed') ? 'passed' : 'skipped'; - const mine = records.filter((r: any) => r?.name === id || r?.case_id === id); + // A record is named by its case id or carries case_id; an E2E record named by its display label ("/qa fix loop") is + // the shard's only case's when the shard ran one case, else it stays unattributed and the trial's cost unknown. + const mine = records.filter((r: any) => r?.name === id || r?.case_id === id || (byCase.size === 1 && !r?.case_id)); const failedRecord = mine.find((r: any) => r.passed === false); const recorded: TrialFailureClass | undefined = !failed ? undefined : failed.failureType === 'TimeoutError' ? 'timeout' : failedRecord ? failureClassOf(failedRecord) : 'assertion'; diff --git a/test/gstack-ship-policy.test.ts b/test/gstack-ship-policy.test.ts index d7a4cf269..5f0893659 100644 --- a/test/gstack-ship-policy.test.ts +++ b/test/gstack-ship-policy.test.ts @@ -8,6 +8,7 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; import { EXIT } from '../lib/headless-artifacts'; import { DEFAULT_POLICY, POLICY_KEYS, globToRegExp, matchesAny, validatePolicy } from '../lib/ship-policy'; +import { fillDue } from '../lib/ship-queue-train'; import { BASE_POLICY, cleanup, makeFixtureRepo, runBin, type FixtureRepo } from './helpers/restamp-fixture'; const repos: FixtureRepo[] = []; @@ -72,6 +73,18 @@ describe("gstack's own .gstack/ship-policy.json", () => { expect(raw.train).toMatchObject({ gate_workflow: 'evals.yml', free_workflow: 'free-tests.yml', full_gate_check: 'gstack/full-gate', record_artifact: 'gstack-train-record' }); for (const wf of [raw.train.gate_workflow, raw.train.free_workflow]) expect(fs.existsSync(path.join(root, '.github/workflows', wf))).toBe(true); }); + + // Value: protects=the fill rule the coordinator relies on: a train dispatches at 3 READY PRs or one hour after the first READY, whichever comes first + // (2026-10-26: 5 PRs / 3 hours left READY PRs waiting most of a day); fails_when=the live block drifts back to a larger or slower fill; why_new=the values were unpinned; seam=the policy file; no_claim=fillDue's arithmetic (ship-queue-train-logic) + test('fills a train at 3 READY PRs or one hour after the first READY, whichever comes first', () => { + const raw = JSON.parse(fs.readFileSync(path.join(import.meta.dir, '..', '.gstack', 'ship-policy.json'), 'utf8')); + expect([raw.train.max_size, raw.train.fill_hours]).toEqual([3, 1]); + const first = '2026-10-26T09:00:00.000Z'; + const slots = (n: number) => Array.from({ length: n }, (_, i) => ({ pr: i + 1, ready_at: new Date(Date.parse(first) + i * 60_000).toISOString() })) as any; + expect(fillDue(slots(2), raw.train, '2026-10-26T09:59:00Z', false)).toBe('2026-10-26T10:00:00.000Z'); + expect(fillDue(slots(2), raw.train, '2026-10-26T10:00:00Z', false)).toBeNull(); + expect(fillDue(slots(3), raw.train, '2026-10-26T09:05:00Z', false)).toBeNull(); + }); }); describe('gstack-ship-policy (bin)', () => { diff --git a/test/paid-report-census.test.ts b/test/paid-report-census.test.ts index 60cd9e0c8..32292dd54 100644 --- a/test/paid-report-census.test.ts +++ b/test/paid-report-census.test.ts @@ -97,6 +97,33 @@ describe('census attribution from the 2026-10-03 artifacts', () => { }); }); +describe('a case shard\'s result records are its one case\'s cost (2026-10-26 train 76db1d1f10c4: 54 of 136 trials cost-unknown, verdict $24.41 under a $32.92 run)', () => { + const shard = (dir: string, slug: string) => path.join(dir, 'paid-slice-1-a1', 'shards', slug); + const results = (dir: string, slug: string, tests: Array>) => + fs.writeFileSync(path.join(shard(dir, slug), 'e2e-2026-10-26.json'), JSON.stringify({ tests, total_cost_usd: tests.reduce((sum, t) => sum + Number(t.cost_usd), 0) })); + const collector = (dir: string) => JSON.parse(fs.readFileSync(path.join(dir, 'collector-outcomes.json'), 'utf8')); + + test('a record named by its display label ("/qa fix loop") belongs to the shard\'s only case, so the trial and the verdict carry its cost', () => { + const r = report('periodic', dir => results(dir, 'skill-e2e-qa-workflow--qa-fix-loop', + [{ name: '/qa fix loop', suite: 'QA skill E2E', tier: 'e2e', passed: true, duration_ms: 1000, cost_usd: 1.23 }])); + const trial = r.history.find(record => record.case === 'qa-fix-loop'); + expect(trial).toMatchObject({ outcome: 'passed', cost_usd: 1.23 }); + expect(trial.cost_known).toBeUndefined(); + expect(collector(r.dir).verdict.costUsd).toBe(1.23); + expect(r.out).toContain('cost $1.23 known + 6 of 7 trial(s) cost unknown'); + }); + + test('a multi-case shard whose records carry display labels keeps those trials cost-unknown instead of guessing', () => { + const r = report('periodic', dir => results(dir, 'skill-e2e-cso', + [{ name: '/cso full audit', suite: 'cso', tier: 'e2e', passed: true, duration_ms: 1000, cost_usd: 0.5 }, + { name: 'cso-infra-scope', suite: 'cso', tier: 'e2e', passed: true, duration_ms: 1000, cost_usd: 0.25 }])); + expect(r.history.find(record => record.case === 'cso-full-audit')).toMatchObject({ cost_usd: 0, cost_known: false }); + expect(r.history.find(record => record.case === 'cso-infra-scope')).toMatchObject({ cost_usd: 0.25 }); + expect(r.history.find(record => record.case === 'cso-infra-scope').cost_known).toBeUndefined(); + expect(collector(r.dir).verdict.costUsd).toBe(0.75); + }); +}); + describe('record builders attach ledger sessions and failure causes (plan 0.1/0.2)', () => { const noOp = (dir: string) => path.join(dir, 'paid-slice-1-a1', 'shards', 'skill-e2e-plan-mode-no-op'); const failNoOp = (dir: string) => { diff --git a/test/ship-queue-train-cost.test.ts b/test/ship-queue-train-cost.test.ts new file mode 100644 index 000000000..1499c47d4 --- /dev/null +++ b/test/ship-queue-train-cost.test.ts @@ -0,0 +1,40 @@ +/** + * The train verdict's cost line: the known sum plus how many of the record's + * cases carry no cost, so a partial sum is never read as the run's spend. + * Train 76db1d1f10c4 (2026-10-26) printed cost=$24.41 while its slice report + * said "cost $32.92 known + 54 of 136 trial(s) cost unknown": 54 E2E records + * named by display label had no case attributed (scripts/lib/paid-report.ts). + */ +import { afterEach, describe, expect, test } from 'bun:test'; +import { EXIT } from '../lib/headless-artifacts'; +import { cleanup } from './helpers/restamp-fixture'; +import { GREEN_GATE, PASS, TRAIN_POLICY, line, nonceOf, openPr, queue, readyReceipt, trainFixture, type Rule, type TrainFixture } from './helpers/ship-queue-train-fixture'; + +const fixtures: TrainFixture[] = []; +const make = () => { const f = trainFixture(TRAIN_POLICY); fixtures.push(f); return f; }; +afterEach(() => { while (fixtures.length) cleanup(fixtures.pop()!.repo); }); + +function greenTrain(fx: TrainFixture, gate: Rule) { + const h1 = openPr(fx, '1'); + const h2 = openPr(fx, '2'); + readyReceipt(fx, '1', h1); + readyReceipt(fx, '2', h2); + fx.write(d => { d.rules['evals.yml'] = gate; }); + return nonceOf(queue(fx, ['train', 'build', '--prs', `1@${h1},2@${h2}`, '--free-timeout', '1']).out); +} + +describe('train verdict cost line', () => { + // Value: protects=a train verdict never presents a partial sum as the run's cost (train 76db1d1f10c4 on 2026-10-26: verdict $24.41, the run's slice reports $32.92 known with 54 trials unattributed); + // fails_when=the ATTEMPT line prints cost=$ without naming the cost-unknown cases, or names them when every panel has a cost; why_new=the line only had the known sum; seam=the gate record's panels; no_claim=why a trial's cost is unknown + test('the verdict prints the known cost and how many cases have none, and nothing extra when every panel is priced', () => { + const fx = make(); + const unpriced = { ...GREEN_GATE, record: { ...GREEN_GATE.record, panels: [PASS('c1'), { ...PASS('c2'), cost_usd: null }, { ...PASS('j1', 'judge', 0.1), cost_usd: null }] } }; + const nonce = greenTrain(fx, unpriced); + const v = queue(fx, ['train', 'verdict', '--train', nonce]); + expect([v.status, line(v.out, 'ATTEMPT:')]).toEqual([EXIT.ok, 'ATTEMPT: a1 gate run 101 GREEN cost=$0.70 known + 2 of 3 case(s) cost unknown (rule 2/2, judge 1/1)']); + const all = make(); + const priced = greenTrain(all, GREEN_GATE); + expect(line(queue(all, ['train', 'verdict', '--train', priced]).out, 'ATTEMPT:')).toBe('ATTEMPT: a1 gate run 101 GREEN cost=$1.50 (rule 2/2, judge 1/1)'); + }, 120_000); + +});