From a3fd25df32d280b17e7ba2bba0f4db2f686c9b98 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 24 Oct 2026 21:23:18 +0000 Subject: [PATCH] fix(eval-bg): local lanes export EVALS_RUN_ID so native-capture cases start (GSTA-70) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CHANGELOG.md | 10 +++++++++ scripts/eval-bg.ts | 21 +++++++++++++++-- test/eval-detach-timeout-floor.test.ts | 31 +++++++++++++++++++++++++- 3 files changed, 59 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9652e71fd..9d71be157 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## [Unreleased] + +**Local paid runs from `eval:bg` now set `EVALS_RUN_ID`, so the native-capture cases start instead of failing in 0 seconds.** CI sets `EVALS_RUN_ID` per slice, but `bun run eval:bg:pr --local` launched without it, and the qa-callers, ship-docsync and qa-functional cases refuse to start without it. In GSTA-70's local /ship run, 17 of the 26 selected behaviors failed that way for $0. A failure like that is a launch failure, not a verdict, so the local run told you nothing about those cases. + +### Fixed +- **`scripts/eval-bg.ts`**: the local backend launches through a new `launchLocal`. It exports `EVALS_RUN_ID` as the run-scoped log's name, reduced to `[A-Za-z0-9_.-]`, and keeps a value you exported yourself. The id is written on the log's second line, so you can find the run's native captures. `--help` says so. Dispatched CI runs are unchanged. + +### Tests +- `test/eval-detach-timeout-floor.test.ts`: a detached child launched by `launchLocal` sees a safe `EVALS_RUN_ID` named after its log, and an exported id is kept. A source check confirms that the local backend launches through `launchLocal`. + ## [1.91.186.0] - 2026-10-24 **Approving an autoplan run no longer fails one time in ten because its run id looks like a card number.** Today's daily flaky-fix pass found `test/capy-captain-flow.test.ts` failing once and passing on the serial retry in 7 of the 57 Free Tests runs from 2026-10-21 to 2026-10-23, one of them a push to main (39878113535). Every failure had the same assertion: `gstack-approvals import` refused the approval row with "field ask: pii.cc". The default autoplan run id is `run-YYYYMMDDHHMMSS`, the import writes it into the row's ask and source, and the credit-card rule saw those 14 digits as a card number whenever they passed the Luhn check, which a timestamp does about one time in ten. A real user's approval import failed the same way. diff --git a/scripts/eval-bg.ts b/scripts/eval-bg.ts index b14709771..bb44b1ae4 100644 --- a/scripts/eval-bg.ts +++ b/scripts/eval-bg.ts @@ -295,6 +295,22 @@ export function launchDetached(opts: { log: string; label: string; timeoutSecond return result.stdout.trim(); } +/** The run id a local lane exports: the run-scoped log's name, reduced to [A-Za-z0-9_.-] as the native-capture cases require. */ +export function localRunId(log: string): string { + return path.basename(log, '.log').replace(/[^\w.-]/g, '-'); +} + +/** + * The local backend: CI sets EVALS_RUN_ID per slice, and the native-capture + * cases refuse to start without it, so a local lane exports one too (an + * exported value is kept). The id is logged under the header. + */ +export function launchLocal(opts: { log: string; label: string; timeoutSeconds: number; header: string; command: string[]; env?: NodeJS.ProcessEnv }): string { + const env = opts.env ?? process.env; + const runId = env.EVALS_RUN_ID || localRunId(opts.log); + return launchDetached({ ...opts, header: `${opts.header}\n[eval-bg] EVALS_RUN_ID=${runId}`, env: { ...env, EVALS_RUN_ID: runId } }); +} + const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); const pollMs = () => Number(process.env.GSTACK_EVAL_BG_POLL_MS) || 30_000; @@ -362,7 +378,8 @@ log; it ends with "### gstack-detach EXIT= ###" (0 passed, 1 failed, Backend: a clean HEAD pushed to ${CANONICAL_REPO} dispatches the CI workflow and follows it; anything else runs locally under bin/gstack-detach, capped at -ceil(1.5 x planned serial seconds / EVALS_JOBS) + 20 min (at most 4 h). +ceil(1.5 x planned serial seconds / EVALS_JOBS) + 20 min (at most 4 h), with +EVALS_RUN_ID set to the log's name unless one is exported. --local / --dispatch force a backend (also GSTACK_EVAL_BG_MODE=local|dispatch); --dispatch refuses instead of falling back --timeout SECS override the computed local cap @@ -417,7 +434,7 @@ async function main(argv: string[]): Promise { const header = `[eval-bg] backend=local lane=${lane} tested=${tested} cap=${cap.seconds}s jobs=${jobs}${timeoutOverride !== null ? ' (--timeout)' : ''} (${choice.reason})`; console.log(header); if (cap.capped) console.log(`[eval-bg] WARNING: planned work exceeds the ${LOCAL_CAP_SECONDS}s local cap; the tail may not run. Dispatch to CI or raise EVALS_JOBS.`); - console.log(launchDetached({ log, label, timeoutSeconds: cap.seconds, header, command: ['bun', 'run', LANE_SPECS[lane].script] })); + console.log(launchLocal({ log, label, timeoutSeconds: cap.seconds, header, command: ['bun', 'run', LANE_SPECS[lane].script] })); return 0; } diff --git a/test/eval-detach-timeout-floor.test.ts b/test/eval-detach-timeout-floor.test.ts index 382dd0ad7..bfa3ed553 100644 --- a/test/eval-detach-timeout-floor.test.ts +++ b/test/eval-detach-timeout-floor.test.ts @@ -20,7 +20,7 @@ import * as path from 'node:path'; import { CANONICAL_REPO, DISPATCH_FOLLOW_SECONDS, EVAL_BG_LANES, LANE_SPECS, LOCAL_CAP_SECONDS, UNRECORDED_SHARD_MS, chooseBackend, conclusionExitCode, dedupeKey, dispatchBlockers, dispatchPlan, findDuplicateRun, launchDetached, - localCapSeconds, makeNonce, plannedSerialMs, statusOf, type CheckoutFacts, type RunSummary, + launchLocal, localCapSeconds, localRunId, makeNonce, plannedSerialMs, statusOf, type CheckoutFacts, type RunSummary, } from '../scripts/eval-bg'; import { buildRunManifest, loadPaidTestDurations, recordedShardMs } from '../scripts/test-paid-shards'; @@ -158,6 +158,35 @@ describe('completion contract on both backends (DX-2)', () => { } finally { fs.rmSync(dir, { recursive: true, force: true }); } }, 40_000); + // GSTA-70: the native-capture cases (qa-callers, ship-docsync, qa-functional) + // refuse to start without EVALS_RUN_ID. CI sets it per slice; the local lane + // used to launch without it, so 17 of /ship's PR cases failed in 0 s. + test('local: the lane exports a safe EVALS_RUN_ID named after its log, and keeps an exported one', async () => { + const { dir, env } = sandbox(); + try { + delete env.EVALS_RUN_ID; + const probe = (out: string) => [process.execPath, '-e', + `const id = process.env.EVALS_RUN_ID ?? ''; require('fs').writeFileSync(${JSON.stringify(out)}, id); process.exit(/^[\\w.-]+$/.test(id) ? 0 : 4)`]; + const log = path.join(dir, 'evals-pr-gstack-feature+x-20261024-200431-1.log'); + launchLocal({ log, label: 'evals-test', timeoutSeconds: 60, header: '[eval-bg] backend=local lane=pr tested=abc', env, command: probe(path.join(dir, 'id')) }); + const text = await waitForExit(log); + expect(localRunId(log)).toBe('evals-pr-gstack-feature-x-20261024-200431-1'); + expect(fs.readFileSync(path.join(dir, 'id'), 'utf8')).toBe(localRunId(log)); + expect(text.split('\n').slice(0, 2)).toEqual(['[eval-bg] backend=local lane=pr tested=abc', `[eval-bg] EVALS_RUN_ID=${localRunId(log)}`]); + expect(text).toMatch(/### gstack-detach EXIT=0 ###/); + const kept = path.join(dir, 'kept.log'); + launchLocal({ log: kept, label: 'evals-test', timeoutSeconds: 60, header: 'h', env: { ...env, EVALS_RUN_ID: 'mine-1' }, command: probe(path.join(dir, 'kept')) }); + expect(await waitForExit(kept)).toMatch(/### gstack-detach EXIT=0 ###/); + expect(fs.readFileSync(path.join(dir, 'kept'), 'utf8')).toBe('mine-1'); + } finally { fs.rmSync(dir, { recursive: true, force: true }); } + }, 40_000); + + test('the local backend launches through launchLocal', () => { + const source = fs.readFileSync(path.join(ROOT, 'scripts/eval-bg.ts'), 'utf8'); + expect(source).toMatch(/console\.log\(launchLocal\(\{ log, label, timeoutSeconds: cap\.seconds, header, command: \['bun', 'run', LANE_SPECS\[lane\]\.script\] \}\)\)/); + expect(source.match(/launchDetached\(\{ log, label/g)?.length).toBe(1); + }); + for (const [conclusion, code, word] of [['failure', 1, 'failed'], ['cancelled', 130, 'cancelled'], ['success', 0, 'passed']] as const) { test(`dispatch: a followed run that ends ${conclusion} writes EXIT=${code}`, async () => { const { dir, env } = sandbox();