From 7f2cfa6b8192ea0852b2d3573342b39f7642901b Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Mon, 2 Nov 2026 11:00:05 -0800 Subject: [PATCH 01/25] v1.93.13.0 fix(test): never hand an exported live-brain URL to gbrain in tests (#3466) (#3876) * test: an exported live-brain URL must not reach a temp-HOME gbrain child (#3466) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(test): strip the variables that point gbrain at a live brain in the test preload (#3466) Co-authored-by: AndreIntelas <126392195+AndreIntelas@users.noreply.github.com> Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> Co-authored-by: AndreIntelas <126392195+AndreIntelas@users.noreply.github.com> --- CHANGELOG.md | 10 ++++++++ VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- package.json | 2 +- test-setup.ts | 9 +++++++ test/state-root-config-merge.test.ts | 38 ++++++++++++++++++++++++++++ 6 files changed, 60 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 09d3cac92..5f1a13758 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## [1.93.13.0] - 2026-11-02 + +**gstack's own tests no longer write into a developer's real gbrain.** If you export `GBRAIN_DATABASE_URL` for your working brain and run gstack's test suite, tests that spawn bins with a temp `HOME` used to pass that URL straight through. With no gbrain config under the temp `HOME`, the real `gbrain` on your PATH then added `brain-sync-remote-*` sources and test pages to your live brain. From #3466 by @AndreIntelas. + +### Fixed +- **The test preload (`test-setup.ts`) strips the variables gbrain uses to pick a brain** before any test module loads: `GBRAIN_DATABASE_URL`, `GBRAIN_DIRECT_DATABASE_URL`, `DATABASE_URL` and `GBRAIN_HOME`. It prints one line on stderr for each variable it strips. This covers `bun test ` and every shard of `bun run test`. + +### Tests +- `test/state-root-config-merge.test.ts`: a `bun test` run with all four variables exported spawns a child under a temp `HOME` that calls `buildGbrainEnv`, and the child sees none of them. It fails on main. + ## [1.93.12.0] - 2026-11-02 **The Opus 5.5 `lite` skills now keep the install's gbrain blocks, disabled skills and the Claims Need Evidence rule (GST-944).** The lite tree Capy publishes was rendered without the install's settings. On a machine with gbrain detected, Opus 5.5 sessions lost the brain context-load and save-results blocks that the full install serves, and skills turned off with `disabled_skills` showed up again in the lite router. `./setup --loadout lite` was not affected. diff --git a/VERSION b/VERSION index 7f209dd81..c2fc44987 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.93.12.0 +1.93.13.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 513f2a765..923c3f576 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.93.12.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.93.13.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/package.json b/package.json index 098af7b3f..4c4f97952 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.93.12", + "version": "1.93.13", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/test-setup.ts b/test-setup.ts index 31234cc63..a9c6fc020 100644 --- a/test-setup.ts +++ b/test-setup.ts @@ -39,6 +39,15 @@ for (const name of ['GSTACK_STATE_ROOT', 'GSTACK_STATE_DIR']) { // an unattended Capy session it never set up. The shard engine strips the same // list for the lanes; this reaches direct runs too. for (const name of CAPY_LAUNCH_MARKERS) delete process.env[name]; +// The variables gbrain reads to pick a brain (#3466). Tests spawn bins with +// { ...process.env, HOME: }; with no gbrain config under the temp HOME, +// buildGbrainEnv passes an exported URL through and a real gbrain on PATH +// writes test pages and sources into the developer's live brain. +for (const name of ['GBRAIN_DATABASE_URL', 'GBRAIN_DIRECT_DATABASE_URL', 'DATABASE_URL', 'GBRAIN_HOME']) { + if (process.env[name] === undefined) continue; + delete process.env[name]; + process.stderr.write(`test-setup: stripped inherited ${name}; tests must not reach a live gbrain\n`); +} process.env.GSTACK_TEST_LEGACY_ROOT = path.join(os.tmpdir(), `gstack-test-legacy-root-${process.pid}-${Date.now()}-unused`); // Workflow leases (bin/gstack-lease) default to the installation's git dir; // for the dev checkout under test that is this repo's .git, and a live test diff --git a/test/state-root-config-merge.test.ts b/test/state-root-config-merge.test.ts index a0e5cd73e..d6a1a25e5 100644 --- a/test/state-root-config-merge.test.ts +++ b/test/state-root-config-merge.test.ts @@ -163,4 +163,42 @@ describe('test preload hermeticity', () => { expect(r.stderr).toContain('test-setup: stripped inherited GSTACK_STATE_DIR; tests isolate state via GSTACK_HOME'); expect(r.stderr.match(/test-setup: stripped/g)?.length).toBe(2); }); + + test('bun test strips an inherited live-brain URL, so a temp-HOME gbrain child never sees it (#3466)', () => { + const live = path.join(tmp, 'live-gbrain'); + fs.mkdirSync(live, { recursive: true }); + fs.writeFileSync(path.join(live, 'config.json'), JSON.stringify({ database_url: 'postgresql://live-config.invalid/brain' })); + const childHome = path.join(tmp, 'child-home'); + fs.mkdirSync(childHome, { recursive: true }); + const probe = path.join(tmp, 'brain-probe.test.ts'); + fs.writeFileSync(probe, [ + "import { test, expect } from 'bun:test';", + "import { spawnSync } from 'child_process';", + "test('probe', () => {", + " const script = \"const { buildGbrainEnv } = require(\" + JSON.stringify(" + JSON.stringify(path.join(ROOT, 'lib', 'gbrain-exec.ts')) + ") + \"); const e = buildGbrainEnv(); console.log(['GBRAIN_DATABASE_URL','GBRAIN_DIRECT_DATABASE_URL','DATABASE_URL','GBRAIN_HOME'].map(k => k + '=' + (e[k] ?? '')).join(' '));\";", + " const r = spawnSync(process.execPath, ['--no-env-file', '--config=/dev/null', '-e', script], {", + " env: { ...process.env, HOME: " + JSON.stringify(childHome) + " }, encoding: 'utf-8', timeout: 30_000,", + " });", + " expect(r.stdout.trim()).toBe('GBRAIN_DATABASE_URL= GBRAIN_DIRECT_DATABASE_URL= DATABASE_URL= GBRAIN_HOME=');", + "});", + ].join('\n')); + const r = spawnSync('bun', ['test', '--preload', path.join(ROOT, 'test-setup.ts'), probe], { + cwd: tmp, + env: { + ...env, GSTACK_HOME: resolved, + GBRAIN_DATABASE_URL: 'postgresql://live-brain.invalid/brain', + GBRAIN_DIRECT_DATABASE_URL: 'postgresql://live-brain.invalid/direct', + DATABASE_URL: 'postgresql://live-brain.invalid/fallback', + GBRAIN_HOME: live, + }, + encoding: 'utf-8', + timeout: 60_000, + }); + expect(r.stderr).not.toContain('live-brain.invalid'); + expect(r.stderr).not.toContain('live-config.invalid'); + expect(r.status).toBe(0); + for (const name of ['GBRAIN_DATABASE_URL', 'GBRAIN_DIRECT_DATABASE_URL', 'DATABASE_URL', 'GBRAIN_HOME']) { + expect(r.stderr).toContain(`test-setup: stripped inherited ${name}; tests must not reach a live gbrain`); + } + }); }); From 4b43d4fd43442f07588bdb6266c45d2340bd0856 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Mon, 2 Nov 2026 11:11:00 -0800 Subject: [PATCH 02/25] v1.93.14.0 ci: each free-suite job's timeout follows the plan's runner wall (#3850) (#3877) * fix(free-tests): each free-suite job's timeout comes from the plan (runner wall + flaky retry + setup), not a fixed 20 minutes (#3850) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs: free-suite job timeout follows the plan (#3850) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(free-tests): narrow the CI matrix in the #3850 wiring test; keep free-suite job names Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- .github/workflows/free-tests.yml | 12 +++++- CHANGELOG.md | 4 ++ VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- docs/TESTING_INTERNALS.md | 11 +++++- package.json | 2 +- scripts/test-free-shards.ts | 52 ++++++++++++++++++++++--- test/free-tests-workflow-wiring.test.ts | 30 ++++++++++++++ test/test-free-shards.test.ts | 29 ++++++++++++++ 9 files changed, 133 insertions(+), 11 deletions(-) diff --git a/.github/workflows/free-tests.yml b/.github/workflows/free-tests.yml index 0d3cff19e..b7b8f291b 100644 --- a/.github/workflows/free-tests.yml +++ b/.github/workflows/free-tests.yml @@ -52,10 +52,16 @@ on: # that free-suite reads. FREE_SUITE_JOBS is k in `--ci-plan --shard-jobs k`: # each shard runs min(k, its parallel files) sandboxed sub-shards side by side # (scripts/lib/shard-jobs.ts); the exclusive host-state shard stays one child. +# FREE_SUITE_SETUP_SECONDS is a free-suite job's time outside the "Run free +# suite" step. --ci-plan adds it to each shard's runner wall and flaky-retry +# wall to emit that shard's job timeout (#3850), so GitHub never kills a job +# before the runner can name a hung file. Measured over 110 free-suite jobs in +# PR runs 41467261350..41478988224: p50 60 s, p90 80 s, max 142 s. env: FREE_SUITE_RUNNER: ubicloud-standard-8 FREE_SUITE_SHARDS: '10' FREE_SUITE_JOBS: '4' + FREE_SUITE_SETUP_SECONDS: '300' # Keyed on the PR number, not head_ref: a bare branch name carries no fork # prefix, so same-name branches from two forks would share one group and a @@ -149,7 +155,7 @@ jobs: if [ "$runner" != "$FREE_SUITE_RUNNER" ] || [ "$shards" != "$FREE_SUITE_SHARDS" ] || [ "$jobs" != "$FREE_SUITE_JOBS" ]; then echo "- Runner-sizing experiment: $shards shards on $runner at $jobs sub-shards each (defaults $FREE_SUITE_SHARDS on $FREE_SUITE_RUNNER at $FREE_SUITE_JOBS); not a PR or train Free Tests result" >> "$GITHUB_STEP_SUMMARY" fi - matrix=$(bun run scripts/test-free-shards.ts --ci-plan "$RUNNER_TEMP/free-plan.json" --shards "$shards" --shard-jobs "$jobs") + matrix=$(bun run scripts/test-free-shards.ts --ci-plan "$RUNNER_TEMP/free-plan.json" --shards "$shards" --shard-jobs "$jobs" --job-setup-seconds "$FREE_SUITE_SETUP_SECONDS") echo "matrix=$matrix" >> "$GITHUB_OUTPUT" echo "runner=$runner" >> "$GITHUB_OUTPUT" - name: Seed growth ratchet (60 s ceiling, shrink-only allowlist) @@ -434,10 +440,12 @@ jobs: run: xvfb-run -a bun run test:quick free-suite: + name: free-suite (${{ matrix.shard }}) needs: [free-plan, reuse] if: ${{ !cancelled() && needs.free-plan.result == 'success' && (needs.reuse.result != 'success' || needs.reuse.outputs.reused != 'true') }} runs-on: ${{ needs.free-plan.outputs.runner }} - timeout-minutes: 20 + # Per shard, from the plan: runner wall + flaky retry + setup (#3850). + timeout-minutes: ${{ matrix.timeout_minutes }} strategy: fail-fast: false max-parallel: 20 diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f1a13758..f1e7f5cef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## [1.93.14.0] - 2026-11-02 + +Free Tests no longer goes red with no named file when one test hangs. Each `free-suite` job's GitHub timeout used to be a fixed 20 minutes, shorter than the runner's own wall for a 10-shard plan (about 37 minutes), so GitHub killed the job before the runner could name the hung file or retry it, and no shard result was uploaded. The plan now emits each shard's job timeout as its runner wall plus the flaky-retry pass plus the job's measured setup (`--ci-plan --job-setup-seconds`), 49 minutes per shard today, and fails loudly above 90 minutes instead of truncating. See docs/TESTING_INTERNALS.md#runners-how-the-suites-execute-2026-08-overhaul. Closes #3850. + ## [1.93.13.0] - 2026-11-02 **gstack's own tests no longer write into a developer's real gbrain.** If you export `GBRAIN_DATABASE_URL` for your working brain and run gstack's test suite, tests that spawn bins with a temp `HOME` used to pass that URL straight through. With no gbrain config under the temp `HOME`, the real `gbrain` on your PATH then added `brain-sync-remote-*` sources and test pages to your live brain. From #3466 by @AndreIntelas. diff --git a/VERSION b/VERSION index c2fc44987..5bc28cc73 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.93.13.0 +1.93.14.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 923c3f576..98abcfbee 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.93.13.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.93.14.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/docs/TESTING_INTERNALS.md b/docs/TESTING_INTERNALS.md index bc3451c68..10dc9ae12 100644 --- a/docs/TESTING_INTERNALS.md +++ b/docs/TESTING_INTERNALS.md @@ -206,7 +206,16 @@ GSTA-203 chose them by a 3-run A/B against 20 × `ubicloud-standard-4` at k = 1 (the PR that set them has the table). A `workflow_dispatch` run may override them with `free_runner` (`ubicloud-standard-2`, `-4` or `-8`), `free_shards` (1–40) and `free_jobs` (1–8) for runner-sizing experiments only; -its job summary says so, and it is never cited as a PR or train Free Tests result. Each shard prints +its job summary says so, and it is never cited as a PR or train Free Tests result. +Each `free-suite` job's GitHub timeout comes from the plan, not a fixed number +(#3850): `--ci-plan --job-setup-seconds $FREE_SUITE_SETUP_SECONDS` emits every +shard's `timeout_minutes` as the shard's runner wall (the packed wall above), +plus the flaky-retry pass's wall (at most 5 files, so the 6-minute base), plus +the job's setup and teardown (`ciShardJobTimeoutMinutes`). So GitHub never kills +a hung file before the runner names it and retries it. A shard that would need +more than 90 minutes fails the plan with `FREE_JOB_TIMEOUT_OVER_CAP`; plan more +shards or a higher k instead. The plan step prints the emitted timeouts and adds +them to its job summary. Each shard prints `RESOURCES: peak_used_mib=… min_free_disk_mib=… samples=…` from a memory and disk sample every 2 seconds, even when the suite fails. The public `TREE_MUTATING` map now classifies exclusive host-state fixtures; its sole diff --git a/package.json b/package.json index 4c4f97952..ae43c22ed 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.93.13", + "version": "1.93.14", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/scripts/test-free-shards.ts b/scripts/test-free-shards.ts index b4012a60a..c875b450e 100755 --- a/scripts/test-free-shards.ts +++ b/scripts/test-free-shards.ts @@ -201,6 +201,24 @@ export function wallTimeoutForPackedShard(predictedMs: number, baseMs = DEFAULT_ // tighter — it keeps the per-file floor the runner has always guaranteed. return Math.max(baseMs, Math.ceil(predictedMs * 3), fileCount * PER_FILE_WALL_MS); } +/** The flaky-retry pass re-runs at most this many failing files, once, serially, in the same CI step. */ +export const FLAKY_RETRY_CAP = 5; +/** No free-suite GitHub job gets a longer timeout; a plan whose shard would need more fails instead. */ +export const MAX_CI_JOB_TIMEOUT_MINUTES = 90; +/** + * The GitHub job timeout for one planned CI shard (#3850): the wall --ci-run + * gives the shard, plus the flaky-retry pass's wall, plus the job's measured + * setup and teardown. GitHub must never kill the job before the runner can + * name a hung file and retry it; that kill uploads no result. + */ +export function ciShardJobTimeoutMinutes(shard: { shard: number; files: string[]; predictedMs: number }, setupSeconds: number): number { + const runnerMs = wallTimeoutForPackedShard(shard.predictedMs, DEFAULT_WALL_TIMEOUT_MS, shard.files.length) + wallTimeoutForShard(FLAKY_RETRY_CAP); + const minutes = Math.ceil((runnerMs / 1000 + setupSeconds) / 60); + if (minutes > MAX_CI_JOB_TIMEOUT_MINUTES) { + throw new Error(`FREE_JOB_TIMEOUT_OVER_CAP: CI shard ${shard.shard} needs a ${minutes}-minute job (runner wall plus retry plus ${setupSeconds} s setup), over the ${MAX_CI_JOB_TIMEOUT_MINUTES}-minute cap; plan more shards or a higher --shard-jobs instead of a longer job`); + } + return minutes; +} /** * Full-suite parallelism: use all available CPUs, with a floor of one and a * per-platform cap (maxFullSuiteJobs). Shards stay serial internally; separate @@ -453,6 +471,16 @@ export function createFreeCiPlan(files: string[], count: number, durations: Reco return { ...body, id: planDigest(body) }; } +/** + * --ci-plan's stdout, the GitHub matrix. With --job-setup-seconds each entry + * carries the shard's job timeout (ciShardJobTimeoutMinutes); without it, the + * bare shard list the Windows lane reads. + */ +export function ciPlanMatrix(plan: FreeCiPlan, setupSeconds?: number): { shard: number[] } | { include: Array<{ shard: number; timeout_minutes: number }> } { + if (setupSeconds === undefined) return { shard: plan.shards.map(shard => shard.shard) }; + return { include: plan.shards.map(shard => ({ shard: shard.shard, timeout_minutes: ciShardJobTimeoutMinutes(shard, setupSeconds) })) }; +} + export function validateFreeCiPlan(plan: FreeCiPlan, files: string[], revision: string): void { const { id, version, shards } = plan; if (version !== 1 || plan.revision !== revision || !Array.isArray(shards) || !shards.length @@ -575,6 +603,8 @@ type CliOptions = { jobs?: number; /** Set only by --shard-jobs (with --ci-plan); absent means k = 1 in every shard. */ shardJobs?: number; + /** Set only by --job-setup-seconds (with --ci-plan): the matrix then carries each shard's job timeout. */ + jobSetupSeconds?: number; }; export function parseCliOptions(argv: string[]): CliOptions { @@ -591,6 +621,7 @@ export function parseCliOptions(argv: string[]): CliOptions { let attributeHome = false; let jobs: number | undefined; let shardJobs: number | undefined; + let jobSetupSeconds: number | undefined; const paths: Record<'ciPlan' | 'ciRun' | 'ciVerify' | 'result' | 'results' | 'filesFrom', string | null> = { ciPlan: null, ciRun: null, ciVerify: null, result: null, results: null, filesFrom: null, }; @@ -616,6 +647,11 @@ export function parseCliOptions(argv: string[]): CliOptions { '--files-from': pathFlag('--files-from', 'filesFrom'), '--jobs': (next) => { jobs = parseJobsFlag(next()); }, '--shard-jobs': (next) => { shardJobs = parseJobsFlag(next(), '--shard-jobs'); }, + '--job-setup-seconds': (next) => { + const value = next() ?? ''; + jobSetupSeconds = /^\d+$/.test(value) ? Number(value) : NaN; + if (!(jobSetupSeconds > 0 && jobSetupSeconds <= 3600)) throw new Error('--job-setup-seconds needs a whole number of seconds from 1 to 3600'); + }, '--shards': (next) => { const value = next(); if (!value) throw new Error('Missing value for --shards'); @@ -641,8 +677,9 @@ export function parseCliOptions(argv: string[]): CliOptions { if (paths.ciVerify && !paths.results) throw new Error('--ci-verify requires --results'); if (jobs !== undefined && !paths.ciRun) rejectJobsWithoutCiRun(); if (shardJobs !== undefined && !paths.ciPlan) rejectShardJobsWithoutCiPlan(); + if (jobSetupSeconds !== undefined && !paths.ciPlan) throw new Error('--job-setup-seconds applies only to --ci-plan'); if (quick && (recordDurations || windowsOnly || shardIndex !== null)) throw new Error('--quick cannot change recording, Windows or shard selection'); - return { dryRun, listOnly, recordDurations, windowsOnly, verbose, shardCount, shardIndex, wallTimeoutMs, wallTimeoutExplicit, quick, attributeHome, ...paths, ...(jobs === undefined ? {} : { jobs }), ...(shardJobs === undefined ? {} : { shardJobs }) }; + return { dryRun, listOnly, recordDurations, windowsOnly, verbose, shardCount, shardIndex, wallTimeoutMs, wallTimeoutExplicit, quick, attributeHome, ...paths, ...(jobs === undefined ? {} : { jobs }), ...(shardJobs === undefined ? {} : { shardJobs }), ...(jobSetupSeconds === undefined ? {} : { jobSetupSeconds }) }; } const describeLines = (lines: string[]) => `${lines.slice(0, 5).join(', ')}${lines.length > 5 ? ` (+${lines.length - 5} more)` : ''}`; @@ -1891,7 +1928,6 @@ async function retryFailedFreeFiles( // Exists for syscall-supervised sandboxes (see fullSuiteJobs) where a run // lands 0-1 spurious browser-timing failures under an otherwise-green // suite. Capped so a genuinely broken tree never masquerades as flaky. - const RETRY_CAP = 5; if ( worst !== 0 && process.env.GSTACK_FREE_RETRY_FLAKY === '1' @@ -1906,7 +1942,7 @@ async function retryFailedFreeFiles( // mask the unattributable evidence as a FLAKY-PASS. const allAttributed = outcomes.every((o) => hasScopedFailureAttribution(o) && (o.status === 'passed' || (o.failingFiles.length > 0 && o.unattributedFailures === 0))); - if (allAttributed && flakyFiles.length > 0 && flakyFiles.length <= RETRY_CAP) { + if (allAttributed && flakyFiles.length > 0 && flakyFiles.length <= FLAKY_RETRY_CAP) { console.log(`[test:free] flaky-retry: re-running ${flakyFiles.length} failing file(s) once, serially: ${flakyFiles.join(', ')}`); const retryOutcome = await runFreeShard(flakyFiles, totalShards + 1, totalShards + 1, { wallTimeoutMs: shardTimeout(flakyFiles.length), @@ -1942,7 +1978,7 @@ async function retryFailedFreeFiles( console.error('[test:free] flaky-retry FAILED — the failures reproduce serially; not flaky.'); } } else { - console.log(`[test:free] flaky-retry skipped: ${allAttributed ? `${flakyFiles.length} failing file(s) exceeds cap ${RETRY_CAP}` : 'failures not fully attributed'}.`); + console.log(`[test:free] flaky-retry skipped: ${allAttributed ? `${flakyFiles.length} failing file(s) exceeds cap ${FLAKY_RETRY_CAP}` : 'failures not fully attributed'}.`); } } return { exitCode: worst, retry }; @@ -1978,8 +2014,14 @@ async function main(): Promise { warnUnseededFreeFiles(ciFiles, durations); const plan = createFreeCiPlan(ciFiles, options.shardCount, durations, revision, options.shardJobs); validateFreeCiPlan(plan, ciFiles, revision); + const matrix = ciPlanMatrix(plan, options.jobSetupSeconds); + if ('include' in matrix) { + const line = `job timeout per shard (runner wall + flaky retry + ${options.jobSetupSeconds} s setup): ${matrix.include.map(entry => `${entry.shard}=${entry.timeout_minutes}m`).join(' ')}`; + console.error(`[test:free] ${line}`); + appendStepSummary(line); + } writeJson(options.ciPlan, plan); - console.log(JSON.stringify({ shard: plan.shards.map(shard => shard.shard) })); + console.log(JSON.stringify(matrix)); return 0; } const plan = JSON.parse(fs.readFileSync((options.ciRun ?? options.ciVerify)!, 'utf8')) as FreeCiPlan; diff --git a/test/free-tests-workflow-wiring.test.ts b/test/free-tests-workflow-wiring.test.ts index 0b36507d7..71a93c7c1 100644 --- a/test/free-tests-workflow-wiring.test.ts +++ b/test/free-tests-workflow-wiring.test.ts @@ -19,6 +19,10 @@ import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; import { spawnSync } from 'node:child_process'; +import { + DEFAULT_WALL_TIMEOUT_MS, FLAKY_RETRY_CAP, ciPlanMatrix, collectFreeTestFiles, createFreeCiPlan, + loadFreeTestDurations, wallTimeoutForPackedShard, wallTimeoutForShard, +} from '../scripts/test-free-shards'; const WORKFLOW = path.resolve(import.meta.dir, '..', '.github', 'workflows', 'free-tests.yml'); @@ -235,4 +239,30 @@ describe('free-tests workflow wiring', () => { expect(source).toMatch(/permissions:\s*\n\s*contents:\s*read/); expect(source).toMatch(/persist-credentials:\s*false/); }); + test('every free-suite job timeout covers its shard\'s runner wall, flaky retry and setup (#3850)', () => { + // GitHub must never kill a free-suite job before the runner's own wall can + // name the hung file and its flaky-retry pass can run: that kill uploads no + // result and leaves red CI with no attribution. The job timeout therefore + // follows the real plan for the committed duration seed and the live + // FREE_SUITE_SHARDS / FREE_SUITE_JOBS, never a hand-picked number. + const workflow = Bun.YAML.parse(source) as any; + const setupSeconds = Number(workflow.env.FREE_SUITE_SETUP_SECONDS); + expect(Number.isInteger(setupSeconds) && setupSeconds > 0).toBe(true); + const plan = createFreeCiPlan(collectFreeTestFiles(), Number(workflow.env.FREE_SUITE_SHARDS), + loadFreeTestDurations() ?? {}, 'wiring', Number(workflow.env.FREE_SUITE_JOBS)); + const declared = workflow.jobs['free-suite']['timeout-minutes']; + const emitted = ciPlanMatrix(plan, setupSeconds); + if (!('include' in emitted)) throw new Error('the CI matrix carries no per-shard timeouts'); + const short = plan.shards.flatMap(shard => { + const neededMs = wallTimeoutForPackedShard(shard.predictedMs, DEFAULT_WALL_TIMEOUT_MS, shard.files.length) + + wallTimeoutForShard(FLAKY_RETRY_CAP) + setupSeconds * 1000; + const minutes = declared === '${{ matrix.timeout_minutes }}' + ? emitted.include.find(entry => entry.shard === shard.shard)?.timeout_minutes ?? 0 + : Number(declared); + return minutes * 60_000 >= neededMs ? [] : [`shard ${shard.shard}: ${minutes} min < ${Math.ceil(neededMs / 1000)} s`]; + }); + expect(short).toEqual([]); + const planStep = workflow.jobs['free-plan'].steps.find((step: any) => step.id === 'plan'); + expect(planStep.run).toContain('--job-setup-seconds "$FREE_SUITE_SETUP_SECONDS"'); + }); }); diff --git a/test/test-free-shards.test.ts b/test/test-free-shards.test.ts index 3d84740c3..265f0b75f 100644 --- a/test/test-free-shards.test.ts +++ b/test/test-free-shards.test.ts @@ -29,6 +29,9 @@ import { loadFreeTestDurations, packShardsByDuration, wallTimeoutForPackedShard, + ciShardJobTimeoutMinutes, + ciPlanMatrix, + MAX_CI_JOB_TIMEOUT_MINUTES, createFreeCiPlan, validateFreeCiPlan, verifyFreeCiResults, @@ -1101,4 +1104,30 @@ describe('test-free-shards: duration-aware packing (full-suite LPT)', () => { // When the prediction is the larger bound, it still wins. expect(wallTimeoutForPackedShard(600_000, WALL_BASE_MS, 10)).toBe(1_800_000); }); + + test('a shard\'s job timeout is its runner wall plus the flaky-retry wall plus setup, rounded up (#3850)', () => { + // 750 s predicted -> 2250 s wall, + 360 s retry, + 300 s setup = 2910 s -> 49 min. + expect(ciShardJobTimeoutMinutes({ shard: 1, files: ['a'], predictedMs: 750_000 }, 300)).toBe(49); + // A tiny shard still covers the 6-minute base wall and the retry pass. + expect(ciShardJobTimeoutMinutes({ shard: 2, files: ['a'], predictedMs: 1_000 }, 60)).toBe(13); + }); + + test('a shard that would need a job over the cap fails the plan instead of being truncated', () => { + const predictedMs = MAX_CI_JOB_TIMEOUT_MINUTES * 60_000 / 3; + expect(() => ciShardJobTimeoutMinutes({ shard: 4, files: ['a'], predictedMs }, 300)).toThrow(/FREE_JOB_TIMEOUT_OVER_CAP: CI shard 4 needs a 101-minute job/); + }); + + test('the matrix carries per-shard timeouts only when the plan is given a setup margin', () => { + const plan = createFreeCiPlan(['a.test.ts', 'b.test.ts'], 2, { 'a.test.ts': 750_000, 'b.test.ts': 1_000 }, 'rev'); + expect(ciPlanMatrix(plan)).toEqual({ shard: [1, 2] }); + expect(ciPlanMatrix(plan, 300)).toEqual({ include: [{ shard: 1, timeout_minutes: 49 }, { shard: 2, timeout_minutes: 17 }] }); + }); + + test('--job-setup-seconds is a --ci-plan flag with whole seconds from 1 to 3600', () => { + expect(parseCliOptions(['--ci-plan', 'p.json', '--job-setup-seconds', '300']).jobSetupSeconds).toBe(300); + expect(() => parseCliOptions(['--shards', '2', '--job-setup-seconds', '300'])).toThrow('--job-setup-seconds applies only to --ci-plan'); + for (const bad of ['0', '3601', '1.5', 'x']) { + expect(() => parseCliOptions(['--ci-plan', 'p.json', '--job-setup-seconds', bad])).toThrow('--job-setup-seconds needs a whole number of seconds from 1 to 3600'); + } + }); }); From 90ccf30e2a30f31a84910e0f0ac973e02683d583 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Mon, 2 Nov 2026 11:23:49 -0800 Subject: [PATCH 03/25] v1.93.15.0 fix(pregate): pytest, Playwright and mixed-language runners; literals stops flagging stray digits (#3598, #3492) (#3741) * fix(pregate): runner detection for pytest, Playwright and mixed-language roots; literals binds one-character values A missing pytest is PREGATE_ENVIRONMENT, not the diff's failure (#3598). Playwright specs run under playwright test, with missing browsers reported as the machine's gap (#3629). Root tests in two languages each get their runner, and a repo's bun/vitest/jest claims only JS/TS (#3600). literals counts a one-character old value only in a statement naming the constant, and a call-site name also needs the module (#3492, gstack-evals#97). Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: pre-landing review fixes Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs(pregate): name the import-type shapes the Playwright detection skips Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: escape Playwright file filters; skip type-only Playwright imports; pytest probe and summary read hardening Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(pregate): linear Playwright import scan; comments and no-space imports detected Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs(pregate): Playwright missing-browsers exception and literals continuation rule Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CHANGELOG.md | 13 +++ VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- docs/pregate.md | 20 ++++- docs/state-root.md | 4 +- docs/troubleshooting.md | 5 +- lib/pregate/checks/lanes.ts | 96 ++++++++++++++++++---- lib/pregate/checks/literals.ts | 22 +++++- lib/pregate/init-draft.ts | 3 +- package.json | 2 +- test/pregate-environment.test.ts | 80 +++++++++++++++++++ test/pregate-non-js.test.ts | 2 +- test/pregate-selection.test.ts | 132 +++++++++++++++++++++++++++++++ 13 files changed, 353 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f1e7f5cef..17c0eceee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## [1.93.15.0] - 2026-11-02 + +**Pregate stops blaming a clean branch for a missing pytest, Playwright specs and mixed-language root tests, and the literals check stops flagging stray digits (#3598, #3492, gstack-evals#97).** A machine without pytest reported "fix the in-branch failure". Playwright specs in a vitest repo went to vitest, which found no tests and failed. Root tests in `.mjs` and `.py` got no runner at all. And any lone `2` in a file that named a changed constant (a comment, a `$2` in a shell stub, a fixture) failed `literals`. A wrapper bump from 3 to 4 would have raised 5 false hits here, and gstack-evals#97 hit 11. Rewritten from outside PRs #3639 and #3600 by @vilano93 and #3629 by @nfitzsimmons. + +### Fixed +- **Missing pytest is the machine's gap.** Before planning `python3 -m pytest`, `lanes` asks that python3 whether it can import pytest (10 s probe, from the repo root). If it can't, the file is unclaimed and the lane is `PREGATE_ENVIRONMENT` (missing-binary, `pytest` or `python3`): "install pytest on this machine and rerun, or run the lane remotely; the diff is not at fault". +- **Playwright specs run under Playwright.** A JS/TS test file that imports `@playwright/test` at the start of a line runs as ` playwright test --reporter=list` from its nearest package. Other tests keep their runner. A declared `runner` still takes every file. Playwright's summary is read count line by count line. Failed and interrupted count as failed, flaky and passed as passed, and skipped as not run. A run where every test fails and Playwright says its browser is not installed is `PREGATE_ENVIRONMENT` naming `npx playwright install`. +- **Mixed-language root tests each get a runner.** A repo's bun, vitest or jest now claims only JS/TS files. When no one runner fits every root test, they are split by language and each part is detected on its own. So `.mjs` with `.py` runs `node --test` and pytest, and a `.py` test in a vitest repo gets pytest instead of vitest. `gstack-pregate init` drops its "a selection mixing these languages needs runner set" note. +- **`literals` binds a one-character old value to its statement.** The value counts on a line that names the constant, or on up to three lines that continue that statement, never on a comment line. Version-history rows count as before. A lowercase call-site name (`attempts`, `limit`) also needs the changed module named in the file. Values of two or more characters are unchanged. + +### Changed +- **docs/pregate.md** describes the new detection and the literals binding. **docs/troubleshooting.md** names the Playwright missing-browsers case under `PREGATE_ENVIRONMENT`. **docs/state-root.md** lists `pregate/runs//`. + ## [1.93.14.0] - 2026-11-02 Free Tests no longer goes red with no named file when one test hangs. Each `free-suite` job's GitHub timeout used to be a fixed 20 minutes, shorter than the runner's own wall for a 10-shard plan (about 37 minutes), so GitHub killed the job before the runner could name the hung file or retry it, and no shard result was uploaded. The plan now emits each shard's job timeout as its runner wall plus the flaky-retry pass plus the job's measured setup (`--ci-plan --job-setup-seconds`), 49 minutes per shard today, and fails loudly above 90 minutes instead of truncating. See docs/TESTING_INTERNALS.md#runners-how-the-suites-execute-2026-08-overhaul. Closes #3850. diff --git a/VERSION b/VERSION index 5bc28cc73..854f27d1b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.93.14.0 +1.93.15.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 98abcfbee..bd3ae8bb9 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.93.14.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.93.15.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/docs/pregate.md b/docs/pregate.md index 4e223e0cf..09d1afe7d 100644 --- a/docs/pregate.md +++ b/docs/pregate.md @@ -74,7 +74,7 @@ the byte-pinned `/ship` renders in `test/fixtures/golden/`). | Key | Meaning | |---|---| -| `runner` | the local test command with `{files}` (gstack: `bun test --timeout=30000 {files}`, the 30 s per-test timeout CI uses); unset, it is detected (bun, vitest, jest, pytest, `node --test`, and `flutter test`/`dart test` per Dart package root, through fvm when that package or an ancestor pins it with `.fvmrc` or `.fvm/fvm_config.json` and fvm is on PATH; `cargo test -p ` per Rust crate, run from the root of the workspace the crate is a member of (Cargo's `members` globs, `exclude` and `package.workspace`), else the crate directory; a JS/TS test whose nearest package.json is a nested package declaring vitest or jest (a dependency or its `test` script) runs from that package, with package-relative paths, through the package manager its `packageManager` field or the nearest lockfile up to the repo root names: `pnpm exec`, `yarn`, `bun x`, else `npx`; every other JS/TS test keeps the root detection); nothing detected, a Dart SDK, cargo or package manager missing from PATH, a `.dart` file with no `pubspec.yaml` above it, a `.rs` file outside any crate, or one in a crate whose name Cargo would reject (anything but letters, digits, `_` and `-`), is `incomplete` (the runnable groups still run, so a real failure still fails) | +| `runner` | the local test command with `{files}` (gstack: `bun test --timeout=30000 {files}`, the 30 s per-test timeout CI uses); unset, it is detected (bun, vitest or jest for JS/TS files, pytest when python3 can import it (else `pytest` or `python3` is named missing), `node --test`, and `flutter test`/`dart test` per Dart package root, through fvm when that package or an ancestor pins it with `.fvmrc` or `.fvm/fvm_config.json` and fvm is on PATH; `cargo test -p ` per Rust crate, run from the root of the workspace the crate is a member of (Cargo's `members` globs, `exclude` and `package.workspace`), else the crate directory; a JS/TS test whose nearest package.json is a nested package declaring vitest or jest (a dependency or its `test` script) runs from that package, with package-relative paths, through the package manager its `packageManager` field or the nearest lockfile up to the repo root names: `pnpm exec`, `yarn`, `bun x`, else `npx`; a JS/TS test file that imports `@playwright/test` at the start of a line runs as `playwright test --reporter=list` from its nearest package through that same package manager, whatever the unit runner is, a type-only import (`import type …`, or one whose every specifier is `type X`) does not count, file arguments are passed as escaped regexes anchored to the path (Playwright reads them as regexes), and a run where every test fails and Playwright says its browser is not installed is `PREGATE_ENVIRONMENT` (a partial install, where some tests pass, stays the diff's failure); every other test keeps the root detection, split by language when no one runner fits them all, so `.mjs` and `.py` root tests each get their own runner); nothing detected, a Dart SDK, cargo or package manager missing from PATH, a `.dart` file with no `pubspec.yaml` above it, a `.rs` file outside any crate, or one in a crate whose name Cargo would reject (anything but letters, digits, `_` and `-`), is `incomplete` (the runnable groups still run, so a real failure still fails) | | `dependencies` | glob → test files for subprocess targets, runtime reads and generated inputs the import graph cannot see; `[]` declares "no lane needed" (docs, licences). A `[]` that matches a deleted test does not skip it: the deleted test still runs its owning suite, and a file naming it that `[]` covers (a durations list) counts as covered. With this key present, even empty, the policy [declares lanes](#lane-requirements) | | `empty_lanes` | extra globs for the docs/no-tests empty lane. Any Markdown file already qualifies with no pin file when no test depends on it (the same import graph as other files: a quoted path, a `join()` of path segments, an import, or a `dependencies` entry). reStructuredText and AsciiDoc count as Markdown here. README, CHANGELOG, CONTRIBUTING, LICENSE, AGENTS and CLAUDE qualify even without `.md`, and so does licence text under any vendored name (`roboto-LICENSE.txt`, `LICENSE-MIT`, `licenses/*.txt`). Source code and files under a test directory never qualify. A test that reads the file still selects that test | | `selection` | `workflow.yml/job` → a command printing the lane's file list, for matrices a planner computes (gstack: `scripts/test-free-shards.ts --list`) | @@ -220,6 +220,14 @@ Two stages with separate clocks, both printed: other line holding the old value still fail. Only the row itself is exempt: the old value after its closing bracket on the same line still fails. Rows in object form (`{ v: 45 }`) or spread over several lines are not recognized as history rows. + A one-character old value (`2`) counts only in a statement that names the + constant: on the name's line, or on up to three lines that continue it (the + line before ends with `(`, `[`, `{`, `,`, `.`, `=` or another operator, or the + line starts with `.`, `)`, `]`, `}` or an operator), never on a comment line, plus the history rows + above. A `$2` in a shell stub, a comment, or `export default 2` in a fixture + elsewhere in the file is not a consumer. A call-site name such as `attempts` + or `limit` is an ordinary word, so for it the file must also name the changed + module. - `timing`: added or changed test lines that compare an elapsed wall-clock or CPU time against a bound, or sleep and then assert, print @@ -437,7 +445,7 @@ test-tier command: Per-file lines come from a progress adapter where the runner's reporter provides them: `bun test` (its `:` headers) is the only adapter today; a one-file run (each serial file) reports its file from the launch for any - runner. Other runners (node --test, vitest, jest, pytest, flutter test, dart test, cargo test) get heartbeats + runner. Other runners (node --test, vitest, jest, pytest, playwright test, flutter test, dart test, cargo test) get heartbeats only. A heartbeat follows each 60 s with no progress line; - is killed as **stalled** when its log does not grow for the stall timeout (`--stall-timeout `, else `stall_timeout_s`, else 300; `0` disables): the @@ -994,7 +1002,7 @@ failure the branch caused carries `cause: "diff"` (`PREGATE_LANE_FAILED`, | Kind | Evidence | |---|---| -| `missing-binary` | exit 127, `command not found`, a spawn `ENOENT`, a runner tool `not on PATH` (dart, flutter, cargo, a package manager) | +| `missing-binary` | exit 127, `command not found`, a spawn `ENOENT`, a runner tool `not on PATH` (dart, flutter, cargo, a package manager, `python3`, or `pytest` when python3 cannot import it), Playwright's browsers not installed | | `network` | `ENOTFOUND`, `ECONNREFUSED`, `ECONNRESET`, `ETIMEDOUT`, `EAI_AGAIN`, `Could not resolve host`, `fetch failed`, a TLS handshake or certificate failure | | `quota` | `quota`, `429 Too Many Requests`, `rate limit`, `insufficient_quota` (a vCPU quota on the remote backend) | | `disk` | `ENOSPC`, `No space left on device`, `EDQUOT` | @@ -1005,7 +1013,11 @@ The classifier is conservative. A local lane that exited non-zero is read from its output **only when its runner reported no test summary** (nothing ran) and the runner was not seen starting a test file; a run whose summary counts `N ran, M failed` is the diff's failure whatever its log says, so a test that -failed on `ECONNREFUSED` is still a red test. A remote gate that exited +failed on `ECONNREFUSED` is still a red test. One exception: when every +failed run is a Playwright run in which every test failed and Playwright says +its browsers are not installed, the lane is `missing-binary` with `proof: +"output"` (a partial install, where some tests pass, stays the diff's +failure). A remote gate that exited without a `PREGATE_REMOTE:` line is read from its **preamble only** (its first 40 lines, before any test output could stream): a VM refused for quota or an unreachable API there never ran the tests, so it is environment and the diff --git a/docs/state-root.md b/docs/state-root.md index 97f0328b4..7f75b9b14 100644 --- a/docs/state-root.md +++ b/docs/state-root.md @@ -105,7 +105,9 @@ lives in `projects//`. `bin/gstack-slug` picks the slug from the git - Without a remote, the project root's directory name is used. A bucket can also hold a user-owned `pregate.json` policy, for a repo that keeps -`.gstack/` out of git (see [pregate v1](pregate.md#pregate-v1)). +`.gstack/` out of git (see [pregate v1](pregate.md#pregate-v1)), and +`pregate/runs//`, the scratch each pregate run writes (lane and guard +logs, the remote gate's file list); old runs are pruned when a new one starts. The first slug gstack resolves in a directory sticks (it is cached under `slug-cache/`), so a project keeps its bucket when it adds a remote later. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index e7db68e52..86e0a0a66 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -3932,7 +3932,10 @@ and the table prints `environment: — `. A test that failed an whose log also names a connection error is **not** this code: the runner's own summary counted a failure, so it is `PREGATE_LANE_FAILED` with `cause: "diff"`; nor is a runner, guard or gate script whose own binary is present but -exits 127. `proof: "launch"` is certain (the command could not start, its +exits 127. One exception: a Playwright lane where every test failed and +Playwright says its browsers are not installed is `missing-binary` with +`proof: "output"` (`npx playwright install`); a partial install stays the +diff's failure. `proof: "launch"` is certain (the command could not start, its binary is off PATH, the runtime or a gstack tool refused); `proof: "output"` means a rule matched the command's text before any test was counted, which the diff's own output could have written, so the line says `(inferred from output)` diff --git a/lib/pregate/checks/lanes.ts b/lib/pregate/checks/lanes.ts index f900babe1..1de603914 100644 --- a/lib/pregate/checks/lanes.ts +++ b/lib/pregate/checks/lanes.ts @@ -27,6 +27,7 @@ * Incidents: p0-4-pipe-hidden-failure (`false | tail` read as green), * p0-4-windows-lane-unrun (a Windows-only change merged with its lane unrun). */ +import { spawnSync } from 'node:child_process'; import * as fs from 'node:fs'; import * as path from 'node:path'; import { baseInputs, sha256 } from '../context'; @@ -43,17 +44,21 @@ import { verifyFromRun } from '../../run-evidence'; import { matchesAny } from '../../ship-policy'; import type { CheckDef, CheckResult, PregateContext, RemoteObligation } from '../types'; -/** The test command with `{files}`; detected from the repo when the config does not name one. */ +/** The runner a `.py`-only selection gets; it needs the pytest module importable by python3 (`pytestMissing`). */ +const PYTEST = 'python3 -m pytest {files}'; + +/** The test command with `{files}`; detected from the repo when the config does not name one. The repo's bun, vitest or jest claims JS/TS files only. */ export function detectRunner(ctx: Pick, files: readonly string[]): string | null { if (ctx.config.runner) return ctx.config.runner; const has = (f: string) => fs.existsSync(path.join(ctx.repoRoot, f)); let pkg: { devDependencies?: Record; dependencies?: Record; scripts?: Record } = {}; try { pkg = JSON.parse(fs.readFileSync(path.join(ctx.repoRoot, 'package.json'), 'utf8')); } catch { /* no package.json */ } const deps = { ...pkg.dependencies, ...pkg.devDependencies }; - if (has('bunfig.toml') || has('bun.lock') || has('bun.lockb')) return 'bun test {files}'; - if (deps.vitest) return 'npx vitest run {files}'; - if (deps.jest) return 'npx jest {files}'; - if (files.every(f => /\.py$/.test(f)) && files.length) return 'python3 -m pytest {files}'; + const jsts = files.every(f => /\.[cm]?[jt]sx?$/.test(f)); + if (jsts && (has('bunfig.toml') || has('bun.lock') || has('bun.lockb'))) return 'bun test {files}'; + if (jsts && deps.vitest) return 'npx vitest run {files}'; + if (jsts && deps.jest) return 'npx jest {files}'; + if (files.every(f => /\.py$/.test(f)) && files.length) return PYTEST; if (files.every(f => /\.[cm]?js$/.test(f)) && files.length) return 'node --test {files}'; if (files.every(f => /\.[cm]?[jt]sx?$/.test(f)) && files.length && Bun.which('bun')) return 'bun test {files}'; return null; @@ -113,6 +118,18 @@ export function dartRunner(repoRoot: string, root: string, env: NodeJS.ProcessEn return { runner: `${fvm ? 'fvm ' : ''}${tool} test {files}` }; } +/** + * The tool `python3 -m pytest` lacks: `python3` off the PATH of `env`, or `pytest` when that python3, run from the repo root as the + * runner is, cannot import it (a `pytest` script on PATH may belong to another interpreter); null when both are there, or when the + * probe itself times out or dies, so the run decides. + */ +function pytestMissing(repoRoot: string, env: NodeJS.ProcessEnv): string | null { + const python = Bun.which('python3', { PATH: env.PATH ?? '' }); + if (!python) return 'python3'; + const r = spawnSync(python, ['-c', 'import importlib.util, sys; sys.exit(3 if importlib.util.find_spec("pytest") is None else 0)'], { cwd: repoRoot, env, stdio: 'ignore', timeout: 10_000 }); + return r.status === 3 ? 'pytest' : null; +} + /** * `cargo test -p ` for a crate (cargo runs a crate's tests, never a file list), or the missing tool when cargo is * not on PATH. The name comes from the repo's Cargo.toml and reaches a shell and the `{files}` expansion: a name outside @@ -163,6 +180,23 @@ function packageRunner(repoRoot: string, dir: string, env: NodeJS.ProcessEnv): { return { runner: `${PM_EXEC[pm]} ${tool} {files}` }; } +/** + * Whether a JS/TS test imports a value from Playwright's runner in a statement that starts a line: `import … from '@playwright/test'` + * (one line or several), a bare import, or `require`. A type-only import (`import type …`, or every specifier `type X`) and a string or + * fixture body that mentions the import mid-line do not count. + */ +function importsPlaywright(text: string): boolean { + const code = text.replace(/\/\*[\s\S]*?\*\/|\/\/[^\n]*/g, ''); + if (/^[\t ]*(?:import[\t ]*|(?:const|let|var)\b[^=\n]*=[\t ]*require\([\t ]*)['"]@playwright\/test['"]/m.test(code)) return true; + for (const m of code.matchAll(/^[\t ]*import(?=[\s{*])([^'"`;]*?)\bfrom[\t ]*['"]@playwright\/test['"]/gm)) { + const spec = m[1]!.trim(); + if (!/^type(?=[\s{*])/.test(spec) && spec.replace(/[{}]/g, '').split(',').some(s => s.trim() && !/^type\s/.test(s.trim()))) return true; + } + return false; +} +/** Playwright's own text when its browsers are not installed on this machine. */ +const PLAYWRIGHT_NO_BROWSERS = /^.*(?:Executable doesn't exist at|download new browsers).*$/m; + /** The run plan: the groups that can run, the files no runner claims, tools missing from PATH, and `.dart` files with no pubspec.yaml above. */ export interface RunPlan { groups: RunGroup[]; unclaimed: string[]; missing: string[]; noPubspec: string[] } @@ -171,8 +205,11 @@ export interface RunPlan { groups: RunGroup[]; unclaimed: string[]; missing: str * Dart tests run from their own package root (a sub-package's imports resolve * against its own pubspec); Rust tests run per crate as `cargo test -p ` * from the crate's workspace root; JS/TS tests owned by a nested package that - * declares vitest or jest run from that package; the rest go to the detected - * runner from the repo root. Files no runner claims (a tool missing from PATH, + * declares vitest or jest run from that package; a JS/TS test that imports + * `@playwright/test` runs through `playwright test` from its nearest package; + * the rest go to the detected runner from the repo root, split by language + * when no one runner fits them all (`.mjs` + `.py` runs both), and pytest only + * when python3 can import it. Files no runner claims (a tool missing from PATH, * a `.dart` file outside any package, a `.rs` file outside any crate or in a * crate whose name Cargo would reject, no detected runner) are `unclaimed`: the lane is `incomplete`, never a pass. */ @@ -182,6 +219,7 @@ export function planRuns(ctx: PregateContext, files: readonly string[]): RunPlan const dart = new Map(); const cargo = new Map(); const packages = new Map(); + const playwright = new Map(); const seen = new Map(); const seenPackage = new Map(); let model: CargoModel | null = null; @@ -204,6 +242,7 @@ export function planRuns(ctx: PregateContext, files: readonly string[]): RunPlan continue; } const pkg = /\.[cm]?[jt]sx?$/.test(f) ? nearestRoot(ctx.repoRoot, f, 'package.json', seenPackage) : null; + if (pkg !== null && importsPlaywright(readRepoFile(ctx.repoRoot, f) ?? '')) { add(playwright, pkg, under(pkg, f)); continue; } if (pkg) { if (!pkgRunners.has(pkg)) pkgRunners.set(pkg, packageRunner(ctx.repoRoot, pkg, ctx.env)); const r = pkgRunners.get(pkg); @@ -225,12 +264,24 @@ export function planRuns(ctx: PregateContext, files: readonly string[]): RunPlan if (r) missing.add(r.missing); unclaimed.push(...c.files.map(f => (c.cwd ? `${c.cwd}/${f}` : f))); } - if (rest.length) { - const runner = detectRunner(ctx, rest); - if (runner) groups.push({ runner, cwd: '', files: rest }); - else unclaimed.push(...rest); + const language = (f: string) => (/\.py$/.test(f) ? 'py' : /\.[cm]?js$/.test(f) ? 'js' : /\.[cm]?[jt]sx?$/.test(f) ? 'ts' : f); + const parts = !rest.length ? [] : detectRunner(ctx, rest) ? [rest] : [...Map.groupBy(rest, language).values()]; + const byRunner = new Map(); + for (const part of parts) { + const runner = detectRunner(ctx, part); + const lacking = runner === PYTEST ? pytestMissing(ctx.repoRoot, ctx.env) : null; + if (lacking) missing.add(lacking); + if (runner && !lacking) for (const f of part) add(byRunner, runner, f); + else unclaimed.push(...part); } + for (const [runner, list] of byRunner) groups.push({ runner, cwd: '', files: list }); for (const [root, list] of [...packages].sort(([a], [b]) => a.localeCompare(b))) groups.push({ runner: (pkgRunners.get(root) as { runner: string }).runner, cwd: root, files: list }); + for (const [root, list] of [...playwright].sort(([a], [b]) => a.localeCompare(b))) { + const pm = packageManager(ctx.repoRoot, root); + const bin = pm === 'npm' ? 'npx' : pm; + if (onPath(bin, ctx.env)) groups.push({ runner: `${PM_EXEC[pm]} playwright test --reporter=list {files}`, cwd: root, files: list }); + else { missing.add(bin); unclaimed.push(...list.map(f => (root ? `${root}/${f}` : f))); } + } return { groups, unclaimed: [...unclaimed, ...noPubspec], missing: [...missing].sort(), noPubspec }; } @@ -269,11 +320,20 @@ function parseCargo(output: string): RunnerSummary | null { return { ran: lines.reduce((n, m) => n + Number(m[1]), 0) + failed, failed }; } -/** The runner's own summary line: bun, node --test, vitest, jest, pytest, bats, dart/flutter test, cargo test. A bats, dart/flutter or cargo runner's summary is tried first (cargo's only there: its `N passed` lines would misread as another runner's); for any other runner the bats and dart formats are a last resort. `ran: null` when not recognised. Colour codes are dropped and a carriage return ends a line, so a coloured or redrawn verdict still reads. */ +/** playwright test: the last `N failed`, `N interrupted`, `N flaky`, `N passed` and `N skipped` lines (its closing summary, with test names between them; a test's own output earlier does not count); skipped tests did not run. */ +function parsePlaywright(output: string): RunnerSummary | null { + if (!/^\s*\d+ (?:passed|failed|flaky|interrupted|skipped)\b/m.test(output)) return null; + const count = (word: string) => Number([...output.matchAll(new RegExp(`^\\s*(\\d+) ${word}\\b`, 'gm'))].pop()?.[1] ?? 0); + const failed = count('failed') + count('interrupted'); + return { ran: count('passed') + count('flaky') + failed, failed }; +} + +/** The runner's own summary line: bun, node --test, vitest, jest, pytest, bats, dart/flutter test, cargo test, playwright test. A bats, dart/flutter, playwright or cargo runner's summary is tried first (cargo's only there: its `N passed` lines would misread as another runner's); for any other runner the bats and dart formats are a last resort. `ran: null` when not recognised. Colour codes are dropped and a carriage return ends a line, so a coloured or redrawn verdict still reads. */ export function parseSummary(raw: string, runner = ''): RunnerSummary { const output = raw.replace(/\x1b\[[0-9;?]*[A-Za-z]/g, '').replace(/\r/g, '\n'); if (/(? `'${f.replace(/'/g, `'\\''`)}'`; async function runOnce(ctx: PregateContext, env: NodeJS.ProcessEnv, g: RunGroup, files: string[], log: string, counter: Counter): Promise { - const joined = files.map(quote).join(' '); + // playwright test reads each file argument as a case-insensitive regex: escape it and anchor it to a path segment, so `a[1].spec.ts` runs that file, not `a1.spec.ts`. + const args = /(? `(?:^|/)${f.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}$`) : files; + const joined = args.map(quote).join(' '); const run = g.runner.replace('{files}', () => joined); const command = g.cwd ? `(cd ${quote(g.cwd)} && ${run})` : run; const shown = g.cwd ? files.map(f => `${g.cwd}/${f}`) : files; const r = await supervise({ command: run, cwd: path.join(ctx.repoRoot, g.cwd), env: { ...env, CI: env.CI ?? '1' }, log, deadline: Date.now() + ctx.timeoutMs, stallMs: ctx.stallTimeoutMs, label: `lanes (${shown.length === 1 ? shown[0] : `${shown.length} files`})`, files, adapter: adapterFor(g.runner), counter, progress: ctx.progress }); const output = readRange(log, r.start, r.end); const summary = parseSummary(output, g.runner); - return { exit: r.exit, outcome: r.outcome, signal: r.signal, observedStart: r.observedStart, ...summary, tail: output.trim().split('\n').slice(-25), runner: g.runner, command, files: shown, lastFile: r.lastFile && g.cwd ? `${g.cwd}/${r.lastFile}` : r.lastFile, ms: r.ms, ...(r.error ? { error: r.error } : {}) }; + const noBrowsers = /(? x.exit !== 0); + if (failedRuns.every(x => x.noBrowsers && x.failed !== null && x.failed === x.ran)) return environmentResult(withRun, { kind: 'missing-binary', evidence: `Playwright browsers not installed (npx playwright install): ${failedRuns[0]!.noBrowsers}`, proof: 'output' }, `every Playwright test failed to launch its browser (${r.failed} of ${r.ran}; ${total} files)`); return { ...withRun, status: 'fail', cause: 'diff', code: 'PREGATE_LANE_FAILED', detail: `local lane exited ${r.exit} (${r.failed ?? '?'} failed of ${r.ran ?? '?'} ran; ${total} files; pipefail)`, fix: `read ${r.log}, fix the in-branch failure, rerun gstack-pregate --stage tests` }; } if (r.ran === 0) return { ...withRun, status: 'fail', cause: 'diff', code: 'PREGATE_LANE_FAILED', detail: `ZERO-RUN: the runner reported 0 tests${r.zero ? ` in ${r.zero}` : ''} (${total} selected file(s))`, fix: 'a bad selector, a missing file or a skipped describe; exit 0 is not a pass' }; diff --git a/lib/pregate/checks/literals.ts b/lib/pregate/checks/literals.ts index d3158c481..2e1121a71 100644 --- a/lib/pregate/checks/literals.ts +++ b/lib/pregate/checks/literals.ts @@ -69,7 +69,12 @@ function unquote(v: string): string { return /^['"`]/.test(v) ? v.slice(1, -1) : * `/retry.ts.snap`, or `lib/install.ts` for `install.sh`, is not the module (GST-820): a bare stem such as `install` or `retry` is an * English word in half the test tree, and a one-character old value (`2`) * sits in every test, so that value needs the constant's name (a wrapper - * version 2 → 3 flagged 575 consumers through the stem alone). + * version 2 → 3 flagged 575 consumers through the stem alone), and counts only + * in a statement that names it: on the name's line, or on a line that + * continues it (the line before ends open, `(` `,` `.` `=`, or the line + * starts with `.` `)` `]`), comments excluded (#3492: `$2`, "from revision 2", + * `export default 2`). A call-site name (`attempts`, `limit`) is an ordinary + * word, so for it the file must also name the module (gstack-evals#97). * * A version-history row (`[45, ""]`, the old value first in a * bracketed row) is skipped when the same table, a run of consecutive @@ -94,12 +99,23 @@ export function consumersOf(repoRoot: string, testFiles: readonly string[], chan const moduleRe = new RegExp(`(?:^|[/\\\\])${escape(stem)}(?=[^\\w.-]|$|\\.${dts}(?:${exts})(?![\\w.-])|\\.(?!\\w))|(? /^\s*(?:\/\/|#|\*|\/\*| # /ship driver: run /ship from a parent agent -Read this, not `ship/SKILL.md`, when a parent agent on any host runs /ship +Read this, not `ship/SKILL.md`, when a parent agent runs /ship (`SESSION_KIND: unattended`). Commands live in `/bin/` (`` holds `ship/`); run each with the `GSTACK_LAUNCH` assignments your installer printed (`. `), never `export $(...)`, `env`, `printenv` or a bare `set` in a tool fence. **Rules.** Each phase below is one call: one program (Capy: an execute program; else a script) that runs the bins, reads, dispatches and waits, returning only result lines and reader -outputs; print them verbatim. The model keeps only lead judgment, fixes, CHANGELOG, PR prose, -gates and the report. `STOP:` (exit 3) means nothing after it ran; `NEXT:` names +outputs, printed verbatim. The model keeps only lead judgment, fixes, CHANGELOG, PR prose, +gates, the report. `STOP:` (exit 3) means nothing after it ran; `NEXT:` names the next act. Read a section only when a phase or the list at the end sends you, never up front. A STOP or decision this file does not cover: read `ship/SKILL.md` at that step, finish it there, come back. Unattended, take a question's recommended option (log it); gate @@ -20,15 +20,14 @@ phase: each answered or auto-taken question, verbatim), **Reviews** (Phases 3-5: start tokens, `Pre-Landing Review:`, `Gate edits:`, the reviewed tree), **Release** (Phases 5-7: version, CHANGELOG, commits), **Documentation** (Step 14.5) and **Next steps** (every phase: the next act). -**Reviewers are the sections' own.** Every audit, specialist, Red Team, QA, fix-check, adversarial, -outside-voice and documentation brief, its severity and confidence rules and gates come from its -section verbatim (placeholders filled); this file +**Reviewers are the sections' own.** Every reviewer brief, its severity and confidence rules +and gates come from its section verbatim (placeholders filled); this file never shortens a reviewer's brief. A blocking reader's prompt is its `gstack-review-dispatch render` file, unchanged; its findings reach `merged.jsonl` only via `collect`; `judge` refuses a drop or downgrade. Audit readers (Steps 7, 8, 14.5) cannot block, so each runs at low effort on a small model, Step 14.5 once on Sonnet 5.5; findings go to the PR body. **Calls.** One pass with no fixes is 9 calls: Phases 1-8 and the report; at most 12 with a STOP's -section read or a fix-check, plus a keep-alive per 200 s waited. +section read or fix-check, plus a keep-alive per 200 s waited. **Budget: observe.** `gstack-ship-budget start --mode observe` records, never admits: each program ends with `gstack-ship-budget note --step --reading R --credits C --record `, R and C read in that program @@ -38,7 +37,7 @@ A dispatching program takes `BEFORE` (R and C) as readers start and passes `--dispatched-at --dispatched-credits --readers ` with the reading after the last settles. **Waits.** No call waits past 200 s: a program returns what still runs 200 s after its first line; the next -call resumes the poll. Lanes, pregate and `finish` run in background. Never end the turn while +call resumes the poll. Lanes, pregate, `finish` run in background. Never end the turn while readers run (each wakes it). A small waiting child saves nothing. Past 100 minutes, stop. Returns: ≤32 KB per program; split longer reads at headings, each byte once, before first use: phase programs end with `gstack-ship-parts next --run --phase --room <32000 - its output>`; on @@ -46,12 +45,12 @@ end with `gstack-ship-parts next --run --phase --room <32000 - i **One program per pass.** Each review pass, Step 11 and the Step 14.5 audit is one program, and the parent makes no other call while it runs: (1) `gstack-review-dispatch pass`: `gstack-review-plan --base ` -with the pass's flags, then its staged manifest (Step 14.5 has one reader, `docs`); (2) `gstack-review-log --start +with the pass's flags, then its staged manifest (Step 14.5: one reader, `docs`); (2) `gstack-review-log --start ` once per row the step logs; (3) `BEFORE`; (4) `sent` (`BRIEF_MISMATCH` voids it), then one subagent per reader on its rendered file; (5) a poll until every reader settles (status `done` or `failed`, or a final summary; a fresh `idle` has not run); (6) `note` with the reading pair; a later batch (the Red Team, -given the specialists' collected lines) repeats (3)-(6); (7) the collect: return only each reader's +on the specialists' collected lines) repeats (3)-(6); (7) the collect: return only each reader's final output and status, via `collect`; it never walks the dispatch steps call by call. ## 1. Load: Step 0 @@ -63,12 +62,12 @@ Install check, this file. `gstack-ship-budget start --record --mode observe [--token-cap ] --reading R --credits C`, then `gstack-ship-preflight --base --lane