/** * Static tripwire for .github/workflows/free-tests.yml — the Linux free-suite * lane. Pins the three properties that made the lane worth having: * * 1. It invokes the CANONICAL runner (bun run test:free), not a raw * `bun test ` glob — the runner owns TEST_ROOTS and strict-output * classification, so a truncated run can't report green. * 2. It is SECRETLESS: free tests make no API calls, and keeping keys out * means fork PRs get real signal here. Any `secrets.` reference is a * regression. * 3. It triggers on `pull_request` (never `pull_request_target`, which * would hand a fork PR the base repo's context). * * Same wiring-tripwire class as test/hermetic-wiring.test.ts. */ import { describe, test, expect } from 'bun:test'; import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; import { spawnSync } from 'node:child_process'; const WORKFLOW = path.resolve(import.meta.dir, '..', '.github', 'workflows', 'free-tests.yml'); describe('free-tests workflow wiring', () => { const source = fs.readFileSync(WORKFLOW, 'utf-8'); test('workflow exists and invokes the canonical runner', () => { expect(source).toContain('bun run test:free'); expect(source).not.toMatch(/run:\s*bun test\s/); }); test('secretless: no secrets reach the free lane', () => { expect(source).not.toContain('secrets.'); expect(source).not.toContain('ANTHROPIC_API_KEY'); expect(source).not.toContain('OPENAI_API_KEY'); }); test('pull_request trigger, never pull_request_target', () => { expect(source).toContain('pull_request:'); expect(source).not.toContain('pull_request_target'); }); test('the isolated matrix consumes one plan and the required aggregate verifies all receipts', () => { const workflow = Bun.YAML.parse(source) as any; const planner = workflow.jobs['free-plan']; const suite = workflow.jobs['free-suite']; const aggregate = workflow.jobs['free-tests']; expect(planner.steps.find((step: any) => step.id === 'plan').run).toContain('--ci-plan'); expect(suite.needs).toBe('free-plan'); expect(suite.strategy.matrix).toBe('${{ fromJSON(needs.free-plan.outputs.matrix) }}'); expect(suite.strategy['fail-fast']).toBe(false); expect(suite.strategy['max-parallel']).toBe(20); expect(suite.steps.find((step: any) => step.name === 'Run free suite').run).toContain('--ci-run'); expect(suite.steps.find((step: any) => step.name === 'Upload strict shard result').if).toBe('always()'); expect(aggregate.if).toBe('always()'); expect(aggregate.needs).toContain('free-suite'); expect(aggregate.steps.some((step: any) => step.run?.includes('--ci-verify'))).toBe(true); expect(aggregate.needs).toContain('typecheck'); const gate = aggregate.steps.find((step: any) => step.env?.TYPECHECK_RESULT); expect(gate.env.TYPECHECK_RESULT).toBe('${{ needs.typecheck.result }}'); expect(gate.run).toContain('test "$TYPECHECK_RESULT" = success'); const typecheck = workflow.jobs.typecheck.steps.map((step: any) => step.run).filter(Boolean); expect(typecheck).toEqual(expect.arrayContaining(['bun run typecheck', 'bun run typecheck:test', 'bun run format:cso:check'])); expect(source).not.toContain('--quick'); }); test('zsh is installed so the bash+zsh portability arms run instead of skipping (#2669)', () => { const suite = (Bun.YAML.parse(source) as any).jobs['free-suite']; const apt = suite.steps.find((step: any) => step.name === 'Install Xvfb + X11 utilities + gate tools').run; expect(apt.split(/\s+/)).toContain('zsh'); const ubicloud = fs.readFileSync(path.resolve(import.meta.dir, '..', 'scripts', 'ubicloud', 'setup-free-suite.sh'), 'utf-8'); const ubicloudApt = ubicloud.match(/apt-get install(?:[^\n]*\\\n)*[^\n]*/)![0]; expect(ubicloudApt.split(/\s+/)).toContain('zsh'); }); test('the aggregate summarizes every shard flake ledger before strict verification (W7a)', () => { const steps = (Bun.YAML.parse(source) as any).jobs['free-tests'].steps; const download = steps.findIndex((step: any) => step.with?.pattern === 'flake-ledger-*'); const summary = steps.findIndex((step: any) => step.run?.includes('scripts/test-health-report.ts flake-summary "$RUNNER_TEMP/flake-ledgers" >> "$GITHUB_STEP_SUMMARY"')); const verify = steps.findIndex((step: any) => step.run?.includes('--ci-verify')); expect(steps[download].with.path).toBe('${{ runner.temp }}/flake-ledgers'); expect(steps[download].with['merge-multiple']).toBeUndefined(); expect(download).toBeLessThan(summary); expect(summary).toBeLessThan(verify); }); test('the plan job runs the seed ratchet against the merge-base with full history', () => { const planner = (Bun.YAML.parse(source) as any).jobs['free-plan']; expect(planner.steps[0].with['fetch-depth']).toBe(0); const ratchet = planner.steps.find((step: any) => step.name?.startsWith('Seed growth ratchet')); expect(ratchet.run).toContain('GSTACK_FREE_SEED_BASE="$(git merge-base HEAD origin/main)"'); expect(ratchet.run).toContain('test/free-seed-ratchet.test.ts'); }); test('flake telemetry stays wired: retry flag, single-writer ledger, unconditional artifact', () => { // WS1: a timing flake must not red the required lane, but every // flaky-pass must be recorded and uploaded — a green run is exactly when // the evidence matters. Removing any of these silently returns flakes to // either merge-blocking (flag off) or invisibility (ledger/artifact off). expect(source).toMatch(/GSTACK_FREE_RETRY_FLAKY:\s*"1"/); expect(source).toMatch(/GSTACK_FLAKE_LEDGER:\s*\$\{\{ runner\.temp \}\}\/flake-ledger\.jsonl/); expect(source).toContain('name: flake-ledger'); expect(source).toMatch(/name: Upload flake ledger\s*\n\s*if: always\(\)/); }); test.skipIf(!Bun.which('bash'))('a recovered retry retains its original detailed spool', () => { const steps = (Bun.YAML.parse(source) as any).jobs['free-suite'].steps; const probe = steps.find((step: any) => step.id === 'flake_spool'); const upload = steps.find((step: any) => step.with?.name === 'free-test-shard-logs-${{ matrix.shard }}'); expect(probe.if).toBe('always()'); expect(upload.if).toBe("failure() || steps.flake_spool.outputs.present == 'true'"); expect(upload.with.path).toBe('.context/free-test-logs/gstack-free-test-*.log'); expect(upload.with['include-hidden-files']).toBe(true); const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'free spool ')); const output = path.join(directory, 'step-output'); const ledger = path.join(directory, 'flake-ledger.jsonl'); try { for (const contents of [null, '', '{"kind":"flaky-pass","file":"test/example.test.ts"}\n']) { if (contents !== null) fs.writeFileSync(ledger, contents); fs.writeFileSync(output, ''); const result = spawnSync('bash', ['-e', '-c', probe.run], { // Git Bash accepts C:/... paths; native backslashes are not shell paths. env: { ...process.env, RUNNER_TEMP: directory.split(path.sep).join('/'), GITHUB_OUTPUT: output.split(path.sep).join('/') }, encoding: 'utf8', timeout: 5000, }); expect(result.status, result.stderr).toBe(0); expect(fs.readFileSync(output, 'utf8')).toBe(contents ? 'present=true\n' : ''); } } finally { fs.rmSync(directory, { recursive: true, force: true }); } }); test('every Linux Playwright install has a version-keyed browser cache, 5-minute attempts and one retry of the install alone (run 39641712129)', () => { const jobs = (Bun.YAML.parse(source) as any).jobs; const installing = Object.keys(jobs).filter(name => jobs[name].steps.some((step: any) => step.run?.includes('playwright install --with-deps'))); expect(installing.sort()).toEqual(['free-suite', 'quick-bun-default']); for (const name of installing) { const steps = jobs[name].steps; const version = steps.findIndex((step: any) => step.id === 'playwright'); const cache = steps.findIndex((step: any) => step.with?.path === '~/.cache/ms-playwright'); const install = steps.findIndex((step: any) => step.id === 'playwright_install'); const retry = steps.findIndex((step: any) => step.if === "steps.playwright_install.outcome == 'failure'"); expect(steps[version].run, name).toContain('require("playwright-core/package.json").version'); expect(steps[cache].with.key, name).toBe('linux-playwright-chromium-${{ steps.playwright.outputs.version }}'); expect(steps[cache].with['restore-keys'], name).toBeUndefined(); expect(version, name).toBeLessThan(cache); expect(cache, name).toBeLessThan(install); expect(retry, name).toBe(install + 1); for (const step of [steps[install], steps[retry]]) { expect(step['timeout-minutes'], name).toBe(5); expect(step.run, name).toBe('npx playwright install --with-deps chromium'); } expect(steps[install]['continue-on-error'], name).toBe(true); expect(steps.filter((step: any) => step['continue-on-error']), name).toEqual([steps[install]]); } }); test('test jobs give PR and dispatch runs the local main that push runs get from checkout (#3237)', () => { const jobs = (Bun.YAML.parse(source) as any).jobs; for (const name of ['free-suite', 'quick-bun-default']) { const [checkout, guard] = jobs[name].steps; expect(checkout.uses, name).toStartWith('actions/checkout@'); expect(guard.if, name).toBe("github.ref != 'refs/heads/main'"); expect(guard.run, name).toBe('git checkout -q -B main'); } }); test('least-privilege token: contents read-only, credentials not persisted', () => { // The job executes PR-controlled code (install lifecycle scripts + the // suite itself). A default-grant GITHUB_TOKEN persisted into .git/config // by checkout would hand that code whatever the repo default allows. expect(source).toMatch(/permissions:\s*\n\s*contents:\s*read/); expect(source).toMatch(/persist-credentials:\s*false/); }); });