diff --git a/.github/workflows/free-tests.yml b/.github/workflows/free-tests.yml index 948b597ee..897e5b458 100644 --- a/.github/workflows/free-tests.yml +++ b/.github/workflows/free-tests.yml @@ -23,6 +23,22 @@ on: push: branches: [main] workflow_dispatch: + inputs: + free_runner: + description: 'Runner-sizing experiments only: Ubicloud runner for the free-suite shards (ubicloud-standard-2, -4 or -8; empty keeps FREE_SUITE_RUNNER). Never cite such a run as a PR or train Free Tests result.' + type: string + default: '' + free_shards: + description: 'Runner-sizing experiments only: shard count, 1-40 (empty keeps FREE_SUITE_SHARDS)' + type: string + default: '' + +# The one source of the free-suite runner and shard count. pull_request and +# push runs never see workflow_dispatch inputs, so free-plan resolves these +# (or a dispatch override) into outputs that free-suite reads. +env: + FREE_SUITE_RUNNER: ubicloud-standard-8 + FREE_SUITE_SHARDS: '20' # Keyed on the PR number, not head_ref: a bare branch name carries no fork # prefix, so same-name branches from two forks would share one group and a @@ -43,6 +59,7 @@ jobs: timeout-minutes: 5 outputs: matrix: ${{ steps.plan.outputs.matrix }} + runner: ${{ steps.plan.outputs.runner }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: @@ -56,9 +73,27 @@ jobs: - id: plan name: Inventory and balance the complete free suite # Warns in the job summary when more than 5 files are unseeded. + env: + RUNNER_OVERRIDE: ${{ inputs.free_runner }} + SHARDS_OVERRIDE: ${{ inputs.free_shards }} run: | - matrix=$(bun run scripts/test-free-shards.ts --ci-plan "$RUNNER_TEMP/free-plan.json" --shards 20) + set -euo pipefail + runner="${RUNNER_OVERRIDE:-$FREE_SUITE_RUNNER}" + shards="${SHARDS_OVERRIDE:-$FREE_SUITE_SHARDS}" + case "$runner" in + ubicloud-standard-2|ubicloud-standard-4|ubicloud-standard-8) ;; + *) echo "::error::free_runner '$runner' is not ubicloud-standard-2, -4 or -8"; exit 1 ;; + esac + case "$shards" in + ''|*[!0-9]*) echo "::error::free_shards '$shards' is not a whole number"; exit 1 ;; + esac + if [ "$shards" -lt 1 ] || [ "$shards" -gt 40 ]; then echo "::error::free_shards $shards is outside 1-40"; exit 1; fi + if [ "$runner" != "$FREE_SUITE_RUNNER" ] || [ "$shards" != "$FREE_SUITE_SHARDS" ]; then + echo "- Runner-sizing experiment: $shards shards on $runner (defaults $FREE_SUITE_SHARDS on $FREE_SUITE_RUNNER); not a PR or train Free Tests result" >> "$GITHUB_STEP_SUMMARY" + fi + matrix=$(bun run scripts/test-free-shards.ts --ci-plan "$RUNNER_TEMP/free-plan.json" --shards "$shards") echo "matrix=$matrix" >> "$GITHUB_OUTPUT" + echo "runner=$runner" >> "$GITHUB_OUTPUT" - name: Seed growth ratchet (60 s ceiling, shrink-only allowlist) run: | GSTACK_FREE_SEED_BASE="$(git merge-base HEAD origin/main)" bun test test/free-seed-ratchet.test.ts @@ -314,7 +349,7 @@ jobs: free-suite: needs: free-plan - runs-on: ubicloud-standard-8 + runs-on: ${{ needs.free-plan.outputs.runner }} timeout-minutes: 20 strategy: fail-fast: false @@ -446,6 +481,18 @@ jobs: - name: Build trusted CSO helper used by free contract tests run: bun run build:cso + # Runner sizing: the suite runs one test at a time, so a shard needs RAM + # and disk more than cores. Sample both while it runs and report the peak. + - name: Start the memory and disk sampler + run: | + cat > "$RUNNER_TEMP/sample-resources.sh" <<'EOF' + while :; do + printf '%s %s %s\n' "$(date +%s)" "$(free -k | awk 'NR == 2 { print $3 }')" "$(df -Pk / | awk 'NR == 2 { print $4 }')" + sleep 2 + done + EOF + nohup sh "$RUNNER_TEMP/sample-resources.sh" > "$RUNNER_TEMP/resources.log" 2>&1 & + # GSTACK_EXPECT_BINARIES=1 arms make-pdf/test/e2e/ci-prereqs.test.ts: # if a future edit drops the gate build (or poppler), the lane FAILS # instead of the gates silently self-skipping back to false green. @@ -464,6 +511,12 @@ jobs: GSTACK_FREE_RETRY_FLAKY: "1" GSTACK_FLAKE_LEDGER: ${{ runner.temp }}/flake-ledger.jsonl + - name: Report peak memory and minimum free disk + if: always() + run: | + if [ ! -s "$RUNNER_TEMP/resources.log" ]; then echo "RESOURCES: no samples"; exit 0; fi + awk '{ if ($2 > used) used = $2; if (free == "" || $3 < free) free = $3 } END { printf "RESOURCES: peak_used_mib=%d min_free_disk_mib=%d samples=%d\n", used / 1024, free / 1024, NR }' "$RUNNER_TEMP/resources.log" | tee -a "$GITHUB_STEP_SUMMARY" + - name: Upload strict shard result if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 diff --git a/test/free-durations-record-workflow.test.ts b/test/free-durations-record-workflow.test.ts index a86d0fcbc..9bd99d135 100644 --- a/test/free-durations-record-workflow.test.ts +++ b/test/free-durations-record-workflow.test.ts @@ -29,9 +29,10 @@ test('the recorder runs weekly before test-health and on dispatch, read-only exc expect(recorder.jobs.propose.permissions).toEqual({ contents: 'write', 'pull-requests': 'write', actions: 'write' }); }); -test('Linux records on the free-suite runner, turns red on any red file, and uploads only a complete recording', () => { +test('Linux records on the free-suite runner image, turns red on any red file, and uploads only a complete recording', () => { const linux = recorder.jobs.linux; - expect(linux['runs-on']).toBe(freeTests.jobs['free-suite']['runs-on']); + expect(linux['runs-on']).toMatch(/^ubicloud-standard-\d+$/); + expect(freeTests.env.FREE_SUITE_RUNNER).toMatch(/^ubicloud-standard-\d+$/); const runs = linux.steps.map((step: any) => step.run).filter(Boolean).join('\n'); expect(runs).toContain('bash scripts/ubicloud/setup-free-suite.sh'); const record = linux.steps.find((step: any) => step.name === 'Time every free file alone'); diff --git a/test/free-tests-workflow-wiring.test.ts b/test/free-tests-workflow-wiring.test.ts index cd8322971..1d33e4d47 100644 --- a/test/free-tests-workflow-wiring.test.ts +++ b/test/free-tests-workflow-wiring.test.ts @@ -166,6 +166,65 @@ describe('free-tests workflow wiring', () => { } }); + test.skipIf(!Bun.which('bash'))('free-plan resolves the one runner and shard constant for every event, and validates dispatch overrides', () => { + const workflow = Bun.YAML.parse(source) as any; + const planner = workflow.jobs['free-plan']; + const plan = planner.steps.find((step: any) => step.id === 'plan'); + expect(workflow.env.FREE_SUITE_RUNNER).toMatch(/^ubicloud-standard-(2|4|8)$/); + expect(workflow.env.FREE_SUITE_SHARDS).toMatch(/^[1-9][0-9]?$/); + expect(planner.outputs.runner).toBe('${{ steps.plan.outputs.runner }}'); + expect(workflow.jobs['free-suite']['runs-on']).toBe('${{ needs.free-plan.outputs.runner }}'); + expect(plan.env).toEqual({ RUNNER_OVERRIDE: '${{ inputs.free_runner }}', SHARDS_OVERRIDE: '${{ inputs.free_shards }}' }); + expect(plan.run).not.toContain('${{'); + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'free plan ')); + const bin = path.join(directory, 'bin'); + fs.mkdirSync(bin); + fs.writeFileSync(path.join(bin, 'bun'), '#!/bin/sh\nprintf "%s\\n" "$*" > "$RUNNER_TEMP/bun-args"\necho \'{"shard":[1]}\'\n', { mode: 0o755 }); + const toShell = (p: string) => p.split(path.sep).join('/'); + const resolve = (overrides: Record) => { + const output = path.join(directory, 'output'); + fs.writeFileSync(output, ''); + const result = spawnSync('bash', ['-c', plan.run], { + env: { ...process.env, ...workflow.env, RUNNER_OVERRIDE: '', SHARDS_OVERRIDE: '', ...overrides, + PATH: `${toShell(bin)}:${process.env.PATH}`, RUNNER_TEMP: toShell(directory), + GITHUB_OUTPUT: toShell(output), GITHUB_STEP_SUMMARY: toShell(path.join(directory, 'summary')) }, + encoding: 'utf8', timeout: 5000, + }); + const outputs = Object.fromEntries(fs.readFileSync(output, 'utf8').trim().split('\n').filter(Boolean).map(line => line.split(/=(.*)/s).slice(0, 2))); + const args = fs.existsSync(path.join(directory, 'bun-args')) ? fs.readFileSync(path.join(directory, 'bun-args'), 'utf8') : ''; + fs.rmSync(path.join(directory, 'bun-args'), { force: true }); + return { status: result.status, outputs, args }; + }; + try { + const unset = resolve({}); + expect(unset.status).toBe(0); + expect(unset.outputs.runner).toBe(workflow.env.FREE_SUITE_RUNNER); + expect(unset.args).toContain(`--shards ${workflow.env.FREE_SUITE_SHARDS}`); + const override = resolve({ RUNNER_OVERRIDE: 'ubicloud-standard-4', SHARDS_OVERRIDE: '16' }); + expect(override.status).toBe(0); + expect(override.outputs.runner).toBe('ubicloud-standard-4'); + expect(override.args).toContain('--shards 16'); + for (const bad of [{ RUNNER_OVERRIDE: 'ubuntu-24.04' }, { RUNNER_OVERRIDE: 'ubicloud-standard-8 x' }, { SHARDS_OVERRIDE: '0' }, { SHARDS_OVERRIDE: '41' }, { SHARDS_OVERRIDE: '16; true' }]) { + const refused = resolve(bad); + expect(refused.status, JSON.stringify(bad)).not.toBe(0); + expect(refused.outputs.runner, JSON.stringify(bad)).toBeUndefined(); + expect(refused.args, JSON.stringify(bad)).toBe(''); + } + } finally { fs.rmSync(directory, { recursive: true, force: true }); } + }); + + test('every shard reports its peak memory and minimum free disk, even when the suite fails', () => { + const steps = (Bun.YAML.parse(source) as any).jobs['free-suite'].steps; + const start = steps.findIndex((step: any) => step.name === 'Start the memory and disk sampler'); + const suite = steps.findIndex((step: any) => step.name === 'Run free suite'); + const report = steps.findIndex((step: any) => step.name === 'Report peak memory and minimum free disk'); + expect(start).toBeGreaterThan(-1); + expect(start).toBeLessThan(suite); + expect(report).toBeGreaterThan(suite); + expect(steps[report].if).toBe('always()'); + expect(steps[report].run).toContain('RESOURCES: peak_used_mib='); + }); + test('least-privilege token: contents read-only, credentials not persisted', () => { // The job executes PR-controlled code (install lifecycle scripts + the // suite itself). A default-grant GITHUB_TOKEN persisted into .git/config