ci(free-tests): one runner and shard constant for every event, dispatch overrides for sizing, per-shard memory and disk peak

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

.github/workflows/free-tests.yml

@@ -23,6 +23,22 @@ on:
   push:
     branches: [main]
   workflow_dispatch:
+    inputs:
+      free_runner:
+        description: 'Runner-sizing experiments only: Ubicloud runner for the free-suite shards (ubicloud-standard-2, -4 or -8; empty keeps FREE_SUITE_RUNNER). Never cite such a run as a PR or train Free Tests result.'
+        type: string
+        default: ''
+      free_shards:
+        description: 'Runner-sizing experiments only: shard count, 1-40 (empty keeps FREE_SUITE_SHARDS)'
+        type: string
+        default: ''
+
+# The one source of the free-suite runner and shard count. pull_request and
+# push runs never see workflow_dispatch inputs, so free-plan resolves these
+# (or a dispatch override) into outputs that free-suite reads.
+env:
+  FREE_SUITE_RUNNER: ubicloud-standard-8
+  FREE_SUITE_SHARDS: '20'
 
 # Keyed on the PR number, not head_ref: a bare branch name carries no fork
 # prefix, so same-name branches from two forks would share one group and a
@@ -43,6 +59,7 @@ jobs:
     timeout-minutes: 5
     outputs:
       matrix: ${{ steps.plan.outputs.matrix }}
+      runner: ${{ steps.plan.outputs.runner }}
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
         with:
@@ -56,9 +73,27 @@ jobs:
       - id: plan
         name: Inventory and balance the complete free suite
         # Warns in the job summary when more than 5 files are unseeded.
+        env:
+          RUNNER_OVERRIDE: ${{ inputs.free_runner }}
+          SHARDS_OVERRIDE: ${{ inputs.free_shards }}
         run: |
-          matrix=$(bun run scripts/test-free-shards.ts --ci-plan "$RUNNER_TEMP/free-plan.json" --shards 20)
+          set -euo pipefail
+          runner="${RUNNER_OVERRIDE:-$FREE_SUITE_RUNNER}"
+          shards="${SHARDS_OVERRIDE:-$FREE_SUITE_SHARDS}"
+          case "$runner" in
+            ubicloud-standard-2|ubicloud-standard-4|ubicloud-standard-8) ;;
+            *) echo "::error::free_runner '$runner' is not ubicloud-standard-2, -4 or -8"; exit 1 ;;
+          esac
+          case "$shards" in
+            ''|*[!0-9]*) echo "::error::free_shards '$shards' is not a whole number"; exit 1 ;;
+          esac
+          if [ "$shards" -lt 1 ] || [ "$shards" -gt 40 ]; then echo "::error::free_shards $shards is outside 1-40"; exit 1; fi
+          if [ "$runner" != "$FREE_SUITE_RUNNER" ] || [ "$shards" != "$FREE_SUITE_SHARDS" ]; then
+            echo "- Runner-sizing experiment: $shards shards on $runner (defaults $FREE_SUITE_SHARDS on $FREE_SUITE_RUNNER); not a PR or train Free Tests result" >> "$GITHUB_STEP_SUMMARY"
+          fi
+          matrix=$(bun run scripts/test-free-shards.ts --ci-plan "$RUNNER_TEMP/free-plan.json" --shards "$shards")
           echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
+          echo "runner=$runner" >> "$GITHUB_OUTPUT"
       - name: Seed growth ratchet (60 s ceiling, shrink-only allowlist)
         run: |
           GSTACK_FREE_SEED_BASE="$(git merge-base HEAD origin/main)" bun test test/free-seed-ratchet.test.ts
@@ -314,7 +349,7 @@ jobs:
 
   free-suite:
     needs: free-plan
-    runs-on: ubicloud-standard-8
+    runs-on: ${{ needs.free-plan.outputs.runner }}
     timeout-minutes: 20
     strategy:
       fail-fast: false
@@ -446,6 +481,18 @@ jobs:
       - name: Build trusted CSO helper used by free contract tests
         run: bun run build:cso
 
+      # Runner sizing: the suite runs one test at a time, so a shard needs RAM
+      # and disk more than cores. Sample both while it runs and report the peak.
+      - name: Start the memory and disk sampler
+        run: |
+          cat > "$RUNNER_TEMP/sample-resources.sh" <<'EOF'
+          while :; do
+            printf '%s %s %s\n' "$(date +%s)" "$(free -k | awk 'NR == 2 { print $3 }')" "$(df -Pk / | awk 'NR == 2 { print $4 }')"
+            sleep 2
+          done
+          EOF
+          nohup sh "$RUNNER_TEMP/sample-resources.sh" > "$RUNNER_TEMP/resources.log" 2>&1 &
+
       # GSTACK_EXPECT_BINARIES=1 arms make-pdf/test/e2e/ci-prereqs.test.ts:
       # if a future edit drops the gate build (or poppler), the lane FAILS
       # instead of the gates silently self-skipping back to false green.
@@ -464,6 +511,12 @@ jobs:
           GSTACK_FREE_RETRY_FLAKY: "1"
           GSTACK_FLAKE_LEDGER: ${{ runner.temp }}/flake-ledger.jsonl
 
+      - name: Report peak memory and minimum free disk
+        if: always()
+        run: |
+          if [ ! -s "$RUNNER_TEMP/resources.log" ]; then echo "RESOURCES: no samples"; exit 0; fi
+          awk '{ if ($2 > used) used = $2; if (free == "" || $3 < free) free = $3 } END { printf "RESOURCES: peak_used_mib=%d min_free_disk_mib=%d samples=%d\n", used / 1024, free / 1024, NR }' "$RUNNER_TEMP/resources.log" | tee -a "$GITHUB_STEP_SUMMARY"
+
       - name: Upload strict shard result
         if: always()
         uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7

test/free-durations-record-workflow.test.ts

@@ -29,9 +29,10 @@ test('the recorder runs weekly before test-health and on dispatch, read-only exc
   expect(recorder.jobs.propose.permissions).toEqual({ contents: 'write', 'pull-requests': 'write', actions: 'write' });
 });
 
-test('Linux records on the free-suite runner, turns red on any red file, and uploads only a complete recording', () => {
+test('Linux records on the free-suite runner image, turns red on any red file, and uploads only a complete recording', () => {
   const linux = recorder.jobs.linux;
-  expect(linux['runs-on']).toBe(freeTests.jobs['free-suite']['runs-on']);
+  expect(linux['runs-on']).toMatch(/^ubicloud-standard-\d+$/);
+  expect(freeTests.env.FREE_SUITE_RUNNER).toMatch(/^ubicloud-standard-\d+$/);
   const runs = linux.steps.map((step: any) => step.run).filter(Boolean).join('\n');
   expect(runs).toContain('bash scripts/ubicloud/setup-free-suite.sh');
   const record = linux.steps.find((step: any) => step.name === 'Time every free file alone');

test/free-tests-workflow-wiring.test.ts

@@ -166,6 +166,65 @@ describe('free-tests workflow wiring', () => {
     }
   });
 
+  test.skipIf(!Bun.which('bash'))('free-plan resolves the one runner and shard constant for every event, and validates dispatch overrides', () => {
+    const workflow = Bun.YAML.parse(source) as any;
+    const planner = workflow.jobs['free-plan'];
+    const plan = planner.steps.find((step: any) => step.id === 'plan');
+    expect(workflow.env.FREE_SUITE_RUNNER).toMatch(/^ubicloud-standard-(2|4|8)$/);
+    expect(workflow.env.FREE_SUITE_SHARDS).toMatch(/^[1-9][0-9]?$/);
+    expect(planner.outputs.runner).toBe('${{ steps.plan.outputs.runner }}');
+    expect(workflow.jobs['free-suite']['runs-on']).toBe('${{ needs.free-plan.outputs.runner }}');
+    expect(plan.env).toEqual({ RUNNER_OVERRIDE: '${{ inputs.free_runner }}', SHARDS_OVERRIDE: '${{ inputs.free_shards }}' });
+    expect(plan.run).not.toContain('${{');
+    const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'free plan '));
+    const bin = path.join(directory, 'bin');
+    fs.mkdirSync(bin);
+    fs.writeFileSync(path.join(bin, 'bun'), '#!/bin/sh\nprintf "%s\\n" "$*" > "$RUNNER_TEMP/bun-args"\necho \'{"shard":[1]}\'\n', { mode: 0o755 });
+    const toShell = (p: string) => p.split(path.sep).join('/');
+    const resolve = (overrides: Record<string, string>) => {
+      const output = path.join(directory, 'output');
+      fs.writeFileSync(output, '');
+      const result = spawnSync('bash', ['-c', plan.run], {
+        env: { ...process.env, ...workflow.env, RUNNER_OVERRIDE: '', SHARDS_OVERRIDE: '', ...overrides,
+          PATH: `${toShell(bin)}:${process.env.PATH}`, RUNNER_TEMP: toShell(directory),
+          GITHUB_OUTPUT: toShell(output), GITHUB_STEP_SUMMARY: toShell(path.join(directory, 'summary')) },
+        encoding: 'utf8', timeout: 5000,
+      });
+      const outputs = Object.fromEntries(fs.readFileSync(output, 'utf8').trim().split('\n').filter(Boolean).map(line => line.split(/=(.*)/s).slice(0, 2)));
+      const args = fs.existsSync(path.join(directory, 'bun-args')) ? fs.readFileSync(path.join(directory, 'bun-args'), 'utf8') : '';
+      fs.rmSync(path.join(directory, 'bun-args'), { force: true });
+      return { status: result.status, outputs, args };
+    };
+    try {
+      const unset = resolve({});
+      expect(unset.status).toBe(0);
+      expect(unset.outputs.runner).toBe(workflow.env.FREE_SUITE_RUNNER);
+      expect(unset.args).toContain(`--shards ${workflow.env.FREE_SUITE_SHARDS}`);
+      const override = resolve({ RUNNER_OVERRIDE: 'ubicloud-standard-4', SHARDS_OVERRIDE: '16' });
+      expect(override.status).toBe(0);
+      expect(override.outputs.runner).toBe('ubicloud-standard-4');
+      expect(override.args).toContain('--shards 16');
+      for (const bad of [{ RUNNER_OVERRIDE: 'ubuntu-24.04' }, { RUNNER_OVERRIDE: 'ubicloud-standard-8 x' }, { SHARDS_OVERRIDE: '0' }, { SHARDS_OVERRIDE: '41' }, { SHARDS_OVERRIDE: '16; true' }]) {
+        const refused = resolve(bad);
+        expect(refused.status, JSON.stringify(bad)).not.toBe(0);
+        expect(refused.outputs.runner, JSON.stringify(bad)).toBeUndefined();
+        expect(refused.args, JSON.stringify(bad)).toBe('');
+      }
+    } finally { fs.rmSync(directory, { recursive: true, force: true }); }
+  });
+
+  test('every shard reports its peak memory and minimum free disk, even when the suite fails', () => {
+    const steps = (Bun.YAML.parse(source) as any).jobs['free-suite'].steps;
+    const start = steps.findIndex((step: any) => step.name === 'Start the memory and disk sampler');
+    const suite = steps.findIndex((step: any) => step.name === 'Run free suite');
+    const report = steps.findIndex((step: any) => step.name === 'Report peak memory and minimum free disk');
+    expect(start).toBeGreaterThan(-1);
+    expect(start).toBeLessThan(suite);
+    expect(report).toBeGreaterThan(suite);
+    expect(steps[report].if).toBe('always()');
+    expect(steps[report].run).toContain('RESOURCES: peak_used_mib=');
+  });
+
   test('least-privilege token: contents read-only, credentials not persisted', () => {
     // The job executes PR-controlled code (install lifecycle scripts + the
     // suite itself). A default-grant GITHUB_TOKEN persisted into .git/config