From 12407a3f4daf039e017e229a68076851aaaa10ef Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 31 Oct 2026 22:23:36 +0000 Subject: [PATCH] ci(free): per-shard sub-shard k in the CI plan (--shard-jobs), free_jobs and windows_jobs dispatch inputs (k=1 defaults) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- .github/workflows/free-tests.yml | 26 ++++++++--- .github/workflows/windows-free-tests.yml | 21 ++++++++- scripts/lib/shard-jobs.ts | 31 +++++++++++-- scripts/test-free-shards.ts | 50 ++++++++++++++------- test/free-tests-workflow-wiring.test.ts | 14 +++--- test/shard-jobs.test.ts | 55 ++++++++++++++++++++++-- test/windows-native-workflows.test.ts | 36 +++++++++++++++- 7 files changed, 195 insertions(+), 38 deletions(-) diff --git a/.github/workflows/free-tests.yml b/.github/workflows/free-tests.yml index 443888a40..9b4d18a32 100644 --- a/.github/workflows/free-tests.yml +++ b/.github/workflows/free-tests.yml @@ -32,13 +32,21 @@ on: description: 'Runner-sizing experiments only: shard count, 1-40 (empty keeps FREE_SUITE_SHARDS)' type: string default: '' + free_jobs: + description: 'Runner-sizing experiments only: sub-shards per CI shard (--shard-jobs), 1-8 (empty keeps FREE_SUITE_JOBS)' + type: string + default: '' -# The one source of the free-suite runner and shard count. pull_request and -# push runs never see workflow_dispatch inputs, so free-plan resolves these -# (or a dispatch override) into outputs that free-suite reads. +# The one source of the free-suite runner, shard count and sub-shards per +# shard. pull_request and push runs never see workflow_dispatch inputs, so +# free-plan resolves these (or a dispatch override) into outputs and the plan +# that free-suite reads. FREE_SUITE_JOBS is k in `--ci-plan --shard-jobs k`: +# each shard runs min(k, its parallel files) sandboxed sub-shards side by side +# (scripts/lib/shard-jobs.ts); the exclusive host-state shard stays one child. env: FREE_SUITE_RUNNER: ubicloud-standard-2 FREE_SUITE_SHARDS: '20' + FREE_SUITE_JOBS: '1' # Keyed on the PR number, not head_ref: a bare branch name carries no fork # prefix, so same-name branches from two forks would share one group and a @@ -76,10 +84,12 @@ jobs: env: RUNNER_OVERRIDE: ${{ inputs.free_runner }} SHARDS_OVERRIDE: ${{ inputs.free_shards }} + JOBS_OVERRIDE: ${{ inputs.free_jobs }} run: | set -euo pipefail runner="${RUNNER_OVERRIDE:-$FREE_SUITE_RUNNER}" shards="${SHARDS_OVERRIDE:-$FREE_SUITE_SHARDS}" + jobs="${JOBS_OVERRIDE:-$FREE_SUITE_JOBS}" case "$runner" in ubicloud-standard-2|ubicloud-standard-4|ubicloud-standard-8) ;; *) echo "::error::free_runner '$runner' is not ubicloud-standard-2, -4 or -8"; exit 1 ;; @@ -88,10 +98,14 @@ jobs: ''|*[!0-9]*) echo "::error::free_shards '$shards' is not a whole number"; exit 1 ;; esac if [ "$shards" -lt 1 ] || [ "$shards" -gt 40 ]; then echo "::error::free_shards $shards is outside 1-40"; exit 1; fi - if [ "$runner" != "$FREE_SUITE_RUNNER" ] || [ "$shards" != "$FREE_SUITE_SHARDS" ]; then - echo "- Runner-sizing experiment: $shards shards on $runner (defaults $FREE_SUITE_SHARDS on $FREE_SUITE_RUNNER); not a PR or train Free Tests result" >> "$GITHUB_STEP_SUMMARY" + case "$jobs" in + ''|*[!0-9]*) echo "::error::free_jobs '$jobs' is not a whole number"; exit 1 ;; + esac + if [ "$jobs" -lt 1 ] || [ "$jobs" -gt 8 ]; then echo "::error::free_jobs $jobs is outside 1-8"; exit 1; fi + if [ "$runner" != "$FREE_SUITE_RUNNER" ] || [ "$shards" != "$FREE_SUITE_SHARDS" ] || [ "$jobs" != "$FREE_SUITE_JOBS" ]; then + echo "- Runner-sizing experiment: $shards shards on $runner at $jobs sub-shards each (defaults $FREE_SUITE_SHARDS on $FREE_SUITE_RUNNER at $FREE_SUITE_JOBS); not a PR or train Free Tests result" >> "$GITHUB_STEP_SUMMARY" fi - matrix=$(bun run scripts/test-free-shards.ts --ci-plan "$RUNNER_TEMP/free-plan.json" --shards "$shards") + matrix=$(bun run scripts/test-free-shards.ts --ci-plan "$RUNNER_TEMP/free-plan.json" --shards "$shards" --shard-jobs "$jobs") echo "matrix=$matrix" >> "$GITHUB_OUTPUT" echo "runner=$runner" >> "$GITHUB_OUTPUT" - name: Seed growth ratchet (60 s ceiling, shrink-only allowlist) diff --git a/.github/workflows/windows-free-tests.yml b/.github/workflows/windows-free-tests.yml index 6d19e5b48..08846a45a 100644 --- a/.github/workflows/windows-free-tests.yml +++ b/.github/workflows/windows-free-tests.yml @@ -34,6 +34,10 @@ on: description: Record per-file Windows durations (artifact) instead of running the lane type: boolean default: false + windows_jobs: + description: 'Sub-shard experiments only: sub-shards per Windows shard (--shard-jobs), 1-4 (empty keeps WINDOWS_FREE_JOBS). Never cite such a run as a PR result.' + type: string + default: '' # free-durations-record.yml calls the recording path weekly. workflow_call: inputs: @@ -41,6 +45,11 @@ on: type: boolean default: false +# k in `--ci-plan --shard-jobs k`: each Windows shard runs min(k, its parallel +# files) sandboxed sub-shards side by side (scripts/lib/shard-jobs.ts). +env: + WINDOWS_FREE_JOBS: '1' + concurrency: group: windows-free-${{ github.event.pull_request.number || github.run_id }} cancel-in-progress: true @@ -69,8 +78,18 @@ jobs: # 4-vCPU Windows runner, so the lane lands near 4 min with setup. env: GSTACK_FREE_TEST_DURATIONS: scripts/free-test-durations-windows.json + JOBS_OVERRIDE: ${{ inputs.windows_jobs }} run: | - matrix=$(bun run scripts/test-free-shards.ts --windows-only --ci-plan "$RUNNER_TEMP/windows-plan.json" --shards 6) + set -euo pipefail + jobs="${JOBS_OVERRIDE:-$WINDOWS_FREE_JOBS}" + case "$jobs" in + ''|*[!0-9]*) echo "::error::windows_jobs '$jobs' is not a whole number"; exit 1 ;; + esac + if [ "$jobs" -lt 1 ] || [ "$jobs" -gt 4 ]; then echo "::error::windows_jobs $jobs is outside 1-4"; exit 1; fi + if [ "$jobs" != "$WINDOWS_FREE_JOBS" ]; then + echo "- Sub-shard experiment: $jobs sub-shards per Windows shard (default $WINDOWS_FREE_JOBS); not a PR Windows Free Tests result" >> "$GITHUB_STEP_SUMMARY" + fi + matrix=$(bun run scripts/test-free-shards.ts --windows-only --ci-plan "$RUNNER_TEMP/windows-plan.json" --shards 6 --shard-jobs "$jobs") echo "matrix=$matrix" >> "$GITHUB_OUTPUT" - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: diff --git a/scripts/lib/shard-jobs.ts b/scripts/lib/shard-jobs.ts index c96100e01..d0510fb77 100644 --- a/scripts/lib/shard-jobs.ts +++ b/scripts/lib/shard-jobs.ts @@ -45,16 +45,39 @@ export const FREE_JOBS_INVALID = 'FREE_JOBS_INVALID'; export const FREE_SUBSHARD_TIMEOUT = 'FREE_SUBSHARD_TIMEOUT'; export const FREE_SUBSHARD_REJECTED = 'FREE_SUBSHARD_REJECTED'; const DOCS = 'docs/TESTING_INTERNALS.md#sub-shards-inside-one-ci-job'; -const JOBS_FIX = `--jobs applies only to --ci-run and defaults to 1; GSTACK_FREE_JOBS sets the local full-suite process count and never changes --ci-run. See ${DOCS}`; +const JOBS_FIX = `--jobs applies only to --ci-run and defaults to the plan's per-shard k (1 when the plan has none); --shard-jobs applies only to --ci-plan; GSTACK_FREE_JOBS sets the local full-suite process count and never changes --ci-run. See ${DOCS}`; -/** `--jobs ` at parse time: digits only, at least 1. */ -export function parseJobsFlag(raw: string | undefined): number { +/** `--jobs ` or `--shard-jobs ` at parse time: digits only, at least 1. */ +export function parseJobsFlag(raw: string | undefined, flag = '--jobs'): number { if (raw === undefined || !/^\d+$/.test(raw) || Number(raw) < 1) { - throw new Error(`${FREE_JOBS_INVALID}: --jobs needs a positive whole number; got ${JSON.stringify(raw ?? '')}. ${JOBS_FIX}`); + throw new Error(`${FREE_JOBS_INVALID}: ${flag} needs a positive whole number; got ${JSON.stringify(raw ?? '')}. ${JOBS_FIX}`); } return Number(raw); } +export function rejectShardJobsWithoutCiPlan(): never { + throw new Error(`${FREE_JOBS_INVALID}: --shard-jobs needs --ci-plan. ${JOBS_FIX}`); +} + +/** + * One planned shard's k under `--ci-plan --shard-jobs k`: at most one parallel + * sub-shard per file that may run in parallel, so a shard of only serial files + * (the exclusive TREE_MUTATING shard) keeps k = 1. Undefined means k = 1, and + * the plan then carries no `jobs` key, so its id matches a plan without k. + */ +export function plannedShardJobs(files: readonly string[], shardJobs: number, isSerial: (file: string) => boolean): number | undefined { + const jobs = Math.min(shardJobs, files.filter(file => !isSerial(file)).length); + return jobs > 1 ? jobs : undefined; +} + +/** Why a plan's per-shard k cannot stand; null when absent or valid. */ +export function plannedShardJobsProblem(shard: { shard: number; files: readonly string[]; jobs?: unknown }, isSerial: (file: string) => boolean): string | null { + if (shard.jobs === undefined) return null; + const parallel = shard.files.filter(file => !isSerial(file)).length; + return Number.isInteger(shard.jobs) && (shard.jobs as number) >= 2 && (shard.jobs as number) <= parallel ? null + : `CI shard ${shard.shard} plans --jobs ${JSON.stringify(shard.jobs)} for ${parallel} parallel file(s)`; +} + export function rejectJobsWithoutCiRun(): never { throw new Error(`${FREE_JOBS_INVALID}: --jobs needs --ci-run. ${JOBS_FIX}`); } diff --git a/scripts/test-free-shards.ts b/scripts/test-free-shards.ts index deda761cf..d61ca6112 100755 --- a/scripts/test-free-shards.ts +++ b/scripts/test-free-shards.ts @@ -77,11 +77,15 @@ * bun run scripts/test-free-shards.ts --attribute-home # name real-home writers: each file alone, private HOME * bun run scripts/test-free-shards.ts --ci-plan plan.json --shards 20 * bun run scripts/test-free-shards.ts --ci-run plan.json --shard 1 --result result.json + * bun run scripts/test-free-shards.ts --ci-plan plan.json --shards 20 --shard-jobs 2 + * # the plan stores a per-shard k: min(k, the shard's files that may run in parallel), + * # kept only when above 1, so the exclusive TREE_MUTATING shard stays one child. * bun run scripts/test-free-shards.ts --ci-run plan.json --shard 1 --result result.json --jobs 4 - * # k sub-shards side by side inside one CI job (default 1 = one child, today's behavior; - * # scripts/lib/shard-jobs.ts). --jobs applies only to --ci-run: a positive whole number up - * # to the shard's file count, else FREE_JOBS_INVALID. GSTACK_FREE_JOBS sets the local - * # full-suite process count and never changes --ci-run; --jobs is its only knob. + * # k sub-shards side by side inside one CI job (default: the plan's per-shard k, else 1 = + * # one child, today's behavior; scripts/lib/shard-jobs.ts). --jobs applies only to + * # --ci-run and overrides the plan: a positive whole number up to the shard's file count, + * # else FREE_JOBS_INVALID. --ci-verify requires every result to have run at the plan's k. + * # GSTACK_FREE_JOBS sets the local full-suite process count and never changes --ci-run. * bun run scripts/test-free-shards.ts --ci-verify plan.json --results results/ */ @@ -97,8 +101,8 @@ import { attributeFreeHomeWriters, guardFreeHome, sharedFreeHome, type FreeHomeG import { appendStepSummary, ciHealthSummary, SEED_REFRESH_COMMAND, unseededWarning, windowsCurationLine } from './lib/free-ci-health'; import { curateWindowsSafe, type CurationResult } from './lib/windows-curation'; import { - assertSoloSeedSample, ciRunContext, parseJobsFlag, parseJunitCounts, readJunitReport, rejectJobsWithoutCiRun, runCiShard, SLOW_UNDER_LOAD, subShardRecordProblems, - validateJobsForShard, type ShardTestCounts, type SubShardRecord, + assertSoloSeedSample, ciRunContext, parseJobsFlag, parseJunitCounts, plannedShardJobs, plannedShardJobsProblem, readJunitReport, rejectJobsWithoutCiRun, + rejectShardJobsWithoutCiPlan, runCiShard, SLOW_UNDER_LOAD, subShardRecordProblems, validateJobsForShard, type ShardTestCounts, type SubShardRecord, } from './lib/shard-jobs'; export { curateWindowsSafe, detectWindowsFragility, KNOWN_WINDOWS_INCOMPATIBLE, type CurationResult } from './lib/windows-curation'; import { @@ -424,19 +428,27 @@ export interface FreeCiPlan { version: 1; revision: string; id: string; - shards: Array<{ shard: number; files: string[]; predictedMs: number }>; + /** `jobs` is the shard's k from --shard-jobs, present only when above 1. */ + shards: Array<{ shard: number; files: string[]; predictedMs: number; jobs?: number }>; } +/** Files that run in the trailing serial sub-shard under k > 1, never beside another child. */ +const runsSerialUnderJobs = (file: string) => file in TREE_MUTATING || file in SLOW_UNDER_LOAD; + const planDigest = (plan: Omit): string => createHash('sha256').update(JSON.stringify(plan)).digest('hex'); /** One immutable plan is shared by isolated CI machines; never repack per job. */ -export function createFreeCiPlan(files: string[], count: number, durations: Record, revision: string): FreeCiPlan { +export function createFreeCiPlan(files: string[], count: number, durations: Record, revision: string, shardJobs = 1): FreeCiPlan { const readers = files.filter(file => !(file in TREE_MUTATING)); const exclusive = files.filter(file => file in TREE_MUTATING).sort(); const packed = packShardsByDuration(readers, count, durations); - const shards = packed.shards.map((files, index) => ({ shard: index + 1, files, predictedMs: packed.predictedMs[index] })); - if (exclusive.length) shards.push({ shard: shards.length + 1, files: exclusive, predictedMs: exclusive.reduce((ms, file) => ms + (durations[file] ?? 0), 0) }); + const planned = packed.shards.map((files, index) => ({ files, predictedMs: packed.predictedMs[index] })); + if (exclusive.length) planned.push({ files: exclusive, predictedMs: exclusive.reduce((ms, file) => ms + (durations[file] ?? 0), 0) }); + const shards = planned.map(({ files, predictedMs }, index) => { + const jobs = plannedShardJobs(files, shardJobs, runsSerialUnderJobs); + return { shard: index + 1, files, predictedMs, ...(jobs ? { jobs } : {}) }; + }); const body = { version: 1 as const, revision, shards }; return { ...body, id: planDigest(body) }; } @@ -447,6 +459,8 @@ export function validateFreeCiPlan(plan: FreeCiPlan, files: string[], revision: || id !== planDigest({ version, revision: plan.revision, shards })) throw new Error('CI plan identity or revision mismatch'); if (shards.some((shard, index) => shard.shard !== index + 1 || !Array.isArray(shard.files) || !Number.isFinite(shard.predictedMs) || shard.predictedMs < 0)) throw new Error('Invalid CI shard plan'); + const jobsProblems = shards.map(shard => plannedShardJobsProblem(shard, runsSerialUnderJobs)).filter(Boolean); + if (jobsProblems.length) throw new Error(`Invalid CI shard plan: ${jobsProblems.join('; ')}`); const planned = shards.flatMap(shard => shard.files).sort(); if (new Set(planned).size !== planned.length || JSON.stringify(planned) !== JSON.stringify([...files].sort())) throw new Error('CI plan must cover every free file exactly once'); } @@ -486,6 +500,7 @@ export function verifyFreeCiResults(plan: FreeCiPlan, results: FreeCiResult[]): if (!hasCompleteCiSummary(outcome)) throw new Error('Missing or incomplete CI execution summary'); const subShardProblems = subShardRecordProblems(result); if (subShardProblems.length) throw new Error(`CI shard ${outcome.shard} sub-shards rejected: ${subShardProblems.join('; ')}`); + if ((result.jobs ?? 1) !== (shard.jobs ?? 1)) throw new Error(`CI shard ${outcome.shard} ran at --jobs ${result.jobs ?? 1}; the plan says ${shard.jobs ?? 1}`); if (outcome.status === 'passed') { if (outcome.exitCode !== 0 || outcome.failingFiles.length || outcome.unattributedFailures || result.retry) throw new Error('Inconsistent passing CI result'); } else { @@ -556,8 +571,10 @@ type CliOptions = { result: string | null; results: string | null; filesFrom: string | null; - /** Set only by --jobs (with --ci-run); absent means k = 1. */ + /** Set only by --jobs (with --ci-run); absent means the plan's per-shard k. */ jobs?: number; + /** Set only by --shard-jobs (with --ci-plan); absent means k = 1 in every shard. */ + shardJobs?: number; }; export function parseCliOptions(argv: string[]): CliOptions { @@ -573,6 +590,7 @@ export function parseCliOptions(argv: string[]): CliOptions { let quick = false; let attributeHome = false; let jobs: number | undefined; + let shardJobs: number | undefined; const paths: Record<'ciPlan' | 'ciRun' | 'ciVerify' | 'result' | 'results' | 'filesFrom', string | null> = { ciPlan: null, ciRun: null, ciVerify: null, result: null, results: null, filesFrom: null, }; @@ -597,6 +615,7 @@ export function parseCliOptions(argv: string[]): CliOptions { '--results': pathFlag('--results', 'results'), '--files-from': pathFlag('--files-from', 'filesFrom'), '--jobs': (next) => { jobs = parseJobsFlag(next()); }, + '--shard-jobs': (next) => { shardJobs = parseJobsFlag(next(), '--shard-jobs'); }, '--shards': (next) => { const value = next(); if (!value) throw new Error('Missing value for --shards'); @@ -621,8 +640,9 @@ export function parseCliOptions(argv: string[]): CliOptions { if (paths.ciRun && (shardIndex === null || !paths.result)) throw new Error('--ci-run requires --shard and --result'); if (paths.ciVerify && !paths.results) throw new Error('--ci-verify requires --results'); if (jobs !== undefined && !paths.ciRun) rejectJobsWithoutCiRun(); + if (shardJobs !== undefined && !paths.ciPlan) rejectShardJobsWithoutCiPlan(); if (quick && (recordDurations || windowsOnly || shardIndex !== null)) throw new Error('--quick cannot change recording, Windows or shard selection'); - return { dryRun, listOnly, recordDurations, windowsOnly, verbose, shardCount, shardIndex, wallTimeoutMs, wallTimeoutExplicit, quick, attributeHome, ...paths, ...(jobs === undefined ? {} : { jobs }) }; + return { dryRun, listOnly, recordDurations, windowsOnly, verbose, shardCount, shardIndex, wallTimeoutMs, wallTimeoutExplicit, quick, attributeHome, ...paths, ...(jobs === undefined ? {} : { jobs }), ...(shardJobs === undefined ? {} : { shardJobs }) }; } const describeLines = (lines: string[]) => `${lines.slice(0, 5).join(', ')}${lines.length > 5 ? ` (+${lines.length - 5} more)` : ''}`; @@ -1954,7 +1974,7 @@ async function main(): Promise { appendStepSummary(line); } warnUnseededFreeFiles(ciFiles, durations); - const plan = createFreeCiPlan(ciFiles, options.shardCount, durations, revision); + const plan = createFreeCiPlan(ciFiles, options.shardCount, durations, revision, options.shardJobs); validateFreeCiPlan(plan, ciFiles, revision); writeJson(options.ciPlan, plan); console.log(JSON.stringify({ shard: plan.shards.map(shard => shard.shard) })); @@ -1975,7 +1995,7 @@ async function main(): Promise { } const shard = plan.shards[options.shardIndex! - 1]; if (!shard || shard.shard !== options.shardIndex) throw new Error('CI shard index is outside the plan'); - const jobs = options.jobs ?? 1; + const jobs = options.jobs ?? shard.jobs ?? 1; validateJobsForShard(jobs, shard.files.length, shard.shard); const durations = loadFreeTestDurations() ?? {}; const { result, exitCode } = await runCiShard({ @@ -1986,7 +2006,7 @@ async function main(): Promise { runShard: (files, shardNumber, totalShards, runOptions) => runFreeShard(files, shardNumber, totalShards, runOptions), retry: outcomes => retryFailedFreeFiles(outcomes, plan.shards.length, options), pack: (files, count) => packShardsByDuration(files, count, durations).shards, - isSerial: file => file in TREE_MUTATING || file in SLOW_UNDER_LOAD, + isSerial: runsSerialUnderJobs, terminationRequested: isTerminationRequested, }); writeJson(options.result!, result); diff --git a/test/free-tests-workflow-wiring.test.ts b/test/free-tests-workflow-wiring.test.ts index 613936f0d..d874d6cac 100644 --- a/test/free-tests-workflow-wiring.test.ts +++ b/test/free-tests-workflow-wiring.test.ts @@ -174,7 +174,8 @@ describe('free-tests workflow wiring', () => { expect(workflow.env.FREE_SUITE_SHARDS).toMatch(/^[1-9][0-9]?$/); expect(planner.outputs.runner).toBe('${{ steps.plan.outputs.runner }}'); expect(workflow.jobs['free-suite']['runs-on']).toBe('${{ needs.free-plan.outputs.runner }}'); - expect(plan.env).toEqual({ RUNNER_OVERRIDE: '${{ inputs.free_runner }}', SHARDS_OVERRIDE: '${{ inputs.free_shards }}' }); + expect(workflow.env.FREE_SUITE_JOBS).toMatch(/^[1-8]$/); + expect(plan.env).toEqual({ RUNNER_OVERRIDE: '${{ inputs.free_runner }}', SHARDS_OVERRIDE: '${{ inputs.free_shards }}', JOBS_OVERRIDE: '${{ inputs.free_jobs }}' }); expect(plan.run).not.toContain('${{'); const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'free plan ')); const bin = path.join(directory, 'bin'); @@ -185,7 +186,7 @@ describe('free-tests workflow wiring', () => { const output = path.join(directory, 'output'); fs.writeFileSync(output, ''); const result = spawnSync('bash', ['-c', plan.run], { - env: { ...process.env, ...workflow.env, RUNNER_OVERRIDE: '', SHARDS_OVERRIDE: '', ...overrides, + env: { ...process.env, ...workflow.env, RUNNER_OVERRIDE: '', SHARDS_OVERRIDE: '', JOBS_OVERRIDE: '', ...overrides, PATH: `${toShell(bin)}:${process.env.PATH}`, RUNNER_TEMP: toShell(directory), GITHUB_OUTPUT: toShell(output), GITHUB_STEP_SUMMARY: toShell(path.join(directory, 'summary')) }, encoding: 'utf8', timeout: 5000, @@ -199,12 +200,13 @@ describe('free-tests workflow wiring', () => { const unset = resolve({}); expect(unset.status).toBe(0); expect(unset.outputs.runner).toBe(workflow.env.FREE_SUITE_RUNNER); - expect(unset.args).toContain(`--shards ${workflow.env.FREE_SUITE_SHARDS}`); - const override = resolve({ RUNNER_OVERRIDE: 'ubicloud-standard-4', SHARDS_OVERRIDE: '16' }); + expect(unset.args).toContain(`--shards ${workflow.env.FREE_SUITE_SHARDS} --shard-jobs ${workflow.env.FREE_SUITE_JOBS}`); + const override = resolve({ RUNNER_OVERRIDE: 'ubicloud-standard-4', SHARDS_OVERRIDE: '16', JOBS_OVERRIDE: '3' }); expect(override.status).toBe(0); expect(override.outputs.runner).toBe('ubicloud-standard-4'); - expect(override.args).toContain('--shards 16'); - const refusals: Record[] = [{ RUNNER_OVERRIDE: 'ubuntu-24.04' }, { RUNNER_OVERRIDE: 'ubicloud-standard-8 x' }, { SHARDS_OVERRIDE: '0' }, { SHARDS_OVERRIDE: '41' }, { SHARDS_OVERRIDE: '16; true' }]; + expect(override.args).toContain('--shards 16 --shard-jobs 3'); + const refusals: Record[] = [{ RUNNER_OVERRIDE: 'ubuntu-24.04' }, { RUNNER_OVERRIDE: 'ubicloud-standard-8 x' }, { SHARDS_OVERRIDE: '0' }, { SHARDS_OVERRIDE: '41' }, { SHARDS_OVERRIDE: '16; true' }, + { JOBS_OVERRIDE: '0' }, { JOBS_OVERRIDE: '9' }, { JOBS_OVERRIDE: '2; true' }, { JOBS_OVERRIDE: '-1' }]; for (const bad of refusals) { const refused = resolve(bad); expect(refused.status, JSON.stringify(bad)).not.toBe(0); diff --git a/test/shard-jobs.test.ts b/test/shard-jobs.test.ts index 83714013d..af18dfa54 100644 --- a/test/shard-jobs.test.ts +++ b/test/shard-jobs.test.ts @@ -10,11 +10,11 @@ import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; import { - collectFreeTestFiles, createFreeCiPlan, eligibleFreeRetryFiles, parseCliOptions, runFreeShard, TREE_MUTATING, verifyFreeCiResults, + collectFreeTestFiles, createFreeCiPlan, eligibleFreeRetryFiles, parseCliOptions, runFreeShard, TREE_MUTATING, validateFreeCiPlan, verifyFreeCiResults, type FreeCiResult, type FreeShardOutcome, } from '../scripts/test-free-shards'; import { - assertSoloSeedSample, assignSubShards, ciRunContext, FREE_JOBS_INVALID, mergeSubShardOutcomes, parseJobsFlag, parseJunitCounts, + assertSoloSeedSample, assignSubShards, ciRunContext, FREE_JOBS_INVALID, mergeSubShardOutcomes, parseJobsFlag, parseJunitCounts, plannedShardJobs, plannedShardJobsProblem, readJunitReport, runCiShard, SLOW_UNDER_LOAD, subShardRecordProblems, subShardTimeoutText, validateJobsForShard, type CiShardDeps, type RunShardOptions, } from '../scripts/lib/shard-jobs'; @@ -89,6 +89,52 @@ describe('--jobs validation (FREE_JOBS_INVALID)', () => { }); }); +describe('per-shard k in the plan (--ci-plan --shard-jobs)', () => { + test('--shard-jobs belongs to --ci-plan only and takes the same positive whole numbers', () => { + const base = ['--ci-plan', 'plan.json', '--shards', '4']; + expect(parseCliOptions([...base, '--shard-jobs', '2']).shardJobs).toBe(2); + expect('shardJobs' in parseCliOptions(base)).toBe(false); + for (const raw of ['0', '2.5', 'x']) expect(() => parseCliOptions([...base, '--shard-jobs', raw])).toThrow(`${FREE_JOBS_INVALID}: --shard-jobs needs a positive whole number`); + expect(() => parseCliOptions(['--shard-jobs', '2'])).toThrow(`${FREE_JOBS_INVALID}: --shard-jobs needs --ci-plan`); + expect(() => parseCliOptions(['--ci-run', 'p.json', '--shard', '1', '--result', 'r.json', '--shard-jobs', '2'])).toThrow(FREE_JOBS_INVALID); + }); + + test('each shard plans min(k, its parallel files); the exclusive shard and one-file shards stay one child, and k = 1 keeps the plan id', () => { + const withExclusive = [...files, EXCLUSIVE]; + const plain = createFreeCiPlan(withExclusive, 3, durations, 'revision-a'); + expect(createFreeCiPlan(withExclusive, 3, durations, 'revision-a', 1)).toEqual(plain); + expect(plain.shards.every(shard => !('jobs' in shard))).toBe(true); + const planned = createFreeCiPlan(withExclusive, 3, durations, 'revision-a', 4); + expect(planned.id).not.toBe(plain.id); + expect(planned.shards.map(shard => shard.files)).toEqual(plain.shards.map(shard => shard.files)); + const exclusiveShard = planned.shards.at(-1)!; + expect(exclusiveShard.files).toEqual([EXCLUSIVE]); + expect('jobs' in exclusiveShard).toBe(false); + for (const shard of planned.shards.slice(0, -1)) expect(shard.jobs).toBe(Math.min(4, shard.files.length) > 1 ? Math.min(4, shard.files.length) : undefined); + expect(() => validateFreeCiPlan(planned, withExclusive, 'revision-a')).not.toThrow(); + expect(plannedShardJobs(['a', 'b', 'c'], 2, file => file === 'a')).toBe(2); + expect(plannedShardJobs(['a', 'b'], 4, file => file === 'a')).toBeUndefined(); + expect(plannedShardJobs([], 4, () => false)).toBeUndefined(); + }); + + test('a plan whose per-shard k is out of range is invalid, and every shard result must have run at its planned k', () => { + const isSerial = (file: string) => file === EXCLUSIVE; + expect(plannedShardJobsProblem({ shard: 1, files }, isSerial)).toBeNull(); + expect(plannedShardJobsProblem({ shard: 1, files, jobs: 7 }, isSerial)).toBeNull(); + for (const jobs of [1, 0, 8, 2.5, '2']) expect(plannedShardJobsProblem({ shard: 1, files, jobs }, isSerial)).toContain('CI shard 1 plans --jobs'); + expect(plannedShardJobsProblem({ shard: 2, files: [EXCLUSIVE, files[0]], jobs: 2 }, isSerial)).toContain('for 1 parallel file(s)'); + const plan = createFreeCiPlan(files, 1, durations, 'revision-a', 2); + expect(plan.shards[0].jobs).toBe(2); + const tampered = structuredClone(plan); + tampered.shards[0].jobs = 99; + expect(() => validateFreeCiPlan(tampered, files, 'revision-a')).toThrow(); + const k1: FreeCiResult = { planId: plan.id, revision: 'revision-a', outcome: passed(plan.shards[0].files), retry: null }; + expect(() => verifyFreeCiResults(plan, [k1])).toThrow('CI shard 1 ran at --jobs 1; the plan says 2'); + const plainPlan = createFreeCiPlan(files, 1, durations, 'revision-a'); + expect(() => verifyFreeCiResults(plainPlan, [{ ...k1, planId: plainPlan.id }])).not.toThrow(); + }); +}); + describe('k = 1 golden: PR 2 changes nothing in CI', () => { test('one child over the whole planned shard with the same wall, one retry pass, and the same result keys', async () => { const h = harness(files); @@ -160,7 +206,8 @@ describe('k > 1: merge, rejection and one retry pass per CI shard', () => { summary: { testsRan: 16, filesRan: 8, sawTerminalSummary: true } }); expect(result.subShards!.map(record => record.status)).toEqual(['passed', 'passed', 'passed', 'passed']); expect(h.retries).toHaveLength(1); - expect(() => verifyFreeCiResults(h.plan, [result])).not.toThrow(); + expect(() => verifyFreeCiResults({ ...h.plan, shards: [{ ...h.shard, jobs: 3 }] }, [result])).not.toThrow(); + expect(() => verifyFreeCiResults(h.plan, [result])).toThrow('CI shard 1 ran at --jobs 3; the plan says 1'); }); test('failures from every sub-shard are collected first, so k never multiplies the five-file retry cap', async () => { @@ -263,7 +310,7 @@ describe('k > 1: merge, rejection and one retry pass per CI shard', () => { }); test('--ci-verify rejects tampered sub-shard records', () => { - const plan = createFreeCiPlan(files, 1, durations, 'revision-a'); + const plan = createFreeCiPlan(files, 1, durations, 'revision-a', 2); const shard = plan.shards[0]; const assignments = assignSubShards(plan.id, shard, 2, () => false, packFiles => [packFiles.slice(0, 3), packFiles.slice(3)]); const merged = mergeSubShardOutcomes(shard, assignments.map(assignment => ({ assignment, outcome: passed(assignment.files) })), 10); diff --git a/test/windows-native-workflows.test.ts b/test/windows-native-workflows.test.ts index 269943058..e9dfbb03a 100644 --- a/test/windows-native-workflows.test.ts +++ b/test/windows-native-workflows.test.ts @@ -4,7 +4,9 @@ * that moved out of it. */ import { expect, test } from 'bun:test'; -import { existsSync, readFileSync } from 'node:fs'; +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import os from 'node:os'; import path from 'node:path'; import { collectFreeTestFiles, createFreeCiPlan, curateWindowsSafe, parseCliOptions, validateFreeCiPlan } from '../scripts/test-free-shards'; import { windowsCurationLine } from '../scripts/lib/free-ci-health'; @@ -25,7 +27,7 @@ const WINDOWS_CURATED_FLOOR = 638; test('the Windows lane plans, runs one strict shard per job and verifies every result', () => { expect(Object.keys(windows.on).sort()).toEqual(['pull_request', 'push', 'workflow_call', 'workflow_dispatch']); expect(windows.on.push).toEqual({ branches: ['main'] }); - expect(Object.keys(windows.on.workflow_dispatch.inputs)).toEqual(['record_durations']); + expect(Object.keys(windows.on.workflow_dispatch.inputs)).toEqual(['record_durations', 'windows_jobs']); expect(Object.keys(windows.on.workflow_call.inputs)).toEqual(['record_durations']); const plan = windows.jobs['windows-plan']; expect(plan['runs-on']).toBe('ubuntu-24.04'); @@ -47,6 +49,36 @@ test('the Windows lane plans, runs one strict shard per job and verifies every r expect(aggregate.steps.at(-1).run).toContain('--windows-only --ci-verify "$RUNNER_TEMP/windows-plan.json" --results '); }); +test.skipIf(!Bun.which('bash'))('windows-plan passes the one sub-shard constant, or a validated dispatch override, to --shard-jobs', () => { + const planStep = windows.jobs['windows-plan'].steps.find((step: any) => step.id === 'plan'); + expect(windows.env.WINDOWS_FREE_JOBS).toMatch(/^[1-4]$/); + expect(planStep.env.JOBS_OVERRIDE).toBe('${{ inputs.windows_jobs }}'); + expect(planStep.run).not.toContain('${{'); + const directory = mkdtempSync(path.join(os.tmpdir(), 'windows plan ')); + const bin = path.join(directory, 'bin'); + mkdirSync(bin); + writeFileSync(path.join(bin, 'bun'), '#!/bin/sh\nprintf "%s\\n" "$*" > "$RUNNER_TEMP/bun-args"\necho \'{"shard":[1]}\'\n', { mode: 0o755 }); + const toShell = (p: string) => p.split(path.sep).join('/'); + const resolve = (override: string) => { + rmSync(path.join(directory, 'bun-args'), { force: true }); + const result = spawnSync('bash', ['-c', planStep.run], { + env: { ...process.env, ...windows.env, JOBS_OVERRIDE: override, PATH: `${toShell(bin)}:${process.env.PATH}`, RUNNER_TEMP: toShell(directory), + GITHUB_OUTPUT: toShell(path.join(directory, 'output')), GITHUB_STEP_SUMMARY: toShell(path.join(directory, 'summary')) }, + encoding: 'utf8', timeout: 5000, + }); + return { status: result.status, args: existsSync(path.join(directory, 'bun-args')) ? readFileSync(path.join(directory, 'bun-args'), 'utf8') : '' }; + }; + try { + expect(resolve('')).toEqual({ status: 0, args: expect.stringContaining(`--shards 6 --shard-jobs ${windows.env.WINDOWS_FREE_JOBS}`) }); + expect(resolve('2')).toEqual({ status: 0, args: expect.stringContaining('--shards 6 --shard-jobs 2') }); + for (const bad of ['0', '5', '2; true', 'x']) { + const refused = resolve(bad); + expect(refused.status, bad).not.toBe(0); + expect(refused.args, bad).toBe(''); + } + } finally { rmSync(directory, { recursive: true, force: true }); } +}); + test('a Windows CI plan binds the curated file set, so it never verifies as the full Linux suite', () => { for (const argv of [['--windows-only', '--ci-plan', 'p.json'], ['--windows-only', '--ci-run', 'p.json', '--shard', '1', '--result', 'r.json'], ['--windows-only', '--ci-verify', 'p.json', '--results', 'd']]) { expect(parseCliOptions(argv).windowsOnly).toBe(true);