diff --git a/scripts/test-free-shards.ts b/scripts/test-free-shards.ts index 7e73c048a..17766544d 100755 --- a/scripts/test-free-shards.ts +++ b/scripts/test-free-shards.ts @@ -73,6 +73,7 @@ * bun run scripts/test-free-shards.ts --wall-timeout 600 # override the kill deadline * bun run scripts/test-free-shards.ts --verbose # forward the full child stream * bun run scripts/test-free-shards.ts --quick # explicit fast subset, not acceptance + * bun run scripts/test-free-shards.ts --files-from list.txt # exactly these planned files (one per line) * bun run scripts/test-free-shards.ts --attribute-home # name real-home writers: each file alone, private HOME * bun run scripts/test-free-shards.ts --ci-plan plan.json --shards 20 * bun run scripts/test-free-shards.ts --ci-run plan.json --shard 1 --result result.json @@ -540,6 +541,7 @@ type CliOptions = { ciVerify: string | null; result: string | null; results: string | null; + filesFrom: string | null; }; export function parseCliOptions(argv: string[]): CliOptions { @@ -554,8 +556,8 @@ export function parseCliOptions(argv: string[]): CliOptions { let wallTimeoutExplicit = false; let quick = false; let attributeHome = false; - const paths: Record<'ciPlan' | 'ciRun' | 'ciVerify' | 'result' | 'results', string | null> = { - ciPlan: null, ciRun: null, ciVerify: null, result: null, results: null, + const paths: Record<'ciPlan' | 'ciRun' | 'ciVerify' | 'result' | 'results' | 'filesFrom', string | null> = { + ciPlan: null, ciRun: null, ciVerify: null, result: null, results: null, filesFrom: null, }; const pathFlag = (flag: string, key: keyof typeof paths) => (next: () => string | undefined) => { const value = next(); @@ -576,6 +578,7 @@ export function parseCliOptions(argv: string[]): CliOptions { '--ci-verify': pathFlag('--ci-verify', 'ciVerify'), '--result': pathFlag('--result', 'result'), '--results': pathFlag('--results', 'results'), + '--files-from': pathFlag('--files-from', 'filesFrom'), '--shards': (next) => { const value = next(); if (!value) throw new Error('Missing value for --shards'); @@ -595,6 +598,7 @@ export function parseCliOptions(argv: string[]): CliOptions { }); const ciModes = [paths.ciPlan, paths.ciRun, paths.ciVerify].filter(Boolean).length; + if (paths.filesFrom && (quick || windowsOnly || listOnly || ciModes || recordDurations || attributeHome)) throw new Error('--files-from cannot be combined with --quick, --windows-only, --list, --record-durations, --attribute-home or the --ci-* modes'); if (ciModes > 1 || (ciModes && (quick || listOnly || dryRun || recordDurations || attributeHome))) throw new Error('CI modes cannot be combined with other selection modes'); if (paths.ciRun && (shardIndex === null || !paths.result)) throw new Error('--ci-run requires --shard and --result'); if (paths.ciVerify && !paths.results) throw new Error('--ci-verify requires --results'); @@ -602,6 +606,47 @@ export function parseCliOptions(argv: string[]): CliOptions { return { dryRun, listOnly, recordDurations, windowsOnly, verbose, shardCount, shardIndex, wallTimeoutMs, wallTimeoutExplicit, quick, attributeHome, ...paths }; } +const describeLines = (lines: string[]) => `${lines.slice(0, 5).join(', ')}${lines.length > 5 ? ` (+${lines.length - 5} more)` : ''}`; + +/** + * `--files-from `: one repo-relative path per line (LF or CRLF), blank + * lines ignored. Rejected before any planning: an unreadable file, a malformed + * or repeated line, an empty list, or a path outside the free planned set. + */ +export function readFilesFrom(file: string, planned: readonly string[]): string[] { + let text: string; + try { + text = fs.readFileSync(file, 'utf8'); + } catch (error) { + throw new Error(`--files-from ${file}: cannot read it (${(error as NodeJS.ErrnoException).code ?? String(error)})`); + } + const files: string[] = []; + const malformed: string[] = []; + text.split(/\r?\n/).forEach((line, index) => { + if (line === '') return; + const segments = line.split('/'); + if (line !== line.trim() || line.includes('\\') || line.startsWith('/') || /^[A-Za-z]:/.test(line) + || segments.some(segment => segment === '' || segment === '.' || segment === '..')) { + malformed.push(`line ${index + 1} ${JSON.stringify(line)}`); + } else if (files.includes(line)) { + malformed.push(`line ${index + 1} repeats ${line}`); + } else { + files.push(line); + } + }); + if (malformed.length) throw new Error(`--files-from ${file}: ${malformed.length} malformed line(s): ${describeLines(malformed)}; fix: one repo-relative path per line (forward slashes, no surrounding spaces, no ./ or ../, no repeats)`); + if (!files.length) throw new Error(`--files-from ${file}: the list is empty (blank lines are ignored); nothing to run`); + const plannedSet = new Set(planned); + const unknown = files.filter(f => !plannedSet.has(f)); + if (unknown.length) throw new Error(`--files-from ${file}: ${unknown.length} path(s) not in the free planned set: ${describeLines(unknown)}; fix: use paths from \`bun run scripts/test-free-shards.ts --list\``); + return files; +} + +/** sha256 of the sorted list, one path per line: the digest a remote gate compares against its upload. */ +export function filesDigest(files: readonly string[]): string { + return createHash('sha256').update([...files].sort().map(f => `${f}\n`).join('')).digest('hex'); +} + function formatShardSummary(shards: string[][]): string[] { return shards.map((files, index) => { const preview = files.slice(0, 3).join(', '); @@ -1887,6 +1932,10 @@ async function main(): Promise { } let files = allFiles; + if (options.filesFrom) { + files = readFilesFrom(options.filesFrom, allFiles); + console.log(`[test:free] files-from ${options.filesFrom}: ${files.length} files, sha256=${filesDigest(files)}`); + } if (options.quick) { const missing = QUICK_CORE.filter(file => !allFiles.includes(file)); if (missing.length) throw new Error(`Quick core files missing: ${missing.join(', ')}`); @@ -2015,7 +2064,15 @@ async function main(): Promise { outcomes.push(exclusiveOutcome); } - return Math.max(homeChange ? 1 : 0, (await retryFailedFreeFiles(outcomes, totalShards, options)).exitCode); + const exitCode = Math.max(homeChange ? 1 : 0, (await retryFailedFreeFiles(outcomes, totalShards, options)).exitCode); + if (options.filesFrom) { + // The machine-read result for a remote gate: tests from each shard's own summary; failed counts failing files, plus one per failed shard with none attributed. + const tests = outcomes.reduce((sum, outcome) => sum + (outcome.summary?.testsRan ?? 0), 0); + const failing = outcomes.filter(outcome => outcome.status !== 'passed'); + const failed = exitCode === 0 ? 0 : Math.max(1, new Set(failing.flatMap(outcome => outcome.failingFiles)).size + failing.filter(outcome => !outcome.failingFiles.length).length); + console.log(`[test:free] FILES-FROM RESULT: files=${files.length} tests=${tests} failed=${failed} exit=${exitCode}`); + } + return exitCode; } /** Dirty generated artifacts (SKILL.md / host outputs) after a failed exclusive shard. */ diff --git a/test/fixtures/shard-cli-parity/results.json b/test/fixtures/shard-cli-parity/results.json index b24a4fa80..f8986404f 100644 --- a/test/fixtures/shard-cli-parity/results.json +++ b/test/fixtures/shard-cli-parity/results.json @@ -1,11 +1,12 @@ { - "recordedFrom": "96764e80 (pre-engine base runners); paid lane re-recorded at v1.91.12.0 for the marathon tier and the --slice-budget, --max-parallel, --case and --trials flags (PR 2999); free lane re-recorded for the --attribute-home flag (PR 3014)", + "recordedFrom": "96764e80 (pre-engine base runners); paid lane re-recorded at v1.91.12.0 for the marathon tier and the --slice-budget, --max-parallel, --case and --trials flags (PR 2999); free lane re-recorded for the --attribute-home flag (PR 3014) and the --files-from flag (fix wave 3, W2-A3)", "free": [ { "argv": [], "result": { "ok": { "dryRun": false, + "filesFrom": null, "listOnly": false, "recordDurations": false, "windowsOnly": false, @@ -33,6 +34,7 @@ "result": { "ok": { "dryRun": true, + "filesFrom": null, "listOnly": true, "recordDurations": false, "windowsOnly": false, @@ -59,6 +61,7 @@ "result": { "ok": { "dryRun": false, + "filesFrom": null, "listOnly": true, "recordDurations": false, "windowsOnly": true, @@ -84,6 +87,7 @@ "result": { "ok": { "dryRun": false, + "filesFrom": null, "listOnly": false, "recordDurations": true, "windowsOnly": false, @@ -109,6 +113,7 @@ "result": { "ok": { "dryRun": false, + "filesFrom": null, "listOnly": false, "recordDurations": false, "windowsOnly": false, @@ -137,6 +142,7 @@ "result": { "ok": { "dryRun": false, + "filesFrom": null, "listOnly": false, "recordDurations": false, "windowsOnly": false, @@ -163,6 +169,7 @@ "result": { "ok": { "dryRun": false, + "filesFrom": null, "listOnly": false, "recordDurations": false, "windowsOnly": false, @@ -191,6 +198,7 @@ "result": { "ok": { "dryRun": false, + "filesFrom": null, "listOnly": false, "recordDurations": false, "windowsOnly": false, @@ -221,6 +229,7 @@ "result": { "ok": { "dryRun": false, + "filesFrom": null, "listOnly": false, "recordDurations": false, "windowsOnly": false, @@ -249,6 +258,7 @@ "result": { "ok": { "dryRun": false, + "filesFrom": null, "listOnly": false, "recordDurations": false, "windowsOnly": false, @@ -399,6 +409,7 @@ "result": { "ok": { "dryRun": false, + "filesFrom": null, "listOnly": false, "recordDurations": false, "windowsOnly": false, diff --git a/test/test-free-shards-files-from.test.ts b/test/test-free-shards-files-from.test.ts new file mode 100644 index 000000000..72bd67ca5 --- /dev/null +++ b/test/test-free-shards-files-from.test.ts @@ -0,0 +1,85 @@ +/** + * W2-A3 (fix wave 3): test-free-shards.ts --files-from plans exactly + * the listed files from the free planned set and rejects, before planning, an + * unknown path, an empty list or a malformed line. The remote gate + * (scripts/ubicloud/pregate-remote.sh) compares the printed digest with its + * upload; the full run's FILES-FROM RESULT line is covered there. + */ +import { afterAll, describe, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { collectFreeTestFiles, filesDigest, parseCliOptions, readFilesFrom } from '../scripts/test-free-shards'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'free-files-from-')); +afterAll(() => fs.rmSync(DIR, { recursive: true, force: true })); +let n = 0; +const list = (text: string) => { + const file = path.join(DIR, `list-${++n}.txt`); + fs.writeFileSync(file, text); + return file; +}; +const PLANNED = ['browse/test/a.test.ts', 'test/b.test.ts', 'test/c.test.ts', 'test/d.test.ts', 'test/e.test.ts', 'test/f.test.ts', 'test/g.test.ts']; + +describe('--files-from parsing', () => { + test('one path per line; blank lines and CRLF endings are accepted; order is kept', () => { + expect(readFilesFrom(list('test/c.test.ts\r\n\r\nbrowse/test/a.test.ts\n\n'), PLANNED)).toEqual(['test/c.test.ts', 'browse/test/a.test.ts']); + }); + + test('paths outside the free planned set are rejected with the first five, a count and the --list fix', () => { + const file = list(['test/b.test.ts', ...['x1', 'x2', 'x3', 'x4', 'x5', 'x6', 'x7'].map(name => `test/${name}.test.ts`)].join('\n')); + expect(() => readFilesFrom(file, PLANNED)).toThrow( + `--files-from ${file}: 7 path(s) not in the free planned set: test/x1.test.ts, test/x2.test.ts, test/x3.test.ts, test/x4.test.ts, test/x5.test.ts (+2 more); fix: use paths from \`bun run scripts/test-free-shards.ts --list\``); + }); + + test('an empty list, an unreadable file and every malformed line shape are rejected', () => { + const empty = list('\n\r\n\n'); + expect(() => readFilesFrom(empty, PLANNED)).toThrow(`--files-from ${empty}: the list is empty (blank lines are ignored); nothing to run`); + expect(() => readFilesFrom(path.join(DIR, 'absent.txt'), PLANNED)).toThrow('cannot read it (ENOENT)'); + const bad = [' test/b.test.ts', 'test/c.test.ts ', './test/d.test.ts', 'test/../test/e.test.ts', '/abs/test/f.test.ts', 'test\\g.test.ts', 'C:/x.test.ts', 'test//b.test.ts']; + for (const line of bad) { + expect(() => readFilesFrom(list(`test/b.test.ts\n${line}\n`), PLANNED), line).toThrow(`1 malformed line(s): line 2 ${JSON.stringify(line)}; fix: one repo-relative path per line`); + } + expect(() => readFilesFrom(list('test/b.test.ts\ntest/c.test.ts\ntest/b.test.ts\n'), PLANNED)).toThrow('line 3 repeats test/b.test.ts'); + }); + + test('the digest is the sorted list, so a reordered upload still matches', () => { + expect(filesDigest(['test/c.test.ts', 'browse/test/a.test.ts'])).toBe(filesDigest(['browse/test/a.test.ts', 'test/c.test.ts'])); + expect(filesDigest(['test/c.test.ts'])).not.toBe(filesDigest(['test/c.test.ts', 'test/b.test.ts'])); + }); + + test('it conflicts with --quick, --windows-only, --list, --record-durations, --attribute-home and every --ci-* mode', () => { + expect(parseCliOptions(['--files-from', 'x.txt', '--dry-run']).filesFrom).toBe('x.txt'); + expect(() => parseCliOptions(['--files-from'])).toThrow('Missing path for --files-from'); + for (const extra of [['--quick'], ['--windows-only'], ['--list'], ['--record-durations'], ['--attribute-home'], ['--ci-plan', 'p.json'], ['--ci-run', 'p.json', '--shard', '1', '--result', 'r.json'], ['--ci-verify', 'p.json', '--results', 'r']]) { + expect(() => parseCliOptions(['--files-from', 'x.txt', ...extra]), extra[0]).toThrow('--files-from cannot be combined with --quick, --windows-only, --list, --record-durations, --attribute-home or the --ci-* modes'); + } + }); +}); + +describe('--files-from planning through the CLI', () => { + const cli = (...args: string[]) => { + const r = Bun.spawnSync([process.execPath, path.join(ROOT, 'scripts/test-free-shards.ts'), ...args], { cwd: ROOT, timeout: 60_000 }); + return { exit: r.exitCode, out: r.stdout.toString() + r.stderr.toString() }; + }; + + test('a dry run plans exactly the listed files and prints their digest', () => { + const planned = collectFreeTestFiles(ROOT); + const chosen = [planned[3]!, planned[0]!]; + const file = list(chosen.join('\n') + '\n'); + const r = cli('--files-from', file, '--dry-run'); + expect(r.exit, r.out).toBe(0); + expect(r.out).toContain(`[test:free] files-from ${file}: 2 files, sha256=${filesDigest(chosen)}`); + expect(r.out).toContain('Would run 2 files across'); + const listed = r.out.split('\n').filter(line => line.startsWith('Shard ') && line.includes('->')).flatMap(line => line.split('-> ')[1]!.split(', ')); + expect(listed.sort()).toEqual([...chosen].sort()); + }, 60_000); + + test('a rejected list exits 1 before planning', () => { + const r = cli('--files-from', list('test/not-a-real-file.test.ts\n'), '--dry-run'); + expect(r.exit).toBe(1); + expect(r.out).toContain('1 path(s) not in the free planned set: test/not-a-real-file.test.ts'); + expect(r.out).not.toContain('Would run'); + }, 60_000); +});