diff --git a/changelog.d/gsta258-h-per-file-adoption.md b/changelog.d/gsta258-h-per-file-adoption.md new file mode 100644 index 000000000..13c28ab77 --- /dev/null +++ b/changelog.d/gsta258-h-per-file-adoption.md @@ -0,0 +1,4 @@ +- gstack's own CI carry now runs the test files CI skipped entirely. Free Tests records `skipped_files` (the planned files whose every test skipped on the runner) in its `tested-tree` record, so a /ship carry runs those files here through the `bun run test` lane's `per_file` runner instead of counting them as run (GST-1190). A file that skips here too is not owed (GST-1314), so the carry no longer falls back to the full lane. + +### Tests +- `test/ci-tree-reuse.test.ts`: only a file whose every test skipped is listed, a partly skipped file stays covered, and a run without complete per-file counts records no list. diff --git a/docs/pregate.md b/docs/pregate.md index 678682a71..8621a0947 100644 --- a/docs/pregate.md +++ b/docs/pregate.md @@ -795,10 +795,9 @@ once: does a file whose log hit the 2 MB cap, since a cut log's last summary may not be the run's. Any of these refusals runs the whole lane here. An entry with no `per_file` refuses as `inventory_skipped`, naming the files, and the lane runs here whole. A partly - skipped file ran tests and stays covered. gstack's `bun run test` lane - declares `per_file`, and its Free Tests writer starts emitting the list in a - follow-up pull request, so the runner is in place on the base before any run - lists a skipped file. A `tree.json` + skipped file ran tests and stays covered. gstack's own Free Tests writer + (`scripts/ci-tree-reuse.ts record`) emits the list from complete per-file + counts, and gstack's `bun run test` lane declares `per_file`. A `tree.json` without the key (older writers, other repos) is read as before, with every listed file counted as run; add the list to your writer to close that gap. The remote gate's clearance does not use it: that gate runs on a Linux runner diff --git a/scripts/ci-tree-reuse.ts b/scripts/ci-tree-reuse.ts index 49256933f..cc7e605df 100755 --- a/scripts/ci-tree-reuse.ts +++ b/scripts/ci-tree-reuse.ts @@ -6,7 +6,8 @@ * The producer. A Free Tests or Windows Free Tests aggregate runs it only after its strict verification * passed, and uploads the file as the `tested-tree` artifact: run_id, run_attempt, event, sha (GITHUB_SHA), * head_sha (the pull request head for pull_request, else GITHUB_SHA), tree (git's tree of GITHUB_SHA), - * workflow, os (the OSes the run's test jobs ran on), files (every test file in the plan) and tests, plus + * workflow, os (the OSes the run's test jobs ran on), files (every test file in the plan), tests and, from complete + * per-file counts, skipped_files (the planned files whose every test skipped on the runner; GST-1190), plus * sizing_experiment: true when a dispatch overrode the runner, shards or sub-shards (SIZING_OVERRIDE), which * verifyRun never accepts as evidence. * @@ -134,7 +135,8 @@ export function findPrReuse(deps: PrReuseDeps, o: { repo: string; base: string; } /** tree.json for the `tested-tree` artifact; throws on any missing or malformed input, so a broken record fails the aggregate instead of uploading. */ -export function treeRecord(env: NodeJS.ProcessEnv, o: { plan: { shards: Array<{ files: string[] }> }; results: Array<{ outcome?: { summary?: { testsRan?: number } } }>; os: string[]; tree: string }) { +type FileCounts = { tests: number; pass: number; fail: number }; +export function treeRecord(env: NodeJS.ProcessEnv, o: { plan: { shards: Array<{ files: string[] }> }; results: Array<{ outcome?: { summary?: { testsRan?: number }; counts?: { complete?: boolean; files?: Record } } }>; os: string[]; tree: string }) { const int = (name: string) => { const v = Number(env[name]); if (!Number.isSafeInteger(v) || v < 1) throw new Error(`${name} is not a positive integer`); return v; }; const sha = env.GITHUB_SHA ?? ''; const event = env.GITHUB_EVENT_NAME ?? ''; @@ -149,7 +151,11 @@ export function treeRecord(env: NodeJS.ProcessEnv, o: { plan: { shards: Array<{ if (!files.length) throw new Error('the plan lists no test files'); const counts = o.results.map(r => r.outcome?.summary?.testsRan); const tests = o.results.length && counts.every(n => Number.isSafeInteger(n) && n! >= 0) ? counts.reduce((a, b) => a! + b!, 0)! : files.length; - return { run_id: int('GITHUB_RUN_ID'), run_attempt: int('GITHUB_RUN_ATTEMPT'), event, sha, head_sha: head, tree: o.tree, workflow, os: o.os, files, tests, ...(env.SIZING_OVERRIDE ? { sizing_experiment: true } : {}) }; + // skipped_files (GST-1190): planned files whose every test skipped on this runner, so a carry runs them where they might not + // skip. A partly skipped file ran tests and stays covered. Only from complete per-file counts; otherwise left out (unknown). + const counted = o.results.length > 0 && o.results.every(r => r.outcome?.counts?.complete === true && r.outcome.counts.files); + const skipped_files = counted ? files.filter(f => o.results.some(r => { const c = r.outcome!.counts!.files![f]; return !!c && c.tests > 0 && c.pass === 0 && c.fail === 0; })) : undefined; + return { run_id: int('GITHUB_RUN_ID'), run_attempt: int('GITHUB_RUN_ATTEMPT'), event, sha, head_sha: head, tree: o.tree, workflow, os: o.os, files, tests, ...(skipped_files ? { skipped_files } : {}), ...(env.SIZING_OVERRIDE ? { sizing_experiment: true } : {}) }; } function flag(args: string[], name: string): string | undefined { const i = args.indexOf(name); return i >= 0 ? args[i + 1] : undefined; } @@ -164,7 +170,7 @@ function record(args: string[]): number { const out = need(args, '--out'); fs.mkdirSync(path.dirname(path.resolve(out)), { recursive: true }); fs.writeFileSync(out, JSON.stringify(rec, null, 2) + '\n'); - console.log(`TESTED_TREE: run ${rec.run_id}/${rec.run_attempt} ${rec.event} head ${rec.head_sha.slice(0, 12)} tree ${rec.tree.slice(0, 12)} files=${rec.files.length} tests=${rec.tests} os=${rec.os.join(',')}`); + console.log(`TESTED_TREE: run ${rec.run_id}/${rec.run_attempt} ${rec.event} head ${rec.head_sha.slice(0, 12)} tree ${rec.tree.slice(0, 12)} files=${rec.files.length} tests=${rec.tests} skipped_files=${rec.skipped_files?.length ?? 'unknown'} os=${rec.os.join(',')}`); return 0; } diff --git a/test/ci-tree-reuse.test.ts b/test/ci-tree-reuse.test.ts index 0c697dd3a..c5ec0cc9c 100644 --- a/test/ci-tree-reuse.test.ts +++ b/test/ci-tree-reuse.test.ts @@ -212,6 +212,19 @@ describe('treeRecord: the tested-tree producer', () => { expect(treeRecord(env({ SIZING_OVERRIDE: '1' }), { plan, results, os: ['Linux'], tree: TREE })).toMatchObject({ sizing_experiment: true }); expect(treeRecord(env({ SIZING_OVERRIDE: '' }), { plan, results, os: ['Linux'], tree: TREE })).not.toHaveProperty('sizing_experiment'); }); + // Value: protects=a carry knows which files ran nothing on CI (GST-1190, harness-evals: 75 tests skipped on the runner); fails_when=a fully + // skipped file is not listed, a partly skipped one is, or a run without complete counts claims a list + test('skipped_files lists only files whose every test skipped; a partly skipped file stays covered; incomplete counts leave it out', () => { + const n = (tests: number, pass: number, skip: number) => ({ tests, pass, fail: 0, skip, todo: 0, expects: pass }); + const counted: Array<{ outcome: { summary: { testsRan: number }; counts: { complete: boolean; files: Record> } } }> = [ + { outcome: { summary: { testsRan: 7 }, counts: { complete: true, files: { 'test/a.test.ts': n(3, 0, 3), 'test/b.test.ts': n(4, 2, 2) } } } }, + { outcome: { summary: { testsRan: 5 }, counts: { complete: true, files: { 'test/c.test.ts': n(5, 5, 0) } } } }, + ]; + expect(treeRecord(env(), { plan, results: counted, os: ['Linux'], tree: TREE }).skipped_files).toEqual(['test/a.test.ts']); + expect(treeRecord(env(), { plan, results, os: ['Linux'], tree: TREE })).not.toHaveProperty('skipped_files'); + const partial = [counted[0]!, { outcome: { ...counted[1]!.outcome, counts: { complete: false, files: {} } } }]; + expect(treeRecord(env(), { plan, results: partial, os: ['Linux'], tree: TREE })).not.toHaveProperty('skipped_files'); + }); test('missing or malformed inputs throw, so the aggregate fails instead of uploading a record', () => { expect(() => treeRecord(env({ GITHUB_EVENT_NAME: 'pull_request', GITHUB_SHA: MERGE }), { plan, results, os: ['Linux'], tree: TREE })).toThrow(/PR_HEAD_SHA/); expect(() => treeRecord(env({ GITHUB_RUN_ID: '' }), { plan, results, os: ['Linux'], tree: TREE })).toThrow(/GITHUB_RUN_ID/);