From 7f36eacbd28187389f1f7230d40eaa439590c94e Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Mon, 31 Aug 2026 04:50:31 +0000 Subject: [PATCH] =?UTF-8?q?feat:=20free-lane=20flake=20ledger=20=E2=80=94?= =?UTF-8?q?=20retry=20ON=20in=20CI,=20flaky-passes=20recorded=20and=20uplo?= =?UTF-8?q?aded?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The runner's attribution-gated flaky-retry pass (cap 5, truncation veto) was OFF in the required lane and its FLAKY-PASS evidence was console-only — so a single timing flake red the merge gate while repeat offenders stayed unenumerable. free-tests.yml now sets GSTACK_FREE_RETRY_FLAKY=1 and points GSTACK_FLAKE_LEDGER at runner.temp; every flaky-pass appends a JSONL entry (SINGLE writer: the parent runner — no concurrent-append hazard by construction; fail-open with a loud warning so a broken ledger can never red the lane) and the artifact uploads UNCONDITIONALLY — a flaky-pass run is green, which is exactly when the evidence matters. Wiring pinned by free-tests-workflow-wiring; ledger behavior unit-tested incl. the fail-open path. Matches 2026 industry practice (retry for data, quarantine out of merge-blocking but never out of logging) with the repo's own receipts. Co-Authored-By: Claude Fable 5 --- .github/workflows/free-tests.yml | 21 +++++++++ scripts/test-free-shards.ts | 51 +++++++++++++++++++++ test/flake-ledger.test.ts | 59 +++++++++++++++++++++++++ test/free-tests-workflow-wiring.test.ts | 11 +++++ 4 files changed, 142 insertions(+) create mode 100644 test/flake-ledger.test.ts diff --git a/.github/workflows/free-tests.yml b/.github/workflows/free-tests.yml index 76b579d27..f7dd8e7ab 100644 --- a/.github/workflows/free-tests.yml +++ b/.github/workflows/free-tests.yml @@ -129,6 +129,27 @@ jobs: run: xvfb-run -a bun run test:free env: GSTACK_EXPECT_BINARIES: "1" + # WS1 flake telemetry: a single timing flake must not red the only + # required lane — the runner's attribution-gated retry pass (cap 5, + # truncation veto) re-runs failing files once, serially, and a + # clean retry downgrades to a LOUD flaky-pass. Every flaky-pass is + # appended to the ledger (single writer: the parent runner) and + # uploaded below, so repeat offenders are an enumerable series — + # recorded and ranked, never masked. Pinned by + # test/free-tests-workflow-wiring.test.ts. + GSTACK_FREE_RETRY_FLAKY: "1" + GSTACK_FLAKE_LEDGER: ${{ runner.temp }}/flake-ledger.jsonl + + # Uploaded unconditionally (not just on failure): a flaky-pass run is + # GREEN — that's the point — so its evidence must survive green runs. + - name: Upload flake ledger + if: always() + uses: actions/upload-artifact@v7 + with: + name: flake-ledger + path: ${{ runner.temp }}/flake-ledger.jsonl + if-no-files-found: ignore + retention-days: 90 # The runner streams the full child output to per-run logs under the OS # tmpdir and prints only the quiet contract to the console. Without this diff --git a/scripts/test-free-shards.ts b/scripts/test-free-shards.ts index 7fd8e6814..9b010405e 100755 --- a/scripts/test-free-shards.ts +++ b/scripts/test-free-shards.ts @@ -1022,6 +1022,44 @@ export function buildRunEpilogue( export type FreeShardStatus = 'passed' | 'failed' | 'timed-out'; +// ─── Flake ledger (WS1 telemetry) ─────────────────────────────────────────── +// Single-writer JSONL: ONLY this parent runner appends (never shards, never +// tests — no concurrent-append hazard by construction). CI points +// GSTACK_FLAKE_LEDGER at $RUNNER_TEMP and uploads it as an artifact every +// run, so repeat offenders become an enumerable series instead of console +// scrollback. Fail-open with a loud stderr warning: a broken ledger must +// never red the only required lane. + +export interface FlakeLedgerEntry { + ts: string; + runner: 'free'; + kind: 'flaky-pass'; + file: string; + /** Shard the original failure surfaced in, when attributable. */ + shard?: number; +} + +export function flakeLedgerPath(env: NodeJS.ProcessEnv = process.env): string { + return env.GSTACK_FLAKE_LEDGER || path.join(os.tmpdir(), 'gstack-flake-ledger.jsonl'); +} + +export function appendFlakeLedger( + entries: FlakeLedgerEntry[], + ledgerPath: string, + warn: (line: string) => void = (line) => console.error(line), +): boolean { + if (entries.length === 0) return true; + try { + fs.mkdirSync(path.dirname(ledgerPath), { recursive: true }); + fs.appendFileSync(ledgerPath, entries.map((e) => JSON.stringify(e)).join('\n') + '\n'); + return true; + } catch (error) { + warn(`[test:free] WARNING: could not append flake ledger at ${ledgerPath} ` + + `(${error instanceof Error ? error.message : String(error)}) — flaky-pass telemetry lost for this run, verdict unaffected`); + return false; + } +} + export interface FreeShardOutcome { shard: number; files: string[]; @@ -1511,6 +1549,19 @@ async function main(): Promise { if (retryOutcome.status === 'passed') { console.log(`[test:free] FLAKY-PASS — ${flakyFiles.length} file(s) failed once and passed on serial retry: ${flakyFiles.join(', ')}`); console.log('[test:free] treat repeat offenders as real flakes worth fixing, not noise.'); + // Durable record (WS1): console lines vanish with the scrollback; the + // ledger makes repeat offenders rankable across runs (eval:flake-rank). + const ts = new Date().toISOString(); + appendFlakeLedger( + flakyFiles.map((file) => ({ + ts, + runner: 'free' as const, + kind: 'flaky-pass' as const, + file, + shard: outcomes.find((o) => o.failingFiles.includes(file))?.shard, + })), + flakeLedgerPath(), + ); worst = 0; } else { console.error('[test:free] flaky-retry FAILED — the failures reproduce serially; not flaky.'); diff --git a/test/flake-ledger.test.ts b/test/flake-ledger.test.ts new file mode 100644 index 000000000..aa6c37902 --- /dev/null +++ b/test/flake-ledger.test.ts @@ -0,0 +1,59 @@ +/** + * WS1 flake-ledger unit tests: the free runner's FLAKY-PASS events become a + * durable JSONL series (single writer: the parent runner). Fail-open is the + * contract — a broken ledger warns loudly but must never turn a real verdict + * into a failure on the only required lane. + */ +import { describe, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { appendFlakeLedger, flakeLedgerPath, type FlakeLedgerEntry } from '../scripts/test-free-shards'; + +const entry = (file: string): FlakeLedgerEntry => ({ + ts: '2026-08-31T00:00:00.000Z', + runner: 'free', + kind: 'flaky-pass', + file, + shard: 2, +}); + +describe('flake ledger', () => { + test('appends one JSONL line per entry, creating parent dirs', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'flake-ledger-')); + const ledger = path.join(dir, 'nested', 'ledger.jsonl'); + expect(appendFlakeLedger([entry('test/a.test.ts')], ledger)).toBe(true); + expect(appendFlakeLedger([entry('test/b.test.ts'), entry('test/c.test.ts')], ledger)).toBe(true); + const lines = fs.readFileSync(ledger, 'utf-8').trim().split('\n'); + expect(lines).toHaveLength(3); + expect(JSON.parse(lines[0])).toMatchObject({ runner: 'free', kind: 'flaky-pass', file: 'test/a.test.ts', shard: 2 }); + fs.rmSync(dir, { recursive: true, force: true }); + }); + + test('empty entry list is a no-op success (no file created)', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'flake-ledger-')); + const ledger = path.join(dir, 'ledger.jsonl'); + expect(appendFlakeLedger([], ledger)).toBe(true); + expect(fs.existsSync(ledger)).toBe(false); + fs.rmSync(dir, { recursive: true, force: true }); + }); + + test('FAIL-OPEN: an unwritable path warns and returns false, never throws', () => { + const warnings: string[] = []; + // A path whose parent is a FILE cannot be mkdir'd — deterministic EEXIST/ENOTDIR. + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'flake-ledger-')); + const blocker = path.join(dir, 'blocker'); + fs.writeFileSync(blocker, 'not a dir'); + const ledger = path.join(blocker, 'ledger.jsonl'); + const ok = appendFlakeLedger([entry('test/a.test.ts')], ledger, (l) => warnings.push(l)); + expect(ok).toBe(false); + expect(warnings).toHaveLength(1); + expect(warnings[0]).toContain('verdict unaffected'); + fs.rmSync(dir, { recursive: true, force: true }); + }); + + test('env override wins over the tmpdir default', () => { + expect(flakeLedgerPath({ GSTACK_FLAKE_LEDGER: '/x/y.jsonl' } as NodeJS.ProcessEnv)).toBe('/x/y.jsonl'); + expect(flakeLedgerPath({} as NodeJS.ProcessEnv)).toContain('gstack-flake-ledger.jsonl'); + }); +}); diff --git a/test/free-tests-workflow-wiring.test.ts b/test/free-tests-workflow-wiring.test.ts index 736b391b9..30e774ef9 100644 --- a/test/free-tests-workflow-wiring.test.ts +++ b/test/free-tests-workflow-wiring.test.ts @@ -54,6 +54,17 @@ describe('free-tests workflow wiring', () => { } }); + test('flake telemetry stays wired: retry flag, single-writer ledger, unconditional artifact', () => { + // WS1: a timing flake must not red the required lane, but every + // flaky-pass must be recorded and uploaded — a green run is exactly when + // the evidence matters. Removing any of these silently returns flakes to + // either merge-blocking (flag off) or invisibility (ledger/artifact off). + expect(source).toMatch(/GSTACK_FREE_RETRY_FLAKY:\s*"1"/); + expect(source).toMatch(/GSTACK_FLAKE_LEDGER:\s*\$\{\{ runner\.temp \}\}\/flake-ledger\.jsonl/); + expect(source).toContain('name: flake-ledger'); + expect(source).toMatch(/name: Upload flake ledger\s*\n\s*if: always\(\)/); + }); + test('least-privilege token: contents read-only, credentials not persisted', () => { // The job executes PR-controlled code (install lifecycle scripts + the // suite itself). A default-grant GITHUB_TOKEN persisted into .git/config