mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-10 23:19:09 +02:00
feat: free-lane flake ledger — retry ON in CI, flaky-passes recorded and uploaded
The runner's attribution-gated flaky-retry pass (cap 5, truncation veto) was OFF in the required lane and its FLAKY-PASS evidence was console-only — so a single timing flake red the merge gate while repeat offenders stayed unenumerable. free-tests.yml now sets GSTACK_FREE_RETRY_FLAKY=1 and points GSTACK_FLAKE_LEDGER at runner.temp; every flaky-pass appends a JSONL entry (SINGLE writer: the parent runner — no concurrent-append hazard by construction; fail-open with a loud warning so a broken ledger can never red the lane) and the artifact uploads UNCONDITIONALLY — a flaky-pass run is green, which is exactly when the evidence matters. Wiring pinned by free-tests-workflow-wiring; ledger behavior unit-tested incl. the fail-open path. Matches 2026 industry practice (retry for data, quarantine out of merge-blocking but never out of logging) with the repo's own receipts. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
2dabc02447
commit
7f36eacbd2
@@ -0,0 +1,59 @@
|
||||
/**
|
||||
* WS1 flake-ledger unit tests: the free runner's FLAKY-PASS events become a
|
||||
* durable JSONL series (single writer: the parent runner). Fail-open is the
|
||||
* contract — a broken ledger warns loudly but must never turn a real verdict
|
||||
* into a failure on the only required lane.
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { appendFlakeLedger, flakeLedgerPath, type FlakeLedgerEntry } from '../scripts/test-free-shards';
|
||||
|
||||
const entry = (file: string): FlakeLedgerEntry => ({
|
||||
ts: '2026-08-31T00:00:00.000Z',
|
||||
runner: 'free',
|
||||
kind: 'flaky-pass',
|
||||
file,
|
||||
shard: 2,
|
||||
});
|
||||
|
||||
describe('flake ledger', () => {
|
||||
test('appends one JSONL line per entry, creating parent dirs', () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'flake-ledger-'));
|
||||
const ledger = path.join(dir, 'nested', 'ledger.jsonl');
|
||||
expect(appendFlakeLedger([entry('test/a.test.ts')], ledger)).toBe(true);
|
||||
expect(appendFlakeLedger([entry('test/b.test.ts'), entry('test/c.test.ts')], ledger)).toBe(true);
|
||||
const lines = fs.readFileSync(ledger, 'utf-8').trim().split('\n');
|
||||
expect(lines).toHaveLength(3);
|
||||
expect(JSON.parse(lines[0])).toMatchObject({ runner: 'free', kind: 'flaky-pass', file: 'test/a.test.ts', shard: 2 });
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test('empty entry list is a no-op success (no file created)', () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'flake-ledger-'));
|
||||
const ledger = path.join(dir, 'ledger.jsonl');
|
||||
expect(appendFlakeLedger([], ledger)).toBe(true);
|
||||
expect(fs.existsSync(ledger)).toBe(false);
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test('FAIL-OPEN: an unwritable path warns and returns false, never throws', () => {
|
||||
const warnings: string[] = [];
|
||||
// A path whose parent is a FILE cannot be mkdir'd — deterministic EEXIST/ENOTDIR.
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'flake-ledger-'));
|
||||
const blocker = path.join(dir, 'blocker');
|
||||
fs.writeFileSync(blocker, 'not a dir');
|
||||
const ledger = path.join(blocker, 'ledger.jsonl');
|
||||
const ok = appendFlakeLedger([entry('test/a.test.ts')], ledger, (l) => warnings.push(l));
|
||||
expect(ok).toBe(false);
|
||||
expect(warnings).toHaveLength(1);
|
||||
expect(warnings[0]).toContain('verdict unaffected');
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test('env override wins over the tmpdir default', () => {
|
||||
expect(flakeLedgerPath({ GSTACK_FLAKE_LEDGER: '/x/y.jsonl' } as NodeJS.ProcessEnv)).toBe('/x/y.jsonl');
|
||||
expect(flakeLedgerPath({} as NodeJS.ProcessEnv)).toContain('gstack-flake-ledger.jsonl');
|
||||
});
|
||||
});
|
||||
@@ -54,6 +54,17 @@ describe('free-tests workflow wiring', () => {
|
||||
}
|
||||
});
|
||||
|
||||
test('flake telemetry stays wired: retry flag, single-writer ledger, unconditional artifact', () => {
|
||||
// WS1: a timing flake must not red the required lane, but every
|
||||
// flaky-pass must be recorded and uploaded — a green run is exactly when
|
||||
// the evidence matters. Removing any of these silently returns flakes to
|
||||
// either merge-blocking (flag off) or invisibility (ledger/artifact off).
|
||||
expect(source).toMatch(/GSTACK_FREE_RETRY_FLAKY:\s*"1"/);
|
||||
expect(source).toMatch(/GSTACK_FLAKE_LEDGER:\s*\$\{\{ runner\.temp \}\}\/flake-ledger\.jsonl/);
|
||||
expect(source).toContain('name: flake-ledger');
|
||||
expect(source).toMatch(/name: Upload flake ledger\s*\n\s*if: always\(\)/);
|
||||
});
|
||||
|
||||
test('least-privilege token: contents read-only, credentials not persisted', () => {
|
||||
// The job executes PR-controlled code (install lifecycle scripts + the
|
||||
// suite itself). A default-grant GITHUB_TOKEN persisted into .git/config
|
||||
|
||||
Reference in New Issue
Block a user