mirror of
https://github.com/garrytan/gstack.git
synced 2026-08-31 10:20:42 +02:00
Garry TanandClaude Fable 5
9f2ee58d38
evals-periodic.yml re-platforms onto the sharded runner: planner manifest → 6 executor slices → FAIL-CLOSED report. This IS the coverage contract: all ~70 periodic-tier files weekly (EVALS_ALL=1), killing the silent-rot class where a hard-coded 9-file matrix left ~57 files running NOWHERE (the autoplan E2E rotted invisibly for months). - test/helpers/periodic-exclude-data.ts: reasoned exclusions in their OWN literals file (deliberately not touchfiles-data — map-diff evaluates old versions of that file standalone). Every entry carries reason + tracking with a re-entry condition; the runner surfaces each exclusion per run; policy test pins real-file + non-empty fields. Initial: ship-idempotency + brain-privacy-gate (documented-red, never green) and skill-e2e-ios (manual hardware). The TODOS 'sidebar E2E trio' turned out already deleted — only tombstone tests remain. - gate-census job: weekly EVALS_ALL gate-tier run — PR lanes are diff-billed, so without this the full gate census might never execute anywhere; with the hollow-shard guard it is a census-health check (exit 0 + zero executed tests fails), not just a test run. - failure notification is a concrete gh issue UPSERT (one tracking issue, commented per red week — never issue-per-week spam), with issues:write scoped to the report job. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
46 lines
2.0 KiB
TypeScript
46 lines
2.0 KiB
TypeScript
/**
|
|
* The periodic exclude list is a set of DECISIONS, not a place tests go to
|
|
* die: every entry names a real file (a deleted/renamed file must drop its
|
|
* entry) and carries a non-empty reason + tracking pointer (the re-entry
|
|
* condition lives there). The runner surfaces each exclusion per run, and
|
|
* removing an entry re-activates the file on the next weekly lane.
|
|
*/
|
|
import { describe, expect, test } from 'bun:test';
|
|
import * as fs from 'node:fs';
|
|
import * as path from 'node:path';
|
|
|
|
import { PERIODIC_CI_EXCLUDE } from './helpers/periodic-exclude-data';
|
|
import { isPaidTestFile } from './helpers/paid-test-set';
|
|
import { selectPaidTestFiles } from '../scripts/test-paid-shards';
|
|
|
|
const ROOT = path.resolve(__dirname, '..');
|
|
|
|
describe('periodic exclude policy', () => {
|
|
test('every entry names a real paid file and carries reason + tracking', () => {
|
|
const entries = Object.entries(PERIODIC_CI_EXCLUDE);
|
|
expect(entries.length).toBeGreaterThan(0);
|
|
for (const [file, meta] of entries) {
|
|
expect(fs.existsSync(path.join(ROOT, file)), `stale exclude entry: ${file}`).toBe(true);
|
|
expect(isPaidTestFile(file), `${file} is not a paid file — exclusion is meaningless`).toBe(true);
|
|
expect(meta.reason.length, `${file}: empty reason`).toBeGreaterThan(20);
|
|
expect(meta.tracking.length, `${file}: empty tracking pointer`).toBeGreaterThan(5);
|
|
}
|
|
});
|
|
|
|
test('exclusions apply to the periodic tier only, with the reason surfaced', () => {
|
|
const files = Object.keys(PERIODIC_CI_EXCLUDE);
|
|
const periodic = selectPaidTestFiles(files, 'periodic');
|
|
expect(periodic.selected).toEqual([]);
|
|
for (const { reason } of periodic.excluded) {
|
|
expect(reason).toStartWith('excluded: ');
|
|
expect(reason).toContain('[');
|
|
}
|
|
// Gate tier ignores the list (these files are periodic-tier anyway; the
|
|
// list must never leak into gate semantics).
|
|
const gate = selectPaidTestFiles(files, 'gate');
|
|
for (const { reason } of gate.excluded) {
|
|
expect(reason).not.toStartWith('excluded: ');
|
|
}
|
|
});
|
|
});
|