feat(ci): weekly periodic lane runs EVERY periodic test + gate census backstop

evals-periodic.yml re-platforms onto the sharded runner: planner
manifest → 6 executor slices → FAIL-CLOSED report. This IS the coverage
contract: all ~70 periodic-tier files weekly (EVALS_ALL=1), killing the
silent-rot class where a hard-coded 9-file matrix left ~57 files
running NOWHERE (the autoplan E2E rotted invisibly for months).

- test/helpers/periodic-exclude-data.ts: reasoned exclusions in their
  OWN literals file (deliberately not touchfiles-data — map-diff
  evaluates old versions of that file standalone). Every entry carries
  reason + tracking with a re-entry condition; the runner surfaces each
  exclusion per run; policy test pins real-file + non-empty fields.
  Initial: ship-idempotency + brain-privacy-gate (documented-red,
  never green) and skill-e2e-ios (manual hardware). The TODOS 'sidebar
  E2E trio' turned out already deleted — only tombstone tests remain.
- gate-census job: weekly EVALS_ALL gate-tier run — PR lanes are
  diff-billed, so without this the full gate census might never execute
  anywhere; with the hollow-shard guard it is a census-health check
  (exit 0 + zero executed tests fails), not just a test run.
- failure notification is a concrete gh issue UPSERT (one tracking
  issue, commented per red week — never issue-per-week spam), with
  issues:write scoped to the report job.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-08-29 05:48:42 +00:00
co-authored by Claude Fable 5
parent 056bc61a26
commit 9f2ee58d38
4 changed files with 357 additions and 49 deletions
+34
View File
@@ -0,0 +1,34 @@
/**
* Periodic-lane exclusions — LITERALS ONLY (own file, deliberately NOT in
* touchfiles-data.ts: that file is evaluated standalone by map-diff against
* old git versions, and its contract must not grow unrelated exports).
*
* The weekly periodic CI lane runs EVERY periodic-tier file (EVALS_ALL=1) so
* tests can't rot invisibly — the coverage contract. A file lands here only
* when running it weekly is KNOWN waste (documented-red or requires manual
* hardware), and every entry must carry a tracking pointer with a re-entry
* condition, so an exclusion is a decision with an owner, not a place tests
* go to die. Pinned by test/periodic-exclude-policy.test.ts: entries must
* name real files and carry non-empty reason + tracking.
*
* Removing an entry re-activates the file on the next weekly run — that IS
* the re-entry mechanism.
*/
export const PERIODIC_CI_EXCLUDE: Record<string, { reason: string; tracking: string }> = {
'test/skill-e2e-ship-idempotency.test.ts': {
reason:
'documented-red: the PTY child sits at the Claude Code welcome screen for the full budget '
+ '(readiness/typing race vs CLI 2.1.x); never green since it was born in v1.63',
tracking: 'TODOS.md "periodic tier — three documented-red tests need structural repair" (1 of 3 resolved: sidebar trio already deleted)',
},
'test/skill-e2e-brain-privacy-gate.test.ts': {
reason:
'documented-red: the artifacts-sync stop-gate preconditions do not survive the hermetic env '
+ 'even with per-test HOME/GSTACK_HOME injection; never green anywhere',
tracking: 'TODOS.md "periodic tier — three documented-red tests need structural repair"',
},
'test/skill-e2e-ios.test.ts': {
reason: 'requires a live iOS device/simulator toolchain (xcodebuild, devicectl) — manual hardware, not a CI runner capability',
tracking: 'TODOS.md "skill-e2e-ios CI story" (device/runner decision)',
},
};
+45
View File
@@ -0,0 +1,45 @@
/**
* The periodic exclude list is a set of DECISIONS, not a place tests go to
* die: every entry names a real file (a deleted/renamed file must drop its
* entry) and carries a non-empty reason + tracking pointer (the re-entry
* condition lives there). The runner surfaces each exclusion per run, and
* removing an entry re-activates the file on the next weekly lane.
*/
import { describe, expect, test } from 'bun:test';
import * as fs from 'node:fs';
import * as path from 'node:path';
import { PERIODIC_CI_EXCLUDE } from './helpers/periodic-exclude-data';
import { isPaidTestFile } from './helpers/paid-test-set';
import { selectPaidTestFiles } from '../scripts/test-paid-shards';
const ROOT = path.resolve(__dirname, '..');
describe('periodic exclude policy', () => {
test('every entry names a real paid file and carries reason + tracking', () => {
const entries = Object.entries(PERIODIC_CI_EXCLUDE);
expect(entries.length).toBeGreaterThan(0);
for (const [file, meta] of entries) {
expect(fs.existsSync(path.join(ROOT, file)), `stale exclude entry: ${file}`).toBe(true);
expect(isPaidTestFile(file), `${file} is not a paid file — exclusion is meaningless`).toBe(true);
expect(meta.reason.length, `${file}: empty reason`).toBeGreaterThan(20);
expect(meta.tracking.length, `${file}: empty tracking pointer`).toBeGreaterThan(5);
}
});
test('exclusions apply to the periodic tier only, with the reason surfaced', () => {
const files = Object.keys(PERIODIC_CI_EXCLUDE);
const periodic = selectPaidTestFiles(files, 'periodic');
expect(periodic.selected).toEqual([]);
for (const { reason } of periodic.excluded) {
expect(reason).toStartWith('excluded: ');
expect(reason).toContain('[');
}
// Gate tier ignores the list (these files are periodic-tier anyway; the
// list must never leak into gate semantics).
const gate = selectPaidTestFiles(files, 'gate');
for (const { reason } of gate.excluded) {
expect(reason).not.toStartWith('excluded: ');
}
});
});