mirror of
https://github.com/garrytan/gstack.git
synced 2026-08-31 18:30:39 +02:00
feat(ci): weekly periodic lane runs EVERY periodic test + gate census backstop
evals-periodic.yml re-platforms onto the sharded runner: planner manifest → 6 executor slices → FAIL-CLOSED report. This IS the coverage contract: all ~70 periodic-tier files weekly (EVALS_ALL=1), killing the silent-rot class where a hard-coded 9-file matrix left ~57 files running NOWHERE (the autoplan E2E rotted invisibly for months). - test/helpers/periodic-exclude-data.ts: reasoned exclusions in their OWN literals file (deliberately not touchfiles-data — map-diff evaluates old versions of that file standalone). Every entry carries reason + tracking with a re-entry condition; the runner surfaces each exclusion per run; policy test pins real-file + non-empty fields. Initial: ship-idempotency + brain-privacy-gate (documented-red, never green) and skill-e2e-ios (manual hardware). The TODOS 'sidebar E2E trio' turned out already deleted — only tombstone tests remain. - gate-census job: weekly EVALS_ALL gate-tier run — PR lanes are diff-billed, so without this the full gate census might never execute anywhere; with the hollow-shard guard it is a census-health check (exit 0 + zero executed tests fails), not just a test run. - failure notification is a concrete gh issue UPSERT (one tracking issue, commented per red week — never issue-per-week spam), with issues:write scoped to the report job. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
056bc61a26
commit
9f2ee58d38
@@ -0,0 +1,34 @@
|
||||
/**
|
||||
* Periodic-lane exclusions — LITERALS ONLY (own file, deliberately NOT in
|
||||
* touchfiles-data.ts: that file is evaluated standalone by map-diff against
|
||||
* old git versions, and its contract must not grow unrelated exports).
|
||||
*
|
||||
* The weekly periodic CI lane runs EVERY periodic-tier file (EVALS_ALL=1) so
|
||||
* tests can't rot invisibly — the coverage contract. A file lands here only
|
||||
* when running it weekly is KNOWN waste (documented-red or requires manual
|
||||
* hardware), and every entry must carry a tracking pointer with a re-entry
|
||||
* condition, so an exclusion is a decision with an owner, not a place tests
|
||||
* go to die. Pinned by test/periodic-exclude-policy.test.ts: entries must
|
||||
* name real files and carry non-empty reason + tracking.
|
||||
*
|
||||
* Removing an entry re-activates the file on the next weekly run — that IS
|
||||
* the re-entry mechanism.
|
||||
*/
|
||||
export const PERIODIC_CI_EXCLUDE: Record<string, { reason: string; tracking: string }> = {
|
||||
'test/skill-e2e-ship-idempotency.test.ts': {
|
||||
reason:
|
||||
'documented-red: the PTY child sits at the Claude Code welcome screen for the full budget '
|
||||
+ '(readiness/typing race vs CLI 2.1.x); never green since it was born in v1.63',
|
||||
tracking: 'TODOS.md "periodic tier — three documented-red tests need structural repair" (1 of 3 resolved: sidebar trio already deleted)',
|
||||
},
|
||||
'test/skill-e2e-brain-privacy-gate.test.ts': {
|
||||
reason:
|
||||
'documented-red: the artifacts-sync stop-gate preconditions do not survive the hermetic env '
|
||||
+ 'even with per-test HOME/GSTACK_HOME injection; never green anywhere',
|
||||
tracking: 'TODOS.md "periodic tier — three documented-red tests need structural repair"',
|
||||
},
|
||||
'test/skill-e2e-ios.test.ts': {
|
||||
reason: 'requires a live iOS device/simulator toolchain (xcodebuild, devicectl) — manual hardware, not a CI runner capability',
|
||||
tracking: 'TODOS.md "skill-e2e-ios CI story" (device/runner decision)',
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,45 @@
|
||||
/**
|
||||
* The periodic exclude list is a set of DECISIONS, not a place tests go to
|
||||
* die: every entry names a real file (a deleted/renamed file must drop its
|
||||
* entry) and carries a non-empty reason + tracking pointer (the re-entry
|
||||
* condition lives there). The runner surfaces each exclusion per run, and
|
||||
* removing an entry re-activates the file on the next weekly lane.
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
|
||||
import { PERIODIC_CI_EXCLUDE } from './helpers/periodic-exclude-data';
|
||||
import { isPaidTestFile } from './helpers/paid-test-set';
|
||||
import { selectPaidTestFiles } from '../scripts/test-paid-shards';
|
||||
|
||||
const ROOT = path.resolve(__dirname, '..');
|
||||
|
||||
describe('periodic exclude policy', () => {
|
||||
test('every entry names a real paid file and carries reason + tracking', () => {
|
||||
const entries = Object.entries(PERIODIC_CI_EXCLUDE);
|
||||
expect(entries.length).toBeGreaterThan(0);
|
||||
for (const [file, meta] of entries) {
|
||||
expect(fs.existsSync(path.join(ROOT, file)), `stale exclude entry: ${file}`).toBe(true);
|
||||
expect(isPaidTestFile(file), `${file} is not a paid file — exclusion is meaningless`).toBe(true);
|
||||
expect(meta.reason.length, `${file}: empty reason`).toBeGreaterThan(20);
|
||||
expect(meta.tracking.length, `${file}: empty tracking pointer`).toBeGreaterThan(5);
|
||||
}
|
||||
});
|
||||
|
||||
test('exclusions apply to the periodic tier only, with the reason surfaced', () => {
|
||||
const files = Object.keys(PERIODIC_CI_EXCLUDE);
|
||||
const periodic = selectPaidTestFiles(files, 'periodic');
|
||||
expect(periodic.selected).toEqual([]);
|
||||
for (const { reason } of periodic.excluded) {
|
||||
expect(reason).toStartWith('excluded: ');
|
||||
expect(reason).toContain('[');
|
||||
}
|
||||
// Gate tier ignores the list (these files are periodic-tier anyway; the
|
||||
// list must never leak into gate semantics).
|
||||
const gate = selectPaidTestFiles(files, 'gate');
|
||||
for (const { reason } of gate.excluded) {
|
||||
expect(reason).not.toStartWith('excluded: ');
|
||||
}
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user