mirror of
https://github.com/garrytan/gstack.git
synced 2026-08-31 18:30:39 +02:00
feat(ci): weekly periodic lane runs EVERY periodic test + gate census backstop
evals-periodic.yml re-platforms onto the sharded runner: planner manifest → 6 executor slices → FAIL-CLOSED report. This IS the coverage contract: all ~70 periodic-tier files weekly (EVALS_ALL=1), killing the silent-rot class where a hard-coded 9-file matrix left ~57 files running NOWHERE (the autoplan E2E rotted invisibly for months). - test/helpers/periodic-exclude-data.ts: reasoned exclusions in their OWN literals file (deliberately not touchfiles-data — map-diff evaluates old versions of that file standalone). Every entry carries reason + tracking with a re-entry condition; the runner surfaces each exclusion per run; policy test pins real-file + non-empty fields. Initial: ship-idempotency + brain-privacy-gate (documented-red, never green) and skill-e2e-ios (manual hardware). The TODOS 'sidebar E2E trio' turned out already deleted — only tombstone tests remain. - gate-census job: weekly EVALS_ALL gate-tier run — PR lanes are diff-billed, so without this the full gate census might never execute anywhere; with the hollow-shard guard it is a census-health check (exit 0 + zero executed tests fails), not just a test run. - failure notification is a concrete gh issue UPSERT (one tracking issue, commented per red week — never issue-per-week spam), with issues:write scoped to the report job. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
056bc61a26
commit
9f2ee58d38
@@ -63,6 +63,7 @@ import {
|
||||
strictTestExitCode,
|
||||
} from './test-strict-output';
|
||||
import { PAID_TEST_GLOBS, isPaidTestFile } from '../test/helpers/paid-test-set';
|
||||
import { PERIODIC_CI_EXCLUDE } from '../test/helpers/periodic-exclude-data';
|
||||
import { getProjectEvalDir } from '../test/helpers/eval-store';
|
||||
import { preflightAnthropicApi } from '../test/helpers/anthropic-preflight';
|
||||
import {
|
||||
@@ -75,6 +76,7 @@ import {
|
||||
} from '../test/helpers/touchfiles';
|
||||
|
||||
export { PAID_TEST_GLOBS, isPaidTestFile };
|
||||
export { PERIODIC_CI_EXCLUDE };
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
|
||||
@@ -142,7 +144,17 @@ export interface TierSelection {
|
||||
export function selectPaidTestFiles(files: string[], tier: PaidTier, rootDir = ROOT): TierSelection {
|
||||
const selected: string[] = [];
|
||||
const excluded: Array<{ file: string; reason: string }> = [];
|
||||
// Periodic-lane exclusions (documented-red / manual-hardware files): a
|
||||
// known-red weekly shard is triage waste locally AND in CI, so the list
|
||||
// applies to every periodic run, with the reason surfaced per file.
|
||||
const ciExcluded = (file: string): { reason: string; tracking: string } | undefined =>
|
||||
tier === 'periodic' ? PERIODIC_CI_EXCLUDE[normalizeRelativePath(file)] : undefined;
|
||||
for (const file of files) {
|
||||
const exclusion = ciExcluded(file);
|
||||
if (exclusion) {
|
||||
excluded.push({ file, reason: `excluded: ${exclusion.reason} [${exclusion.tracking}]` });
|
||||
continue;
|
||||
}
|
||||
const source = fs.readFileSync(path.join(rootDir, file), 'utf8');
|
||||
const classification = classifyPaidTestFile(source, tier);
|
||||
if (classification.included) selected.push(file);
|
||||
|
||||
Reference in New Issue
Block a user