mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-03 01:46:55 +02:00
fix(evals): plan CI-unrunnable cases as excluded entries, not empty case shards
design-review-fix drives the Aside browser and registers test.skip on Linux runners, so its case shard executed zero cases and failed the exact-one-case check in proof census 36597762183 (eval-slices 6). CASE_CI_EXCLUDE (reason + tracking, beside PERIODIC_CI_EXCLUDE) now turns such cases into excluded manifest entries that --list and the manifest surface; every planned case shard still must execute exactly its case.
This commit is contained in:
1 parent
45cdfab2a0
commit
0554f3d959
3 files changed
+69
-7
No files matched your search
@@ -44,3 +44,18 @@ export const PERIODIC_CI_EXCLUDE: Record<string, { reason: string; tracking: str
|
||||
tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)',
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* Case-level exclusions for case-sharded files (`<file>#<case id>`), same
|
||||
* contract as above: a case lands here only when a CI runner cannot execute it
|
||||
* (it self-skips), with reason + tracking. The planner records each as an
|
||||
* excluded manifest entry instead of an empty case shard, so the exact
|
||||
* one-case check stays strict for every planned case. Pinned by
|
||||
* test/periodic-exclude-policy.test.ts.
|
||||
*/
|
||||
export const CASE_CI_EXCLUDE: Record<string, { reason: string; tracking: string }> = {
|
||||
'test/skill-e2e-design.test.ts#design-review-fix': {
|
||||
reason: '/design-review drives the Aside browser; CI runners are Linux without Aside, so the case registers test.skip("needs Aside")',
|
||||
tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)',
|
||||
},
|
||||
};
|
||||
Reference in new issue
Block a user