test: record the detector widening as non-behavioral; lower plan-eng-review size pins (#3349)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed 2026-10-25 02:24:27 +00:00
1 parent 3a15cb0f40
commit e549d90cb3
3 files changed
+13 -13

No files matched your search

+1 -1
View File
@@ -11,7 +11,7 @@
- `test/eng-first-review.test.ts`: the two captured plan-eng-review first-review questions (`eng-annotated-cache-au`, `eng-declarative-as`) still open review when their `[P1]` label becomes `[CRITICAL]` or `[INFORMATIONAL]`. They stay closed for `[HIGH]`, `[critical]`, `[Critical]`, `[P4]`, `[CRITICAL|INFORMATIONAL]` and no label.
### Gate integrity
- One accepted-input change to review: the paid-test detectors `test/helpers/eng-cache-writer-decision.ts` and `test/helpers/pty/boundaries.ts` accept `[CRITICAL]` and `[INFORMATIONAL]` alongside `[P0]`-`[P3]` in a plan finding's annotation, because the skill now asks for those labels. Without it, both captures relabelled `[CRITICAL]` are rejected. Verdict: not a weakening. The label is metadata, and every substantive check (decision, assessment, actors, options) is unchanged. Mutation checks: restoring `P[0-3]` alone fails the positive cases; `[^\]]+`, `P[0-9]` or a case-insensitive CRITICAL each fail the negative controls. No threshold, budget, timeout, trial count or case kind changed.
- One accepted-input change to review: the paid-test detectors `test/helpers/eng-cache-writer-decision.ts` and `test/helpers/pty/boundaries.ts` accept `[CRITICAL]` and `[INFORMATIONAL]` alongside `[P0]`-`[P3]` in a plan finding's annotation, because the skill now asks for those labels. Without it, both captures relabelled `[CRITICAL]` are rejected. Verdict: not a weakening. The label is metadata, and every substantive check (decision, assessment, actors, options) is unchanged. Mutation checks: restoring `P[0-3]` alone fails the positive cases; `[^\]]+`, `P[0-9]` or a case-insensitive CRITICAL each fail the negative controls. `scripts/harness-version.json` records `test/helpers/pty/boundaries.ts` as non-behavioral at HARNESS_VERSION 6: only the changed plan-eng-review prompt asks for the new labels, and the cases that run it already start their own series. The 11 plan-eng-review size pins in `test/fixtures/skill-md-size-allowlist.json` come down by 75 characters. No threshold, budget, timeout, trial count or case kind changed.
## [1.91.191.0] - 2026-10-25
+1 -1
View File
@@ -34,7 +34,7 @@
"test/helpers/pty-screen.ts": {"blob":"4f77569f9ce11f9cee9f347216e468cd7c956820","since":6},
"test/helpers/pty/auq.ts": {"blob":"16e5ea2bb5b9f28f44808a1a09024dbd3e6f28c4","since":6},
"test/helpers/pty/binary.ts": {"blob":"404d7515076d5c00804f89e23ea69296312c00e1","since":6},
"test/helpers/pty/boundaries.ts": {"blob":"8c87100c9dd0cfc28c02598f250726fd93bfbdea","since":6,"nonBehavioral":"Removes the plan-design 0D focus picker from the shared harness; floor.ts gains an optional pickSetupChoice hook that no other caller passes, so no other case's input or verdict can change. plan-design-finding-floor, the only case whose seed raises that menu, passes the picker from its owned fixture test/fixtures/design-floor-focus-picker.ts, so its owned bytes start its own new series."},
"test/helpers/pty/boundaries.ts": {"blob":"c5520118da36f96f60a3c9720751bb57a669865c","since":6,"nonBehavioral":"Widens the plan-eng-review first-review severity annotation from [P0-3] to [P0-3|CRITICAL|INFORMATIONAL] (#3349). Only a question carrying the new labels can change verdict, and only the changed plan-eng-review prompt asks for them; every case that runs it owns that generated SKILL.md, which this change also edits, so those cases already start their own series. No other case's input or verdict can change."},
"test/helpers/pty/classify.ts": {"blob":"3f2daa12ff6484f96b8d28d910a50231c4bb3ae8","since":6},
"test/helpers/pty/fake-session.ts": {"blob":"554e064ef259b8444280eddfe9b0db93c66a8202","since":6},
"test/helpers/pty/judge.ts": {"blob":"81c5bb4c48ab928aca2f186a1ade9df7169d7fbe","since":6},
+11 -11
View File
@@ -8,7 +8,7 @@
"agy/gstack-land-and-deploy": 116611,
"agy/gstack-plan-design-review": 140013,
"agy/gstack-plan-devex-review": 135760,
"agy/gstack-plan-eng-review": 152701,
"agy/gstack-plan-eng-review": 152626,
"agy/gstack-review": 124218,
"claude/design-review": 136531,
"codex/gstack-autoplan": 130628,
@@ -18,7 +18,7 @@
"codex/gstack-office-hours": 102339,
"codex/gstack-plan-design-review": 142466,
"codex/gstack-plan-devex-review": 138649,
"codex/gstack-plan-eng-review": 155247,
"codex/gstack-plan-eng-review": 155172,
"codex/gstack-review": 125122,
"copilot/gstack-autoplan": 131637,
"copilot/gstack-codex": 100235,
@@ -28,7 +28,7 @@
"copilot/gstack-office-hours": 100495,
"copilot/gstack-plan-design-review": 140884,
"copilot/gstack-plan-devex-review": 136912,
"copilot/gstack-plan-eng-review": 153426,
"copilot/gstack-plan-eng-review": 153351,
"copilot/gstack-review": 125489,
"cursor/gstack-autoplan": 129278,
"cursor/gstack-design-consultation": 107222,
@@ -36,7 +36,7 @@
"cursor/gstack-land-and-deploy": 115550,
"cursor/gstack-plan-design-review": 138968,
"cursor/gstack-plan-devex-review": 134591,
"cursor/gstack-plan-eng-review": 151555,
"cursor/gstack-plan-eng-review": 151480,
"cursor/gstack-review": 140750,
"factory/gstack-autoplan": 131562,
"factory/gstack-codex": 100655,
@@ -46,34 +46,34 @@
"factory/gstack-office-hours": 101374,
"factory/gstack-plan-design-review": 141749,
"factory/gstack-plan-devex-review": 137960,
"factory/gstack-plan-eng-review": 154669,
"factory/gstack-plan-eng-review": 154594,
"factory/gstack-review": 144529,
"gbrain/gstack-autoplan": 129815,
"gbrain/gstack-design-review": 132842,
"gbrain/gstack-land-and-deploy": 116072,
"gbrain/gstack-plan-design-review": 127855,
"gbrain/gstack-plan-devex-review": 115129,
"gbrain/gstack-plan-eng-review": 134287,
"gbrain/gstack-plan-eng-review": 134212,
"hermes/gstack-autoplan": 129905,
"hermes/gstack-design-review": 132949,
"hermes/gstack-land-and-deploy": 116174,
"hermes/gstack-plan-design-review": 127964,
"hermes/gstack-plan-devex-review": 115215,
"hermes/gstack-plan-eng-review": 134372,
"hermes/gstack-plan-eng-review": 134297,
"kiro/gstack-autoplan": 129040,
"kiro/gstack-design-consultation": 107064,
"kiro/gstack-design-review": 142321,
"kiro/gstack-land-and-deploy": 115440,
"kiro/gstack-plan-design-review": 138808,
"kiro/gstack-plan-devex-review": 134431,
"kiro/gstack-plan-eng-review": 151405,
"kiro/gstack-plan-eng-review": 151330,
"kiro/gstack-review": 140562,
"openclaw/gstack-autoplan": 129520,
"openclaw/gstack-design-review": 131647,
"openclaw/gstack-land-and-deploy": 115660,
"openclaw/gstack-plan-design-review": 126706,
"openclaw/gstack-plan-devex-review": 113968,
"openclaw/gstack-plan-eng-review": 132684,
"openclaw/gstack-plan-eng-review": 132609,
"opencode/gstack-autoplan": 131640,
"opencode/gstack-design-consultation": 109357,
"opencode/gstack-design-review": 144705,
@@ -81,7 +81,7 @@
"opencode/gstack-office-hours": 100329,
"opencode/gstack-plan-design-review": 140738,
"opencode/gstack-plan-devex-review": 136812,
"opencode/gstack-plan-eng-review": 153307,
"opencode/gstack-plan-eng-review": 153232,
"opencode/gstack-review": 142985,
"slate/gstack-autoplan": 129159,
"slate/gstack-design-consultation": 107143,
@@ -89,7 +89,7 @@
"slate/gstack-land-and-deploy": 115495,
"slate/gstack-plan-design-review": 138888,
"slate/gstack-plan-devex-review": 134511,
"slate/gstack-plan-eng-review": 151480,
"slate/gstack-plan-eng-review": 151405,
"slate/gstack-review": 140656
}
}