From e549d90cb3bf1a9ec9f03ceff9142b3516a6399c Mon Sep 17 00:00:00 2001 From: garrytan Date: Sun, 25 Oct 2026 02:24:27 +0000 Subject: [PATCH] test: record the detector widening as non-behavioral; lower plan-eng-review size pins (#3349) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CHANGELOG.md | 2 +- scripts/harness-version.json | 2 +- test/fixtures/skill-md-size-allowlist.json | 22 +++++++++++----------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96bd92c78..82c51a4ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ - `test/eng-first-review.test.ts`: the two captured plan-eng-review first-review questions (`eng-annotated-cache-au`, `eng-declarative-as`) still open review when their `[P1]` label becomes `[CRITICAL]` or `[INFORMATIONAL]`. They stay closed for `[HIGH]`, `[critical]`, `[Critical]`, `[P4]`, `[CRITICAL|INFORMATIONAL]` and no label. ### Gate integrity -- One accepted-input change to review: the paid-test detectors `test/helpers/eng-cache-writer-decision.ts` and `test/helpers/pty/boundaries.ts` accept `[CRITICAL]` and `[INFORMATIONAL]` alongside `[P0]`-`[P3]` in a plan finding's annotation, because the skill now asks for those labels. Without it, both captures relabelled `[CRITICAL]` are rejected. Verdict: not a weakening. The label is metadata, and every substantive check (decision, assessment, actors, options) is unchanged. Mutation checks: restoring `P[0-3]` alone fails the positive cases; `[^\]]+`, `P[0-9]` or a case-insensitive CRITICAL each fail the negative controls. No threshold, budget, timeout, trial count or case kind changed. +- One accepted-input change to review: the paid-test detectors `test/helpers/eng-cache-writer-decision.ts` and `test/helpers/pty/boundaries.ts` accept `[CRITICAL]` and `[INFORMATIONAL]` alongside `[P0]`-`[P3]` in a plan finding's annotation, because the skill now asks for those labels. Without it, both captures relabelled `[CRITICAL]` are rejected. Verdict: not a weakening. The label is metadata, and every substantive check (decision, assessment, actors, options) is unchanged. Mutation checks: restoring `P[0-3]` alone fails the positive cases; `[^\]]+`, `P[0-9]` or a case-insensitive CRITICAL each fail the negative controls. `scripts/harness-version.json` records `test/helpers/pty/boundaries.ts` as non-behavioral at HARNESS_VERSION 6: only the changed plan-eng-review prompt asks for the new labels, and the cases that run it already start their own series. The 11 plan-eng-review size pins in `test/fixtures/skill-md-size-allowlist.json` come down by 75 characters. No threshold, budget, timeout, trial count or case kind changed. ## [1.91.191.0] - 2026-10-25 diff --git a/scripts/harness-version.json b/scripts/harness-version.json index d95e24d5a..bcf395dd3 100644 --- a/scripts/harness-version.json +++ b/scripts/harness-version.json @@ -34,7 +34,7 @@ "test/helpers/pty-screen.ts": {"blob":"4f77569f9ce11f9cee9f347216e468cd7c956820","since":6}, "test/helpers/pty/auq.ts": {"blob":"16e5ea2bb5b9f28f44808a1a09024dbd3e6f28c4","since":6}, "test/helpers/pty/binary.ts": {"blob":"404d7515076d5c00804f89e23ea69296312c00e1","since":6}, - "test/helpers/pty/boundaries.ts": {"blob":"8c87100c9dd0cfc28c02598f250726fd93bfbdea","since":6,"nonBehavioral":"Removes the plan-design 0D focus picker from the shared harness; floor.ts gains an optional pickSetupChoice hook that no other caller passes, so no other case's input or verdict can change. plan-design-finding-floor, the only case whose seed raises that menu, passes the picker from its owned fixture test/fixtures/design-floor-focus-picker.ts, so its owned bytes start its own new series."}, + "test/helpers/pty/boundaries.ts": {"blob":"c5520118da36f96f60a3c9720751bb57a669865c","since":6,"nonBehavioral":"Widens the plan-eng-review first-review severity annotation from [P0-3] to [P0-3|CRITICAL|INFORMATIONAL] (#3349). Only a question carrying the new labels can change verdict, and only the changed plan-eng-review prompt asks for them; every case that runs it owns that generated SKILL.md, which this change also edits, so those cases already start their own series. No other case's input or verdict can change."}, "test/helpers/pty/classify.ts": {"blob":"3f2daa12ff6484f96b8d28d910a50231c4bb3ae8","since":6}, "test/helpers/pty/fake-session.ts": {"blob":"554e064ef259b8444280eddfe9b0db93c66a8202","since":6}, "test/helpers/pty/judge.ts": {"blob":"81c5bb4c48ab928aca2f186a1ade9df7169d7fbe","since":6}, diff --git a/test/fixtures/skill-md-size-allowlist.json b/test/fixtures/skill-md-size-allowlist.json index 9083336de..54f7538cf 100644 --- a/test/fixtures/skill-md-size-allowlist.json +++ b/test/fixtures/skill-md-size-allowlist.json @@ -8,7 +8,7 @@ "agy/gstack-land-and-deploy": 116611, "agy/gstack-plan-design-review": 140013, "agy/gstack-plan-devex-review": 135760, - "agy/gstack-plan-eng-review": 152701, + "agy/gstack-plan-eng-review": 152626, "agy/gstack-review": 124218, "claude/design-review": 136531, "codex/gstack-autoplan": 130628, @@ -18,7 +18,7 @@ "codex/gstack-office-hours": 102339, "codex/gstack-plan-design-review": 142466, "codex/gstack-plan-devex-review": 138649, - "codex/gstack-plan-eng-review": 155247, + "codex/gstack-plan-eng-review": 155172, "codex/gstack-review": 125122, "copilot/gstack-autoplan": 131637, "copilot/gstack-codex": 100235, @@ -28,7 +28,7 @@ "copilot/gstack-office-hours": 100495, "copilot/gstack-plan-design-review": 140884, "copilot/gstack-plan-devex-review": 136912, - "copilot/gstack-plan-eng-review": 153426, + "copilot/gstack-plan-eng-review": 153351, "copilot/gstack-review": 125489, "cursor/gstack-autoplan": 129278, "cursor/gstack-design-consultation": 107222, @@ -36,7 +36,7 @@ "cursor/gstack-land-and-deploy": 115550, "cursor/gstack-plan-design-review": 138968, "cursor/gstack-plan-devex-review": 134591, - "cursor/gstack-plan-eng-review": 151555, + "cursor/gstack-plan-eng-review": 151480, "cursor/gstack-review": 140750, "factory/gstack-autoplan": 131562, "factory/gstack-codex": 100655, @@ -46,34 +46,34 @@ "factory/gstack-office-hours": 101374, "factory/gstack-plan-design-review": 141749, "factory/gstack-plan-devex-review": 137960, - "factory/gstack-plan-eng-review": 154669, + "factory/gstack-plan-eng-review": 154594, "factory/gstack-review": 144529, "gbrain/gstack-autoplan": 129815, "gbrain/gstack-design-review": 132842, "gbrain/gstack-land-and-deploy": 116072, "gbrain/gstack-plan-design-review": 127855, "gbrain/gstack-plan-devex-review": 115129, - "gbrain/gstack-plan-eng-review": 134287, + "gbrain/gstack-plan-eng-review": 134212, "hermes/gstack-autoplan": 129905, "hermes/gstack-design-review": 132949, "hermes/gstack-land-and-deploy": 116174, "hermes/gstack-plan-design-review": 127964, "hermes/gstack-plan-devex-review": 115215, - "hermes/gstack-plan-eng-review": 134372, + "hermes/gstack-plan-eng-review": 134297, "kiro/gstack-autoplan": 129040, "kiro/gstack-design-consultation": 107064, "kiro/gstack-design-review": 142321, "kiro/gstack-land-and-deploy": 115440, "kiro/gstack-plan-design-review": 138808, "kiro/gstack-plan-devex-review": 134431, - "kiro/gstack-plan-eng-review": 151405, + "kiro/gstack-plan-eng-review": 151330, "kiro/gstack-review": 140562, "openclaw/gstack-autoplan": 129520, "openclaw/gstack-design-review": 131647, "openclaw/gstack-land-and-deploy": 115660, "openclaw/gstack-plan-design-review": 126706, "openclaw/gstack-plan-devex-review": 113968, - "openclaw/gstack-plan-eng-review": 132684, + "openclaw/gstack-plan-eng-review": 132609, "opencode/gstack-autoplan": 131640, "opencode/gstack-design-consultation": 109357, "opencode/gstack-design-review": 144705, @@ -81,7 +81,7 @@ "opencode/gstack-office-hours": 100329, "opencode/gstack-plan-design-review": 140738, "opencode/gstack-plan-devex-review": 136812, - "opencode/gstack-plan-eng-review": 153307, + "opencode/gstack-plan-eng-review": 153232, "opencode/gstack-review": 142985, "slate/gstack-autoplan": 129159, "slate/gstack-design-consultation": 107143, @@ -89,7 +89,7 @@ "slate/gstack-land-and-deploy": 115495, "slate/gstack-plan-design-review": 138888, "slate/gstack-plan-devex-review": 134511, - "slate/gstack-plan-eng-review": 151480, + "slate/gstack-plan-eng-review": 151405, "slate/gstack-review": 140656 } }