From 0fe616e3ed944803e3d83a2991145a0d8f0f1f06 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 17 Oct 2026 22:31:47 +0000 Subject: [PATCH] fix(qa): input-rule probes are per field, and a value breaking two rules covers neither (W3-9) PR gate 39073934326 (93449bc) failed qa-functional-cli-report with no observed unannounced defect, the same class as 38983306684 and 39001107574. The agent read 'allowed characters' as the ID rule only, counted -5 as its sign probe and never probed the amount's digits-only rule with an otherwise valid value, so the seeded parseInt defect stayed hidden. The stop rule now says each field's documented input rule needs a probe breaking only it, and that a signed amount breaks sign and digits-only and covers neither; the functional table row names input rules per field. The qa union stays inside its parity ratio. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CHANGELOG.md | 2 +- qa-only/sections/exploratory.md | 2 +- qa/sections/exploratory.md | 2 +- qa/sections/system-functional.md | 2 +- scripts/resolvers/qa.ts | 4 ++-- test/qa-functional-prompt.test.ts | 6 ++++-- 6 files changed, 10 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2d5284b8..0ae1772c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ - **The State Machines specialist runs when its detector fails.** The /review and /ship scope block now runs `gstack-state-matrix detect` itself and uses its output only on exit 0. A failure prints `SCOPE_STATE_MACHINE=unknown` with the detector's stderr, and unknown dispatches the specialist. Its output reaches `gstack-state-matrix check` as a file the agent writes, never as text in a shell command. - **Consent options are lettered B, C and D when Aside is absent**, matching the question's answer keys. - **Plan-reality tier 1 rows are never n/a.** A run no longer loops repairing `deferred-asks n/a` past its deadline. -- **/qa's exploratory stop rule needs one probe per documented input rule.** Rules left uncovered at the probe or time cap go to limits. After a fix, every earlier capture is superseded: rerun each one the verdict relies on, or report it as open. +- **/qa's exploratory stop rule needs one probe per documented input rule of each field.** A value that breaks two rules, such as a signed amount under a digits-only rule, covers neither. Rules left uncovered at the probe or time cap go to limits. After a fix, every earlier capture is superseded: rerun each one the verdict relies on, or report it as open. - **`gstack-docs-candidate snapshot --help` and `compare --help` exit 0** before any side effect, and a compare reject names the path, each bad field and a fix that never re-snapshots over the pre-audit baseline. - **The docsync fault actor uses the production candidate protocol.** It writes candidates through the installed snapshot, and a failed snapshot reports a stable reason with one retry under a fresh audit id. - **The required-probe reminder counts only captures after the last source edit** when the caller declares `GSTACK_QA_REQUIRED_AFTER_EDIT`. Pre-repair captures no longer look like post-repair coverage. Without the declaration nothing changes. diff --git a/qa-only/sections/exploratory.md b/qa-only/sections/exploratory.md index 0c59c8a01..bb3927e74 100644 --- a/qa-only/sections/exploratory.md +++ b/qa-only/sections/exploratory.md @@ -36,7 +36,7 @@ For /qa and /qa-only: - Functional Full, Quick and Regression have no default total timer. Set SECONDS to the shorter mode/caller limit; an unlimited mode uses the caller's bound. Without a total time limit, do not create DEADLINE_FILE; announce finite command timeouts. -Stop when scoped contracts are tested or blocked: each documented input rule needs its own probe breaking only that rule; rules left uncovered at a probe or time cap go in limits. +Stop when scoped contracts are tested or blocked: each field's documented input rule needs a probe breaking only it (a signed amount breaks sign and digits-only, covering neither); rules uncovered at a probe/time cap go in limits. Clocks/checkpoints use REPORT_DIR; mixed standalone runs use REPORT_DIR/browser and REPORT_DIR/functional, with one final report at REPORT_DIR. Caller paths win. Names used below (quote every path): - PROBE_DIR: this surface's owned probe directory per the line above. DEADLINE_FILE: `PROBE_DIR/deadline.json`, which exists only for a bounded run. diff --git a/qa/sections/exploratory.md b/qa/sections/exploratory.md index dd842bb68..1f1ee6d7b 100644 --- a/qa/sections/exploratory.md +++ b/qa/sections/exploratory.md @@ -30,7 +30,7 @@ For /qa and /qa-only: - Functional Full, Quick and Regression have no default total timer. Set SECONDS to the shorter mode/caller limit; an unlimited mode uses the caller's bound. Without a total time limit, do not create DEADLINE_FILE; announce finite command timeouts. -Stop when scoped contracts are tested or blocked: each documented input rule needs its own probe breaking only that rule; rules left uncovered at a probe or time cap go in limits. +Stop when scoped contracts are tested or blocked: each field's documented input rule needs a probe breaking only it (a signed amount breaks sign and digits-only, covering neither); rules uncovered at a probe/time cap go in limits. Clocks/checkpoints use REPORT_DIR; mixed standalone runs use REPORT_DIR/browser and REPORT_DIR/functional, with one final report at REPORT_DIR. Caller paths win. Names used below (quote every path): - PROBE_DIR: this surface's owned probe directory per the line above. DEADLINE_FILE: `PROBE_DIR/deadline.json`, which exists only for a bounded run. diff --git a/qa/sections/system-functional.md b/qa/sections/system-functional.md index 3268189ba..50b281ad7 100644 --- a/qa/sections/system-functional.md +++ b/qa/sections/system-functional.md @@ -30,7 +30,7 @@ pass/fail/blocked/not run/inconclusive/not applicable (reason). | Contract | Observe | |---|---| | Successful execution | Expected return/output and final business effect, not just launch/acceptance | -| Invalid/missing input | One probe per documented input rule (allowed characters, format, sign, range, arity), each input breaking only that rule so no other check rejects it first: declared rejection, correct status and no forbidden state change | +| Invalid/missing input | One probe per documented input rule of each field (allowed characters, format, sign, range, arity) breaking only it: declared rejection, correct status and no forbidden state change | | Authentication/authorization | Valid identity, missing/invalid identity, wrong owner/role and durable no-effect boundary | | CLI process contract | Exact exit code, stdout and stderr separately; resulting file/state changes | | State transitions | Initial, intermediate and completed/failed states and their permitted transitions | diff --git a/scripts/resolvers/qa.ts b/scripts/resolvers/qa.ts index 314c744a3..babc12329 100644 --- a/scripts/resolvers/qa.ts +++ b/scripts/resolvers/qa.ts @@ -77,7 +77,7 @@ For /qa and /qa-only: - Functional Full, Quick and Regression have no default total timer. Set SECONDS to the shorter mode/caller limit; an unlimited mode uses the caller's bound. Without a total time limit, do not create DEADLINE_FILE; announce finite command timeouts. -Stop when scoped contracts are tested or blocked: each documented input rule needs its own probe breaking only that rule; rules left uncovered at a probe or time cap go in limits. +Stop when scoped contracts are tested or blocked: each field's documented input rule needs a probe breaking only it (a signed amount breaks sign and digits-only, covering neither); rules uncovered at a probe/time cap go in limits. Clocks/checkpoints use REPORT_DIR; mixed standalone runs use REPORT_DIR/browser and REPORT_DIR/functional, with one final report at REPORT_DIR. Caller paths win. Names used below (quote every path): - PROBE_DIR: this surface's owned probe directory per the line above. DEADLINE_FILE: \`PROBE_DIR/deadline.json\`, which exists only for a bounded run. @@ -185,7 +185,7 @@ pass/fail/blocked/not run/inconclusive/not applicable (reason). | Contract | Observe | |---|---| | Successful execution | Expected return/output and final business effect, not just launch/acceptance | -| Invalid/missing input | One probe per documented input rule (allowed characters, format, sign, range, arity), each input breaking only that rule so no other check rejects it first: declared rejection, correct status and no forbidden state change | +| Invalid/missing input | One probe per documented input rule of each field (allowed characters, format, sign, range, arity) breaking only it: declared rejection, correct status and no forbidden state change | | Authentication/authorization | Valid identity, missing/invalid identity, wrong owner/role and durable no-effect boundary | | CLI process contract | Exact exit code, stdout and stderr separately; resulting file/state changes | | State transitions | Initial, intermediate and completed/failed states and their permitted transitions | diff --git a/test/qa-functional-prompt.test.ts b/test/qa-functional-prompt.test.ts index e5adefb29..202f62228 100644 --- a/test/qa-functional-prompt.test.ts +++ b/test/qa-functional-prompt.test.ts @@ -302,7 +302,9 @@ test('fix runs rerun every pre-repair capture their verdict relies on; supersede }); test('exploratory sections state one stop rule tied to per-rule input probes, and fix runs close superseded captures', () => { - const stop = 'Stop when scoped contracts are tested or blocked: each documented input rule needs its own probe breaking only that rule; rules left uncovered at a probe or time cap go in limits.'; + // PR gates 38983306684, 39001107574 and 39073934326 (#3260 at 93449bc): cli-report counted `-5` as its sign probe and never + // probed the amount's digits-only rule with an otherwise valid value, so it missed the seeded non-digit defect. + const stop = "Stop when scoped contracts are tested or blocked: each field's documented input rule needs a probe breaking only it (a signed amount breaks sign and digits-only, covering neither); rules uncovered at a probe/time cap go in limits."; const fix = 'After a fix, every earlier capture is superseded, even from a client that prints no snapshot: rerun each one your verdict relies on, or report it as an open remainder.'; for (const rel of ['qa/sections/exploratory.md', 'qa-only/sections/exploratory.md']) { const section = readFileSync(join(import.meta.dir, '..', rel), 'utf8'); @@ -311,7 +313,7 @@ test('exploratory sections state one stop rule tied to per-rule input probes, an expect(section.match(/input rule/g)).toHaveLength(1); expect(section.includes(fix)).toBe(rel.startsWith('qa/')); } - expect(readFileSync(join(import.meta.dir, '../qa/sections/system-functional.md'), 'utf8')).toContain('One probe per documented input rule'); + expect(readFileSync(join(import.meta.dir, '../qa/sections/system-functional.md'), 'utf8')).toContain('One probe per documented input rule of each field (allowed characters, format, sign, range, arity) breaking only it:'); }); test('fix-stage checkpoint provenance survives intervening native regression tests', () => {