From ed27163db4dce1b6c4bece98648fd02daea9d0da Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 3 Oct 2026 22:25:32 +0000 Subject: [PATCH] I1: shared-libs plan actor reads one exclusion grammar for scope lists Periodic run 37151477069 refused 'Existing-caller migration and helper hardening stay out of scope' as a hardening proposal: the actor had two partial exclusion rules (unanchored is/are/remains outside, and an anchored list that only stays unchanged), the third refusal at this boundary after 36633323521 and 36776104571. One anchored predicate now covers a bare list of scope nouns (hyphenated compounds included) with is/are/stays/remains and unchanged/untouched/excluded/out of scope/outside/not part; any other verb in the clause is still read as a proposal. Adds the real capture and two adversarial controls. --- ...plan-callers-out-of-scope-37151477069.json | 23 +++++++++++++++++++ test/helpers/shared-libs-plan-actor.ts | 19 ++++++++++----- test/shared-libs-plan-actor.test.ts | 12 ++++++++++ 3 files changed, 48 insertions(+), 6 deletions(-) create mode 100644 test/fixtures/shared-libs-plan-callers-out-of-scope-37151477069.json diff --git a/test/fixtures/shared-libs-plan-callers-out-of-scope-37151477069.json b/test/fixtures/shared-libs-plan-callers-out-of-scope-37151477069.json new file mode 100644 index 000000000..f9b68ec88 --- /dev/null +++ b/test/fixtures/shared-libs-plan-callers-out-of-scope-37151477069.json @@ -0,0 +1,23 @@ +{ + "questions": [ + { + "question": "D1 — Parser source for the two new Retry-After callers\nProject/branch/task: main — plan review for src/import-worker.ts and src/sync-route.ts (PLAN.md), neither written yet.\nELI10: The plan says each new file should get its own hand-written copy of the code that turns a Retry-After header into a wait time. The repo already has that exact code in lib/retry-after.ts, tested and used by the scheduler. Two more copies means three-plus places that must stay identical forever, and the header parsing has odd corners (past dates, huge numbers, strings like \"1.5\") that are easy to get subtly wrong in a copy.\nStakes if we pick wrong: a copy that drifts from the scheduler produces different wait times for the same header, which the plan explicitly forbids, and the bug only shows up as mis-timed retries in production.\nRecommendation: A because the proven helper already matches the fixed contract, costs one import line per caller, and the shared-contract test is the one place to prove the edge cases both callers need.\nCompleteness: A=10/10, B=7/10, C=5/10\nPros / cons:\nA) Reuse lib helper + full proof (recommended) (human: ~1 h / CC: ~5 min)\n ✅ Zero parser logic added; both callers import retrySeconds exactly like src/scheduler.ts:1 does today\n ✅ Shared-contract test gains the edge cases the new callers rely on (past date → 0, date ceiling 3600, blank → fallback, unsafe integer → fallback, caller fallback honored) plus one integration test per caller\n ❌ Any future change to lib/retry-after.ts now affects the scheduler and both new callers at once (blast radius of three callers)\nB) Reuse lib helper, caller tests only (human: ~30 min / CC: ~3 min)\n ✅ Same reuse and same one-line wiring as A, no parser logic duplicated anywhere\n ✅ Smallest test diff: only the two planned caller integration tests, no shared-test edits\n ❌ Edge cases the callers depend on (past dates, date-path ceiling, blank header) stay unproven anywhere in the suite\nC) Local copy per caller, as drafted (human: ~2 h / CC: ~10 min)\n ✅ Each caller is fully self-contained; a change to one cannot alter the others\n ✅ No edits to lib/ or its test file, matching the plan text as written\n ❌ Adds ~30 implementation lines of duplicated logic plus duplicated contract tests, and every copy must reproduce engine-specific Date.parse quirks to stay scheduler-compatible\nNet: one import line per caller plus a few shared test cases buys guaranteed scheduler parity; copies buy isolation nobody asked for at the price of permanent drift risk.", + "header": "Parser source", + "multiSelect": false, + "options": [ + { + "label": "Reuse lib helper + full proof", + "description": "Both new callers import retrySeconds from lib/retry-after.ts (as src/scheduler.ts does) instead of defining a parser. Extend test/retry-after.test.ts with the edge cases the callers rely on: past date → 0, date-path ceiling 3600, whitespace-only → fallback, unsafe integer → fallback, caller fallback honored on null. Keep one integration test per caller. Existing-caller migration and helper hardening stay out of scope. ✅ Zero parser logic added. ✅ Edge cases proven once in the shared test. ❌ Three callers now share one helper's blast radius. Effort human ~1 h / CC ~5 min." + }, + { + "label": "Reuse lib helper, caller tests only", + "description": "Both new callers import retrySeconds from lib/retry-after.ts; no local parser. Only the two planned caller integration tests are added; test/retry-after.test.ts is left unchanged. Existing-caller migration and helper hardening stay out of scope. ✅ Same one-line wiring as A with no duplicated logic. ✅ Smallest test diff. ❌ Past-date, date-ceiling and blank-header behaviors remain unproven in the suite. Effort human ~30 min / CC ~3 min." + }, + { + "label": "Local copy per caller, as drafted", + "description": "Keep the plan as written: each new caller implements its own parser reproducing the scheduler semantics, with per-caller contract and integration tests. lib/retry-after.ts and its test are untouched. ✅ Each caller is self-contained. ✅ Matches the plan text as written. ❌ ~30 duplicated implementation lines plus duplicated tests, and each copy must reproduce engine-specific Date.parse quirks exactly. Effort human ~2 h / CC ~10 min." + } + ] + } + ] +} diff --git a/test/helpers/shared-libs-plan-actor.ts b/test/helpers/shared-libs-plan-actor.ts index 21d6d736c..65d3de5ba 100644 --- a/test/helpers/shared-libs-plan-actor.ts +++ b/test/helpers/shared-libs-plan-actor.ts @@ -1,11 +1,22 @@ import type { SharedQuestionSelector } from './shared-libs-eval-fixture'; +/** + * A whole clause that only lists excluded work: a bare list of scope nouns and + * an exclusion predicate, e.g. "Existing copies and helper hardening stay + * unchanged", "Hardening is outside this decision", "Existing-caller migration + * and helper hardening stay out of scope". A clause with any other verb, such + * as "Harden helper parsing so behavior stays unchanged", is not an exclusion. + */ +const SCOPE_ITEM = String.raw`(?:(?:the|existing|current|its|all|both|helper|parser|lib|shared|callers?|scheduler)[\s-]+)*(?:copies|callers?|hardening|migrations?|semantics|behaviou?r|contract|helper|parser)`; +const SCOPE_EXCLUSION = new RegExp(String.raw`^${SCOPE_ITEM}(?:\s*,\s*${SCOPE_ITEM})*(?:,?\s+and\s+${SCOPE_ITEM})?\s+(?:is|are|stays?|remains?)\s+` + + String.raw`(?:unchanged|untouched|excluded|out of scope|outside|not part)(?:\s+(?:of\s+)?(?:this|the)\s+(?:decision|scope|plan|change))?[.!]?$`, 'i'); + /** Separate explicit exclusions from proposals; do not erase a following "but" clause. */ function affirmativeCommitments(text: string): string { return text.split(/\n|;|(?<=[.!?])\s+|\s+but\s+|\s+however,?\s+/i).map(raw => { let clause = raw.replace(/^[✅❌\s]+/, '').replace(/\s*\((?:no|not|without|never)\b[^()]*\)/gi, '').trim(); if (/^(?:do not|don't|never|no\b|without\b)/i.test(clause)) return ''; - if (/\b(?:is|are|remains?)\s+(?:outside\b|out of scope\b|excluded\b|not part\b)/i.test(clause)) return ''; + if (SCOPE_EXCLUSION.test(clause)) return ''; clause = clause.replace(/\b(?:without|do not|don't|never)\b.*$/i, ''); return clause; }).filter(Boolean).join('\n'); @@ -87,11 +98,7 @@ export function createSharedPlanReuseSelector(): SharedQuestionSelector { } // Inspect the question as well as the selected option: a harmless label must // not authorize an extra commitment hidden in its brief or description. - // "Existing copies and helper hardening stay unchanged" names excluded work. - // Only a bare list of those nouns qualifies; a verb such as "Harden" does not. - const item = String.raw`(?:(?:the|existing|current|its|all|both|helper|parser|lib|shared|caller|scheduler)\s+)*(?:copies|callers|hardening|migrations?|semantics|behaviou?r|contract|helper|parser)`; - const unchangedScope = new RegExp(String.raw`^${item}(?:\s*,\s*${item})*(?:,?\s+and\s+${item})?\s+(?:stays?|remains?)\s+(?:unchanged|untouched)[.!]?$`, 'i'); - const proposed = affirmativeCommitments(context + '\n' + commitment).split('\n').filter(clause => !unchangedScope.test(clause.trim())).join('\n'); + const proposed = affirmativeCommitments(context + '\n' + commitment); const expansions = [ /\b(?:harden\w*|tighten\w*|strict(?:er)?|saniti[sz]\w*|coerc\w*)\b/i, /\b(?:add(?:s|ing)?|insert(?:s|ing)?|introduc(?:e|es|ing)|implement(?:s|ing)?|appl(?:y|ies|ying)|enabl(?:e|es|ing)|creat(?:e|es|ing))\s+(?:(?:a|an|the|one|new|shared|extra|explicit|validation|numeric|malformed|input|parser)\s+)*(?:guard|validator|validation|normalization)\b/i, diff --git a/test/shared-libs-plan-actor.test.ts b/test/shared-libs-plan-actor.test.ts index 61a7a3af5..170873d47 100644 --- a/test/shared-libs-plan-actor.test.ts +++ b/test/shared-libs-plan-actor.test.ts @@ -4,6 +4,7 @@ import { createSharedPlanReuseSelector } from './helpers/shared-libs-plan-actor' import { createSharedInteractiveToolHandler } from './helpers/shared-libs-eval-fixture'; import capturedNoHardening from './fixtures/shared-libs-plan-callers-no-hardening-36633323521.json'; import capturedParity from './fixtures/shared-libs-plan-callers-parity-36776104571.json'; +import capturedOutOfScope from './fixtures/shared-libs-plan-callers-out-of-scope-37151477069.json'; // Exact native R1 from the September 22 timeout. R2 was saved in an Edit, but // never sent as a native AUQ; its public draft fields are reconstructed below. @@ -372,10 +373,21 @@ describe('bounded shared-code planning actor', () => { expect(run.answers).toEqual([]); }); + // Census 37151477069 refused this exact public question: "Existing-caller migration + // and helper hardening stay out of scope" excludes work; it does not propose it. + test('actual native out-of-scope exclusion keeps unchanged-helper reuse answerable', async () => { + const run = actor(); + const result = await run.callback('AskUserQuestion', capturedOutOfScope); + expect(result.updatedInput.answers).toEqual({ [capturedOutOfScope.questions[0].question]: capturedOutOfScope.questions[0].options[0].label }); + expect(run.refusals).toEqual([]); + }); + test.each([ 'Harden helper parsing so behavior stays unchanged.', 'Existing copies stay unchanged and tighten helper validation.', 'Migrate existing retry-worker.ts; helper hardening stays unchanged.', + 'Existing-caller migration stays out of scope and harden the helper.', + 'Helper hardening is out of scope, but tighten helper validation.', ])('an unchanged-scope clause cannot hide an expansion: %s', async extra => { const input = structuredClone(capturedParity); input.questions[0].options[0].description += '\n' + extra;