From d00590f2ec715dff9480b99afd2ff906d5852587 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Fri, 11 Sep 2026 03:57:45 +0000 Subject: [PATCH] test: recognize valid review decisions and pending native cards --- test/autoplan-command-prefix-au.test.ts | 54 +++++++ test/ceo-hold-posture-ag.test.ts | 81 ++++++++++ test/design-count-review.test.ts | 149 +++++++++++++++++++ test/devex-seed-coverage.test.ts | 105 +++++++++++++ test/eng-library-hooks-aq.test.ts | 101 +++++++++++++ test/helpers/autoplan-artifact-permission.ts | 24 ++- test/helpers/ceo-mode-option.ts | 24 ++- test/helpers/claude-pty-runner.ts | 48 +++++- test/helpers/design-count-review.ts | 43 ++++-- test/helpers/devex-seed-coverage.ts | 46 +++--- 10 files changed, 633 insertions(+), 42 deletions(-) diff --git a/test/autoplan-command-prefix-au.test.ts b/test/autoplan-command-prefix-au.test.ts index de9470113..fe8156c30 100644 --- a/test/autoplan-command-prefix-au.test.ts +++ b/test/autoplan-command-prefix-au.test.ts @@ -40,6 +40,57 @@ type Replay = ReturnType; const pick = (r: Replay, seen = new Set()) => permission.pendingAutoplanArtifactPermissionInput(r.viewport, r.context, seen); const panel = (viewport: string) => viewport.slice(viewport.search(/^[─╌]{8,}\n {0,3}Edit file/m)); +// Exact AY public native prefix; only its owned archive path is relocated onto +// this existing digest fixture. The unpublished Bash body is not reconstructed. +function nativeCards(r: Replay): string { + const relative = path.relative(r.context.ownedStateRoot, r.file).split(path.sep).join('/'); + return [ + `● Update(~/.gstack/${relative})`, ' ', '● Updated plan', ' ', '● Updated plan', ' ', + '● Bash(mkdir -p ~/.gstack/analytics', + ` echo '{"skill":"plan-ceo-review","via":"autoplan","ts":"'$(date -u`, + ` +%Y-%m-%dT%H:%M:%SZ)'","iterations":3,"issues_found":56,"issues_…)`, + ' ⎿  Waiting…', '', '', '', + ].join('\n') + panel(r.viewport); +} + +test('native plan redraws and a queued command preserve only the digest-bound pending Edit', () => { + const r = replay(); try { + r.viewport = nativeCards(r); + const granted = pick(r); + expect(granted).toEqual({ input: '1\r', signature: `${r.context.pending!.sessionId}:${r.context.pending!.toolUseId}`, file: r.file }); + expect(permission.autoplanArtifactPermissionInput(r.viewport, r.context, new Set())).toBeNull(); + expect(permission.publishedAutoplanArtifactPermissionInput(r.viewport, r.context, new Set())).toBeNull(); + expect(pick(r, new Set([granted!.signature]))).toBeNull(); + expect(pick(r, new Set([permission.autoplanArtifactMenuKey(r.viewport)]))).toBeNull(); + } finally { r.dispose(); } +}); + +const nativeScreens: Array<[string, (s: string) => string]> = [ + ['foreign Update title', s => s.replace('Update(~/.gstack/', 'Update(/foreign/')], + ['unbound redraw', s => s.replace('● Updated plan', '● Updated another file')], + ['second Update', s => s.replace('● Updated plan', '● Update(/foreign/plan.md)')], + ['second Bash', s => s.replace('● Updated plan', '● Bash(echo another…)')], + ['no native redraw', s => s.replaceAll('● Updated plan', '')], + ['completed command', s => s.replace('Waiting…', 'Done')], + ['missing Waiting marker', s => s.replace(' ⎿  Waiting…', '')], + ['unclosed command card', s => s.replace('"issues_…)', '"issues_…')], + ['unindented command continuation', s => s.replace(' echo ', 'echo ')], + ['competing permission', s => s.replace(' echo ', ' Do you want to proceed? ')], + ['indented native action', s => s.replace(' echo ', ' ● Read ')], + ['indented question', s => s.replace(' echo ', ' ❯ 1. ')], + ['source prefix', s => 'Source:\n' + s], + ['quoted pane', s => s.split('\n').map(row => '> ' + row).join('\n')], + ['code pane', s => '```text\n' + s + '\n```'], + ['second edit panel', s => s + '\n' + panel(s)], + ['foreign active panel', s => s.replace('projects/gstack-autoplan-chain-zmFsqo/', 'projects/foreign/')], + ['foreign menu', s => s.replace('edit to 2026-09-10-user-dashboard.md?', 'edit to other.md?')], + ['persistent approval', s => s.replace('❯ 1. Yes', '❯ 2. Yes')], + ['changed digest-bound addition', s => s.replace('zero before advancing', 'ten before advancing')], +]; +for (const [name, change] of nativeScreens) test(`native batch cards cannot hide another authority: ${name}`, () => { + const r = replay(); try { r.viewport = change(nativeCards(r)); expect(pick(r)).toBeNull(); } finally { r.dispose(); } +}); + test('the exact public command display preserves only the current unpublished Edit approval', () => { const r = replay(); try { expect(r.context.transcriptStatus).toBe('ready'); expect(r.context.publicTools).toHaveLength(2); @@ -125,6 +176,9 @@ const bindings: Array<[string, (r: Replay) => void]> = [ for (const [name, change] of bindings) test(`pending authorization is retained: ${name}`, () => { const r = replay(); try { change(r); expect(pick(r)).toBeNull(); } finally { r.dispose(); } }); +for (const [name, change] of bindings) test(`native cards retain pending authorization: ${name}`, () => { + const r = replay(); try { r.viewport = nativeCards(r); change(r); expect(pick(r)).toBeNull(); } finally { r.dispose(); } +}); test('only the Autoplan workflow selects this fixture and behavioral regression', () => { for (const file of ['test/autoplan-command-prefix-au.test.ts', 'test/fixtures/autoplan-command-prefix-au.json']) expect(selectTests([file], E2E_TOUCHFILES, []).selected).toEqual(['autoplan-chain-pty']); diff --git a/test/ceo-hold-posture-ag.test.ts b/test/ceo-hold-posture-ag.test.ts index fcd1b5a8a..b57acc228 100644 --- a/test/ceo-hold-posture-ag.test.ts +++ b/test/ceo-hold-posture-ag.test.ts @@ -129,3 +129,84 @@ test('new evidence controls select only the existing mode paid owner', () => { expect(selectTests([file], E2E_TOUCHFILES, []).selected).toEqual(['plan-ceo-mode-routing']); } }); + +// Exact public AY parent narration after the answered HOLD SCOPE mode AUQ. +// Its native ownership controls use the existing PLAN.md / approved-approach-B fixture. +const ambiguityNarration = "I'm holding strictly to the plan's approved scope (Approach B, private-only views) and flagging any ambiguities the sketch leaves undecided as targeted questions rather than expanding scope. First up: what happens when a saved view's filters reference something that's been deleted.\n\n"; +const ambiguityReplay = () => { + const transcript = replay(); + transcript.assistantMessages[0]!.text = ambiguityNarration; + return transcript; +}; +const ambiguityMatches = (text = ambiguityNarration) => { + const transcript = ambiguityReplay(); transcript.assistantMessages[0]!.text = text; + return matches(transcript); +}; + +test('approved scope plus targeted ambiguity questions applies HOLD without naming the mode', () => { + expect(posture.test(ambiguityNarration)).toBe(false); + expect(ambiguityMatches()).toBe(true); + for (const text of [ + ambiguityNarration.replace("I'm holding", 'We are keeping'), + ambiguityNarration.replace("I'm holding", 'I will hold'), + ambiguityNarration.replace('the sketch leaves undecided', 'in the plan').replace('flagging', 'surfacing'), + ambiguityNarration.replace("plan's", "PLAN.md's"), + ambiguityNarration.replace("I'm", 'I’m').replace("plan's", 'plan’s'), + ]) expect(ambiguityMatches(text), text).toBe(true); +}); + +test('ambiguity wording must adopt every obligation without quoting, negating or deferring it', () => { + for (const text of [ + '> ' + ambiguityNarration, '"' + ambiguityNarration.trim() + '"', + '```text\n' + ambiguityNarration + '```', '~~~text\n' + ambiguityNarration + '~~~', + 'Example only: ' + ambiguityNarration, 'The user said: ' + ambiguityNarration, + 'Read(file)\n' + ambiguityNarration, 'If approved, ' + ambiguityNarration, + ambiguityNarration.replace("I'm holding", 'I would hold'), + ambiguityNarration.replace("I'm holding", 'I will later hold'), + ambiguityNarration.replace("I'm holding", "I'm not holding"), + ambiguityNarration.replace('and flagging', 'and not flagging'), + ambiguityNarration.replace('approved scope', 'proposed scope'), + ambiguityNarration.replace("plan's", "OTHER.md's"), + ambiguityNarration.replace('private-only views', 'OTHER.md views'), + ambiguityNarration.replace('Approach B', 'Approach C'), + ambiguityNarration.replace('as targeted questions rather than expanding scope', 'as optional improvements'), + ambiguityNarration.replace('rather than expanding scope', 'while expanding scope'), + ambiguityNarration.replace('ambiguities the sketch leaves undecided', 'word choice and formatting'), + ]) expect(ambiguityMatches(text), text).toBe(false); + for (const correction of [ + 'I am expanding scope to include sharing.', + 'Correction: I will add defaults to scope.', + 'The previously excluded sharing feature is now in scope.', + 'Correction: I am no longer holding scope to this plan.', + "Correction: I am not holding strictly to the plan's approved scope.", + 'Correction: I am no longer flagging ambiguities as targeted questions.', + 'Correction: this posture is withdrawn.', + 'This posture is no longer current.', + ]) { + expect(ambiguityMatches(ambiguityNarration + correction), correction).toBe(false); + expect(ambiguityMatches(ambiguityNarration + '> ' + correction), correction).toBe(true); + } +}); + +test('ambiguity posture stays bound to the approved plan and its actual native answer', () => { + for (const change of [ + (t: PlanCountTranscript) => { t.calls[0]!.answered = false; }, + (t: PlanCountTranscript) => { t.calls[0]!.failed = true; }, + (t: PlanCountTranscript) => { t.calls[0]!.answers![t.calls[0]!.questions[0]!.question] = 'Scope Expansion'; }, + (t: PlanCountTranscript) => { t.assistantMessages[0]!.sessionId = 'foreign'; }, + (t: PlanCountTranscript) => { t.assistantMessages[0]!.timestamp = t.calls[0]!.answeredAt!; }, + (t: PlanCountTranscript) => { t.assistantMessages[0]!.timestamp = 'invalid'; }, + (t: PlanCountTranscript) => { t.assistantMessages[0]!.timestamp = new Date(Date.now() + 60_000).toISOString(); }, + ]) { const transcript = ambiguityReplay(); change(transcript); expect(matches(transcript)).toBe(false); } + for (const [from, to] of [ + ['PLAN.md', 'PLAN.md and OTHER.md'], + ['approved.', 'not approved.'], + ['approved.', 'approved if accepted.'], + ['approved.', 'discussed.'], + ]) { + const transcript = ambiguityReplay(); const q = transcript.calls[0]!.questions[0]!; + const before = q.question; q.question = before.replace(from!, to!); + transcript.calls[0]!.answers = { [q.question]: transcript.calls[0]!.answers![before]! }; + expect(matches(transcript), to).toBe(false); + } +}); diff --git a/test/design-count-review.test.ts b/test/design-count-review.test.ts index 15dd1c26e..a15b3f22c 100644 --- a/test/design-count-review.test.ts +++ b/test/design-count-review.test.ts @@ -41,6 +41,155 @@ function replay(input: NativePlanQuestionCall[], first = isDesignCountFirstRevie return { ...counts, started, phases }; } +describe('A descriptive hierarchy header owns its primary and peer controls', () => { + // Minimal public excerpt of AY D3: retain its question, current gap and native + // options, without copying the full review or its repeated option prose. + const first = (): NativePlanQuestionCall => { + const question = 'D3 — Issue 1: How should Save be distinguished from Reset, Cancel, and Export in the header?\n' + + 'Project/branch/task: settings on main, design review of PLAN.md.\n' + + 'ELI10: Right now all four header buttons look identical.'; + const options = [ + {label: '1A Filled primary + ghosts (recommended)', description: 'Save is #1d4ed8 with white text; Reset, Cancel, Export are neutral ghost buttons per DESIGN.md.'}, + {label: '1B Also move Export out', description: 'Primary + ghosts, plus relocate Export below the header; changes accepted DOM order.'}, + {label: '1C Bold label only', description: 'Keep identical buttons, bold the Save text. Weak signal, off-token.'}, + ]; + return {sessionId: 'ay-design', toolUseId: 'hierarchy', questions: [{header: 'Hierarchy', question, multiSelect: false, options}], + answered: true, failed: false, unansweredQuestionIndices: [], answeredAt: '2026-09-11T03:10:11.660Z', + answers: {[question]: options[0]!.label}}; + }; + const retry = (): NativePlanQuestionCall => { + const c = first(), q = c.questions[0]!; + q.header = 'Issue 1'; + q.question = q.question.replace('be distinguished from Reset, Cancel, and Export in the header', 'stand out from Reset, Cancel and Export') + .replace('all four', 'the four') + + ' DESIGN.md already names the answer: Save is the only filled primary button, the other three are neutral ghost buttons.'; + q.options = [ + {label: '1A Filled primary Save (recommended)', description: '✅ Save becomes the only filled button (#1d4ed8, white text); Reset/Cancel/Export use the existing neutral ghost variant (human: ~1h / CC: ~5min). ✅ Matches DESIGN.md exactly and reuses existing Button variants, no new styles.'}, + {label: '1B Position only, no fill', description: '✅ Keeps all four buttons visually calm with Save separated by a 16px gap from the secondaries. ❌ Violates DESIGN.md and still forces label reading.'}, + {label: '1C Leave as-is', description: "✅ Zero implementation work in this update. ✅ No visual change for users who already learned the layout. ❌ Ships a known DESIGN.md violation and the plan's own Visual Hierarchy gap stays open."}, + ]; + c.answers = {[q.question]: q.options[0]!.label}; + return c; + }; + const edit = (change: (c: NativePlanQuestionCall) => void, source = first) => { + const c = source(); change(c); + if (c.answers && Object.keys(c.answers).length) c.answers = {[c.questions[0]!.question]: c.questions[0]!.options[0]!.label}; + return fingerprint(c); + }; + test('a current gap, complete style and opposed partial fix start review for any offered answer', () => { + const c = first(), q = c.questions[0]!; + for (const o of q.options) { + c.answers = {[q.question]: o.label}; + expect(isDesignCountFirstReview(fingerprint(c))).toBe(true); + } + expect(isDesignCountSetup(fingerprint(c))).toBe(false); + expect(isDesignCompletionHandoff(fingerprint(c))).toBe(false); + }); + test('names, palette, peer order, numeric count and severity metadata may vary consistently', () => { + expect(isDesignCountFirstReview(edit(c => { + const q = c.questions[0]!; + q.header = 'Visual Hierarchy'; + q.question = q.question.replaceAll('Save', 'Publish').replace('all four', 'all 4').replace('\nELI10:', '\n[P1]\nELI10:'); + q.options = q.options.map(o => ({...o, description: o.description?.replaceAll('Save', 'Publish') + .replace('#1d4ed8 with white', '#ffee22 with black').replace('Reset, Cancel, Export are', 'Export, Reset, Cancel are')})); + q.options.reverse(); + }))).toBe(true); + }); + test('retry stand-out wording binds existing variants and an owned open hierarchy gap', () => { + const c = retry(), q = c.questions[0]!; + for (const option of q.options) { + c.answers = {[q.question]: option.label}; + expect(isDesignCountFirstReview(fingerprint(c))).toBe(true); + } + expect(isDesignCountFirstReview(edit(c => { + const q = c.questions[0]!; + q.question = q.question.replaceAll('Save', 'Publish').replace('the four', 'the 4'); + q.options = q.options.map(o => ({...o, label: o.label.replaceAll('Save', 'Publish'), + description: o.description?.replaceAll('Save', 'Publish').replace('#1d4ed8, white', '#ffee22, black') + .replace('Reset/Cancel/Export', 'Export, Reset and Cancel')})); + }, retry))).toBe(true); + }); + test('retry variants, authority, primary and peers must remain in the same native option', () => { + const changes: Array<(c: NativePlanQuestionCall) => void> = [ + c => {c.questions[0]!.options[0]!.label = '1A Filled primary Reset (recommended)';}, + c => {c.questions[0]!.options[0]!.description = c.questions[0]!.options[0]!.description!.replace('Save becomes', 'Publish becomes');}, + c => {c.questions[0]!.options[0]!.description = c.questions[0]!.options[0]!.description!.replace('Reset/Cancel/Export', 'Reset/Cancel/Archive');}, + c => {c.questions[0]!.options[0]!.description = c.questions[0]!.options[0]!.description!.replace('neutral ghost variant', 'filled primary variant');}, + c => {c.questions[0]!.options[0]!.description = c.questions[0]!.options[0]!.description!.split(' ✅ Matches')[0]!;}, + c => {c.questions[0]!.options[0]!.description = c.questions[0]!.options[0]!.description!.replace('✅ Matches DESIGN.md exactly', '✅ If approved, matches DESIGN.md exactly');}, + c => {c.questions[0]!.options[0]!.description = c.questions[0]!.options[0]!.description!.replace('✅ Matches DESIGN.md exactly', '✅ "Matches DESIGN.md exactly"');}, + c => {c.questions[0]!.options[1]!.description = c.questions[0]!.options[0]!.description; c.questions[0]!.options[0]!.description = 'Prepare the review.';}, + c => {c.questions[0]!.options[2]!.description = c.questions[0]!.options[2]!.description!.replace('gap stays open', 'gap is closed');}, + c => {c.questions[0]!.options[2]!.description = 'Historical source excerpt:\n' + c.questions[0]!.options[2]!.description;}, + c => {c.questions[0]!.options[2]!.description = c.questions[0]!.options[2]!.description!.replace('Ships a known', 'Does not ship a known');}, + c => {c.questions[0]!.options[2]!.description = c.questions[0]!.options[2]!.description!.replace('Visual Hierarchy gap', 'account permission gap');}, + ]; + for (const change of changes) expect(isDesignCountFirstReview(edit(change, retry))).toBe(false); + }); + const invalid: Array<[string, (c: NativePlanQuestionCall) => void]> = [ + ['unanswered native call', c => {c.answered = false;}], + ['failed native call', c => {c.failed = true;}], + ['no recorded answer', c => {c.answers = {};}], + ['missing completion timestamp', c => {delete c.answeredAt;}], + ['unanswered member', c => {c.unansweredQuestionIndices = [0];}], + ['multiple native questions', c => {c.questions.push(structuredClone(c.questions[0]!));}], + ['multi-select', c => {c.questions[0]!.multiSelect = true;}], + ['duplicate native label', c => {c.questions[0]!.options[1]!.label = c.questions[0]!.options[0]!.label;}], + ['wrong choice issue', c => {c.questions[0]!.options[0]!.label = '2A Filled primary + ghosts (recommended)';}], + ['setup header', c => {c.questions[0]!.header = 'Routing';}], + ['foreign Issue header', c => {c.questions[0]!.header = 'Issue 2';}], + ['no D-numbered finding', c => {c.questions[0]!.question = c.questions[0]!.question.replace('D3 — ', '');}], + ['source-framed question', c => {c.questions[0]!.question = 'Historical example:\n' + c.questions[0]!.question;}], + ['conditional current gap', c => {c.questions[0]!.question = c.questions[0]!.question.replace('ELI10: Right now', 'ELI10: If right now');}], + ['quoted current gap', c => {c.questions[0]!.question = c.questions[0]!.question.replace('ELI10:', '> ELI10:');}], + ['negated current gap', c => {c.questions[0]!.question = c.questions[0]!.question.replace('look identical', 'do not look identical');}], + ['duplicate assessment', c => {c.questions[0]!.question += '\nELI10: Right now all four header buttons look identical.';}], + ['foreign pre-assessment prose', c => {c.questions[0]!.question = c.questions[0]!.question.replace('\nELI10:', '\nSource excerpt:\nELI10:');}], + ['conditional metadata', c => {c.questions[0]!.question = c.questions[0]!.question.replace('Project/branch/task: settings', 'Project/branch/task: If settings');}], + ['wrong control count', c => {c.questions[0]!.question = c.questions[0]!.question.replace('all four', 'all three');}], + ['duplicate peer', c => {c.questions[0]!.question = c.questions[0]!.question.replace('Reset, Cancel, and Export', 'Reset, Cancel, and Cancel');}], + ['primary also a peer', c => {c.questions[0]!.question = c.questions[0]!.question.replace('Reset, Cancel, and Export', 'Save, Cancel, and Export');}], + ['foreign remedy primary', c => {c.questions[0]!.options[0]!.description = c.questions[0]!.options[0]!.description!.replace('Save is', 'Publish is');}], + ['foreign remedy peer', c => {c.questions[0]!.options[0]!.description = c.questions[0]!.options[0]!.description!.replace('Export are', 'Archive are');}], + ['no filled role in native label', c => {c.questions[0]!.options[0]!.label = '1A Prepare a review';}], + ['no concrete color', c => {c.questions[0]!.options[0]!.description = c.questions[0]!.options[0]!.description!.replace('#1d4ed8', 'blue');}], + ['no design authority', c => {c.questions[0]!.options[0]!.description = c.questions[0]!.options[0]!.description!.replace('per DESIGN.md', 'per an archived example');}], + ['split label and remedy owners', c => {c.questions[0]!.options[1]!.description = c.questions[0]!.options[0]!.description; c.questions[0]!.options[0]!.description = 'Prepare the design review.';}], + ['quoted remedy', c => {c.questions[0]!.options[0]!.description = '"' + c.questions[0]!.options[0]!.description + '"';}], + ['conditional remedy', c => {c.questions[0]!.options[0]!.description = 'If approved: ' + c.questions[0]!.options[0]!.description;}], + ['contradicted remedy', c => {c.questions[0]!.options[0]!.description += ' This amendment keeps all four buttons identical.';}], + ['control named Fix cannot bypass owned style checks', c => { + const q = c.questions[0]!; + q.question = q.question.replaceAll('Save', 'Fix'); + q.options[0]!.description = 'This applies the next review step.'; + }], + ['wrong declined control', c => {c.questions[0]!.options[2]!.description = c.questions[0]!.options[2]!.description!.replace('Save text', 'Publish text');}], + ['no retained equality', c => {c.questions[0]!.options[2]!.description = 'Make Save a filled primary button.';}], + ['no retained violation', c => {c.questions[0]!.options[2]!.description = c.questions[0]!.options[2]!.description!.replace('Weak signal, off-token.', 'Strong signal, on-token.');}], + ['quoted deferral', c => {c.questions[0]!.options[2]!.description = '> ' + c.questions[0]!.options[2]!.description;}], + ['conditional deferral', c => {c.questions[0]!.options[2]!.description = 'If accepted: ' + c.questions[0]!.options[2]!.description;}], + ['cancelled deferral', c => {c.questions[0]!.options[2]!.description += ' Correction: do not keep identical buttons.';}], + ]; + test.each(invalid)('%s cannot provide current finding evidence', (_, change) => { + expect(isDesignCountFirstReview(edit(change))).toBe(false); + }); + test('withdrawal and approval status stay local even after a style or a partial-fix match', () => { + for (const source of [first, retry]) for (const target of ['question', 'remedy', 'decline']) { + for (const status of [' This issue is withdrawn.', ' This issue is "withdrawn".', ' This gap is now closed.', ' If approval is granted, use this option.']) { + expect(isDesignCountFirstReview(edit(c => { + const q = c.questions[0]!; + if (target === 'question') q.question += status; + else q.options[target === 'remedy' ? 0 : 2]!.description += status; + }, source))).toBe(false); + } + } + const fp = fingerprint(first()); + expect(isDesignCountFirstReview({...fp, signature: 'foreign:call'})).toBe(false); + expect(isDesignCountFirstReview({...fp, nativeQuestionIndex: 1})).toBe(false); + expect(isDesignCountFirstReview({...fp, options: fp.options.slice(1)})).toBe(false); + }); +}); + describe('Z numbered gap starts review at the actual plan amendment', () => { const actual = () => structuredClone(gapCalls.calls) as NativePlanQuestionCall[]; const first = () => actual()[0]!; diff --git a/test/devex-seed-coverage.test.ts b/test/devex-seed-coverage.test.ts index 81962ac42..85a3db64c 100644 --- a/test/devex-seed-coverage.test.ts +++ b/test/devex-seed-coverage.test.ts @@ -13,6 +13,111 @@ function extra(id: string, sessionId: string): NativePlanQuestionCall { return {sessionId,toolUseId:id,questions:[{header:'Extra',question,multiSelect:false,options:[{label:'Add command',description:'Add the command after the beta.'},{label:'Defer',description:'Defer the command.'}]}],answered:true,failed:false,answers:{[question]:'Defer'},unansweredQuestionIndices:[],answeredAt:'2026-09-09T20:23:00Z'}; } +// Exact public AY headings and signature trace, applied to the existing native +// completion fixture. Full public replay remains separate from paid-run credit. +const tracedAyTitles = [ + 'D5 — Journey stage: INSTALL / HELLO WORLD. The README quickstart points at a file that is not shipped.', + 'D4 — Journey stage: HELLO WORLD. The mandatory 5-minute remote CI check before the first local result.', + 'D7 — Journey stage: REAL USAGE. Two sibling functions take the same two arguments in opposite order.', + 'D6 — Journey stage: DEBUG. The authentication error says nothing.', + "D8 — Journey stage: UPGRADE. v1's Client.evaluate() vanishes in v2 with no warning, alias, or guide.", +]; +function tracedAyTranscript(): PlanCountTranscript { + const t = transcript(); + for (const [i, c] of t.calls.entries()) { + const q = c.questions[0]!, lines = q.question.split('\n'); lines[0] = tracedAyTitles[i]!; + if (i === 2) { + lines[1] = 'Project/branch/task: EvalKit 2.0.0b1 beta, branch main; docs/api.md:3-9.'; + lines.splice(2, 0, 'I traced the first real integration after the demo. docs/api.md lists the two evaluation functions: `run_eval(dataset, evaluator)` and `run_batch(evaluator, dataset)`.'); + q.options[0] = { + label: 'Fix in plan: same order + keyword-only for both (recommended)', + description: '✅ run_eval(*, dataset, evaluator) and run_batch(*, dataset, evaluator); wrong order becomes a TypeError naming the parameter at the call site', + }; + } + q.question = lines.join('\n'); c.answers = { [q.question]: q.options[0]!.label }; + } + return t; +} + +describe('DX current traced journey decisions', () => { + test('five observed title forms retain distinct completed seed decisions', () => { + const t = tracedAyTranscript(), result = devexSeedCoverage(t); + expect(result.complete).toBe(true); expect(result.missing).toEqual([]); + expect(new Set(Object.values(result.decisions).flat()).size).toBe(5); + for (let i = 0; i < 5; i++) { + const copy = structuredClone(t); copy.calls.splice(i, 1); + expect(devexSeedCoverage(copy).missing).toHaveLength(1); + for (const option of t.calls[i]!.questions[0]!.options) { + const alternate = structuredClone(t), c = alternate.calls[i]!; + c.answers = { [c.questions[0]!.question]: option.label }; + expect(devexSeedCoverage(alternate).complete).toBe(true); + } + } + }); + test('quoted, hypothetical, healthy and withdrawn titles cannot supply these findings', () => { + const healthy = [ + (s: string) => s.replace('is not shipped', 'is shipped'), + (s: string) => s.replace('mandatory', 'optional'), + (s: string) => s.replace('opposite order', 'the same order'), + (s: string) => s.replace('says nothing', 'explains the cause and fix'), + (s: string) => s.replace('vanishes in v2 with no warning, alias, or guide', 'remains in v2 as a compatibility alias'), + ]; + for (let i = 0; i < 5; i++) for (const edit of [ + (s: string) => '> ' + s, (s: string) => 'Quoted source: ' + s, + (s: string) => 'If approved, ' + s, + (s: string) => s.replace('Project/branch/task: ', 'Project/branch/task: Historical assessment: '), + (s: string) => s + '\nCorrection: this finding is withdrawn.', + (s: string) => s.replace(tracedAyTitles[i]!, healthy[i]!(tracedAyTitles[i]!)), + ]) { + const t = tracedAyTranscript(); changeDeclaration(t, i, edit); + expect(devexSeedCoverage(t).complete, `${i}: ${edit(tracedAyTitles[i]!)}`).toBe(false); + } + }); + test('signature identity and a current same-function remedy must belong to the trace', () => { + for (const [from, to] of [ + ['I traced the first real integration', 'The source says I traced the first real integration'], + ['docs/api.md lists', 'docs/other.md lists'], + ['`run_batch(evaluator, dataset)`', '`run_batch(dataset, evaluator)`'], + ['I traced the first real integration', '> I traced the first real integration'], + ]) { + const t = tracedAyTranscript(); changeDeclaration(t, 2, s => s.replace(from!, to!)); + expect(devexSeedCoverage(t).missing, to).toContain('reversed-arguments'); + } + for (const i of [2, 3, 4]) for (const mode of ['quoted', 'withdrawn', 'foreign']) { + const t = tracedAyTranscript(), c = t.calls[i]!, q = c.questions[0]!; + q.options = q.options.map(o => mode === 'quoted' ? { label: '"' + o.label + '"', description: '"' + o.description + '"' } + : mode === 'withdrawn' ? { ...o, description: o.description + '\nThis option is withdrawn.' } + : { ...o, description: o.description?.replaceAll('run_batch', 'other_batch').replaceAll('AuthError', 'OtherError').replaceAll('Client.evaluate', 'OtherClient.evaluate') }); + c.answers = { [q.question]: q.options[0]!.label }; + expect(devexSeedCoverage(t).complete, `${i}: ${mode}`).toBe(false); + } + }); + test('the asserted signature trace remains current before ELI10', () => { + for (const status of [ + 'Correction: these functions are now aligned.', + 'These signatures are historical.', + 'These signatures are no longer current.', + 'This trace applies only if approved.', + 'This trace is historical.', + 'This trace is withdrawn.', + ]) for (const quoted of [false, true]) { + const t = tracedAyTranscript(); + changeDeclaration(t, 2, text => text.replace(/^(I traced[^\n]*)$/m, + '$1 ' + (quoted ? JSON.stringify(status) : status))); + expect(devexSeedCoverage(t).complete, `${quoted ? 'quoted' : 'current'}: ${status}`).toBe(quoted); + } + }); + test('new title wording cannot bypass native completion or session ownership', () => { + for (let i = 0; i < 5; i++) for (const mutate of [ + (c: NativePlanQuestionCall) => { c.answered = false; }, + (c: NativePlanQuestionCall) => { c.failed = true; }, + (c: NativePlanQuestionCall) => { c.sessionId = 'foreign'; }, + (c: NativePlanQuestionCall) => { c.unansweredQuestionIndices = [0]; }, + (c: NativePlanQuestionCall) => { c.answers = { [c.questions[0]!.question]: 'Not offered' }; }, + ]) { const t = tracedAyTranscript(); mutate(t.calls[i]!); expect(devexSeedCoverage(t).complete).toBe(false); } + }); +}); + describe('DX seeded-gap coverage', () => { for (const [i, attempt] of fixture.attempts.entries()) test(`actual attempt ${attempt.attempt} has five distinct completed seed decisions`, () => { const result = devexSeedCoverage(transcript(i)); diff --git a/test/eng-library-hooks-aq.test.ts b/test/eng-library-hooks-aq.test.ts index f6e07310c..6b4456504 100644 --- a/test/eng-library-hooks-aq.test.ts +++ b/test/eng-library-hooks-aq.test.ts @@ -64,3 +64,104 @@ describe('AQ library-hooks choice opens batching review on its current remedy',( expect(classify(text(s=>s+'\nCorrection: retry scheduling no longer runs inside each worker.'))).toBe(false); }); }); + +describe('AY scheduler choices retain the current gap and opposed native remedies', () => { + const publicCall=(retry=false):NativePlanQuestionCall=>{ + // Minimal excerpts from the two public calls; no transcript/report corpus. + const question=retry + ? "D1 — Custom inline backoff scheduler vs the job library's built-in retry hooks\nProject/branch/task: main — background job retry framework (PLAN.md:6-8).\nELI10: The plan says (PLAN.md:6-8) to ignore it and hand-roll a scheduler inside each of the 5 workers, same shape as the library version." + : "D3 — Issue 1: custom inline scheduler per worker, or the job library's retry hook with a custom curve?\nProject/branch/task: main, PLAN.md §Architecture — background job retry framework.\nELI10: The plan writes its own \"wait, then try again\" loop inside each of the 5 workers. If that process dies mid-wait, the retry is gone and nobody knows."; + const options=retry?[ + {label:'1A) Library hooks + shared backoff fn (recommended)',description:"Register the library's retry hook in each worker, pass one shared pure backoffDelay(attempt) for the curve. Completeness 9/10."}, + {label:'1B) Custom scheduler as one shared module',description:'Roll your own, but once, with persisted retry state. You own a second job system.'}, + {label:'1C) Proceed as planned (inline in 5 workers)',description:'Keep the plan as written. Completeness 4/10. Retries die with the process; five copies drift.'}, + ]:[ + {label:'1A: Library hook + custom curve (recommended)',description:'✅ Retry state persisted by the library: survives worker crash, deploy, and restart (human: ~1 day / CC: ~20 min).'}, + {label:'1B: Custom inline scheduler as planned',description:'✅ No dependency on library hook semantics. ❌ Retry state lives in process memory: any crash mid-backoff silently drops the job; you rebuild max-attempts, dead-letter, and metrics by hand.'}, + {label:'1C: Hybrid: library hook, but custom scheduler for one worker',description:'Library persistence for 4 workers today; two retry systems remain.'}, + ]; + return {sessionId:'ay-public',toolUseId:retry?'retry':'first',questions:[{header:retry?'Architecture':'Arch 1',question,multiSelect:false,options}], + answered:true,failed:false,unansweredQuestionIndices:[],answeredAt:'2026-09-11T03:22:05.503Z',answers:{[question]:options[0]!.label}}; + }; + const edit=(retry:boolean,change:(c:NativePlanQuestionCall)=>void)=>{ + const c=publicCall(retry);change(c); + if(c.answers&&Object.keys(c.answers).length)c.answers={[c.questions[0]!.question]:c.questions[0]!.options[0]!.label}; + return c; + }; + test('both public forms start review on the same answered native choice',()=>{ + for(const retry of [false,true]){ + const c=publicCall(retry),q=c.questions[0]!; + for(const o of q.options){c.answers={[q.question]:o.label};expect(classify(c)).toBe(true);} + expect(engSetupAUQ(fp(c))).toBe(false); + } + }); + test('worker counts and native option order may vary consistently',()=>{ + for(const retry of [false,true])expect(classify(edit(retry,c=>{ + const q=c.questions[0]!;q.question=q.question.replace('5 workers','7 workers'); + q.options=q.options.map(o=>({...o,label:o.label.replace('5 workers','7 workers'),description:o.description?.replace('five copies','seven copies')})); + q.options.reverse(); + }))).toBe(true); + }); + test('same-owner native completion, metadata and menu are mandatory',()=>{ + const bad:Array<(c:NativePlanQuestionCall)=>void>=[ + c=>{c.answered=false;},c=>{c.failed=true;},c=>{delete c.answeredAt;},c=>{c.answers={};},c=>{c.unansweredQuestionIndices=[0];}, + c=>{c.questions[0]!.header='Routing';},c=>{c.questions[0]!.options[0]!.label='2A: Library hook + custom curve';}, + c=>{c.questions[0]!.question='Source excerpt:\n'+c.questions[0]!.question;}, + c=>{c.questions[0]!.question=c.questions[0]!.question.replace('ELI10: ','ELI10: If approved, ');}, + c=>{c.questions[0]!.question=c.questions[0]!.question.replace('Project/branch/task: ','Project/branch/task: If approved, ');}, + c=>{c.questions[0]!.question=c.questions[0]!.question.replace('ELI10:','> ELI10:');}, + c=>{c.questions[0]!.question+='\nELI10: The plan writes its own loop inside each of the 5 workers.';}, + c=>{c.questions[0]!.question+='\nCorrection: retry scheduling no longer runs inside each worker.';}, + ]; + for(const retry of [false,true])for(const change of bad)expect(classify(edit(retry,change))).toBe(false); + for(const retry of [false,true])expect(engFirstReviewAUQ({...fp(publicCall(retry)),signature:'foreign:call'})).toBe(false); + }); + test('the proposed library mechanism cannot borrow from another native option',()=>{ + for(const retry of [false,true]){ + const keep=retry?2:1; + for(const change of [ + (c:NativePlanQuestionCall)=>{[c.questions[0]!.options[0]!.description,c.questions[0]!.options[keep]!.description]=[c.questions[0]!.options[keep]!.description,c.questions[0]!.options[0]!.description];}, + (c:NativePlanQuestionCall)=>{c.questions[0]!.options[0]!.description='The library could be evaluated later.';}, + (c:NativePlanQuestionCall)=>{c.questions[0]!.options[0]!.description='Source excerpt: '+c.questions[0]!.options[0]!.description;}, + (c:NativePlanQuestionCall)=>{c.questions[0]!.options[0]!.description='"'+c.questions[0]!.options[0]!.description+'"';}, + (c:NativePlanQuestionCall)=>{c.questions[0]!.options[0]!.description+='\nThe library will not own persistence.';}, + (c:NativePlanQuestionCall)=>{c.questions[0]!.options[keep]!.description='Keep the current design; no retries are lost.';}, + (c:NativePlanQuestionCall)=>{c.questions[0]!.options[keep]!.description='If approved, '+c.questions[0]!.options[keep]!.description;}, + (c:NativePlanQuestionCall)=>{c.questions[0]!.options[keep]!.description+='\nThe scheduler is now crash-safe.';}, + ])expect(classify(edit(retry,change))).toBe(false); + } + expect(classify(edit(false,c=>{c.questions[0]!.options[0]!.description=c.questions[0]!.options[0]!.description!.replace('survives worker','never survives worker');}))).toBe(false); + expect(classify(edit(false,c=>{c.questions[0]!.options[1]!.description=c.questions[0]!.options[1]!.description!.replace('silently drops','never drops');}))).toBe(false); + expect(classify(edit(true,c=>{c.questions[0]!.options[2]!.description=c.questions[0]!.options[2]!.description!.replace('five copies','two copies');}))).toBe(false); + }); + test('quoted owner status and approval conditions remain current after matched outcomes',()=>{ + for(const retry of [false,true])for(const target of ['question','remedy','unchanged']){ + for(const status of ["This finding is 'withdrawn'.",'This option is conditional on approval.','If approved, proceed with this option.']){ + expect(classify(edit(retry,c=>{ + const q=c.questions[0]!; + if(target==='question')q.question+='\n'+status; + else q.options[target==='remedy'?0:retry?2:1]!.description+='\n'+status; + }))).toBe(false); + } + } + }); + test('approval clauses remain binding after option tradeoffs',()=>{ + for(const retry of [false,true])for(const option of [0,retry?2:1]){ + for(const clause of ['Assuming approval, proceed with this option.','Provided approval, keep this option.']){ + const add=(c:NativePlanQuestionCall,quoted=false)=>{c.questions[0]!.options[option]!.description+=' ❌ Additional integration effort.\n'+(quoted?'"Earlier assessment: '+clause+'"':clause);}; + expect(classify(edit(retry,c=>add(c)))).toBe(false); + expect(classify(edit(retry,c=>add(c,true)))).toBe(true); + } + } + }); + test('a crash premise cannot erase an owned approval condition',()=>{ + for(const retry of [false,true]){ + expect(classify(edit(retry,c=>{c.questions[0]!.question+='\nIf that process dies, this finding applies only if approved.';}))).toBe(false); + expect(classify(edit(retry,c=>{c.questions[0]!.question+='\n"Earlier assessment: If that process dies, this finding applies only if approved."';}))).toBe(true); + expect(classify(edit(retry,c=>{ + const q=c.questions[0]!,consequence='If the worker process crashes, the retry is gone and nobody knows.'; + q.question=retry?q.question+'\n'+consequence:q.question.replace('If that process dies mid-wait, the retry is gone and nobody knows.',consequence); + }))).toBe(true); + } + }); +}); diff --git a/test/helpers/autoplan-artifact-permission.ts b/test/helpers/autoplan-artifact-permission.ts index 58fba5f9f..36d997c7f 100644 --- a/test/helpers/autoplan-artifact-permission.ts +++ b/test/helpers/autoplan-artifact-permission.ts @@ -371,11 +371,31 @@ function queuedCommandViewport(viewport: string, current: NativePublicToolEvent, /** A native command description can remain above an unpublished Edit panel. * Its text supplies no command identity, completion, or approval authority. * Only the digest-bound pending path may discard this one display prefix. */ -function pendingCommandDisplayViewport(viewport: string): string { +function pendingCommandDisplayViewport(viewport: string, file: string, ownedStateRoot?: string): string { const text = viewport.replace(/\r\n?/g, '\n'); const panels = [...text.matchAll(/^[─╌]{8,}\n {0,3}Edit file[ \t]*\n/gm)]; if (panels.length !== 1 || panels[0]!.index === 0) return viewport; const prefix = text.slice(0, panels[0]!.index).split('\n').filter(line => line.trim()); + // An unpublished batch can leave the current Update title, plan redraws, + // and a queued Bash card above the panel. These cards grant no authority: + // only the one current Edit's owned path and complete digest below do so. + const update = /^[●⏺] Update\(([^\n]+)\)$/.exec(prefix[0] ?? ''); + if (update && ownedStateRoot) { + const relative = path.relative(ownedStateRoot, file).split(path.sep).join('/'); + const alias = path.basename(ownedStateRoot) === '.gstack' ? `~/.gstack/${relative}` : undefined; + const bash = prefix.findIndex(row => /^[●⏺] Bash\(/.test(row)); + const command = prefix.slice(bash, -1); + if ((update[1] === file || update[1] === alias) && bash > 1 && + prefix.slice(1, bash).every(row => /^[●⏺] Updated plan$/.test(row)) && + /^ {2}⎿[ \u00a0]+Waiting…$/.test(prefix.at(-1) ?? '') && command.length > 0 && + command.every((row, i) => (i === 0 ? /^[●⏺] Bash\(\S.*$/ : /^ {6}\S.*$/).test(row)) && + command.at(-1)!.endsWith('…)') && + !command.slice(1).some(row => /^ {6}[●⏺❯☐□>]|^ {6}(?:`{3,}|~{3,})/.test(row)) && + !/(?:Do you want|Would you like|Bash command[^\n]*permission|requested permissions?|allow all edits|always allow access|Esc to cancel|Edit file)/i.test(command.join('\n'))) { + return text.slice(panels[0]!.index); + } + return viewport; + } const title = /^[●⏺] ([^\n]+)$/.exec(prefix[0] ?? '')?.[1]; if (!title || /^(?:["'`“‘]|(?:source|example|quoted|history|historical|hypothetical|previous|earlier)\b)/i.test(title) || !/^ {2}⎿[ \u00a0]+\$ \S.*$/.test(prefix[1] ?? '') || @@ -421,7 +441,7 @@ export function pendingAutoplanArtifactPermissionInput(viewport: string, !mutations.some(e => e.input?.file_path === p.file && results.get(e.toolUseId)?.isError === false && Date.parse(results.get(e.toolUseId)!.timestamp) <= pendingTime)) return null; try { - const currentViewport = p.editDigest ? pendingCommandDisplayViewport(viewport) : viewport; + const currentViewport = p.editDigest ? pendingCommandDisplayViewport(viewport, p.file, context.ownedStateRoot) : viewport; const text = currentViewport.replace(/\r\n?/g, '\n'); const menu = /^ {0,3}Do you want to make this edit to ([^\n?]+)\? *\n {0,3}❯ *1\. Yes *\n {0,3}2\. Yes, and switch to accept edits \(auto-approve file edits and common file commands\) for this session(?: \(shift\+tab\))? *\n {0,3}3\. No *\n\s*Esc to cancel [·•] Tab to amend\s*$/m.exec(text); if (!menu || menu.index + menu[0].length !== text.length || menu[1] !== path.basename(p.file)) return null; diff --git a/test/helpers/ceo-mode-option.ts b/test/helpers/ceo-mode-option.ts index ae998519a..c90770ffa 100644 --- a/test/helpers/ceo-mode-option.ts +++ b/test/helpers/ceo-mode-option.ts @@ -158,19 +158,33 @@ function hasCurrentHoldScopePosture(text: string, selected: NativePlanQuestionCa if (/\b(?:not|never|won't|can't|don't|isn't|aren't|if|unless|until|may|might|could|would|will|later|tomorrow|eventually|future|example|hypothetical|after|once|when|whenever|following|pending|provided|assuming)\b|\b(?:next (?:week|month|year)|subject to)\b/i.test(sentence)) return false; const declaration = /^(?:I'm|I am|We're|We are) (?:locking|keeping|holding) (?:the )?scope (?:to|at) ([^,\n]+),\s*(?:flagging|treating|marking) (?:anything|everything) (?:beyond|outside) that(?: \([^()\n]+\))? as out of scope,? and (?:hunting|checking|looking) for (?:silent )?(?:failure modes|errors|edge cases)\b([^.!?\n]*)\.$/i.exec(sentence); const pressureTest = /^(?:I'm|I am|We're|We are) (?:locking|keeping|holding) (?:the )?scope fixed to ([^,\n]+),\s*pressure[- ]testing every stated behavior for failure modes, ([^.!?\n]+?) while deferring (?:anything|everything) extra rather than adding it silently\.$/i.exec(sentence); + const ambiguityReview = /^(?:I'm|I am|We're|We are) (?:keeping|holding) strictly to (?:the )?(plan|[\w./-]+\.md)'s approved scope(?: \((Approach [A-Z])(?:, [^()\n]+)?\))? and (?:flagging|surfacing|raising) (?:any|the) ambiguities (?:the (?:sketch|plan) leaves undecided|in the (?:plan|sketch)) as targeted questions rather than expanding scope\.$/i.exec(sentence); const commitment = declaration ?? pressureTest; - if (!commitment) return false; + if (!commitment && !ambiguityReview) return false; // A later current correction can withdraw the declaration. Quoted examples // cannot; the opening declaration was matched before removing quoted blocks. - const currentProse = plain.replace(/```[\s\S]*?(?:```|$)|~~~[\s\S]*?(?:~~~|$)/g, '') + // Only this recognized opening's explicit exclusion is negative expansion; + // keep later corrections and every other scope statement in the withdrawal check. + const current = ambiguityReview + ? opening.replace(/ rather than expanding scope\.$/i, '.') + plain.slice(opening.length) : plain; + const currentProse = current.replace(/```[\s\S]*?(?:```|$)|~~~[\s\S]*?(?:~~~|$)/g, '') .replace(/^\s*>.*$/gm, '').replace(/"[^"\n]*"|“[^”\n]*”/g, ''); + if (ambiguityReview && (/\b(?:no longer|not)\s+(?:flagging|surfacing|raising)\s+(?:(?:any|the)\s+)?ambiguities\b/i.test(currentProse) || + /\b(?:this|the) posture (?:is|was|has been) (?:withdrawn|rejected|cancelled|canceled|superseded|(?:not|no longer) current)\b/i.test(currentProse))) return false; if (/\b(?:expand\w*|widen\w*|reduc(?:e|ing)|shrink\w*)\s+(?:the\s+)?scope\b/i.test(currentProse) || /\b(?:add|adding)\b[^.!?\n]*\b(?:to|into)\s+(?:the\s+)?scope\b/i.test(currentProse) || - /\b(?:no longer|not)\s+(?:locking|keeping|holding|lock|keep|hold)\s+(?:the\s+)?scope\b/i.test(currentProse) || + /\b(?:no longer|not)\s+(?:locking|keeping|holding|lock|keep|hold)\s+(?:(?:the\s+)?scope\b|strictly to (?:the )?(?:plan|[\w./-]+\.md)'s approved scope\b)/i.test(currentProse) || /\b(?:previously|formerly) excluded\b[^.!?\n]*\b(?:now )?in scope\b/i.test(currentProse)) return false; const context = selected.questions.map(q => /^Project\/branch\/task:([^\n]*)/im.exec(q.question)?.[1] ?? '').join(' '); const plans = new Set(context.match(/\b[\w./-]+\.md\b/gi) ?? []); - const baseline = commitment[1]!.trim(); + if (ambiguityReview) { + if (plans.size !== 1 || (sentence.match(/\b[\w./-]+\.md\b/gi) ?? []).some(plan => !plans.has(plan))) return false; + // Parenthetical approach labels must belong to the approved current plan. + const approach = ambiguityReview[2]; + const approval = approach ? new RegExp(`\\b${approach}\\b[^.!?\\n]*\\bapproved\\b`, 'i') : /\bapproved\b/i; + return approval.test(context) && !/\b(?:not|no|unapproved|rejected|hypothetical|if|unless|until|after|once|when|whenever|following|pending|provided|assuming)\b|\bsubject to\b/i.test(context); + } + const baseline = commitment![1]!.trim(); const namedPlan = /^(?:the )?(?:(?:\d+|one|two|three|four|five|six|seven|eight|nine|ten) )?([\w./-]+\.md) (?:bullets|requirements|scope)(?: from approach [A-Z])?$/i.exec(baseline); const possessivePlan = /^(?:the )?([\w./-]+\.md)'s (?:(?:\d+|one|two|three|four|five|six|seven|eight|nine|ten) )?(?:bullets|requirements|scope)( plus the approved schema)?$/i.exec(baseline); const plan = namedPlan ?? possessivePlan; @@ -179,7 +193,7 @@ function hasCurrentHoldScopePosture(text: string, selected: NativePlanQuestionCa if (possessivePlan?.[2] && (!/\bschema\b[^.!?\n]{0,80}\bapproved\b|\bapproved\b[^.!?\n]{0,80}\bschema\b/i.test(context) || /\b(?:not|no|unapproved|rejected|hypothetical|if|unless|until|after|once|when|whenever|following|pending|provided|assuming)\b|\bsubject to\b/i.test(context))) return false; // Concrete failure surfaces distinguish review rigor from merely retaining scope. - const hardening = commitment[2]!; + const hardening = commitment![2]!; return [/\bconstraints\b/i, /\b(?:error handling|errors)\b/i, /\bedge cases\b/i, /\baccess(?:-rule)? leaks\b/i, /\bsecurity\b/i, /\btest(?:ing|s)\b/i, /\bproduction visibility\b/i] .filter(surface => surface.test(hardening)).length >= 2; diff --git a/test/helpers/claude-pty-runner.ts b/test/helpers/claude-pty-runner.ts index 9833b83db..d1254e889 100644 --- a/test/helpers/claude-pty-runner.ts +++ b/test/helpers/claude-pty-runner.ts @@ -3237,17 +3237,19 @@ function engNumberedFindingAUQ(fp: AskUserQuestionFingerprint): boolean { // The category may precede the issue number. Bind this library choice to // the current scheduling defect and both concrete outcomes, not its label. const declaredLibraryHooks = /^Issue ([1-9]\d*): Custom inline scheduler vs\. the job library's built-in retry hooks \([A-Za-z][\w./-]*:[1-9]\d*(?:-[1-9]\d*)?\)$/i.exec(title); - const libraryHooks = /^Architecture issue ([1-9]\d*): custom inline scheduler vs the job library's built-in retry hooks\?$/i.exec(title) ?? declaredLibraryHooks; + const implicitLibraryHooks = /^D([1-9]\d*)\s*[—–:-]\s*Custom inline backoff scheduler vs the job library's built-in retry hooks$/i.exec(q.question.split('\n')[0]!); + const scopedLibraryHooks = /^Issue ([1-9]\d*): custom inline scheduler per worker, or the job library's retry hook with a custom curve\?$/i.exec(title) ?? implicitLibraryHooks; + const libraryHooks = /^Architecture issue ([1-9]\d*): custom inline scheduler vs the job library's built-in retry hooks\?$/i.exec(title) ?? declaredLibraryHooks ?? scopedLibraryHooks; if (libraryHooks) { - if (!new RegExp(`^${declaredLibraryHooks ? 'Issue' : 'Arch(?:itecture)?'} ${libraryHooks[1]}$`, 'i').test(q.header.trim())) return false; - const ordinal = declaredLibraryHooks && /^D([1-9]\d*)\s*[—–:-]/.exec(q.question)?.[1]; + if (!(implicitLibraryHooks ? /^Architecture$/i : new RegExp(`^${declaredLibraryHooks ? 'Issue' : 'Arch(?:itecture)?'} ${libraryHooks[1]}$`, 'i')).test(q.header.trim())) return false; + const ordinal = (declaredLibraryHooks || scopedLibraryHooks) && /^D([1-9]\d*)\s*[—–:-]/.exec(q.question)?.[1]; const declaredOwner = `(?:(?:this|the|that) (?:finding|issue|gap|remedy|amendment|assessment|option|deferral|(?:unchanged )?risk)|Issue ${libraryHooks[1]}${ordinal ? `|D${ordinal}` : ''})`; const declaredBoundary = '(?:^|[.!?;]\\s+|\\n|[✅❌]\\s*)(?:Correction:\\s*)?'; const scalarOwner = new RegExp(`${declaredBoundary}${declaredOwner} (?:is|was|has been) (?:(?:now|already) )?$`, 'i'); const current = (text: string) => { const prose = text.replace(/```[\s\S]*?(?:```|$)|~~~[\s\S]*?(?:~~~|$)/g, '') .replace(/^(?:\s*>| {4}|\t).*$/gm, ''); - if (declaredLibraryHooks) return prose.replace(/\*\*/g, '') + if (declaredLibraryHooks || scopedLibraryHooks) return prose.replace(/\*\*/g, '') .replace(/"[^"\n]*"|“[^”\n]*”|(? { const status = /^["“'‘`](withdrawn|superseded|rejected|cancelled|canceled|resolved|closed|hypothetical|not current|no longer current)["”'’`]$/i.exec(quoted); @@ -3264,16 +3266,48 @@ function engNumberedFindingAUQ(fp: AskUserQuestionFingerprint): boolean { const contexts = lines.filter(line => /^Project\/branch\/task: \S/.test(line)); const assessments = [...text.matchAll(/^ELI10: (.+)$/gm)]; const preface = text.slice(0, assessments[0]?.index ?? 0).split('\n').filter(line => line.trim()).slice(1); + // A crash consequence explains the current defect; approval conditions + // still suspend the owned decision and are checked below. + const framingText = scopedLibraryHooks ? text.replace(/(?:^|[.!?]\s+)If (?:that|the|this) (?:worker )?process (?:dies|restarts|crashes)\b[^.!?\n]*[.!?]?/gi, '') : text; if (contexts.length !== 1 || assessments.length !== 1 || preface.length !== 1 || preface[0] !== contexts[0] || - framed.test(text) || closed.test(text) || /\b(?:the|this) plan no longer rebuilds retry scheduling\b|\bretry scheduling no longer runs inside each worker\b/i.test(text)) return false; + framed.test(framingText) || closed.test(text) || /\b(?:the|this) plan no longer rebuilds retry scheduling\b|\bretry scheduling no longer runs inside each worker\b/i.test(text)) return false; const assessment = assessments[0]![1]!; // A declarative issue with a source location can own the same concrete // scheduling decision. Bind the assessment and each offered outcome; // the issue number and source location alone do not begin review. - if (declaredLibraryHooks) { + if (declaredLibraryHooks || scopedLibraryHooks) { const ownClosed = new RegExp(`${declaredBoundary}${declaredOwner} (?:is|was|has been) (?:(?:now|already) )?(?:withdrawn|superseded|rejected|cancelled|canceled|resolved|closed|hypothetical|not current|no longer current)\\b`, 'i'); - const conditional = new RegExp(`${declaredBoundary}(?:${declaredOwner} (?:(?:applies|holds) (?:only )?(?:if|when|once|unless) (?:approved|accepted)|is (?:conditional|contingent|dependent) on (?:approval|acceptance)|requires (?:approval|acceptance))|(?:if|when|once) (?:approved|accepted),? (?:use|adopt|accept|keep|proceed|choose|preserve)\\b)`, 'i'); + // Read approval clauses in the original text, including an owned + // condition after a crash premise or after an option's tradeoffs. + const approvalBoundary = scopedLibraryHooks ? `(?:${declaredBoundary}|,\\s+)` : declaredBoundary; + const approvalPremise = scopedLibraryHooks ? '(?:(?:if|when|once) (?:approved|accepted)|(?:assuming|provided) (?:approval|acceptance))' : '(?:if|when|once) (?:approved|accepted)'; + const conditional = new RegExp(`${approvalBoundary}(?:${declaredOwner} (?:(?:applies|holds) (?:only )?(?:if|when|once|unless) (?:approved|accepted)|is (?:conditional|contingent|dependent) on (?:approval|acceptance)|requires (?:approval|acceptance))|${approvalPremise},? (?:use|adopt|accept|keep|proceed|choose|preserve)\\b)`, 'i'); if (ownClosed.test(text) || conditional.test(text)) return false; + if (scopedLibraryHooks) { + const workers = /(?:^|[.!?]\s+)The plan writes its own\s+loop inside each of the ([1-9]\d*) workers\./i.exec(assessment)?.[1] ?? + /(?:^|[.!?]\s+)The plan says \([A-Za-z][\w./-]*:[1-9]\d*(?:-[1-9]\d*)?\) to ignore it and hand-roll a scheduler inside each of the ([1-9]\d*) workers\b/i.exec(assessment)?.[1]; + if (!workers || Number(workers) < 2) return false; + const ids = q.options.map(o => /^([1-9]\d*)([A-D])[).:]\s+(\S[\s\S]*)$/.exec(o.label)); + if (ids.some(id => id?.[1] !== libraryHooks[1]) || new Set(ids.map(id => id![2])).size !== q.options.length) return false; + const actions = ids.map(id => id![3]!.replace(/\s*\(recommended\)$/i, '')); + const repair = actions.findIndex(action => /^Library hooks? \+ (?:custom curve|shared backoff fn)$/i.test(action)); + const keep = actions.findIndex(action => /^Custom inline scheduler as planned$/i.test(action) || + new RegExp(`^Proceed as planned \\(inline in ${workers} workers\\)$`, 'i').test(action)); + if (repair < 0 || keep < 0 || repair === keep) return false; + const remedy = current(q.options[repair]!.description ?? '').trim(), unchanged = current(q.options[keep]!.description ?? '').trim(); + const cancelled = /(?:^|[.!?;]\s+|\n|\bCorrection:\s*)(?:do not|don't|never|skip|cancel|withdraw) (?:use|accept|keep|proceed|adopt|choose|preserve|register)\b/i; + if ([remedy, unchanged].some(value => framed.test(value.split('❌')[0]!) || closed.test(value) || ownClosed.test(value) || conditional.test(value) || cancelled.test(value)) || + /\bthe library will not own (?:persistence|attempt counting|crash safety)\b/i.test(remedy) || + /\b(?:the|this) (?:unchanged |per-worker )?scheduler is (?:now |already )?crash-safe\b|\bretry state no longer lives in-process\b/i.test(unchanged)) return false; + const persisted = /^(?:✅\s*)?Retry state persisted by the library: [^.!?✅❌]*\bsurvives\b[^.!?✅❌]*\b(?:crash|restart|deploy)\b[^.!?✅❌]*/i.exec(remedy); + const registered = /^Register the library's retry hook in each worker, pass one shared pure [A-Za-z_$][\w$]*\(attempt\) for the curve\./i.test(remedy); + const lost = /(?:^|❌\s*)Retry state lives in process memory: [^.!?✅❌]*\b(?:crash|restart)\b[^.!?✅❌]*\b(?:drops|loses)\b[^.!?✅❌]*\b(?:job|retry|retries)\b/i.exec(unchanged); + const drift = /^Keep the plan as written\.[\s\S]*?\bRetries die with the process; ([a-z]+|[1-9]\d*) copies drift\./i.exec(unchanged); + const count = drift && (/^[1-9]\d*$/.test(drift[1]!) ? Number(drift[1]) : + ['zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine', 'ten'].indexOf(drift[1]!.toLowerCase())); + return Boolean((registered || (persisted && !/\b(?:not|never|no longer)\b/i.test(persisted[0]))) && + ((lost && !/\b(?:not|never|no longer)\b/i.test(lost[0])) || (drift && count === Number(workers)))); + } if (!/^The job library already knows how to retry a failed job later; you just tell it how long to wait\. The plan instead rebuilds that waiting-and-rescheduling machinery by hand inside each worker\./.test(assessment) || !/\bjobs get lost \(worker dies mid-sleep\)/.test(assessment) || !/\bno attempt cap or dead-letter path\b/.test(assessment)) return false; diff --git a/test/helpers/design-count-review.ts b/test/helpers/design-count-review.ts index 2c65257a3..9f3cf3f55 100644 --- a/test/helpers/design-count-review.ts +++ b/test/helpers/design-count-review.ts @@ -67,8 +67,10 @@ function ordinaryDesignIssue(fp: AskUserQuestionFingerprint): boolean { // F labels annotate findings; they do not establish review identity alone. const headerActionIssue = /^(?:D[1-9]\d*\s*[—–:-]\s*)?Issue ([1-9]\d*)(?: \(F[1-9]\d*\))?: (How should the header action group establish the primary action)\?$/i.exec(title); const signaledPrimaryIssue = /^(?:D[1-9]\d*\s*[—–:-]\s*)?Issue ([1-9]\d*) \(G[1-9]\d*\): (How should the header action group signal that [A-Za-z][A-Za-z0-9 _-]{0,39} is the primary action)\?$/i.exec(title); + const distinguishedPrimaryIssue = /^D[1-9]\d*\s*[—–:-]\s*Issue ([1-9]\d*): How should ([A-Za-z][A-Za-z0-9 _-]{0,39}) (?:be distinguished|stand out) from ([A-Za-z][A-Za-z0-9 ,_-]{0,119}?)(?: in the header)?\?$/i.exec(title); const issue = /^(?:D[1-9]\d*\s*[—–:-]\s*)?Issue ([1-9]\d*)(?: \((?:(?:G[1-9]\d*|Pass [1-7]), )?(?:Visual Hierarchy|Spacing|Color|Typography|Motion)\))?: ([^?]+)\?$/i.exec(title) ?? headerActionIssue ?? signaledPrimaryIssue; - const descriptivePrimaryHeader = signaledPrimaryIssue && /^(?!(?:focus|scope|setup|routing|learnings|outside voices|next steps?)$)[A-Za-z][A-Za-z _-]{0,39}$/i.test(q.header.trim()); + const descriptivePrimaryHeader = (signaledPrimaryIssue && /^(?!(?:focus|scope|setup|routing|learnings|outside voices|next steps?)$)[A-Za-z][A-Za-z _-]{0,39}$/i.test(q.header.trim())) || + (distinguishedPrimaryIssue && /^(?:Visual )?Hierarchy$/i.test(q.header.trim())); if (!issue || !(new RegExp(`^Issue ${issue[1]}(?:: [A-Za-z][A-Za-z0-9 _-]{0,39})?$`, 'i').test(q.header.trim()) || descriptivePrimaryHeader) || / o.label)).size !== q.options.length || @@ -76,9 +78,10 @@ function ordinaryDesignIssue(fp: AskUserQuestionFingerprint): boolean { !q.options.some(o => o.label === call.answers?.[q.question])) return false; // The numbered headline must ask about a concrete design requirement. // Reviewer participation or workflow navigation can also use Issue labels. - if (!/\b(?:buttons?|primary(?: header)? actions?|primary emphasis|hierarchy|spacing|contrast|colou?rs?|labels?|typography|fonts?|loading|spinner|skeleton|motion)\b/i.test(issue[2]!)) return false; - const opposed = q.options.filter(o => /^(?:[1-9]\d*[A-Z](?:[).:]\s*|\s+))?(?:Defer|Decline|Leave|Keep|Accept the gap)\b/i.test(o.label)); - const repair = !headerActionIssue && /\b(?:fix|resolve|address)\b/i.test(title) && + if (!distinguishedPrimaryIssue && !/\b(?:buttons?|primary(?: header)? actions?|primary emphasis|hierarchy|spacing|contrast|colou?rs?|labels?|typography|fonts?|loading|spinner|skeleton|motion)\b/i.test(issue[2]!)) return false; + const opposed = q.options.filter(o => /^(?:[1-9]\d*[A-Z](?:[).:]\s*|\s+))?(?:Defer|Decline|Leave|Keep|Accept the gap)\b/i.test(o.label) || + (distinguishedPrimaryIssue && /^(?:Keep|Leave)\b/i.test(o.description?.trim() ?? ''))); + const repair = !headerActionIssue && !distinguishedPrimaryIssue && /\b(?:fix|resolve|address)\b/i.test(title) && q.options.some(o => /\b(?:closing|closes|fixes|resolves?|applies?)\b/i.test(o.description ?? '')); // A source citation alone can describe a report or the next reviewer. // Bind the alternate wording to a named control's concrete style amendment @@ -87,8 +90,9 @@ function ordinaryDesignIssue(fp: AskUserQuestionFingerprint): boolean { /^Give ([A-Za-z][A-Za-z0-9 _-]{0,39}) primary emphasis in the header action group$/i.exec(issue[2]!) ?? /^How should the header action group signal that ([A-Za-z][A-Za-z0-9 _-]{0,39}) is the primary action$/i.exec(issue[2]!) ?? /^How should (?:the )?(?:header )?actions establish that ([A-Za-z][A-Za-z0-9 _-]{0,39}) is the primary action$/i.exec(issue[2]!) ?? + (distinguishedPrimaryIssue && /^How should ([A-Za-z][A-Za-z0-9 _-]{0,39}) (?:be distinguished|stand out) from /i.exec(issue[2]!)) ?? (headerActionIssue && new RegExp(`^Issue ${issue[1]}: ([A-Za-z][A-Za-z0-9 _-]{0,39})$`, 'i').exec(q.header.trim())); - const primaryEmphasisIssue = !!signaledPrimaryIssue || /^Give [A-Za-z][A-Za-z0-9 _-]{0,39} primary emphasis in the header action group$/i.test(issue[2]!); + const primaryEmphasisIssue = !!signaledPrimaryIssue || !!distinguishedPrimaryIssue || /^Give [A-Za-z][A-Za-z0-9 _-]{0,39} primary emphasis in the header action group$/i.test(issue[2]!); const scopedPrimaryStatus = !!headerActionIssue || primaryEmphasisIssue; const explicitStyle = primary && `${primary[1]} filled (?:primary )?#[0-9a-f]{6}(?:/| with )(?:white|black)(?: text)?; ` + '[A-Za-z][A-Za-z0-9 ,/_-]{0,99} neutral ghost(?: buttons)?\\.'; @@ -139,7 +143,8 @@ function ordinaryDesignIssue(fp: AskUserQuestionFingerprint): boolean { !sourceAssessment.test(prefix.join(' ')) && !sourceAssessment.test(assessments[0]![1]!) ? assessments[0]![1]! : ''; const headerPeers = primary && headerActionIssue && new RegExp(`^${primary[1]}, ([A-Za-z][A-Za-z0-9 _-]{0,39}(?:, [A-Za-z][A-Za-z0-9 _-]{0,39})*(?:,? and [A-Za-z][A-Za-z0-9 _-]{0,39})?) currently look (?:the same|identical)\\.`, 'i').exec(assessment); - const primaryAssessment = headerActionIssue ? headerPeers?.[0] : + const countedHeader = distinguishedPrimaryIssue && /^(?:Right now|Today) (?:all|the) (two|three|four|five|six|seven|eight|nine|ten|[1-9]\d*) header buttons look (?:the same|identical)\./i.exec(assessment); + const primaryAssessment = distinguishedPrimaryIssue ? countedHeader?.[0] : headerActionIssue ? headerPeers?.[0] : primary && new RegExp(`^(?:Right now|Today) ${primary[1]}(?:, [A-Za-z][A-Za-z0-9 _-]{0,39})+(?:,? and [A-Za-z][A-Za-z0-9 _-]{0,39})? (?:(?:all )?look (?:the same|identical)|are (?:all )?(?:(?:two|three|four|five|six|seven|eight|nine|ten|[1-9]\\d*) )?identical buttons)\\b`, 'i').exec(assessment)?.[0]; const premiseSentence = assessment.split(/[.!?](?:\s|$)/)[0] ?? ''; const currentPrimary = !!primaryAssessment && !/\b(?:not|never|no longer)\b/i.test(primaryAssessment) && @@ -149,8 +154,8 @@ function ordinaryDesignIssue(fp: AskUserQuestionFingerprint): boolean { const numberValue = (value: string) => /^\d+$/.test(value) ? Number(value) : ['zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine', 'ten'].indexOf(value.toLowerCase()); const controlNames = (text: string) => text.toLowerCase().split(/,\s*(?:and\s+)?|\s+and\s+/).map(s => s.trim()).sort(); - const headerControls = headerPeers ? controlNames(headerPeers[1]!) : []; - const otherControls = headerActionIssue ? headerControls.length : primary && primaryAssessment + const headerControls = distinguishedPrimaryIssue ? controlNames(distinguishedPrimaryIssue[3]!) : headerPeers ? controlNames(headerPeers[1]!) : []; + const otherControls = headerActionIssue || distinguishedPrimaryIssue ? headerControls.length : primary && primaryAssessment ? primaryAssessment.replace(new RegExp(`^(?:Right now|Today) ${primary[1]},\\s*`, 'i'), '') .replace(/\s+(?:(?:all )?look (?:the same|identical)|are (?:all )?(?:(?:two|three|four|five|six|seven|eight|nine|ten|[1-9]\d*) )?identical buttons)$/, '') .split(/,\s*(?:and\s+)?|\s+and\s+/).length : 0; @@ -165,7 +170,9 @@ function ordinaryDesignIssue(fp: AskUserQuestionFingerprint): boolean { // Approval conditions suspend this offered decision; explanatory conditions // about user behavior do not make an otherwise current amendment optional. const pendingPrimaryApproval = (text: string) => primaryEmphasisIssue && /(?:^|[.!?;]\s+|\n)(?:[✅❌]\s*)?(?:Correction:\s*)?(?:If|When|Once|Provided|Assuming|Pending)\s+(?:approval|approved|acceptance|accepted|(?:we|you)\s+(?:approve|accept))\b/i.test(text); - const currentHeaderContract = !headerActionIssue || (headerContract && headerControls.length > 0 && + const currentHeaderContract = distinguishedPrimaryIssue ? (countedHeader && headerControls.length > 0 && + new Set(headerControls).size === headerControls.length && !headerControls.includes(primary![1]!.toLowerCase()) && + numberValue(countedHeader[1]!) === headerControls.length + 1 && !conditionalHeader(questionText)) : !headerActionIssue || (headerContract && headerControls.length > 0 && new Set(headerControls).size === headerControls.length && !headerControls.includes(primary![1]!.toLowerCase()) && numberValue(headerContract[1]!) === headerControls.length && !conditionalHeader(questionText)); const statedVariant = (variantContract || namedContract) && @@ -188,7 +195,17 @@ function ordinaryDesignIssue(fp: AskUserQuestionFingerprint): boolean { const body = currentText(amendment.description ?? ''); if (pendingPrimaryApproval(body)) return false; const headerStyle = primary && headerActionIssue && new RegExp(`^(?:✅\\s*)?${primary[1]} becomes the only filled #[0-9a-f]{6} button with (?:white|black) text; ([A-Za-z][A-Za-z0-9 ,_-]{0,119}) become neutral ghost buttons, exactly as DESIGN\\.md states\\.`, 'i').exec(body); - const style = headerActionIssue ? headerStyle?.[0] : amendments.map(pattern => pattern.exec(body)).find(Boolean)?.[0]; + // A descriptive header still owns a concrete primary and every peer. + // The native label supplies the filled role; its own body supplies tokens. + const distinguishedStyle = primary && distinguishedPrimaryIssue && ( + new RegExp(`^(?:✅\\s*)?${primary[1]} is #[0-9a-f]{6} with (?:white|black) text; ([A-Za-z][A-Za-z0-9 ,_-]{0,119}) are neutral ghost buttons per DESIGN\\.md\\.`, 'i').exec(body) ?? + new RegExp(`^(?:✅\\s*)?${primary[1]} becomes the only filled button \\(#[0-9a-f]{6}, (?:white|black) text\\); ([A-Za-z][A-Za-z0-9 ,/_-]{0,119}) use the existing neutral ghost variant` + + '(?: \\(human: ~?[0-9]+(?:\\.[0-9]+)?(?:h|min) / CC: ~?[0-9]+(?:\\.[0-9]+)?(?:h|min)\\))?\\. (?:✅\\s*)?Matches DESIGN\\.md exactly\\b', 'i').exec(body)); + const style = distinguishedPrimaryIssue ? distinguishedStyle?.[0] : headerActionIssue ? headerStyle?.[0] : amendments.map(pattern => pattern.exec(body)).find(Boolean)?.[0]; + if (distinguishedPrimaryIssue && (!distinguishedStyle || conditionalHeader(body) || invalidContract.test(body) || + /(?:^|[.!?;]\s+|\n)(?:Correction:\s*)?(?:the|this) (?:current )?(?:amendment|fix) keeps (?:all )?(?:two|three|four|five|six|seven|eight|nine|ten|[1-9]\d*) (?:header )?buttons identical\b/i.test(body) || + !new RegExp(`^[1-9]\\d*[A-Z][).:]?\\s+Filled primary (?:(?:\\+|and|with) ghosts|${primary![1]})(?: \\(recommended\\))?$`, 'i').test(amendment.label) || + JSON.stringify(controlNames(distinguishedStyle[1]!.replaceAll('/', ','))) !== JSON.stringify(headerControls))) return false; if (headerActionIssue && (!headerStyle || conditionalHeader(body) || invalidContract.test(body) || JSON.stringify(controlNames(headerStyle[1]!)) !== JSON.stringify(headerControls))) return false; const variantLine = /^✅\s*Uses the existing Button primary and ghost variants from DESIGN\.md; no new styles\./m.exec(body); @@ -218,6 +235,12 @@ function ordinaryDesignIssue(fp: AskUserQuestionFingerprint): boolean { const cancelledHeaderDeferral = headerDeferral && /(?:^|[.!?;]\s+|\n)(?:Correction:\s*)?(?:do not|don't|never|skip|cancel|withdraw) (?:keep|leave) (?:the )?header unchanged\b/i.test(declined); const pros = /^(?:✅(?!\s*(?:If|When|Unless|Historical|Hypothetical|Quoted|Source|Example)\b)\s*[^✅❌]+)+❌\s*/i.exec(deferralBody); const remaining = pros && !sourceAssessment.test(pros[0]) ? deferralBody.slice(pros[0].length) : deferralBody; + const retainedEmphasis = distinguishedPrimaryIssue && new RegExp(`^(?:Keep|Leave) identical (?:header )?buttons, bold (?:the )?${primary![1]} (?:text|label)\\. (?:Weak(?: visual)? signal, )?off-token\\.`, 'i').test(remaining); + const retainedHierarchyGap = distinguishedPrimaryIssue && /^(?:❌\s*)?Ships a (?:known|documented) DESIGN\.md violation and the plan['’]s own Visual Hierarchy gap (?:stays|remains) open\./i.test(remaining); + if (distinguishedPrimaryIssue) return defer !== amendment && !!(retainedEmphasis || retainedHierarchyGap) && + !conditionalHeader(declined) && !sourceAssessment.test(declined) && !withdrawn.test(declined) && + !/(?:^|[.!?;]\s+|\n)(?:Correction:\s*)?(?:do not|don't|never|skip|cancel|withdraw) (?:keep|leave) identical (?:header )?buttons\b/i.test(declined) && + !closedGap.test(declined) && !invalidContract.test(declined) && !cancelledStyle.test(declined) && !withdrawnStyles.test(declined); const retainedButtons = /^(?:❌\s*)?Keep all (two|three|four|five|six|seven|eight|nine|ten|[1-9]\d*) (?:header )?buttons identical; gap stays documented\./i.exec(remaining); const cancelledRetainedButtons = /(?:^|[.!?;]\s+|\n)(?:Correction:\s*)?(?:do not|don't|never|skip|cancel|withdraw) (?:keep|leave) (?:all )?(two|three|four|five|six|seven|eight|nine|ten|[1-9]\d*) (?:header )?buttons identical\b/i.exec(declined); if (headerActionIssue) { diff --git a/test/helpers/devex-seed-coverage.ts b/test/helpers/devex-seed-coverage.ts index c9e838170..41f16a2cc 100644 --- a/test/helpers/devex-seed-coverage.ts +++ b/test/helpers/devex-seed-coverage.ts @@ -8,20 +8,23 @@ export type DevexSeededGap = typeof DEVEX_SEEDED_GAPS[number]; /** Bind an unnamed signature question to its own first asserted explanation. */ function explainedReversedSignatures(q: NativePlanQuestion, title: string): boolean { const question = /^(?:Journey stage [A-Z ]+: )?the two public functions take the same two arguments in (?:opposite|reversed) positional order\. How should (?:the plan|we) (?:fix|align|unify) the signatures\?$/i.test(title); - const declaration = /^Journey stage(?: REAL USAGE:|: REAL USAGE\.) The two public evaluation functions take the same two arguments in (?:opposite|reversed) order\.$/i.test(title); + const traced = /^Journey stage: REAL USAGE\. Two sibling functions take the same two arguments in (?:opposite|reversed) order\.$/i.test(title); + const declaration = traced || /^Journey stage(?: REAL USAGE:|: REAL USAGE\.) The two public evaluation functions take the same two arguments in (?:opposite|reversed) order\.$/i.test(title); if (!question && !declaration) return false; const lines = q.question.split('\n'); if (lines[0]!.trim().replace(/^D\s*\d+\s*[—–:-]\s*/i, '') !== title) return false; const explanation = lines.findIndex(line => line.startsWith('ELI10: ')); const context = lines.slice(1, explanation).filter(line => line.trim()); - const project = declaration && context.length === 1 + const project = traced ? /^Project\/branch\/task: [^;\n]+; ([\w./-]+):\d+(?:[-–]\d+)?\.$/.exec(context[0] ?? '') : declaration && context.length === 1 ? /^Project\/branch\/task: [^;\n]+; ([\w./-]+) lines? \d+(?: to |[-–])\d+\.$/.exec(context[0]!) : null; - if (explanation < 1 || (declaration && !project) || context.some(line => + if (explanation < 1 || (declaration && !project) || (!traced && context.some(line => (!declaration && !/^Project\/branch\/task: [^;\n]+; reviewing the public function signatures in [\w./-]+\.$/.test(line)) || - /\b(?:quoted|source excerpt|source example|hypothetical|historical|not (?:a )?current|if approved)\b/i.test(line))) return false; + /\b(?:quoted|source excerpt|source example|hypothetical|historical|not (?:a )?current|if approved)\b/i.test(line)))) return false; // Inline code may name each signature; a quoted/fenced explanation, earlier // unrelated sentence, past definition or hypothetical definition cannot. - const declaredSignatures = declaration && /^ELI10: ([\w./-]+) documents (`?)run_eval\(\s*dataset\s*,\s*evaluator\s*\)\2 and (`?)run_batch\(\s*evaluator\s*,\s*dataset\s*\)\3\. Same two concepts, reversed positional order, and neither function requires keywords\./.exec(lines[explanation]!); + const declaredSignatures = traced + ? /^I traced the first real integration after the demo\. ([\w./-]+) lists the two evaluation functions: (`?)run_eval\(\s*dataset\s*,\s*evaluator\s*\)\2 and (`?)run_batch\(\s*evaluator\s*,\s*dataset\s*\)\3\./.exec(context[1] ?? '') + : declaration && /^ELI10: ([\w./-]+) documents (`?)run_eval\(\s*dataset\s*,\s*evaluator\s*\)\2 and (`?)run_batch\(\s*evaluator\s*,\s*dataset\s*\)\3\. Same two concepts, reversed positional order, and neither function requires keywords\./.exec(lines[explanation]!); if (declaration ? !declaredSignatures || declaredSignatures[1] !== project?.[1] : !/^ELI10: [\w./-]+(?: lines? \d+(?:\s*[-–]\s*\d+)?)? define (`?)run_eval\(\s*dataset\s*,\s*evaluator\s*\)\1 and (`?)run_batch\(\s*evaluator\s*,\s*dataset\s*\)\2\./.test(lines[explanation]!)) return false; const currentProse = (text: string) => { @@ -31,18 +34,23 @@ function explainedReversedSignatures(q: NativePlanQuestion, title: string): bool return !fence && !/^\s*>/.test(line); }).join('\n').replace(/`[^`\n]*`|"[^"\n]*"|“[^”\n]*”/g, ''); }; - const current = currentProse(lines.slice(explanation).join('\n')); + // The traced declaration owns its named signatures before ELI10, so its + // currentness must include that same source paragraph. + const current = currentProse(lines.slice(traced ? 1 : explanation).join('\n')); if ((current.match(/^ELI10:/gm)?.length ?? 0) !== 1) return false; if (declaration && /\b(?:if|once|when|unless) (?:approved|accepted)|\b(?:after|pending) approval\b/i.test(current)) return false; if (declaration && /(?:^|[.!?\n]\s*)(?:Correction:\s*)?(?:these|the) (?:functions|signatures) (?:are (?:now|already)|have been) (?:aligned|consistent)\b/i.test(current)) return false; + if (traced && /(?:^|[.!?\n]\s*)(?:Correction:\s*)?(?:this|that|the) trace (?:is|was|has been) (?:withdrawn|rejected|historical|(?:not|no longer) current)\b/i.test(current)) return false; if (/(?:^|[.!?\n]\s*)(?:Correction:\s*)?(?:(?:this|that|the) (?:finding|explanation)|(?:(?:this|that|the) )?argument[- ]order (?:issue|defect)|these signatures)\b[^.\n]*\b(?:withdrawn|rejected|(?:already )?(?:fixed|resolved)|historical|(?:not|no longer) current)\b/i.test(current) || /(?:^|[.!?\n]\s*)(?:Correction:\s*)?(?:there is|there's) no argument[- ]order (?:issue|defect)\b/i.test(current) || /(?:^|[.!?\n]\s*)(?:Correction:\s*)?run_eval and run_batch now (?:use|take) the same positional order\b/i.test(current)) return false; // A declared reversal may offer a keyword-only repair instead of a swap // guard. It must bind both arguments to both functions in the same option. if (declaration) return q.options.some(option => - /^(?:Align|Unify|Standardize) order \+ keyword-only(?: \(recommended\))?$/i.test(option.label) && - /^Both functions (?:take|accept|use) dataset and evaluator as keyword-only in the same order\./i.test(option.description ?? '') && + (traced ? /^Fix in plan: same order \+ keyword-only for both(?: \(recommended\))?$/i.test(option.label) && + /^✅\s*run_eval\(\*\s*,\s*dataset\s*,\s*evaluator\s*\) and run_batch\(\*\s*,\s*dataset\s*,\s*evaluator\s*\); wrong order becomes a TypeError naming the parameter at the call site\b/i.test(option.description ?? '') + : /^(?:Align|Unify|Standardize) order \+ keyword-only(?: \(recommended\))?$/i.test(option.label) && + /^Both functions (?:take|accept|use) dataset and evaluator as keyword-only in the same order\./i.test(option.description ?? '')) && !/\b(?:if|once|when|unless) (?:approved|accepted)|\b(?:after|pending) approval\b/i.test(currentProse(option.description ?? '')) && !/(?:^|[.!?\n]\s*)(?:Correction:\s*)?(?:(?:do not|don't|never) (?:change|align|unify) (?:either|both|the|these) (?:functions?|signatures?)\b|(?:do not|don't|never) (?:make|require) (?:either|both|the) (?:functions?|signatures?|arguments?) keyword-only\b|(?:this|the) (?:option|correction|action) is (?:withdrawn|rejected|cancelled)\b)/i.test(currentProse(option.description ?? ''))); // The same offered action must align both functions and retain the call-site @@ -67,7 +75,7 @@ function decisionGaps(q: NativePlanQuestion): DevexSeededGap[] { // adjacent touchpoints without changing the subject or who asserts it. const journeyStage = '(?:DISCOVER|INSTALL|HELLO WORLD|REAL USAGE|DEBUG|UPGRADE)'; const stage = title.match(new RegExp(`^Journey stage ${journeyStage}(?:\\s*\\/\\s*${journeyStage})?: (.+)$`, 'i')) - ?? title.match(new RegExp(`^Journey stage: ${journeyStage}\\. (.+)$`, 'i')); + ?? title.match(new RegExp(`^Journey stage: ${journeyStage}(?:\\s*\\/\\s*${journeyStage})?\\. (.+)$`, 'i')); // New field declarations require a canonical stage. Existing direct // questions can still name another touchpoint without normalizing it. if (!stage && (/^Journey stage:/i.test(title) || @@ -78,10 +86,12 @@ function decisionGaps(q: NativePlanQuestion): DevexSeededGap[] { .replace(/, including ([A-Za-z0-9_-]+(?: [A-Za-z0-9_-]+){0,6}),/gi, (aside, subject: string) => /\b(?:if|unless|assuming|provided|except|excluding|only|no|not|never|without|was|were|is|are|has|had|may|might|could|would|historical|earlier|quoted|source|example|hypothetical|fixed|resolved|cancelled|canceled|withdrawn|rejected|superseded)\b/i.test(subject) ? aside : '') : title; + const opaqueAuthentication = /^(?:The )?authentication error says nothing[.?]?$/i.test(assertionTitle); + const vanishingUpgrade = /^v\d+ Client\.evaluate\(\) vanishes in v\d+ with no warning, alias, or guide[.?]?$/i.test(assertionTitle); // A defect heading can assert a prerequisite or compare named signatures // without a finite verb. Keep these semantic families narrow: a topic label, // healthy signature pair or optional check is not an asserted defect. - const nominalDefect = /^(?:Mandatory|Required) (?:\d+(?:\.\d+)?[- ](?:minute|second) )?(?:remote )?CI (?:check|gate) before (?:the )?first local (?:result|evaluation|run)[.?]?$/i.test(assertionTitle) || + const nominalDefect = /^(?:The )?(?:Mandatory|Required) (?:\d+(?:\.\d+)?[- ](?:minute|second) )?(?:remote )?CI (?:check|gate) before (?:the )?first local (?:result|evaluation|run)[.?]?$/i.test(assertionTitle) || /^run_eval\(\s*dataset\s*,\s*evaluator\s*\) (?:vs\.?|versus|and) run_batch\(\s*evaluator\s*,\s*dataset\s*\): (?:reversed|opposite|swapped) (?:positional|argument) order[.?]?$/i.test(assertionTitle); const nominalSubject = /^(?:Mandatory|Required|Optional)\b[^?!\n]*\bCI (?:check|gate)\b/i.test(assertionTitle) || /^run_eval\([^)]+\) (?:vs\.?|versus|and) run_batch\([^)]+\):/i.test(assertionTitle); @@ -100,14 +110,14 @@ function decisionGaps(q: NativePlanQuestion): DevexSeededGap[] { const finiteTitle = signatureDeclaration ? assertionTitle.replace(/\([^)]*\)/g, '') : assertionTitle; // Negative availability asserts a missing referenced file. Bind it to that // object; do not erase a negation of the quickstart's own reference or gate. - const absentReference = /\b(?:points?|references?) (?:at|to) (?:examples\/first_eval\.py|(?:a|the) (?:file|example)),? (?:which|that) (?:is not in (?:the )?(?:package|wheel)(?: or (?:the )?(?:release )?examples archive)?|does not (?:ship|exist))[.?]?$/i.test(assertionTitle); - const newAssertion = nominalDefect || signatureDeclaration || reversedTuples || absentReference; + const absentReference = /\b(?:points?|references?) (?:at|to) (?:examples\/first_eval\.py|(?:a|the) (?:file|example)),? (?:which|that) (?:is not (?:shipped|in (?:the )?(?:package|wheel)(?: or (?:the )?(?:release )?examples archive)?)|does not (?:ship|exist))[.?]?$/i.test(assertionTitle); + const newAssertion = nominalDefect || signatureDeclaration || reversedTuples || absentReference || opaqueAuthentication || vanishingUpgrade; const guardedDeclaration = Boolean(stage || newAssertion || upgradeTransition); const polarityTitle = absentReference ? title.replace(/\bdoes not (ship|exist)([.?]?)$/i, 'is absent$2') : title; // Punctuation cannot route a newly admitted asserted family around its // ownership checks; an offered alternate still resolves the same decision. const declaration = (questionMarks === 0 || (questionMarks === 1 && title.endsWith('?'))) && - (/^(?:[A-Za-z0-9_.]+\s+){1,12}(?:points?|references?|blocks?|requires?|takes?|raises?|removes?|drops?)\b/i.test(finiteTitle) || nominalDefect) && + (/^(?:[A-Za-z0-9_.]+\s+){1,12}(?:points?|references?|blocks?|requires?|takes?|raises?|removes?|drops?)\b/i.test(finiteTitle) || nominalDefect || opaqueAuthentication || vanishingUpgrade) && !/^`[^`]*`$/.test(rawTitle) && !/\b(?:if|unless|suppose|might|may|could|would|previously|earlier|historical|hypothetical|example|quoted|source|never|no longer|does not|do not|did not)\b/i.test(polarityTitle); if (newAssertion && !declaration) return []; @@ -184,8 +194,8 @@ function decisionGaps(q: NativePlanQuestion): DevexSeededGap[] { offered = signatureOptions.map(option => ({ ...option, label: action(option.label), description: action(option.description ?? '') })); } const options = offered.map(o => `${o.label} ${o.description ?? ''}`); - const ownUpgradeAlias = (option: string) => !upgradeTransition || ( - /(? !(upgradeTransition || vanishingUpgrade) || ( + /(? o.label.trim().replace(/\s*\(recommended\)$/i, '').toLowerCase()); const yesNo = labels.length === 2 && labels.includes('yes') && labels.includes('no'); @@ -207,10 +217,10 @@ function decisionGaps(q: NativePlanQuestion): DevexSeededGap[] { (!declaration || reversedTuples || /\b(?:reversed|opposite|swapped|inconsistent)\b/i.test(title)) && (options.some(o => (!reversedTuples || /\bboth functions\b|\brun_eval\b[^\n]*\brun_batch\b/i.test(o)) && /\b(?:align|unify|standardize|keyword|swap)\b/i.test(o) && /\b(?:order|dataset|arguments?|positional)\b/i.test(o)) || directAction('align|unify|standardize|enforce|make')))) found.push('reversed-arguments'); - if (/\bAuthError\b|\binvalid API key\b/i.test(title) && + if ((opaqueAuthentication || /\bAuthError\b|\binvalid API key\b/i.test(title)) && /\b(?:error|message|code|cause|fix|guidance|opaque|explain)\b|request failed/i.test(title) && - (!declaration || /\b(?:no (?:cause|fix|explanation|code)|opaque)\b|request failed/i.test(title)) && - (options.some(o => (/\bcodes?\b/i.test(o) || /^(?:[A-D]\)\s*)?Coded\b/i.test(o)) && /\b(?:cause|fix|link)\b/i.test(o)) || directAction('add|include|explain|replace|report|give'))) found.push('opaque-auth-error'); + (!declaration || opaqueAuthentication || /\b(?:no (?:cause|fix|explanation|code)|opaque)\b|request failed/i.test(title)) && + (options.some(o => (!opaqueAuthentication || /\bAuthError\b/i.test(o)) && (/\bcodes?\b/i.test(o) || /^(?:[A-D]\)\s*)?Coded\b/i.test(o)) && /\b(?:cause|fix|link)\b/i.test(o)) || directAction('add|include|explain|replace|report|give'))) found.push('opaque-auth-error'); if (/Client\.evaluate\b/i.test(title) && /Client\.run\b|\b(?:v\d+|version \d+|alias|deprecation|migration)\b/i.test(title) && (upgradeVocabulary || upgradeTransition) &&