gstack/test/resolver-ask-user-format.test.ts

/**
 * AskUserQuestion Format resolver — gate-tier assertions on the generated
 * Pros/Cons format directive block.
 *
 * v1.7.0.0 introduces Pros/Cons decision-brief formatting:
 * - D<N> numbered header
 * - ELI10 paragraph
 * - Stakes-if-we-pick-wrong line
 * - Recommendation line (mandatory, even for neutral posture)
 * - Pros/Cons block with ✅/❌ per option, min 2 pros + 1 con, ≥40 char bullets
 * - Net: synthesis line
 *
 * This test pins the format contract so a future edit to the resolver
 * can't silently drop a rule. If the resolver stops emitting one of
 * these tokens, bun test catches it in milliseconds instead of waiting
 * for the weekly periodic eval to notice.
 */
import { describe, test, expect } from 'bun:test';
import * as fs from 'fs';
import * as path from 'path';
import type { TemplateContext } from '../scripts/resolvers/types';
import { HOST_PATHS } from '../scripts/resolvers/types';
import { generateAskUserFormat } from '../scripts/resolvers/preamble/generate-ask-user-format';
import { directiveFor } from '../hosts/claude/hooks/auq-error-fallback-hook.ts';
import { expectMentions, expectTokens } from './helpers/prompt-structure';

const UNATTENDED_POINTER = /unattended[- ]session[`'\s]*block|per its printed/i;

function makeCtx(): TemplateContext {
  return {
    skillName: 'test-skill',
    tmplPath: 'test.tmpl',
    host: 'claude',
    paths: HOST_PATHS.claude,
    preambleTier: 2,
  };
}

describe('generateAskUserFormat — v1.7.0.0 Pros/Cons format', () => {
  const out = generateAskUserFormat(makeCtx());

  test('includes AskUserQuestion Format header', () => {
    expect(out).toContain('## AskUserQuestion Format');
  });

  test('documents D-numbered header requirement', () => {
    expect(out).toContain('D<N>');
    expect(out).toMatch(/first question in a skill invocation is `D1`/i);
  });

  test('documents ELI10 requirement', () => {
    expect(out).toContain('ELI10');
    expect(out).toMatch(/plain English.*16-year-old/);
  });

  test('documents Stakes-if-we-pick-wrong line', () => {
    expect(out).toContain('Stakes if we pick wrong');
  });

  test('documents mandatory Recommendation line', () => {
    expect(out).toContain('Recommendation: <choice>');
    expectMentions(out, [['recommendation', 'always', 'present']], 'out');
  });

  test('documents Pros / cons block header', () => {
    expect(out).toContain('Pros / cons:');
  });

  test('documents ✅ pro markers with min count + min length rule', () => {
    expect(out).toContain('✅');
    expect(out).toMatch(/(?:[Mm]inimum |≥)2 pros/);
    expect(out).toMatch(/40 characters|≥40 chars/);
  });

  test('documents ❌ con markers with min count rule', () => {
    expect(out).toContain('❌');
    expect(out).toMatch(/1 con per option|minimum.*1 con|Each real option: [^\n]*≥1 con/i);
  });

  test('documents hard-stop escape with exact phrase', () => {
    // "No cons — this is a hard-stop choice" may span a line break in the
    // rendered resolver text; match across whitespace collapses.
    expectMentions(out, [['no', 'hard-stop', 'choice']], 'out');
  });

  test('documents neutral-posture escape preserving (recommended) label', () => {
    // CT1 resolution: (recommended) label STAYS on default option to preserve
    // AUTO_DECIDE contract. Neutrality expressed in prose only.
    expect(out).toMatch(/taste call/i);
    // `s` flag makes . match newlines — the label + STAYS phrase spans a line break
    expect(out).toMatch(/\(recommended\)[\s\S]*STAYS|STAYS[\s\S]*\(recommended\)/);
    expect(out).toMatch(/AUTO_DECIDE/);
  });

  test('documents Net line for closing synthesis', () => {
    expect(out).toMatch(/^Net:/m);
    expect(out).toMatch(/synthesis|tradeoff/i);
  });

  test('documents Completeness scoring rules (coverage vs kind)', () => {
    expect(out).toContain('Completeness');
    expect(out).toMatch(/10 = complete/);
    expectMentions(out, [['not', 'coverage', 'options']], 'out');
  });

  test('documents tool_use mandate (rule 11)', () => {
    expect(out).toMatch(/tool_use/);
    // "not a question" spans a newline in the rendered text
    expect(out).toMatch(/not a[\s\S]*question|not[\s\S]*interactive/i);
  });

  test('includes self-check before emitting', () => {
    expect(out).toContain('Self-check before emitting');
    expect(out).toMatch(/D<N> header present/);
    expect(out).toContain("`Net:` closes question text");
  });

  test('documents D-numbering as model-level not runtime state', () => {
    // Codex finding #4 caveat: D-numbering is a prompt wish, not a system
    // guarantee. TemplateContext has no counter. This check pins the caveat.
    expect(out).toMatch(/model-level instruction|not a runtime counter|count your own/i);
  });

  test('per-skill override guidance preserved', () => {
    expect(out).toMatch(/Per-skill instructions may add/);
  });
});

describe('generateAskUserFormat — 5+ option split rule (slim inline + docs pointer)', () => {
  const out = generateAskUserFormat(makeCtx());

  // 5 highest-signal pins. The full rule lives in
  // docs/askuserquestion-split.md; this contract only checks what the
  // inline subsection MUST surface so the agent can act without
  // reading the docs file for routine 5-option splits.

  test('forbids dropping options to fit the 4-option cap', () => {
    expect(out).toMatch(/caps every call at \*\*4 options\*\*/);
    expectMentions(out, [['never', 'drop', 'silently defer']], 'out');
  });

  test('names the Include / Defer / Cut / Hold buckets', () => {
    expect(out).toMatch(/A\) Include/);
    expect(out).toMatch(/B\) Defer/);
    expect(out).toMatch(/C\) Cut/);
    expect(out).toMatch(/D\) Hold/);
  });

  test('specifies D<N>.k child numbering and D<N>.final summary', () => {
    expect(out).toContain('D<N>.k');
    expect(out).toContain('D<N>.final');
  });

  test('AUTO_DECIDE is gated at runtime, not just collision-resistance', () => {
    expect(out).toContain('bin/gstack-question-preference');
    expect(out).toContain('*-split-*');
    expect(out).toContain('never AUTO_DECIDE-eligible');
  });

  test('points to docs/askuserquestion-split.md for the full rule', () => {
    expect(out).toContain('docs/askuserquestion-split.md');
    expect(out).toMatch(/Read on demand when N>4/);
  });

  test('dependency repair routes to one candidate without approving a package or a broken set', () => {
    const source = fs.readFileSync(path.resolve(import.meta.dir, '../docs/askuserquestion-split.md'), 'utf8');
    const repair = source.split('**Step 1 — validate dependencies and capacity.**')[1]?.split('**Step 2')[0] ?? '';
    const text = repair.replace(/\s+/g, ' ');
    expect(text).toContain('actual prior answers');
    expectMentions(text, [['no', 'disposition', 'routing']], 'text');
    expectTokens(text, ['`D<N>.revise-<k>`', '**Include / Defer / Cut / Hold**'], 'text');
    expectMentions(text, [['never', 'candidate', 'silently']], 'text');
    expect(text).toContain('A Hold stops the chain');
    expect(text).toContain('report the unresolved blocking conflict');
    expectMentions(text, [['do not', 'incoherent', 'implement']], 'text');
    expect(text).not.toContain('accept the broken state');
  });

  test('regression: orphan "12." prefix removed from CJK rule', () => {
    expect(out).not.toContain('12. **Non-ASCII');
    expect(out).toContain('**Non-ASCII characters');
  });
});

describe('generateAskUserFormat — runtime-failure prose fallback', () => {
  const out = generateAskUserFormat(makeCtx());

  test('documents the unavailable/failed subsection', () => {
    expectMentions(out, [['askuserquestion', 'unavailable', 'fails']], 'out');
  });

  test('carves out the auto-decide denial as NOT a failure', () => {
    expect(out).toContain('[plan-tune auto-decide]');
    expect(out).toMatch(/NOT a failure/i);
    // and explicitly: do not fall back to prose on an auto-decide denial
    expect(out).toMatch(/Do NOT[\s\S]{0,40}fall back to prose|never prose/i);
  });

  test('retries the errored call exactly once before degrading', () => {
    expect(out).toMatch(/retry the SAME call \*\*once\*\*|retry the same call.*once/i);
    // idempotency guard against double-prompting
    expect(out).toMatch(/double-prompt|no answer could have surfaced/i);
  });

  test('branches on SESSION_KIND: spawned / headless / interactive', () => {
    expect(out).toContain('SESSION_KIND');
    expect(out).toMatch(/`spawned`[\s\S]*auto-choose/);
    expect(out).toMatch(/`headless`[\s\S]*BLOCKED/);
    expect(out).toMatch(/`interactive`[\s\S]*prose fallback/);
    // empty/absent SESSION_KIND degrades to interactive
    expect(out).toMatch(/empty\/absent[\s\S]{0,40}interactive/i);
  });

  test('unattended branch keeps destructive or irreversible options pending, like the printed block', () => {
    for (const skillName of ['test-skill', 'plan-eng-review']) {
      const fallback = generateAskUserFormat({ ...makeCtx(), skillName })
        .split('### When AskUserQuestion is unavailable or a call fails')[1] ?? '';
      const unattended = fallback.split('\n').find(line => line.trim().startsWith('- `unattended`')) ?? '';
      expect(unattended, `${skillName}: failure fallback must branch on unattended`).not.toBe('');
      expectMentions(unattended, [
        ['destructive', 'irreversible', 'pending'],
        ['consent', 'unrecommended question', 'approval gate', 'pending'],
      ], `${skillName} unattended fallback branch`);
      // Generated by /ship Step 7 coverage audit pass 1 (GSTA-91, PR #3421).
      // Value: protects=the failure fallback's unattended branch states its rule with no pointer to a section the reader
      // never sees; fails_when=the "(Unattended session block)" pointer returns to that branch; why_new=CQ3 reads item 1
      // only and the hook test reads the hook only; seam=none; no_claim=model behaviour after an AUQ failure (unit only)
      expect(unattended).not.toMatch(UNATTENDED_POINTER);
    }
  });

  // The mandatory triad the user explicitly required for the plain-text output.
  test('prose fallback mandates the triad: issue ELI10', () => {
    expect(out).toContain('ELI10');
  });

  test('prose fallback mandates the triad: per-choice Completeness score', () => {
    expect(out).toMatch(/Completeness scores per choice/i);
    expect(out).toMatch(/Completeness: X\/10.*EACH choice|on EACH choice/i);
  });

  test('prose fallback mandates the triad: recommendation + (recommended) marker', () => {
    expect(out).toMatch(/Recommendation: <choice> because/);
    expect(out).toMatch(/\(recommended\)`? marker on that choice/);
  });

  test('prose fallback is one paragraph per choice, not a bare bullet list', () => {
    expect(out).toMatch(/ONE paragraph per choice/i);
    expect(out).toMatch(/never a bare bullet list/i);
  });

  test('prose fallback tells the user to reply with a letter, then STOP', () => {
    expect(out).toMatch(/reply with a letter/i);
    expect(out).toMatch(/STOP and wait/i);
  });

  test('prose questions carry their checked identity on the reply line only when tuning is enabled', () => {
    const layout = out.slice(out.indexOf('Layout:'), out.indexOf('**Continuation'));
    expect(layout).toContain('listing the offered selectors');
    expect(layout).toContain('With `QUESTION_TUNING: true`');
    expectTokens(layout, ['`<gstack-qid:{question_id}>`'], 'layout');
  });

  // OV2: the former "tool_use, not prose" assertions must carry the qualifier so the
  // fallback is not self-contradicting. Guards against the instruction collision
  // silently returning on a future edit.
  test('OV2: the Format line qualifies "not prose" with the fallback exception', () => {
    expect(out).toMatch(/must be sent as tool_use, not prose — unless the documented failure fallback/);
  });

  test('OV2: the self-check "not writing prose" line carries the Conductor + fallback qualifiers', () => {
    // After the Conductor-default-prose change, the exception is two-pronged:
    // CONDUCTOR_SESSION makes prose the default, OR the documented failure fallback.
    expect(out).toMatch(/not writing prose — unless `CONDUCTOR_SESSION: true`[\s\S]*OR the documented failure fallback applies/);
  });

  // #2733: proactive spawned rule — spawned outranks Conductor. Without it a
  // spawned session's AUQ handling exists only as a failure-reaction path, and
  // a spawned subagent inside a Conductor workspace prose-STOPs with no reader.
  test('Spawned: proactive do-not-call rule present and ordered ABOVE the Conductor rule', () => {
    const spawnedRule = out.indexOf('`SESSION_KIND: spawned` echoed');
    const conductorRule = out.indexOf('`CONDUCTOR_SESSION: true` echoed');
    expect(spawnedRule).toBeGreaterThan(0);
    expect(conductorRule).toBeGreaterThan(0);
    expect(spawnedRule, 'spawned rule must outrank (precede) the Conductor rule').toBeLessThan(conductorRule);
    expect(out).toMatch(/never prose, never BLOCKED/);
    expect(out).toMatch(/outranks the Conductor rule/);
  });

  test('Spawned: destructive-gate carve-out present (conservative-continue, never prose-STOP)', () => {
    expectMentions(out, [['never', 'irreversible', 'conservative']], 'out');
  });

  test('Spawned: self-check carries the never-reach-this-checklist clause', () => {
    expect(out).toMatch(/in `SESSION_KIND: spawned`[\s\S]{0,120}you should never reach this checklist/);
    expect(out).toMatch(/the echoed STATUS line only/);
  });

  test('Spawned: anti-injection — text-sourced spawned claims can never trigger the rule', () => {
    // Echo-only trigger is the strongest form of the anti-injection contract:
    // no TEXT from anywhere (dispatch prompt included) can flip the session
    // to auto-choose; only the preamble's own tool-result STATUS line can.
    expectMentions(out, [['tool output', 'never', 'trigger']], 'out');
  });

  // Periodic-lane regression (v1.76 → v1.78): v1.76's "(or your dispatch
  // prompt marks this session as spawned)" let the model INFER spawned status
  // from a scripted-looking prompt in a CI-looking session and silently
  // auto-decide every review question (reviewCount=0 across the plan-review
  // E2Es). Two pinned-container rounds then showed ANY prose-declaration
  // channel in the eager path keeps counts unstable (intermittent 0s, band
  // overshoot, paired-control breaks in both directions). The trigger is the
  // machine-verifiable STATUS echo ONLY; the dispatch-declaration channel
  // lives exclusively at failure time (the AUQ hooks' spawned escape), which
  // never enters an interactive session's eager reasoning.
  test('Spawned: trigger is the STATUS echo only — no prose channel in the eager path', () => {
    expect(out).not.toContain('marks this session as spawned');
    expect(out).not.toContain('or your dispatch prompt');
    expect(out).toMatch(/The ONLY trigger is the preamble's own `SESSION_KIND: spawned` STATUS echo/);
    expectMentions(out, [['spawned claims', 'dispatch prompt', 'tool output', 'never', 'trigger']], 'out');
  });

  test('Spawned: absence-safe interactive default (no behavioral language)', () => {
    expectMentions(out, [['no', 'interactive', 'automated']], 'out');
    // Every behavioral tail tried skewed question counts somewhere —
    // "when unsure, ask" overshot the 4-7 review band (8); "HOW MANY
    // questions" undershot (1); "never adds, removes, or batches" broke the
    // paired-finding control (5 > 4). The rule classifies; it says nothing
    // about asking behavior.
    expect(out).not.toMatch(/When unsure, ask/);
    expect(out).not.toMatch(/HOW MANY/);
    expect(out).not.toMatch(/adds, removes, or batches/);
    expect(out).not.toMatch(/exactly as written/);
  });

  // Conductor-default-prose contract (the proactive path, distinct from the
  // failure fallback). Guards the Tool-resolution rule + self-check wording.
  test('Conductor: do-not-call rule present in Tool resolution', () => {
    expect(out).toMatch(/CONDUCTOR_SESSION: true/);
    expectMentions(out, [['do not', 'askuserquestion']], 'out');
    expect(out).toMatch(/gstack-question-log/);
  });

  test('Conductor: one-way prose rule + continuation protocol present', () => {
    expect(out).toMatch(/one-way\b[\s\S]*typed confirmation/i);
    expect(out).toMatch(/never proceed on a vague/i);
    expect(out).toMatch(/Continuation — mapping a typed reply/);
  });
});

describe('CQ2 — cross-file invariant: auto-decide prefix matches the hook', () => {
  const out = generateAskUserFormat(makeCtx());
  const hookSrc = fs.readFileSync(
    path.resolve(__dirname, '..', 'hosts', 'claude', 'hooks', 'question-preference-hook.ts'),
    'utf-8',
  );

  test('the hook actually emits the [plan-tune auto-decide] prefix', () => {
    expect(hookSrc).toContain('[plan-tune auto-decide]');
  });

  test('the resolver references the exact same prefix the hook emits', () => {
    // If a future edit reworded the hook reason, this catches the drift: the prose
    // fallback would stop recognizing the auto-decide denial as not-a-failure.
    const PREFIX = '[plan-tune auto-decide]';
    expect(hookSrc.includes(PREFIX) && out.includes(PREFIX)).toBe(true);
  });
});

// Generated by /ship Step 7 coverage audit (GSTA-91, PR #3421).
// Value: protects=item 1 states the unattended rule itself instead of pointing at a section or block; fails_when=the old
// "Unattended session block" pointer or a "per its printed" block pointer returns; why_new=no test read item 1's
// unattended sentence; seam=none; no_claim=model behaviour under unattended (unit only)
describe('CQ3 — Tool resolution item 1 states the unattended rule with no section or block pointer', () => {
  for (const skillName of ['test-skill', 'plan-eng-review']) {
    test(`${skillName}: item 1 names the unattended pending set and carries no pointer`, () => {
      const out = generateAskUserFormat({ ...makeCtx(), skillName });
      const item1 = out.split('\n').find(line => line.startsWith('1. **\`SESSION_KIND: spawned\` echoed**')) ?? '';
      expect(item1, 'Tool resolution item 1 must exist').not.toBe('');
      expectMentions(item1, [['unattended', 'consent', 'destructive', 'irreversible', 'unrecommended question', 'approval gate', 'pending']], `${skillName} item 1`);
      expect(item1).not.toMatch(UNATTENDED_POINTER);
    });
  }
});

describe('CQ4 — the hook directive keeps destructive or irreversible options pending', () => {
  test('directiveFor(unattended) names destructive or irreversible options as pending', () => {
    expectMentions(directiveFor('unattended'), [['destructive', 'irreversible', 'pending']], 'hook unattended directive');
  });
});