import { describe, expect, test } from 'bun:test'; import { readFileSync } from 'node:fs'; import { join } from 'node:path'; import { ALL_HOST_CONFIGS } from '../hosts'; import { RESOLVERS } from '../scripts/resolvers'; import { HOST_PATHS, type TemplateContext } from '../scripts/resolvers/types'; const root = join(import.meta.dir, '..'); const callers = ['qa', 'qa-only', 'review', 'ship']; function context(host: string, skillName: string): TemplateContext { return { host, skillName, tmplPath: '', paths: HOST_PATHS[host] }; } function render(file: string, ctx: TemplateContext): string { let body = readFileSync(join(root, file), 'utf8'); for (let pass = 0; pass < 10; pass++) { const next = body.replace(/\{\{([A-Z_]+)(?::([^}]+))?\}\}/g, (_match, name, args) => { if (!RESOLVERS[name]) throw new Error(`Unknown resolver ${name}`); return RESOLVERS[name](ctx, args?.split(':')); }); if (next === body) return body; body = next; } throw new Error(`Unresolved template ${file}`); } function assertProbeLocalBlocker(body: string): void { expect(body).toContain('QA setup blocker'); expect(body).toContain('affected probes as blocked'); expect(body).toContain('continue other safe probes'); expect(body).toContain('required QA'); expect(body).toContain('independent functional/static checks'); expect(body).not.toContain('stop that workflow'); expect(body).not.toContain('stop all checks'); } function assertSharedBrowserAuthority(body: string): void { const decision = body.slice(body.indexOf('## Browser access decision'), body.indexOf('## BROWSER SETUP')); expect(decision).toContain('invoking workflow, not this file'); const reportOnly = decision.slice(decision.indexOf('**Report-only'), decision.indexOf('**Standalone /qa')); expect(reportOnly).toContain('/qa-only, /review and /ship'); expect(reportOnly).toContain("do not run the fallback's setup/install or cookie-import workflow"); expect(reportOnly).toContain('Never bootstrap or invoke another skill'); expect(reportOnly).toContain('block only the affected browser probes'); expect(reportOnly).toContain('continue independent functional/static checks'); expect(reportOnly).not.toContain('run `cd'); const standalone = decision.slice(decision.indexOf('**Standalone /qa')); expect(standalone).toContain('explicit approval'); expect(standalone).toContain('STOP and wait'); expect(standalone).toContain('Only after approval'); expect(standalone).toContain('`cd && ./setup`'); expect(standalone).toContain('/setup-browser-cookies'); expect(standalone).toContain('declined, unavailable or unsuccessful'); expect(standalone).toContain('blocked'); expect(decision).toContain('Unknown caller'); expect(decision).toContain('report-only'); expect(body).not.toContain('If `NEEDS_SETUP`: tell the user'); expect(body).not.toContain('An authenticated page needs /setup-browser-cookies'); } describe('QA caller authority in pure host renders', () => { for (const host of ALL_HOST_CONFIGS) { test(`${host.name}: a scope handoff requires the actual prior method read`, () => { for (const caller of callers) { const body = RESOLVERS.QA_EXPLORATORY(context(host.name, caller)); expect(body).toContain('Read `sections/scope.md`'); expect(body).toContain('Do not repeat a Read already completed in this invocation'); expect(body).toContain('Complete these Reads in order before writing charters or probing'); const scope = body.indexOf('1. Read `sections/scope.md`'); const selection = body.indexOf('in full and select the surfaces'); const methods = body.indexOf('2. Read the selected surface methods below in full'); expect(scope).toBeGreaterThan(-1); expect(selection).toBeGreaterThan(scope); expect(methods).toBeGreaterThan(selection); expect(body.indexOf('Write a **charter**')).toBeGreaterThan(methods); expect(body).not.toContain('If the caller has not selected surfaces and established isolation'); } }); test(`${host.name}: exploratory scope defines its controller and timing before probes`, () => { for (const caller of callers) { const body = RESOLVERS.QA_EXPLORATORY(context(host.name, caller)); expect(body).toContain('The **caller** (/qa, /qa-only, /review or /ship)'); expect(body).toContain('charter** per behavior'); expect(body).toContain('bun G start D SECONDS [EARLIER_UTC]'); expect(body).toContain('G enforces the deadline'); expect(body).toContain('announce finite command timeouts'); expect(body.indexOf('bun G start D')).toBeLessThan(body.indexOf('1. First demonstrate success')); expect(body).toContain('scoped contracts are tested or blocked'); } }); test(`${host.name}: final QA cannot omit caller-required rechecks as unaffected`, () => { const body = render('qa/SKILL.md.tmpl', { ...context(host.name, 'qa'), tmplPath: 'qa/SKILL.md.tmpl', preambleTier: 4, }); const final = body.slice(body.indexOf('## Phase 9: Final QA'), body.indexOf('## Phase 10: Report')); expect(final).toContain('Re-run affected contracts and adjacent happy paths on the final inputs'); expect(final).toContain('Caller-required rechecks cannot be skipped as unaffected'); expect(final).toContain('blocked/inconclusive rechecks never verify repairs'); }); test(`${host.name}: prior learnings name QA findings without changing review callers`, () => { for (const skill of ['qa', 'qa-only']) { const body = RESOLVERS.LEARNINGS_SEARCH(context(host.name, skill)); expect(body).toContain('When a QA finding'); expect(body).not.toContain('When a review finding'); expect(body).toContain('Prior learning applied'); } for (const skill of ['review', 'ship']) { expect(RESOLVERS.LEARNINGS_SEARCH(context(host.name, skill))).toContain('When a review finding'); } }); test(`${host.name}: report-only learning lookup cannot configure or initialize stores`, () => { const ctx = context(host.name, 'qa-only'); const search = RESOLVERS.LEARNINGS_SEARCH(ctx, ['query=webhook retries']); expect(search).toContain('Read this project\'s existing learnings.jsonl only if its directory is already known'); expect(search).toContain('the caller permits that Read'); expect(search).toContain('Otherwise skip this optional lookup'); expect(search).toContain('Look for notes matching "webhook retries"'); expect(search).toContain('Do not run gstack-learnings-search here'); expect(search).toContain('Reading old notes never requires writing new ones'); expect(search).not.toContain('```bash'); expect(search).not.toContain('gstack-config'); expect(search).not.toContain('AskUserQuestion'); expect(() => RESOLVERS.LEARNINGS_SEARCH(ctx, ['query=$(touch outside)'])).toThrow(); }); test(`${host.name}: missing lazy sections stop affected probes, not independent checks`, () => { for (const skill of ['qa', 'qa-only']) { const ctx = context(host.name, skill); for (const id of ['browser-setup', 'exploratory']) { const sharedSetup = skill === 'qa-only' && id === 'browser-setup'; const pointer = sharedSetup ? RESOLVERS.QA_RESOURCE(ctx, [id]) : RESOLVERS.SECTION(ctx, [id]); assertProbeLocalBlocker(pointer); expect(pointer).toContain(`\`sections/${id}.md\``); expect(pointer).toContain('SKILL.md directory'); expect(pointer).toContain(sharedSetup ? 'No product-directory or cross-host substitutes' : 'never the product working directory'); expect(pointer).not.toContain('## BROWSER SETUP'); expect(pointer).not.toContain('aside repl'); } } }); test(`${host.name}: each installed caller resource uses the same blocker rule`, () => { for (const caller of callers) { const resource = RESOLVERS.QA_RESOURCE(context(host.name, caller), ['browser-setup']); assertProbeLocalBlocker(resource); if (caller === 'review' || caller === 'ship') { expect(resource).toContain(`../${host.name === 'claude' ? 'qa' : 'gstack-qa'}/sections/browser-setup.md`); expect(resource).toContain(`installed /${caller} SKILL.md`); } else { expect(resource).toContain('installed'); expect(resource).toContain('sections/browser-setup.md'); } } }); test(`${host.name}: shared QA setup chooses runtime caller authority before readiness`, () => { const ctx = context(host.name, 'qa'); const body = render('qa/sections/browser-setup.md.tmpl', ctx); assertSharedBrowserAuthority(body); expect(body.indexOf('## Browser access decision')).toBeLessThan(body.indexOf('## BROWSER SETUP')); expect(body.indexOf('## BROWSER SETUP')).toBeLessThan(body.indexOf('## Browser fallback')); expect(body).toContain('Functional-only'); expect(body).toContain('do not probe Aside'); expect(body).toContain("scope section's ownership rules apply even to LOCAL browser targets"); expect(body).toContain('never substitute unit tests or curl for the browser step'); expect(body).toContain('Never type passwords, one-time codes, or payment details'); expect(body).toContain('Never read, screenshot, navigate, or close any other tab'); expect((body.match(/cd && \.\/setup/g) ?? [])).toHaveLength(1); }); test(`${host.name}: QA fallback delegates setup and authentication without new authority`, () => { for (const caller of callers) { const fallback = RESOLVERS.BROWSE_FALLBACK(context(host.name, caller)); expect(fallback).toContain('Browser access decision'); expect(fallback).not.toContain('OK to proceed?'); expect(fallback).not.toContain('run `cd '); expect(fallback).not.toContain('An authenticated page needs /setup-browser-cookies'); expect(fallback).toContain('$B snapshot -i'); expect(fallback).toContain('DIFF_START'); expect(fallback).toContain('CONSOLE_ERRORS='); } }); test(`${host.name}: browser methodology routes auth through the decision instead of importing`, () => { const body = render('qa/sections/qa-patterns.md.tmpl', context(host.name, 'qa')); const auth = body.slice(body.indexOf('### Phase 2:'), body.indexOf('### Phase 3:')); expect(auth).toContain('Browser access decision'); expect(auth).not.toContain('Fallback: /setup-browser-cookies or'); expect(auth).toContain('Never handle credentials'); expect(body).toContain('Run only for selected browser surfaces'); expect(body).toContain('Confirm each issue by retrying once'); }); test(`${host.name}: functional-only and report-only paths preserve independent checks and gates`, () => { const scope = RESOLVERS.QA_SCOPE(context(host.name, 'qa')); expect(scope).toContain('Functional-only runs must not read browser setup, methodology, verification or bootstrap'); expect(scope).not.toContain('command -v aside'); expect(scope).not.toContain('curl -sI'); for (const caller of callers) { const ctx = context(host.name, caller); const body = RESOLVERS.QA_EXPLORATORY(ctx); const compact = body.replace(/\s+/g, ' '); expect(compact).toContain('Missing or unreadable assets, prerequisites or permission block affected probes, not independent safe checks'); expect(compact).toContain('Report QA setup blockers'); expect(compact).not.toContain('stop all checks'); expect(body).toContain('2. Read the selected surface methods below in full'); expect(body.indexOf('2. Read the selected surface methods below in full')).toBeLessThan(body.indexOf('1. First demonstrate success')); const reads = RESOLVERS.QA_METHOD_READS(context(host.name, caller)); expect(reads).toContain('**Functional surfaces:**'); expect(reads).toContain('sections/system-functional.md'); expect(reads).toContain('**Browser surfaces only:**'); expect(reads).toContain('sections/qa-patterns.md'); expect(body).toContain('no workflows, framework installs or publication'); expect(body).toContain('owns decisions, tests, fixes and publication'); expect(body).toContain('Missing prerequisites/expectations/observations, timeouts and refusal never pass'); expect(body).toContain('Pass requires all required current-input contracts to pass with no required remainder'); if (caller !== 'qa-only') { expect(body).toContain('leaves /review incomplete'); expect(body).toContain('unless the user explicitly accepts that named risk'); expect(body).toContain('noninteractive runs return blocked'); expect(body).toContain('test_stub proposals require ASK approval'); } } const review = RESOLVERS.QA_REVIEW(context(host.name, 'review')); const ship = RESOLVERS.QA_REVIEW(context(host.name, 'ship')); expect(review).toContain('a ship waiver cannot complete it'); expect(ship).toContain('explicit named-risk acceptance'); expect(ship.replace(/\s+/g, ' ')).toContain('Report clean/completed only when all required checks pass on current inputs'); expect(ship).toContain('List failed, blocked, inconclusive and not-run checks'); }); test(`${host.name}: non-QA fallback retains its existing setup and human sign-in flow`, () => { const fallback = RESOLVERS.BROWSE_FALLBACK(context(host.name, 'browse')); expect(fallback).toContain('OK to proceed?'); expect(fallback).toContain('STOP for the answer, then run `cd && ./setup`'); expect(fallback).toContain('An authenticated page needs /setup-browser-cookies'); expect(fallback).toContain('$B handoff'); expect(fallback).toContain('$B resume'); expect(fallback).not.toContain('Browser access decision'); }); test(`${host.name}: orchestration declares required probes before execution`, () => { for (const caller of ['review', 'ship']) { const ctx = context(host.name, caller); const body = (caller === 'review' ? RESOLVERS.QA_REVIEW_PREFLIGHT(ctx) : '') + RESOLVERS.QA_REVIEW(ctx); const exploration = body.indexOf('{{QA_RESOURCE:exploratory}}'); expect(exploration).toBeGreaterThan(-1); const resource = RESOLVERS.QA_RESOURCE(ctx, ['exploratory']); expect(resource).toContain(`../${host.name === 'claude' ? 'qa' : 'gstack-qa'}/sections/exploratory.md`); const shared = render('qa/sections/exploratory.md.tmpl', context(host.name, 'qa')); expect(shared).toContain('Read `sections/system-functional.md` in full'); expect(shared.indexOf('in full and select the surfaces')).toBeLessThan(shared.indexOf('Read `sections/system-functional.md`')); const required = body.indexOf(caller === 'review' ? '2. Check readiness and list required checks' : '2. List required checks'); expect(required).toBeGreaterThan(-1); expect(exploration).toBeLessThan(required); expect(body).toContain('one success and the riskiest changed failure/edge'); expect(body).toContain('Smoke: 5 minutes/12 probes'); expect(body).toContain('Required even for small diffs or missing plans/servers'); expect(body).toContain('Required: plan commands/assertions, listed separately'); expect(body).toContain('Other ideas are optional, untested'); expect(body.replace(/\s+/g, ' ')).toContain('Report clean/completed only when all required checks pass on current inputs'); expect(body).toContain('List failed, blocked, inconclusive and not-run checks'); } const ship = RESOLVERS.QA_REVIEW(context(host.name, 'ship')); expect(ship).toContain('Step 9.4 asks: permission/repair'); expect(ship).toContain('explicit named-risk acceptance; otherwise blocked'); }); test(`${host.name}: caller QA selects surfaces directly and links checkpoints in one final section`, () => { for (const caller of ['review', 'ship']) { const ctx = context(host.name, caller); const body = (caller === 'review' ? RESOLVERS.QA_REVIEW_PREFLIGHT(ctx) : '') + RESOLVERS.QA_REVIEW(ctx); const exploration = body.indexOf('{{QA_RESOURCE:exploratory}}'); const shared = render('qa/sections/exploratory.md.tmpl', context(host.name, 'qa')); const scope = shared.indexOf('Read `sections/scope.md`'); const selection = shared.indexOf('in full and select the surfaces'); const methods = shared.indexOf('**Functional surfaces:**'); const probes = body.indexOf(caller === 'review' ? '2. Check readiness and list required checks' : '2. List required checks'); expect(scope).toBeGreaterThan(-1); expect(exploration).toBeGreaterThan(-1); expect(selection).toBeGreaterThan(scope); expect(methods).toBeGreaterThan(selection); expect(shared.indexOf('Write a **charter**')).toBeGreaterThan(methods); expect(exploration).toBeLessThan(probes); expect(body).not.toContain('**Functional surfaces:**'); expect(RESOLVERS.QA_RESOURCE(ctx, ['exploratory'])).toContain(`../${host.name === 'claude' ? 'qa' : 'gstack-qa'}/sections/exploratory.md`); if (caller === 'review') { const flat = body.replace(/\s+/g, ' '); expect(flat).toContain('Title it `## Exploratory QA and Verification Results`'); expect(flat).toContain('keep metadata/outcome tables'); expect(flat).toContain('demote other headings one level'); expect(flat).toContain('include it here under `### Browser results`'); expect(flat).toContain('other headings demoted two levels'); expect(flat).toContain('Link every checkpoint'); expect(flat).toContain('No second report'); expect(flat).toContain('Keep browser/functional scores and outcomes separate'); expect(flat).toContain('save browser baseline/evidence normally'); } else { expect(body.replace(/\s+/g, ' ')).toContain('PR section `## Exploratory QA'); expect(body).toContain('fields as subsections'); expect(body).toContain('Link every checkpoint; no second report'); } expect(body).toContain('templates/functional-report-template.md'); expect(body).not.toContain('not a second report'); } }); test(`${host.name}: orchestration logs reviewer attempts before parent-owned edits`, () => { const body = RESOLVERS.ADVERSARIAL_STEP(context(host.name, 'review')); expect(body).toContain("queued for the parent's Fix-First handling at Step 5; do not edit during Step 4.8"); expect(body).toContain('do not start an inner repair loop'); expect(body.replace(/\s+/g, ' ')).toContain("Keep each token with that attempt; do not overwrite the parent's REVIEW_START"); expect(body.replace(/\s+/g, ' ')).toContain('save one record per source, phase and attempt, before the'); expect(body).toContain('parent applies queued fixes'); expect(body).toContain('Each token is consumed once'); expect(body).not.toContain('address the findings. Re-run the same shared structured invocation'); expect(body).toContain('The native pass is required for Step 5.8 completion'); }); test(`${host.name}: orchestration skips only history matching without user skips`, () => { for (const caller of ['review', 'ship']) { const body = RESOLVERS.CROSS_REVIEW_DEDUP(context(host.name, caller)); if (caller === 'ship') { const flat = body.replace(/\s+/g, ' '); expect(flat).toContain('Combine saved `findings` with the invocation action list, honoring later user decisions'); expect(flat).toContain('If both history and the invocation action list lack decisions, classify normally'); expect(body.indexOf('2. **Read decisions.**')).toBeLessThan(body.indexOf('3. **Match evidence.**')); expect(flat).toContain('Only explicit `skipped` actions qualify, never `fixed`, `auto-fixed` or unanswered questions'); expect(flat).toContain('Report the suppressed count once if nonzero'); } else { expect(body).toContain('skip history matching silently; still classify current findings'); expect(body.indexOf('If no prior reviews exist')).toBeLessThan(body.indexOf('For each JSONL entry')); expect(body).toContain('If N > 0, print once:'); expect(body).toContain('Otherwise skip the summary'); expect(body).toContain('Only suppress `skipped` findings — never `fixed` or `auto-fixed`'); } expect(body).not.toContain('skip this step silently'); } }); test(`${host.name}: orchestration consumes the existing generation allowance at both coverage gates`, () => { const body = RESOLVERS.TEST_COVERAGE_GATE_SHIP(context(host.name, 'ship')); expect(body).toContain("Use Step 7's remaining generation allowance"); expect(body.match(/If A and allowance remains:/g)).toHaveLength(2); expect(body).toContain('At the cap, offer only B/C or stop'); expect(body).toContain('At the cap, offer only B or stop'); expect(body).toContain('At the cap, omit A and recommend stopping'); expect(body).toContain('Minimum = 60%, Target = 80%'); expect(body).not.toContain('Maximum 2 passes total'); }); } test('report-only recommendations cannot dispatch a repair skill during discovery', () => { const skill = readFileSync(join(root, 'qa-only/SKILL.md.tmpl'), 'utf8'); expect(skill).toContain('Never invoke /qa or another skill from this report-only run'); expect(skill).toContain('separate, user-authorized'); expect(skill).toContain('No test framework detected'); expect(skill).toContain('Never commit, stash or bootstrap'); }); test('standalone QA scopes its general test rule around the approved browser bootstrap', () => { const skill = readFileSync(join(root, 'qa/SKILL.md.tmpl'), 'utf8'); const rule = skill.split('\n').find(line => line.startsWith('**Outside an explicitly approved browser bootstrap:**')); expect(skill).not.toMatch(/^13\. /m); expect(rule).toContain('Outside an explicitly approved browser bootstrap'); expect(rule).toContain('Only create tests through authorized codification in Phase 8a.5'); expect(rule).toContain('Never modify CI configuration or weaken existing tests'); const bootstrap = readFileSync(join(root, 'qa/sections/test-bootstrap.md.tmpl'), 'utf8'); expect(bootstrap).toContain('Browser /qa only, never functional/report-only'); expect(bootstrap).toContain('AskUserQuestion and WAIT'); expect(bootstrap).toContain('install only the actual choice'); expect(bootstrap).toContain('Never silently delete a valid red regression'); expect(bootstrap).toContain('create/extend `.github/workflows/test.yml`'); }); test('negative controls reject workflow-wide stopping and ungated installation', () => { const ctx = context('claude', 'qa'); const pointer = RESOLVERS.SECTION(ctx, ['browser-setup']); expect(() => assertProbeLocalBlocker(pointer + '\nstop that workflow')).toThrow(); expect(() => assertProbeLocalBlocker(pointer.replace('continue other safe probes', 'stop all checks'))).toThrow(); const setup = render('qa/sections/browser-setup.md.tmpl', ctx); expect(() => assertSharedBrowserAuthority(setup.replace('Only after approval', 'Immediately'))).toThrow(); expect(() => assertSharedBrowserAuthority(setup.replace('Never bootstrap or invoke another skill', 'Invoke another skill'))).toThrow(); expect(() => assertSharedBrowserAuthority(setup + '\nIf `NEEDS_SETUP`: tell the user')).toThrow(); expect(() => assertSharedBrowserAuthority(setup + '\nAn authenticated page needs /setup-browser-cookies')).toThrow(); }); });