From b24b6d8fcf57db7e9c8ded236d31b8a5d0b4504a Mon Sep 17 00:00:00 2001 From: garrytan Date: Sun, 4 Oct 2026 07:32:13 +0000 Subject: [PATCH] test: $B counts as a browser call only in command position; qa functional prompt defines its classification values --- test/design-detector-source-fixture.test.ts | 9 ++++++--- test/helpers/qa-functional-eval.ts | 1 + test/skill-e2e-design.test.ts | 4 +++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/test/design-detector-source-fixture.test.ts b/test/design-detector-source-fixture.test.ts index c6094853c..daa43614b 100644 --- a/test/design-detector-source-fixture.test.ts +++ b/test/design-detector-source-fixture.test.ts @@ -14,7 +14,7 @@ const ROOT = path.resolve(import.meta.dir, '..'); const source = fs.readFileSync(path.join(ROOT, 'test/skill-e2e-design.test.ts'), 'utf8'); const id = 'design-review-detector-shim'; -type Mode = 'captured' | 'literal' | 'other-variable' | 'wrong-base' | 'no-scan' | 'wrong-files' +type Mode = 'captured' | 'literal' | 'other-variable' | 'base-variable' | 'browser-variable' | 'wrong-base' | 'no-scan' | 'wrong-files' | 'extra-file' | 'no-probe' | 'no-report' | 'no-finding' | 'no-rule' | 'npx' | 'browser' | 'timeout' | 'browse-error' | 'runner-throw' | 'report-read-throw' | 'quoted-path' | 'relative-path' | 'symlink-path' | 'claimed-probe' | 'wrong-probe-host' | 'wrong-probe-cwd'; @@ -125,6 +125,9 @@ Then write /detector-output.md: one FINDING-NNN row per rule in the DETECT else toolCalls.push({ tool: 'Bash', input: { command }, output: 'DETECT_TOP total=6 rules=4\nDETECT_SUMMARY: total=6\nDETECT_EXIT: 2' }); if (mode === 'npx') toolCalls.push({ tool: 'Bash', input: { command: 'npx impeccable detect .' }, output: '' }); if (mode === 'browser') toolCalls.push({ tool: 'Bash', input: { command: '$B goto http://localhost' }, output: '' }); + // PR run 37184467123: the model named its base-branch variable B. + if (mode === 'base-variable') toolCalls.push({ tool: 'Bash', input: { command: 'B=$(gh pr view --json baseRefName -q .baseRefName 2>/dev/null); [ -n "$B" ] || B=$(gh repo view --json defaultBranchRef -q .defaultBranchRef.name 2>/dev/null); echo "GH_BASE=[$B]"' }, output: 'GH_BASE=[]' }); + if (mode === 'browser-variable') toolCalls.push({ tool: 'Bash', input: { command: 'B=~/.claude/skills/gstack/browse/dist/browse; [ -x "$B" ] && "$B" snapshot -i' }, output: '' }); if (mode !== 'no-report') { let report = captured.report; if (mode === 'no-finding') report = report.replaceAll('FINDING-001', 'ROW-001'); @@ -175,7 +178,7 @@ Then write /detector-output.md: one FINDING-NNN row per rule in the DETECT } } -test.each(['captured', 'literal', 'other-variable'] as const)('source detector callback accepts executed main scope: %s', async mode => { +test.each(['captured', 'literal', 'other-variable', 'base-variable'] as const)('source detector callback accepts executed main scope: %s', async mode => { const result = await exercise([mode]); expect(result.outputs.join('\n')).toContain('DETECT_TOP total=6 rules=4'); expect(result.outputs.join('\n')).toContain('DETECT_EXIT: 2'); @@ -185,7 +188,7 @@ test.each(['captured', 'literal', 'other-variable'] as const)('source detector c duration_ms: 42513, turns_used: 5, tokens_used: 93719, model: 'claude-fable-5-1', transcript: [{ type: 'public-replay' }] }); }); -test.each(['wrong-base', 'no-scan', 'wrong-files', 'extra-file', 'no-probe', 'no-report', 'no-finding', 'no-rule', 'npx', 'browser', 'timeout', 'browse-error'] as const)( +test.each(['wrong-base', 'no-scan', 'wrong-files', 'extra-file', 'no-probe', 'no-report', 'no-finding', 'no-rule', 'npx', 'browser', 'browser-variable', 'timeout', 'browse-error'] as const)( 'source detector callback rejects and records invalid evidence once: %s', async mode => { const result = await exercise([mode]); expect(result.errors[0]).toBeDefined(); diff --git a/test/helpers/qa-functional-eval.ts b/test/helpers/qa-functional-eval.ts index ea324985f..e90014201 100644 --- a/test/helpers/qa-functional-eval.ts +++ b/test/helpers/qa-functional-eval.ts @@ -62,6 +62,7 @@ Actions outside this interface are unsupported and fail acceptance; they are not Materialize qa-reports/evidence.json first, then write a concise qa-reports/report.md using the functional report structure. Link the evidence and checkpoint files rather than repeating full probe payloads in Markdown. Both artifacts are required before completion. The resulting evidence.json schema is: { "revision": "", "runtime": "bun ", "cwd": "", "evidence": [{"command":"","contract":"README.md","expected":"","classification":"pass|product-defect|setup-blocked|inconclusive","observed":}], "learning":[], "limits":[""] } +Classify by what the probe observed: setup-blocked whenever it reported a missing optional dependency (exit 69, SETUP_BLOCKED), even where README documents that outcome; product-defect when observed behavior contradicts README; pass when a working path behaved as README states; inconclusive otherwise. Evidence rows contain ONLY complete JSON actually emitted by native probes, including failures and repeats; retain pre-repair results alongside green results. Never synthesize JSON from a tool error or raw test output. Put tests, raw CLI diagnostics, launch failures and timeouts in Markdown with their actual output and limits. The learning array is a summary: choose one completed checkpoint where an observation motivated a different later command, not the required same-command replay. Select that checkpoint ID in annotations.learning; the production helper copies its observationCommand, hypothesis and nextCommand, or a merged note's capture IDs, argv and hypothesis. Both commands must name exact captured probes with different native child commands, never a combined command list or a replay distinguished only by capture ID. This selects existing exploration evidence, not another probe or a duplicate of the complete checkpoint ledger. Preserve every checkpoint and link every checkpoint in Markdown; keep every executed probe and its complete JSON in evidence, including the required replay. Missing dependencies remain setup blockers, not repairs. No browser installation or execution is needed.`; } diff --git a/test/skill-e2e-design.test.ts b/test/skill-e2e-design.test.ts index 02cbb5b97..ec04aef6e 100644 --- a/test/skill-e2e-design.test.ts +++ b/test/skill-e2e-design.test.ts @@ -1125,7 +1125,9 @@ Then write ${repoDir}/detector-output.md: one FINDING-NNN row per rule in the DE const bash = result.toolCalls.filter(c => c.tool === 'Bash').map(c => String(c.input?.command ?? '')); expect(bash.some(c => c.includes('npx impeccable'))).toBe(false); - expect(bash.some(c => /\$B\b|\bbrowse\s|\baside\s+repl\b|\bplaywright\b|\bpuppeteer\b/.test(c))).toBe(false); + // `$B` runs the browse binary only in command position; a shell variable + // named B (a base branch, say) is not a browser step. + expect(bash.some(c => /(?:^|[;&|(\n]|\b(?:then|do|else)\b)\s*"?\$B"?\s+\S|\bbrowse\s|\baside\s+repl\b|\bplaywright\b|\bpuppeteer\b/.test(c))).toBe(false); expect(result.toolCalls.some(c => /browser|browse|playwright|puppeteer/i.test(c.tool))).toBe(false); const calls: Array<{ argv: string[]; cwd: string; exit: number; engine: Array<{ argv: string[]; cwd: string }> }> = fs.readFileSync(receiptPath, 'utf8').trim().split('\n').map(line => JSON.parse(line));