diff --git a/test/auq-matrix-format-gate.test.ts b/test/auq-matrix-format-gate.test.ts deleted file mode 100644 index 48b9f95e3..000000000 --- a/test/auq-matrix-format-gate.test.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { describe, expect, test } from 'bun:test'; -import * as fs from 'fs'; -import * as path from 'path'; -import { auqMachineFormatProblems, scoreAuqFormat } from './helpers/auq-sdk-capture'; - -const briefs = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/auq-matrix/briefs.json'), 'utf8')) as - { known_good: Record; known_bad: Record }; - -describe('AUQ matrix format gate', () => { - test.each(Object.entries(briefs.known_good))('passes %s', (_name, text) => { - expect(auqMachineFormatProblems(text)).toEqual([]); - }); - test.each(Object.entries(briefs.known_bad))('fails %s', (_name, text) => { - expect(auqMachineFormatProblems(text).length).toBeGreaterThan(0); - }); - test('a brief missing only presentation elements no longer fails, but they are still reported', () => { - expect(scoreAuqFormat(briefs.known_good['no-emoji-or-net']!).missing).toEqual(['Pros / cons:', '✅', '❌', 'Net:']); - }); -}); diff --git a/test/auq-native-capture.test.ts b/test/auq-native-capture.test.ts index 3e0824531..8517f5939 100644 --- a/test/auq-native-capture.test.ts +++ b/test/auq-native-capture.test.ts @@ -3,7 +3,7 @@ import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; import { serializeNativeAuq, displayedNativeAuq, nativeAuqPublicError, nativeAuqViewport, NATIVE_AUQ_CAPTURE_MS } from './helpers/auq-native-capture'; -import { scoreAuqFormat } from './helpers/auq-sdk-capture'; +import { auqMachineFormatProblems, scoreAuqFormat } from './helpers/auq-sdk-capture'; import { createPendingQuestionRecorder, recordPendingQuestion, readFirstPendingQuestionForDisplay } from './helpers/plan-count-pending-question'; import type { NativePlanQuestion, NativePlanQuestionCall } from './helpers/plan-count-transcript'; @@ -632,3 +632,21 @@ fs.writeFileSync(path.join(root,'results.json'),JSON.stringify({results,privateR expect(out).toContain('outcome=question_captured workflowCompleted=false'); } finally {fs.rmSync(dir,{recursive:true,force:true});} },30_000); + +// Stored first-AUQ briefs for the matrix's machine-read format gate. +{ + const briefs = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/auq-matrix/briefs.json'), 'utf8')) as + { known_good: Record; known_bad: Record }; + + describe('AUQ matrix format gate', () => { + test.each(Object.entries(briefs.known_good))('passes %s', (_name, text) => { + expect(auqMachineFormatProblems(text)).toEqual([]); + }); + test.each(Object.entries(briefs.known_bad))('fails %s', (_name, text) => { + expect(auqMachineFormatProblems(text).length).toBeGreaterThan(0); + }); + test('a brief missing only presentation elements no longer fails, but they are still reported', () => { + expect(scoreAuqFormat(briefs.known_good['no-emoji-or-net']!).missing).toEqual(['Pros / cons:', '✅', '❌', 'Net:']); + }); + }); +} diff --git a/test/browse-judge-floors.test.ts b/test/browse-judge-floors.test.ts deleted file mode 100644 index 53a1a160f..000000000 --- a/test/browse-judge-floors.test.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { describe, expect, test } from 'bun:test'; -import * as fs from 'fs'; -import * as path from 'path'; -import { browseJudgeFloorsMet } from './helpers/workflow-judge-cache'; - -type Scores = { clarity: number; completeness: number; actionability: number }; -const stored = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/browse-judge/panel-means.json'), 'utf8')) as - { known_good: Record; known_bad: Record }; - -describe('browse judge floors', () => { - test.each(Object.entries(stored.known_good))('passes %s', (_name, scores) => expect(browseJudgeFloorsMet(scores)).toBe(true)); - test.each(Object.entries(stored.known_bad))('fails %s', (_name, scores) => expect(browseJudgeFloorsMet(scores)).toBe(false)); -}); diff --git a/test/ceo-split-floor.test.ts b/test/ceo-split-floor.test.ts deleted file mode 100644 index 135236572..000000000 --- a/test/ceo-split-floor.test.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { expect, test } from 'bun:test'; -import type { AskUserQuestionFingerprint } from './helpers/claude-pty-runner'; -import type { NativeQuestion } from './helpers/plan-skill-questions'; -import { CEO_SCOPE_CANDIDATES } from './helpers/plan-review-cases'; -import { CEO_SPLIT_CALL_FLOOR } from './helpers/ceo-split-question-policy'; -import { validatePlanReviewDecisionResponse, type PlanReviewDecision, type PlanReviewDecisionInput } from './helpers/plan-review-decisions'; - -const PLATFORMS = ['Slack', 'Discord', 'Microsoft Teams', 'Telegram', 'Mattermost']; -const question = (platform: string, n: number): NativeQuestion => ({ - header: platform, multiSelect: false, - question: `D3.${n} — ${platform} integration\nELI10: Decide this integration independently. Recommendation: Include.`, - options: ['Include', 'Defer', 'Cut', 'Hold'].map(label => ({ label, description: `Choose ${label} for ${platform}.` })), -}); -type NativeCall = AskUserQuestionFingerprint & { toolUseId: string; questions: NativeQuestion[] }; -const call = (id: string, questions: NativeQuestion[]) => ({ - toolUseId: id, questions, selectedOptions: questions.map(() => 1), signature: id, - promptSnippet: 'diagnostic', options: [], observedAtMs: 1, preReview: true, -}) as unknown as NativeCall; -const row = (fp: NativeCall, tab: number, target: string): PlanReviewDecision => ({ - toolUseId: fp.toolUseId, questionIndex: tab, kind: 'scope', targetIds: [target], independentDecisions: 1, - evidence: [{ field: 'question', optionIndex: null, quote: fp.questions[tab - 1]!.question.split('\n')[0]! }], - reason: 'This acknowledged question presents the independent decision described by this target.', - optionActions: (['include', 'defer', 'cut', 'hold'] as const).map((action, i) => ({ optionIndex: i + 1, action })), -}); -/** Stored shapes: each inner array is one AskUserQuestion call; numbers index PLATFORMS. */ -function run(groups: number[][]) { - const fingerprints = groups.map((group, i) => call(`native-${i}`, group.map(p => question(PLATFORMS[p]!, p + 1)))); - const input: PlanReviewDecisionInput = { plan: 'Review the five independent integrations in this plan.', kind: 'scope', - targets: CEO_SCOPE_CANDIDATES, fingerprints, floor: CEO_SPLIT_CALL_FLOOR, deadlineAt: Date.now() + 60_000 }; - const questions = groups.flatMap((group, i) => group.map((p, tab) => row(fingerprints[i]!, tab + 1, `E${p + 1}`))); - return () => validatePlanReviewDecisionResponse(input, { questions }); -} - -test('the split floor is the minimum number of calls for five options at four per call', () => { - expect(CEO_SPLIT_CALL_FLOOR).toBe(Math.ceil(PLATFORMS.length / 4)); -}); -test.each([ - ['five sequential calls', [[0], [1], [2], [3], [4]]], - ['two batched calls', [[0, 1, 2, 3], [4]]], - ['three batched calls', [[0, 1], [2, 3], [4]]], -])('known-good split passes: %s', (_name, groups) => { - expect(run(groups)().coveredTargetIds).toEqual(CEO_SCOPE_CANDIDATES.map(c => c.id)); -}); -test.each([ - ['a candidate silently dropped', [[0, 1, 2, 3]], 'missing target decisions'], - ['one call for a dropped fifth candidate', [[0, 1, 2], [3]], 'missing target decisions'], -])('known-bad split fails: %s', (_name, groups, message) => { - expect(run(groups)).toThrow(message); -}); diff --git a/test/plan-format-kind-note.test.ts b/test/plan-format-kind-note.test.ts index f6829da9a..7940289f4 100644 --- a/test/plan-format-kind-note.test.ts +++ b/test/plan-format-kind-note.test.ts @@ -2,11 +2,19 @@ import { describe, expect, test } from 'bun:test'; import * as fs from 'fs'; import * as path from 'path'; import { KIND_NOTE_RE } from './helpers/plan-format-kind-note'; +import { generateAskUserFormat } from '../scripts/resolvers/preamble/generate-ask-user-format'; +import { HOST_PATHS, type TemplateContext } from '../scripts/resolvers/types'; const stored = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/plan-format-kind-notes.json'), 'utf8')) as { known_good: string[]; known_bad: string[] }; describe('plan-format kind note', () => { + test('accepts the kind note the AskUserQuestion format itself prescribes', () => { + const format = generateAskUserFormat({ skillName: 'plan-ceo-review', host: 'claude', paths: HOST_PATHS.claude } as TemplateContext); + const prescribed = /Note: options differ in kind[^`\n]*/.exec(format)?.[0]; + expect(prescribed).toBeDefined(); + expect(prescribed!).toMatch(KIND_NOTE_RE); + }); test.each(stored.known_good)('accepts %s', line => expect(line).toMatch(KIND_NOTE_RE)); test.each(stored.known_bad)('rejects %s', line => expect(line).not.toMatch(KIND_NOTE_RE)); }); diff --git a/test/plan-mode-evidence.test.ts b/test/plan-mode-evidence.test.ts index 3a6f14303..6d2bfc88b 100644 --- a/test/plan-mode-evidence.test.ts +++ b/test/plan-mode-evidence.test.ts @@ -1,8 +1,10 @@ -import { expect, test } from 'bun:test'; +import { afterAll, describe, expect, test } from 'bun:test'; import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; import { CAPTURE_MS } from './helpers/eval-budgets'; +import { assertNoPlanFileDecisions, seededPlanTargeted } from './helpers/plan-mode-evidence'; +import { planFileHasDecisionsSection, type PlanSkillObservation } from './helpers/claude-pty-runner'; const ROOT = path.resolve(import.meta.dir, '..'); const SECRET = 'fixture-private-diagnostic-value'; @@ -112,3 +114,40 @@ await import(${JSON.stringify(path.join(ROOT, PAID_FILE))}); }); } } + +// Seeded-plan targeting and plan-file decision substitutes (stored observations). +{ + const TOKENS = ['ShardManager', 'ResultMerger']; + const base = { outcome: 'asked', summary: '', evidence: '', elapsedMs: 1 } as unknown as PlanSkillObservation; + const targetingDir = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-mode-targeting-')); + afterAll(() => fs.rmSync(targetingDir, { recursive: true, force: true })); + const plan = (name: string, body: string) => { const file = path.join(targetingDir, name); fs.writeFileSync(file, body); return file; }; + + describe('seeded plan targeting', () => { + test.each([ + ['exact announcement observed', { scopeGateAutoSelectObserved: true }], + ['paraphrased announcement with seed content in the review', { scopeGateAutoSelectObserved: false, tokensObserved: { ShardManager: true, ResultMerger: false } }], + ])('known-good: %s', (_name, fields) => { + expect(seededPlanTargeted({ ...base, ...fields } as PlanSkillObservation, TOKENS)).toBe(true); + }); + test.each([ + ['no announcement and no seed content', { scopeGateAutoSelectObserved: false, tokensObserved: { ShardManager: false, ResultMerger: false } }], + ['an untracked token cannot stand in for the seed', { scopeGateAutoSelectObserved: false, tokensObserved: { 'draft-plan': true } }], + ])('known-bad: %s', (_name, fields) => { + expect(seededPlanTargeted({ ...base, ...fields } as PlanSkillObservation, TOKENS)).toBe(false); + }); + }); + + describe('plan-file decision substitute', () => { + test('a plan_ready plan file with ## Decisions fails', () => { + const planFile = plan('decisions.md', '# Plan\n\n## Decisions to confirm\n- A or B?\n\n## GSTACK REVIEW REPORT\n'); + expect(() => assertNoPlanFileDecisions({ ...base, outcome: 'plan_ready', planFile } as PlanSkillObservation, planFileHasDecisionsSection)) + .toThrow('not written to the plan as a substitute'); + }); + test('a plan_ready plan file without ## Decisions passes, and asked runs are not checked', () => { + const planFile = plan('clean.md', '# Plan\n\n## Approach\nShip it.\n\n## GSTACK REVIEW REPORT\n'); + expect(() => assertNoPlanFileDecisions({ ...base, outcome: 'plan_ready', planFile } as PlanSkillObservation, planFileHasDecisionsSection)).not.toThrow(); + expect(() => assertNoPlanFileDecisions({ ...base, outcome: 'asked' } as PlanSkillObservation, planFileHasDecisionsSection)).not.toThrow(); + }); + }); +} diff --git a/test/plan-mode-targeting.test.ts b/test/plan-mode-targeting.test.ts deleted file mode 100644 index bb1dfcbfb..000000000 --- a/test/plan-mode-targeting.test.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { afterAll, describe, expect, test } from 'bun:test'; -import * as fs from 'node:fs'; -import * as os from 'node:os'; -import * as path from 'node:path'; -import { assertNoPlanFileDecisions, seededPlanTargeted } from './helpers/plan-mode-evidence'; -import { planFileHasDecisionsSection } from './helpers/claude-pty-runner'; -import type { PlanSkillObservation } from './helpers/claude-pty-runner'; - -const TOKENS = ['ShardManager', 'ResultMerger']; -const base = { outcome: 'asked', summary: '', evidence: '', elapsedMs: 1 } as unknown as PlanSkillObservation; -const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-mode-targeting-')); -afterAll(() => fs.rmSync(dir, { recursive: true, force: true })); -const plan = (name: string, body: string) => { const file = path.join(dir, name); fs.writeFileSync(file, body); return file; }; - -describe('seeded plan targeting', () => { - test.each([ - ['exact announcement observed', { scopeGateAutoSelectObserved: true }], - ['paraphrased announcement with seed content in the review', { scopeGateAutoSelectObserved: false, tokensObserved: { ShardManager: true, ResultMerger: false } }], - ])('known-good: %s', (_name, fields) => { - expect(seededPlanTargeted({ ...base, ...fields } as PlanSkillObservation, TOKENS)).toBe(true); - }); - test.each([ - ['no announcement and no seed content', { scopeGateAutoSelectObserved: false, tokensObserved: { ShardManager: false, ResultMerger: false } }], - ['an untracked token cannot stand in for the seed', { scopeGateAutoSelectObserved: false, tokensObserved: { 'draft-plan': true } }], - ])('known-bad: %s', (_name, fields) => { - expect(seededPlanTargeted({ ...base, ...fields } as PlanSkillObservation, TOKENS)).toBe(false); - }); -}); - -describe('plan-file decision substitute', () => { - test('a plan_ready plan file with ## Decisions fails', () => { - const planFile = plan('decisions.md', '# Plan\n\n## Decisions to confirm\n- A or B?\n\n## GSTACK REVIEW REPORT\n'); - expect(() => assertNoPlanFileDecisions({ ...base, outcome: 'plan_ready', planFile } as PlanSkillObservation, planFileHasDecisionsSection)) - .toThrow('not written to the plan as a substitute'); - }); - test('a plan_ready plan file without ## Decisions passes, and asked runs are not checked', () => { - const planFile = plan('clean.md', '# Plan\n\n## Approach\nShip it.\n\n## GSTACK REVIEW REPORT\n'); - expect(() => assertNoPlanFileDecisions({ ...base, outcome: 'plan_ready', planFile } as PlanSkillObservation, planFileHasDecisionsSection)).not.toThrow(); - expect(() => assertNoPlanFileDecisions({ ...base, outcome: 'asked' } as PlanSkillObservation, planFileHasDecisionsSection)).not.toThrow(); - }); -}); diff --git a/test/plan-review-cases.test.ts b/test/plan-review-cases.test.ts index f4abd77d8..99bc7d93c 100644 --- a/test/plan-review-cases.test.ts +++ b/test/plan-review-cases.test.ts @@ -1852,3 +1852,55 @@ describe('DX checkpoint optional TODO actor', () => { expect(() => pickDevexCheckpointQuestion(ambiguous)).toThrow('multiple recommended'); }); }); + +// CEO split-overflow call floor: stored call shapes for the scope validator. +{ + const { CEO_SCOPE_CANDIDATES } = await import('./helpers/plan-review-cases'); + const { CEO_SPLIT_CALL_FLOOR } = await import('./helpers/ceo-split-question-policy'); + const { validatePlanReviewDecisionResponse } = await import('./helpers/plan-review-decisions'); + type PlanReviewDecision = import('./helpers/plan-review-decisions').PlanReviewDecision; + type PlanReviewDecisionInput = import('./helpers/plan-review-decisions').PlanReviewDecisionInput; + type AskUserQuestionFingerprint = import('./helpers/claude-pty-runner').AskUserQuestionFingerprint; + const PLATFORMS = ['Slack', 'Discord', 'Microsoft Teams', 'Telegram', 'Mattermost']; + const question = (platform: string, n: number): NativeQuestion => ({ + header: platform, multiSelect: false, + question: `D3.${n} — ${platform} integration\nELI10: Decide this integration independently. Recommendation: Include.`, + options: ['Include', 'Defer', 'Cut', 'Hold'].map(label => ({ label, description: `Choose ${label} for ${platform}.` })), + }); + type NativeCall = AskUserQuestionFingerprint & { toolUseId: string; questions: NativeQuestion[] }; + const call = (id: string, questions: NativeQuestion[]) => ({ + toolUseId: id, questions, selectedOptions: questions.map(() => 1), signature: id, + promptSnippet: 'diagnostic', options: [], observedAtMs: 1, preReview: true, + }) as unknown as NativeCall; + const row = (fp: NativeCall, tab: number, target: string): PlanReviewDecision => ({ + toolUseId: fp.toolUseId, questionIndex: tab, kind: 'scope', targetIds: [target], independentDecisions: 1, + evidence: [{ field: 'question', optionIndex: null, quote: fp.questions[tab - 1]!.question.split('\n')[0]! }], + reason: 'This acknowledged question presents the independent decision described by this target.', + optionActions: (['include', 'defer', 'cut', 'hold'] as const).map((action, i) => ({ optionIndex: i + 1, action })), + }); + /** Stored shapes: each inner array is one AskUserQuestion call; numbers index PLATFORMS. */ + function run(groups: number[][]) { + const fingerprints = groups.map((group, i) => call(`native-${i}`, group.map(p => question(PLATFORMS[p]!, p + 1)))); + const input: PlanReviewDecisionInput = { plan: 'Review the five independent integrations in this plan.', kind: 'scope', + targets: CEO_SCOPE_CANDIDATES, fingerprints, floor: CEO_SPLIT_CALL_FLOOR, deadlineAt: Date.now() + 60_000 }; + const questions = groups.flatMap((group, i) => group.map((p, tab) => row(fingerprints[i]!, tab + 1, `E${p + 1}`))); + return () => validatePlanReviewDecisionResponse(input, { questions }); + } + + test('the split floor is the minimum number of calls for five options at four per call', () => { + expect(CEO_SPLIT_CALL_FLOOR).toBe(Math.ceil(PLATFORMS.length / 4)); + }); + test.each([ + ['five sequential calls', [[0], [1], [2], [3], [4]]], + ['two batched calls', [[0, 1, 2, 3], [4]]], + ['three batched calls', [[0, 1], [2, 3], [4]]], + ])('known-good split passes: %s', (_name, groups) => { + expect(run(groups)().coveredTargetIds).toEqual(CEO_SCOPE_CANDIDATES.map(c => c.id)); + }); + test.each([ + ['a candidate silently dropped', [[0, 1, 2, 3]], 'missing target decisions'], + ['one call for a dropped fifth candidate', [[0, 1, 2], [3]], 'missing target decisions'], + ])('known-bad split fails: %s', (_name, groups, message) => { + expect(run(groups)).toThrow(message); + }); +} diff --git a/test/shared-libs-fixture.test.ts b/test/shared-libs-fixture.test.ts index 2bbbff13e..fc04908ed 100644 --- a/test/shared-libs-fixture.test.ts +++ b/test/shared-libs-fixture.test.ts @@ -1755,3 +1755,23 @@ describe('skip actor: deferred-reuse wording in a Skip option (PR lane run 36641 await expect(answer('Records an explicit Skip so a future review can reuse the helper once snapshot coverage holds.')).rejects.toThrow('No unambiguous no-change option'); }); }); + +// Stored gh/curl endpoint lists for the PR coverage request budget. +{ + const stored = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/shared-libs/pr-coverage-endpoints.json'), 'utf8')) as + { known_good: Record; known_bad: Record }; + + describe('shared-libs PR coverage request budget', () => { + test.each(Object.entries(stored.known_good))('passes %s', (_name, endpoints) => { + expect(prCoverageRequestViolations(endpoints)).toEqual([]); + }); + test.each(Object.entries(stored.known_bad))('fails %s', (_name, endpoints) => { + expect(prCoverageRequestViolations(endpoints).length).toBeGreaterThan(0); + }); + test('file-list pages beyond the stated budget fail', () => { + const pages = Array.from({ length: 52 }, (_, i) => `/repos/fixture/shared-libs/pulls/${i < 2 ? 42 : 100 + i}/files?per_page=100&page=${i < 2 ? i + 1 : 1}`); + expect(prCoverageRequestViolations(pages)).toEqual(['52 file-list pages exceed the budget of 50 plus PR 7']); + expect(prCoverageRequestViolations(pages.slice(0, 51))).toEqual([]); + }); + }); +} diff --git a/test/shared-libs-pr-coverage-budget.test.ts b/test/shared-libs-pr-coverage-budget.test.ts deleted file mode 100644 index b4a0cea1a..000000000 --- a/test/shared-libs-pr-coverage-budget.test.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { describe, expect, test } from 'bun:test'; -import * as fs from 'fs'; -import * as path from 'path'; -import { prCoverageRequestViolations } from './helpers/shared-libs-eval-fixture'; - -const stored = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/shared-libs/pr-coverage-endpoints.json'), 'utf8')) as - { known_good: Record; known_bad: Record }; - -describe('shared-libs PR coverage request budget', () => { - test.each(Object.entries(stored.known_good))('passes %s', (_name, endpoints) => { - expect(prCoverageRequestViolations(endpoints)).toEqual([]); - }); - test.each(Object.entries(stored.known_bad))('fails %s', (_name, endpoints) => { - expect(prCoverageRequestViolations(endpoints).length).toBeGreaterThan(0); - }); - test('file-list pages beyond the stated budget fail', () => { - const pages = Array.from({ length: 52 }, (_, i) => `/repos/fixture/shared-libs/pulls/${i < 2 ? 42 : 100 + i}/files?per_page=100&page=${i < 2 ? i + 1 : 1}`); - expect(prCoverageRequestViolations(pages)).toEqual(['52 file-list pages exceed the budget of 50 plus PR 7']); - expect(prCoverageRequestViolations(pages.slice(0, 51))).toEqual([]); - }); -}); diff --git a/test/ship-triage-labels.test.ts b/test/ship-triage-labels.test.ts index f812f2af4..390cf6e6b 100644 --- a/test/ship-triage-labels.test.ts +++ b/test/ship-triage-labels.test.ts @@ -11,6 +11,13 @@ const correct = (output: string) => { }; describe('ship triage label grader', () => { + test('grades the two labels the ship triage step itself uses', () => { + const triage = fs.readFileSync(path.join(import.meta.dir, '..', 'ship', 'sections', 'tests.md.tmpl'), 'utf8'); + expect(triage).toMatch(/\bin-branch\b/i); + expect(triage).toMatch(/\bpre-existing\b/i); + expect(triageLabels('- truncate (string.test.js): in-branch\n- divide (math.test.js): pre-existing')) + .toEqual({ string: 'in-branch', math: 'pre-existing' }); + }); test.each(['known-good-json.txt', 'known-good-lines.txt'])('accepts %s', (name) => { expect(correct(read(name))).toBe(true); }); diff --git a/test/skill-body-narration.test.ts b/test/skill-body-narration.test.ts deleted file mode 100644 index acac78c97..000000000 --- a/test/skill-body-narration.test.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { describe, expect, test } from 'bun:test'; -import * as fs from 'fs'; -import * as path from 'path'; -import { hasNarrationLeak } from './helpers/skill-body-narration'; - -const DIR = path.join(import.meta.dir, 'fixtures', 'skillify-narration'); -const read = (name: string) => fs.readFileSync(path.join(DIR, name), 'utf8'); -const KNOWN_BAD = fs.readdirSync(DIR).filter(name => name.startsWith('known-bad-')).sort(); - -describe('skillify narration grader', () => { - test('accepts a clean body that says "let me know" in a trigger', () => { - expect(hasNarrationLeak(read('known-good.md'))).toBe(false); - }); - test('stored known-bad fixtures exist', () => { - expect(KNOWN_BAD.length).toBeGreaterThanOrEqual(4); - }); - test.each(KNOWN_BAD)('still rejects %s', (name) => { - expect(hasNarrationLeak(read(name))).toBe(true); - }); - test('the previous grader rejected every known-bad fixture and also the known-good one', () => { - const previous = (body: string) => /^I /m.test(body) || /Let me /i.test(body) || /^I'll /m.test(body); - for (const name of KNOWN_BAD) expect(previous(read(name))).toBe(true); - expect(previous(read('known-good.md'))).toBe(true); - }); -}); diff --git a/test/skill-fixture.test.ts b/test/skill-fixture.test.ts index 6a58562ff..d818e49fd 100644 --- a/test/skill-fixture.test.ts +++ b/test/skill-fixture.test.ts @@ -29,6 +29,7 @@ import { } from './helpers/skill-fixture'; import { E2E_TOUCHFILES, GLOBAL_TOUCHFILES, selectTests } from './helpers/touchfiles'; import { readShippedSkillRouting } from './helpers/shipped-skill-routing'; +import { hasNarrationLeak } from './helpers/skill-body-narration'; const ROOT = path.resolve(import.meta.dir, '..'); @@ -482,3 +483,27 @@ test('routing fixtures read the ## Skill routing section gstack ships', () => { expect(() => readShippedSkillRouting(root)).toThrow('routing-injection heredoc not found'); } finally { fs.rmSync(root, { recursive: true, force: true }); } }); + +// Stored synthesized SKILL.md bodies for the skillify narration grader. +{ + const DIR = path.join(import.meta.dir, 'fixtures', 'skillify-narration'); + const read = (name: string) => fs.readFileSync(path.join(DIR, name), 'utf8'); + const KNOWN_BAD = fs.readdirSync(DIR).filter(name => name.startsWith('known-bad-')).sort(); + + describe('skillify narration grader', () => { + test('accepts a clean body that says "let me know" in a trigger', () => { + expect(hasNarrationLeak(read('known-good.md'))).toBe(false); + }); + test('stored known-bad fixtures exist', () => { + expect(KNOWN_BAD.length).toBeGreaterThanOrEqual(4); + }); + test.each(KNOWN_BAD)('still rejects %s', (name) => { + expect(hasNarrationLeak(read(name))).toBe(true); + }); + test('the previous grader rejected every known-bad fixture and also the known-good one', () => { + const previous = (body: string) => /^I /m.test(body) || /Let me /i.test(body) || /^I'll /m.test(body); + for (const name of KNOWN_BAD) expect(previous(read(name))).toBe(true); + expect(previous(read('known-good.md'))).toBe(true); + }); + }); +} diff --git a/test/test-value-bar.test.ts b/test/test-value-bar.test.ts index 29e1a5941..d115ed184 100644 --- a/test/test-value-bar.test.ts +++ b/test/test-value-bar.test.ts @@ -15,6 +15,7 @@ import { RETENTION_ONE_LINER, RETIREMENT_FIELDS, REVIEW_EVIDENCE_FIELDS, SWEEP_POINTER, TEST_VALUE_BAR_MAX_BYTES, TEST_VALUE_BAR_MODES, WEAK_REASONS, clampCardField, generateTestValueBar, renderValueCard, type TestValueBarMode, } from '../scripts/resolvers/test-value'; +import { recordsNonTestCallerSearch } from './helpers/test-value-fixture'; const ROOT = path.resolve(import.meta.dir, '..'); const FIX = 'Fix: bun run gen:skill-docs && bun test test/test-value-bar.test.ts'; @@ -239,3 +240,18 @@ describe('docs and registration', () => { expect(read('README.md').split('/deslop-shared-libs, /test-audit, /ship').length - 1).toBe(2); }); }); + +// Stored test-only-export evidence for the review test-value caller-search grader. +{ + const stored = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/test-value-seam-evidence.json'), 'utf8')) as + { known_good: Record; known_bad: Record }; + + describe('test-only export caller-search evidence', () => { + test.each(Object.entries(stored.known_good))('accepts %s', (_name, evidence) => { + expect(recordsNonTestCallerSearch({ evidence })).toBe(true); + }); + test.each(Object.entries(stored.known_bad))('rejects %s', (_name, evidence) => { + expect(recordsNonTestCallerSearch({ evidence })).toBe(false); + }); + }); +} diff --git a/test/test-value-seam-evidence.test.ts b/test/test-value-seam-evidence.test.ts deleted file mode 100644 index bf031ee25..000000000 --- a/test/test-value-seam-evidence.test.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { describe, expect, test } from 'bun:test'; -import * as fs from 'fs'; -import * as path from 'path'; -import { recordsNonTestCallerSearch } from './helpers/test-value-fixture'; - -const stored = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/test-value-seam-evidence.json'), 'utf8')) as - { known_good: Record; known_bad: Record }; - -describe('test-only export caller-search evidence', () => { - test.each(Object.entries(stored.known_good))('accepts %s', (_name, evidence) => { - expect(recordsNonTestCallerSearch({ evidence })).toBe(true); - }); - test.each(Object.entries(stored.known_bad))('rejects %s', (_name, evidence) => { - expect(recordsNonTestCallerSearch({ evidence })).toBe(false); - }); -}); diff --git a/test/workflow-judge-cache.test.ts b/test/workflow-judge-cache.test.ts index 5e230bbff..ff8e9ed77 100644 --- a/test/workflow-judge-cache.test.ts +++ b/test/workflow-judge-cache.test.ts @@ -8,7 +8,7 @@ import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; import { execFileSync } from 'node:child_process'; -import { prepareWorkflowJudgeCache, validWorkflowJudgePanel, validWorkflowJudgeScore, workflowJudgeDependencies, type WorkflowCacheOptions } from './helpers/workflow-judge-cache'; +import { browseJudgeFloorsMet, prepareWorkflowJudgeCache, validWorkflowJudgePanel, validWorkflowJudgeScore, workflowJudgeDependencies, type WorkflowCacheOptions } from './helpers/workflow-judge-cache'; import { readWorkflowJudgeInput, buildWorkflowJudgePrompt, QA_DISCOVERY_REFERENCES, WORKFLOW_JUDGE_RESPONSE_SCHEMA } from './helpers/workflow-judge-input'; const roots: string[] = []; @@ -577,3 +577,15 @@ describe('judge panel', () => { expect(source).not.toMatch(/\bscores\.reasoning\b|\bresult\.reasoning\b/); }); }); + +// Stored browse reference panel means against the judge floors. +{ + type Scores = { clarity: number; completeness: number; actionability: number }; + const stored = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/browse-judge/panel-means.json'), 'utf8')) as + { known_good: Record; known_bad: Record }; + + describe('browse judge floors', () => { + test.each(Object.entries(stored.known_good))('passes %s', (_name, scores) => expect(browseJudgeFloorsMet(scores)).toBe(true)); + test.each(Object.entries(stored.known_bad))('fails %s', (_name, scores) => expect(browseJudgeFloorsMet(scores)).toBe(false)); + }); +}