From 35d4efdc01c9aee4e25c0d9acb22e47752882e46 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 3 Oct 2026 03:41:05 +0000 Subject: [PATCH] test: shared-libs evals deliver the skill instructions in context, as a loaded skill does --- test/helpers/shared-libs-eval-fixture.ts | 5 +++++ test/skill-e2e-shared-libs-periodic.test.ts | 8 ++++---- test/skill-e2e-shared-libs.test.ts | 6 +++--- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/test/helpers/shared-libs-eval-fixture.ts b/test/helpers/shared-libs-eval-fixture.ts index d54b70425..faefb039b 100644 --- a/test/helpers/shared-libs-eval-fixture.ts +++ b/test/helpers/shared-libs-eval-fixture.ts @@ -793,6 +793,11 @@ export function standaloneInstructions(f: SharedLibsFixture, codex = false): str return file; } +/** The skill's instructions as a loaded skill delivers them: in context before any tool call. */ +export function loadedInstructions(file: string): string { + return `\n\nThe /deslop-shared-libs instructions (already loaded):\n\n${fs.readFileSync(file, 'utf8')}\n`; +} + export function reviewLifecycleInstructions(f: SharedLibsFixture): string { const root = SHARED_LIBS_ROOT; const core = extractSkillSections(path.join(root, 'review'), [ diff --git a/test/skill-e2e-shared-libs-periodic.test.ts b/test/skill-e2e-shared-libs-periodic.test.ts index 9b65c092f..3545d8f14 100644 --- a/test/skill-e2e-shared-libs-periodic.test.ts +++ b/test/skill-e2e-shared-libs-periodic.test.ts @@ -10,7 +10,7 @@ import { createSharedPlanReuseSelector } from './helpers/shared-libs-plan-actor' import { SHARED_LIBS_ROOT, commitFixture, createSharedLibsFixture, fixtureWrite, installSourceShims, readRequests, runSharedCapture, runSharedInteractive, seedOpportunitySources, - sharedReadOnlyViolations, snapshotFixture, standaloneInstructions, toolCommandTrace, prCoverageRequestViolations, type SharedLibsFixture, + sharedReadOnlyViolations, snapshotFixture, loadedInstructions, standaloneInstructions, toolCommandTrace, prCoverageRequestViolations, type SharedLibsFixture, SharedCaptureAccumulator, type SharedCaptureAttempt, } from './helpers/shared-libs-eval-fixture'; @@ -83,7 +83,7 @@ describeE2E('Shared-code opportunity and coordination judgment (periodic)', () = const instructions = standaloneInstructions(f); const before = snapshotFixture(f.root); await judgedCapture(attempt, 'empty', 'shared-libs-opportunity-judgment', () => runSharedCapture(f, 'shared-libs-opportunity-judgment', - `Run /deslop-shared-libs using ${instructions} and return the report.`, attempt), async result => { + `Run /deslop-shared-libs and return the report.${loadedInstructions(instructions)}`, attempt), async result => { assertReadOnly(f, before, result, 'shared-libs-opportunity-judgment'); await assertJudgment(result.output, { valid_empty: 'There is only README, .gitignore and a unique one-line src/version.ts, and successful empty PR results. It reports no worthwhile sharing opportunities and does not fabricate callers or blame unavailable history/API access.', @@ -116,7 +116,7 @@ describeE2E('Shared-code opportunity and coordination judgment (periodic)', () = const instructions = standaloneInstructions(f); const before = snapshotFixture(f.root); await judgedCapture(attempt, 'opportunity', 'shared-libs-opportunity-judgment', () => runSharedCapture(f, 'shared-libs-opportunity-judgment', - `Run /deslop-shared-libs using ${instructions}. Review the active TypeScript and Python areas and return the requested report.`, attempt), async result => { + `Run /deslop-shared-libs. Review the active TypeScript and Python areas and return the requested report.${loadedInstructions(instructions)}`, attempt), async result => { assertReadOnly(f, before, result, 'shared-libs-opportunity-judgment'); expect(result.output).toContain(f.tip.slice(0, 7)); expect(result.output).toContain('lib/retry-after.ts'); @@ -149,7 +149,7 @@ describeE2E('Shared-code opportunity and coordination judgment (periodic)', () = const instructions = standaloneInstructions(f); const before = snapshotFixture(f.root); await judgedCapture(attempt, 'audit', 'shared-libs-pr-coverage', () => runSharedCapture(f, 'shared-libs-pr-coverage', - `Run /deslop-shared-libs using ${instructions}. Recent PR 7 mentions https://github.com/fixture/shared-libs/pull/42 as related work. Return the report after checking coordination within the skill's budget.`, attempt), async result => { + `Run /deslop-shared-libs. Recent PR 7 mentions https://github.com/fixture/shared-libs/pull/42 as related work. Return the report after checking coordination within the skill's budget.${loadedInstructions(instructions)}`, attempt), async result => { assertReadOnly(f, before, result, 'shared-libs-pr-coverage'); const requests = readRequests(f).filter(row => row.tool === 'gh' || row.tool === 'curl'); // The skill's stated page budget and PR 42 file-set coverage; per-PR diff --git a/test/skill-e2e-shared-libs.test.ts b/test/skill-e2e-shared-libs.test.ts index 73c791ffc..38d3f4798 100644 --- a/test/skill-e2e-shared-libs.test.ts +++ b/test/skill-e2e-shared-libs.test.ts @@ -15,7 +15,7 @@ import { installSourceShims, readRequests, reviewLifecycleInstructions, reviewPrompt, reviewRevalidationPrompt, reviewRecords, runSharedCapture, runSharedInteractive, seedLifecycleFirstPass, seedOpportunitySources, seedReviewSources, seedSkippedAdvisory, snapshotFixture, specialistFixture, - sharedReadOnlyViolations, standaloneInstructions, toolCommandTrace, type SharedLibsFixture, + sharedReadOnlyViolations, loadedInstructions, standaloneInstructions, toolCommandTrace, type SharedLibsFixture, SharedCaptureAccumulator, type SharedCaptureAttempt, SHARED_LIBS_ROOT, } from './helpers/shared-libs-eval-fixture'; import { @@ -108,7 +108,7 @@ describeE2E('Shared-code safety and review lifecycle (gate)', () => { const instructions = standaloneInstructions(f); const before = snapshotFixture(f.root); await recordCapture(attempt, 'audit', 'shared-libs-read-only', () => runSharedCapture(f, 'shared-libs-read-only', - `Run /deslop-shared-libs for this repository using ${instructions}. Include relevant uncommitted source in your audit. Return the skill's report in conversation.`, attempt), result => { + `Run /deslop-shared-libs for this repository. Include relevant uncommitted source in your audit. Return the skill's report in conversation.${loadedInstructions(instructions)}`, attempt), result => { assertReadOnly(f, before, result); expect(result.output).toMatch(/uncommitted|overlay|raw/i); expect(result.output).toContain(f.tip.slice(0, 7)); @@ -131,7 +131,7 @@ describeE2E('Shared-code safety and review lifecycle (gate)', () => { const instructions = standaloneInstructions(f); const before = snapshotFixture(f.root); await recordCapture(attempt, 'audit', 'shared-libs-unsupported-git', () => runSharedCapture(f, 'shared-libs-unsupported-git', - `Run /deslop-shared-libs for this repository using ${instructions}. Return the review report.`, attempt), result => { + `Run /deslop-shared-libs for this repository. Return the review report.${loadedInstructions(instructions)}`, attempt), result => { assertReadOnly(f, before, result); expect(result.output).toMatch(/unavailable|unsupported|cannot|could not|coverage|limited/i); const calls = readRequests(f).filter(row => row.tool === 'git' && !isInternalClaudeGitRequest(row, toolCommandTrace(result)));