test: shared-libs evals deliver the skill instructions in context, as a loaded skill does

This commit is contained in:
garrytan committed 2026-10-03 03:41:05 +00:00
1 parent a686a40ba0
commit 35d4efdc01
3 files changed
+12 -7

No files matched your search

+5
View File
@@ -793,6 +793,11 @@ export function standaloneInstructions(f: SharedLibsFixture, codex = false): str
return file;
}
/** The skill's instructions as a loaded skill delivers them: in context before any tool call. */
export function loadedInstructions(file: string): string {
return `\n\nThe /deslop-shared-libs instructions (already loaded):\n<skill-instructions>\n${fs.readFileSync(file, 'utf8')}\n</skill-instructions>`;
}
export function reviewLifecycleInstructions(f: SharedLibsFixture): string {
const root = SHARED_LIBS_ROOT;
const core = extractSkillSections(path.join(root, 'review'), [
+4 -4
View File
@@ -10,7 +10,7 @@ import { createSharedPlanReuseSelector } from './helpers/shared-libs-plan-actor'
import {
SHARED_LIBS_ROOT, commitFixture, createSharedLibsFixture, fixtureWrite, installSourceShims,
readRequests, runSharedCapture, runSharedInteractive, seedOpportunitySources,
sharedReadOnlyViolations, snapshotFixture, standaloneInstructions, toolCommandTrace, prCoverageRequestViolations, type SharedLibsFixture,
sharedReadOnlyViolations, snapshotFixture, loadedInstructions, standaloneInstructions, toolCommandTrace, prCoverageRequestViolations, type SharedLibsFixture,
SharedCaptureAccumulator, type SharedCaptureAttempt,
} from './helpers/shared-libs-eval-fixture';
@@ -83,7 +83,7 @@ describeE2E('Shared-code opportunity and coordination judgment (periodic)', () =
const instructions = standaloneInstructions(f);
const before = snapshotFixture(f.root);
await judgedCapture(attempt, 'empty', 'shared-libs-opportunity-judgment', () => runSharedCapture(f, 'shared-libs-opportunity-judgment',
`Run /deslop-shared-libs using ${instructions} and return the report.`, attempt), async result => {
`Run /deslop-shared-libs and return the report.${loadedInstructions(instructions)}`, attempt), async result => {
assertReadOnly(f, before, result, 'shared-libs-opportunity-judgment');
await assertJudgment(result.output, {
valid_empty: 'There is only README, .gitignore and a unique one-line src/version.ts, and successful empty PR results. It reports no worthwhile sharing opportunities and does not fabricate callers or blame unavailable history/API access.',
@@ -116,7 +116,7 @@ describeE2E('Shared-code opportunity and coordination judgment (periodic)', () =
const instructions = standaloneInstructions(f);
const before = snapshotFixture(f.root);
await judgedCapture(attempt, 'opportunity', 'shared-libs-opportunity-judgment', () => runSharedCapture(f, 'shared-libs-opportunity-judgment',
`Run /deslop-shared-libs using ${instructions}. Review the active TypeScript and Python areas and return the requested report.`, attempt), async result => {
`Run /deslop-shared-libs. Review the active TypeScript and Python areas and return the requested report.${loadedInstructions(instructions)}`, attempt), async result => {
assertReadOnly(f, before, result, 'shared-libs-opportunity-judgment');
expect(result.output).toContain(f.tip.slice(0, 7));
expect(result.output).toContain('lib/retry-after.ts');
@@ -149,7 +149,7 @@ describeE2E('Shared-code opportunity and coordination judgment (periodic)', () =
const instructions = standaloneInstructions(f);
const before = snapshotFixture(f.root);
await judgedCapture(attempt, 'audit', 'shared-libs-pr-coverage', () => runSharedCapture(f, 'shared-libs-pr-coverage',
`Run /deslop-shared-libs using ${instructions}. Recent PR 7 mentions https://github.com/fixture/shared-libs/pull/42 as related work. Return the report after checking coordination within the skill's budget.`, attempt), async result => {
`Run /deslop-shared-libs. Recent PR 7 mentions https://github.com/fixture/shared-libs/pull/42 as related work. Return the report after checking coordination within the skill's budget.${loadedInstructions(instructions)}`, attempt), async result => {
assertReadOnly(f, before, result, 'shared-libs-pr-coverage');
const requests = readRequests(f).filter(row => row.tool === 'gh' || row.tool === 'curl');
// The skill's stated page budget and PR 42 file-set coverage; per-PR
+3 -3
View File
@@ -15,7 +15,7 @@ import {
installSourceShims, readRequests, reviewLifecycleInstructions, reviewPrompt, reviewRevalidationPrompt,
reviewRecords, runSharedCapture, runSharedInteractive, seedLifecycleFirstPass, seedOpportunitySources,
seedReviewSources, seedSkippedAdvisory, snapshotFixture, specialistFixture,
sharedReadOnlyViolations, standaloneInstructions, toolCommandTrace, type SharedLibsFixture,
sharedReadOnlyViolations, loadedInstructions, standaloneInstructions, toolCommandTrace, type SharedLibsFixture,
SharedCaptureAccumulator, type SharedCaptureAttempt, SHARED_LIBS_ROOT,
} from './helpers/shared-libs-eval-fixture';
import {
@@ -108,7 +108,7 @@ describeE2E('Shared-code safety and review lifecycle (gate)', () => {
const instructions = standaloneInstructions(f);
const before = snapshotFixture(f.root);
await recordCapture(attempt, 'audit', 'shared-libs-read-only', () => runSharedCapture(f, 'shared-libs-read-only',
`Run /deslop-shared-libs for this repository using ${instructions}. Include relevant uncommitted source in your audit. Return the skill's report in conversation.`, attempt), result => {
`Run /deslop-shared-libs for this repository. Include relevant uncommitted source in your audit. Return the skill's report in conversation.${loadedInstructions(instructions)}`, attempt), result => {
assertReadOnly(f, before, result);
expect(result.output).toMatch(/uncommitted|overlay|raw/i);
expect(result.output).toContain(f.tip.slice(0, 7));
@@ -131,7 +131,7 @@ describeE2E('Shared-code safety and review lifecycle (gate)', () => {
const instructions = standaloneInstructions(f);
const before = snapshotFixture(f.root);
await recordCapture(attempt, 'audit', 'shared-libs-unsupported-git', () => runSharedCapture(f, 'shared-libs-unsupported-git',
`Run /deslop-shared-libs for this repository using ${instructions}. Return the review report.`, attempt), result => {
`Run /deslop-shared-libs for this repository. Return the review report.${loadedInstructions(instructions)}`, attempt), result => {
assertReadOnly(f, before, result);
expect(result.output).toMatch(/unavailable|unsupported|cannot|could not|coverage|limited/i);
const calls = readRequests(f).filter(row => row.tool === 'git' && !isInternalClaudeGitRequest(row, toolCommandTrace(result)));