mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-12 19:44:48 +01:00
test: shared-libs evals deliver the skill instructions in context, as a loaded skill does
This commit is contained in:
1 parent
a686a40ba0
commit
35d4efdc01
3 files changed
+12
-7
No files matched your search
@@ -793,6 +793,11 @@ export function standaloneInstructions(f: SharedLibsFixture, codex = false): str
|
||||
return file;
|
||||
}
|
||||
|
||||
/** The skill's instructions as a loaded skill delivers them: in context before any tool call. */
|
||||
export function loadedInstructions(file: string): string {
|
||||
return `\n\nThe /deslop-shared-libs instructions (already loaded):\n<skill-instructions>\n${fs.readFileSync(file, 'utf8')}\n</skill-instructions>`;
|
||||
}
|
||||
|
||||
export function reviewLifecycleInstructions(f: SharedLibsFixture): string {
|
||||
const root = SHARED_LIBS_ROOT;
|
||||
const core = extractSkillSections(path.join(root, 'review'), [
|
||||
|
||||
@@ -10,7 +10,7 @@ import { createSharedPlanReuseSelector } from './helpers/shared-libs-plan-actor'
|
||||
import {
|
||||
SHARED_LIBS_ROOT, commitFixture, createSharedLibsFixture, fixtureWrite, installSourceShims,
|
||||
readRequests, runSharedCapture, runSharedInteractive, seedOpportunitySources,
|
||||
sharedReadOnlyViolations, snapshotFixture, standaloneInstructions, toolCommandTrace, prCoverageRequestViolations, type SharedLibsFixture,
|
||||
sharedReadOnlyViolations, snapshotFixture, loadedInstructions, standaloneInstructions, toolCommandTrace, prCoverageRequestViolations, type SharedLibsFixture,
|
||||
SharedCaptureAccumulator, type SharedCaptureAttempt,
|
||||
} from './helpers/shared-libs-eval-fixture';
|
||||
|
||||
@@ -83,7 +83,7 @@ describeE2E('Shared-code opportunity and coordination judgment (periodic)', () =
|
||||
const instructions = standaloneInstructions(f);
|
||||
const before = snapshotFixture(f.root);
|
||||
await judgedCapture(attempt, 'empty', 'shared-libs-opportunity-judgment', () => runSharedCapture(f, 'shared-libs-opportunity-judgment',
|
||||
`Run /deslop-shared-libs using ${instructions} and return the report.`, attempt), async result => {
|
||||
`Run /deslop-shared-libs and return the report.${loadedInstructions(instructions)}`, attempt), async result => {
|
||||
assertReadOnly(f, before, result, 'shared-libs-opportunity-judgment');
|
||||
await assertJudgment(result.output, {
|
||||
valid_empty: 'There is only README, .gitignore and a unique one-line src/version.ts, and successful empty PR results. It reports no worthwhile sharing opportunities and does not fabricate callers or blame unavailable history/API access.',
|
||||
@@ -116,7 +116,7 @@ describeE2E('Shared-code opportunity and coordination judgment (periodic)', () =
|
||||
const instructions = standaloneInstructions(f);
|
||||
const before = snapshotFixture(f.root);
|
||||
await judgedCapture(attempt, 'opportunity', 'shared-libs-opportunity-judgment', () => runSharedCapture(f, 'shared-libs-opportunity-judgment',
|
||||
`Run /deslop-shared-libs using ${instructions}. Review the active TypeScript and Python areas and return the requested report.`, attempt), async result => {
|
||||
`Run /deslop-shared-libs. Review the active TypeScript and Python areas and return the requested report.${loadedInstructions(instructions)}`, attempt), async result => {
|
||||
assertReadOnly(f, before, result, 'shared-libs-opportunity-judgment');
|
||||
expect(result.output).toContain(f.tip.slice(0, 7));
|
||||
expect(result.output).toContain('lib/retry-after.ts');
|
||||
@@ -149,7 +149,7 @@ describeE2E('Shared-code opportunity and coordination judgment (periodic)', () =
|
||||
const instructions = standaloneInstructions(f);
|
||||
const before = snapshotFixture(f.root);
|
||||
await judgedCapture(attempt, 'audit', 'shared-libs-pr-coverage', () => runSharedCapture(f, 'shared-libs-pr-coverage',
|
||||
`Run /deslop-shared-libs using ${instructions}. Recent PR 7 mentions https://github.com/fixture/shared-libs/pull/42 as related work. Return the report after checking coordination within the skill's budget.`, attempt), async result => {
|
||||
`Run /deslop-shared-libs. Recent PR 7 mentions https://github.com/fixture/shared-libs/pull/42 as related work. Return the report after checking coordination within the skill's budget.${loadedInstructions(instructions)}`, attempt), async result => {
|
||||
assertReadOnly(f, before, result, 'shared-libs-pr-coverage');
|
||||
const requests = readRequests(f).filter(row => row.tool === 'gh' || row.tool === 'curl');
|
||||
// The skill's stated page budget and PR 42 file-set coverage; per-PR
|
||||
|
||||
@@ -15,7 +15,7 @@ import {
|
||||
installSourceShims, readRequests, reviewLifecycleInstructions, reviewPrompt, reviewRevalidationPrompt,
|
||||
reviewRecords, runSharedCapture, runSharedInteractive, seedLifecycleFirstPass, seedOpportunitySources,
|
||||
seedReviewSources, seedSkippedAdvisory, snapshotFixture, specialistFixture,
|
||||
sharedReadOnlyViolations, standaloneInstructions, toolCommandTrace, type SharedLibsFixture,
|
||||
sharedReadOnlyViolations, loadedInstructions, standaloneInstructions, toolCommandTrace, type SharedLibsFixture,
|
||||
SharedCaptureAccumulator, type SharedCaptureAttempt, SHARED_LIBS_ROOT,
|
||||
} from './helpers/shared-libs-eval-fixture';
|
||||
import {
|
||||
@@ -108,7 +108,7 @@ describeE2E('Shared-code safety and review lifecycle (gate)', () => {
|
||||
const instructions = standaloneInstructions(f);
|
||||
const before = snapshotFixture(f.root);
|
||||
await recordCapture(attempt, 'audit', 'shared-libs-read-only', () => runSharedCapture(f, 'shared-libs-read-only',
|
||||
`Run /deslop-shared-libs for this repository using ${instructions}. Include relevant uncommitted source in your audit. Return the skill's report in conversation.`, attempt), result => {
|
||||
`Run /deslop-shared-libs for this repository. Include relevant uncommitted source in your audit. Return the skill's report in conversation.${loadedInstructions(instructions)}`, attempt), result => {
|
||||
assertReadOnly(f, before, result);
|
||||
expect(result.output).toMatch(/uncommitted|overlay|raw/i);
|
||||
expect(result.output).toContain(f.tip.slice(0, 7));
|
||||
@@ -131,7 +131,7 @@ describeE2E('Shared-code safety and review lifecycle (gate)', () => {
|
||||
const instructions = standaloneInstructions(f);
|
||||
const before = snapshotFixture(f.root);
|
||||
await recordCapture(attempt, 'audit', 'shared-libs-unsupported-git', () => runSharedCapture(f, 'shared-libs-unsupported-git',
|
||||
`Run /deslop-shared-libs for this repository using ${instructions}. Return the review report.`, attempt), result => {
|
||||
`Run /deslop-shared-libs for this repository. Return the review report.${loadedInstructions(instructions)}`, attempt), result => {
|
||||
assertReadOnly(f, before, result);
|
||||
expect(result.output).toMatch(/unavailable|unsupported|cannot|could not|coverage|limited/i);
|
||||
const calls = readRequests(f).filter(row => row.tool === 'git' && !isInternalClaudeGitRequest(row, toolCommandTrace(result)));
|
||||
|
||||
Reference in new issue
Block a user