diff --git a/test/fixtures/shared-libs/pr-coverage-endpoints.json b/test/fixtures/shared-libs/pr-coverage-endpoints.json new file mode 100644 index 000000000..a0ee063fe --- /dev/null +++ b/test/fixtures/shared-libs/pr-coverage-endpoints.json @@ -0,0 +1,14 @@ +{ + "_comment": "gh/curl endpoint lists for shared-libs-pr-coverage. known_good must pass prCoverageRequestViolations; known_bad must fail it. R = /repos/fixture/shared-libs.", + "known_good": { + "single-pass": ["/repos/fixture/shared-libs/pulls?state=all&sort=updated&direction=desc&per_page=100&page=1", "/repos/fixture/shared-libs/pulls/7/files?per_page=100&page=1", "/repos/fixture/shared-libs/pulls?state=open&sort=updated&direction=desc&per_page=100&page=1", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=1", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=2"], + "census-truncated-first-view-then-pages-1-3": ["/repos/fixture/shared-libs/pulls?state=open&sort=updated&direction=desc&per_page=100&page=1", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=1", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=1", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=2", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=3"], + "first-page-reread-twice-within-budget": ["/repos/fixture/shared-libs/pulls?state=open&sort=updated&direction=desc&per_page=100&page=1", "/repos/fixture/shared-libs/pulls/42/files?per_page=100", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=1", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=1", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=2"] + }, + "known_bad": { + "first-page-only": ["/repos/fixture/shared-libs/pulls?state=open&sort=updated&direction=desc&per_page=100&page=1", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=1"], + "never-read-pr-42": ["/repos/fixture/shared-libs/pulls?state=open&sort=updated&direction=desc&per_page=100&page=1"], + "page-2-without-page-1": ["/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=2"], + "six-metadata-pages": ["/repos/fixture/shared-libs/pulls?state=open&per_page=100&page=1", "/repos/fixture/shared-libs/pulls?state=open&per_page=100&page=2", "/repos/fixture/shared-libs/pulls?state=open&per_page=100&page=3", "/repos/fixture/shared-libs/pulls?state=open&per_page=100&page=4", "/repos/fixture/shared-libs/pulls?state=open&per_page=100&page=5", "/repos/fixture/shared-libs/pulls?state=open&per_page=100&page=6", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=1", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=2"] + } +} diff --git a/test/helpers/shared-libs-eval-fixture.ts b/test/helpers/shared-libs-eval-fixture.ts index aa6776ab3..68678e97b 100644 --- a/test/helpers/shared-libs-eval-fixture.ts +++ b/test/helpers/shared-libs-eval-fixture.ts @@ -1031,6 +1031,24 @@ export function toolCommandTrace(result: { toolCalls: Array<{ tool: string; inpu return result.toolCalls.filter(call => call.tool === 'Bash').map(call => String(call.input?.command || '')); } +/** + * PR-coverage request check against the skill's stated older-open-PR budget + * (five metadata pages, fifty file-list pages, plus recent PR 7's one page) + * and the coverage outcome: PR 42's file list was read from page 1 through + * page 2. How often a page was re-read within budget is not graded. + */ +export function prCoverageRequestViolations(endpoints: string[]): string[] { + const violations: string[] = []; + const coordination = endpoints.filter(endpoint => /\/pulls\/42\/files/.test(endpoint)); + if (!coordination.some(endpoint => !/[?&]page=/.test(endpoint) || /[?&]page=1(?:&|$)/.test(endpoint))) violations.push('PR 42 first file page not read'); + if (!coordination.some(endpoint => /[?&]page=2(?:&|$)/.test(endpoint))) violations.push('PR 42 second file page not read'); + const openPages = endpoints.filter(endpoint => /\/pulls\?/.test(endpoint) && /state=open/.test(endpoint)).length; + if (openPages > 5) violations.push(`${openPages} open-PR metadata pages exceed the budget of 5`); + const filePages = endpoints.filter(endpoint => /\/pulls\/\d+\/files/.test(endpoint)).length; + if (filePages > 51) violations.push(`${filePages} file-list pages exceed the budget of 50 plus PR 7`); + return violations; +} + /** Whether the first read of a PR's file-list page 1 left no usable file set: * truncated by `head -c`, or a failed filter (e.g. a jq error) that printed no * file entries. One recovery read of page 1 is then legitimate, still charged diff --git a/test/shared-libs-fixture.test.ts b/test/shared-libs-fixture.test.ts index 7affb5de8..2bbbff13e 100644 --- a/test/shared-libs-fixture.test.ts +++ b/test/shared-libs-fixture.test.ts @@ -7,7 +7,7 @@ import { execFileSync, spawnSync } from 'node:child_process'; import { createSharedInteractiveToolHandler, createSharedLibsFixture, fixtureGit, fixtureWrite, installSourceShims, installHostileGitConfig, isGuardedGitRequest, reviewLifecycleInstructions, standaloneInstructions, SHARED_LIBS_ROOT, readRequests, seedOpportunitySources, sharedReadOnlyViolations, shellQuote, snapshotFixture, type SharedLibsFixture, - SharedCaptureAccumulator, type SharedCaptureAttempt, isInternalClaudeGitRequest, SHARED_LIBS_OLDER_OPEN_PRS, incompleteFirstFileView, + SharedCaptureAccumulator, type SharedCaptureAttempt, isInternalClaudeGitRequest, SHARED_LIBS_OLDER_OPEN_PRS, incompleteFirstFileView, prCoverageRequestViolations, } from './helpers/shared-libs-eval-fixture'; import { EvalCollector, type EvalTestEntry } from './helpers/eval-store'; import { collectorOutcomeCounts } from '../scripts/test-paid-shards'; @@ -1075,7 +1075,12 @@ describe('shared-code PR coverage world', () => { test('the maximum authorized open-metadata scan still leaves older open PRs unchecked', () => { const periodic = fs.readFileSync(path.join(import.meta.dir, 'skill-e2e-shared-libs-periodic.test.ts'), 'utf8'); - expect(periodic).toContain('expect(openPages.length).toBeLessThanOrEqual(5);'); + expect(periodic).toContain('prCoverageRequestViolations('); + const sixth = Array.from({ length: 6 }, (_, i) => `/repos/fixture/shared-libs/pulls?state=open&per_page=100&page=${i + 1}`); + expect(prCoverageRequestViolations([...sixth.slice(0, 5), '/repos/fixture/shared-libs/pulls/42/files?page=1', + '/repos/fixture/shared-libs/pulls/42/files?page=2'])).toEqual([]); + expect(prCoverageRequestViolations([...sixth, '/repos/fixture/shared-libs/pulls/42/files?page=1', + '/repos/fixture/shared-libs/pulls/42/files?page=2'])).toEqual(['6 open-PR metadata pages exceed the budget of 5']); const f = createSharedLibsFixture('pr-world-budget'); cleanup.push(f.root); seedOpportunitySources(f); diff --git a/test/shared-libs-pr-coverage-budget.test.ts b/test/shared-libs-pr-coverage-budget.test.ts new file mode 100644 index 000000000..b4a0cea1a --- /dev/null +++ b/test/shared-libs-pr-coverage-budget.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, test } from 'bun:test'; +import * as fs from 'fs'; +import * as path from 'path'; +import { prCoverageRequestViolations } from './helpers/shared-libs-eval-fixture'; + +const stored = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/shared-libs/pr-coverage-endpoints.json'), 'utf8')) as + { known_good: Record; known_bad: Record }; + +describe('shared-libs PR coverage request budget', () => { + test.each(Object.entries(stored.known_good))('passes %s', (_name, endpoints) => { + expect(prCoverageRequestViolations(endpoints)).toEqual([]); + }); + test.each(Object.entries(stored.known_bad))('fails %s', (_name, endpoints) => { + expect(prCoverageRequestViolations(endpoints).length).toBeGreaterThan(0); + }); + test('file-list pages beyond the stated budget fail', () => { + const pages = Array.from({ length: 52 }, (_, i) => `/repos/fixture/shared-libs/pulls/${i < 2 ? 42 : 100 + i}/files?per_page=100&page=${i < 2 ? i + 1 : 1}`); + expect(prCoverageRequestViolations(pages)).toEqual(['52 file-list pages exceed the budget of 50 plus PR 7']); + expect(prCoverageRequestViolations(pages.slice(0, 51))).toEqual([]); + }); +}); diff --git a/test/skill-e2e-shared-libs-periodic.test.ts b/test/skill-e2e-shared-libs-periodic.test.ts index a93037efc..9b65c092f 100644 --- a/test/skill-e2e-shared-libs-periodic.test.ts +++ b/test/skill-e2e-shared-libs-periodic.test.ts @@ -10,7 +10,7 @@ import { createSharedPlanReuseSelector } from './helpers/shared-libs-plan-actor' import { SHARED_LIBS_ROOT, commitFixture, createSharedLibsFixture, fixtureWrite, installSourceShims, readRequests, runSharedCapture, runSharedInteractive, seedOpportunitySources, - sharedReadOnlyViolations, snapshotFixture, standaloneInstructions, toolCommandTrace, incompleteFirstFileView, type SharedLibsFixture, + sharedReadOnlyViolations, snapshotFixture, standaloneInstructions, toolCommandTrace, prCoverageRequestViolations, type SharedLibsFixture, SharedCaptureAccumulator, type SharedCaptureAttempt, } from './helpers/shared-libs-eval-fixture'; @@ -152,25 +152,9 @@ describeE2E('Shared-code opportunity and coordination judgment (periodic)', () = `Run /deslop-shared-libs using ${instructions}. Recent PR 7 mentions https://github.com/fixture/shared-libs/pull/42 as related work. Return the report after checking coordination within the skill's budget.`, attempt), async result => { assertReadOnly(f, before, result, 'shared-libs-pr-coverage'); const requests = readRequests(f).filter(row => row.tool === 'gh' || row.tool === 'curl'); - const endpoints = requests.map(row => row.endpoint || ''); - expect(endpoints.some(endpoint => /\/pulls\/42\/files/.test(endpoint))).toBe(true); - expect(endpoints.some(endpoint => /\/pulls\/42\/files/.test(endpoint) && /[?&]page=2(?:&|$)/.test(endpoint))).toBe(true); - const openPages = endpoints.filter(endpoint => /\/pulls\?/.test(endpoint) && /state=open/.test(endpoint)); - const filePages = endpoints.filter(endpoint => /\/pulls\/\d+\/files/.test(endpoint)); - expect(openPages.length).toBeLessThanOrEqual(5); - // Only recent PR7's one page is outside the additional scan. Known old PR42 - // shares that scan's 50-page budget, including both of its file-list pages. - expect(filePages.length).toBeLessThanOrEqual(51); - const coordinationPages = endpoints.filter(endpoint => /\/pulls\/42\/files/.test(endpoint)); - // An incomplete first page is not a reusable complete file set. The - // observed capture truncated its first response with head, then fetched - // full pages 1–3. Permit that one recovery while charging every request - // to the hard budget and forbidding repeated complete first-page reads. - const truncatedFirstView = incompleteFirstFileView(result, 42); - expect(coordinationPages.length).toBeLessThanOrEqual(truncatedFirstView ? 4 : 3); - const firstPages = coordinationPages.filter(endpoint => !/[?&]page=/.test(endpoint) || /[?&]page=1(?:&|$)/.test(endpoint)); - expect(firstPages.length).toBeGreaterThan(0); - expect(firstPages.length).toBeLessThanOrEqual(truncatedFirstView ? 2 : 1); + // The skill's stated page budget and PR 42 file-set coverage; per-PR + // page ceilings tuned to one observed run are not graded. + expect(prCoverageRequestViolations(requests.map(row => row.endpoint || ''))).toEqual([]); await assertJudgment(result.output, { older_open_pr: 'Identifies open PR 42 from 2020 as existing coordination work despite its activity being outside the 14-day window. The PR already proposes migrating retry-worker to the proven parser.', partial_overlap: 'Does not count the worker migration already covered by PR 42 as a new opportunity. It may recommend the independent remaining retry-route migration if it explains the narrowed scope and adjusted savings.',