test: shared-libs PR coverage checks the stated page budget, not per-PR ceilings from one run (W7d shared-libs per-endpoint page caps)

prCoverageRequestViolations keeps the skill's stated older-open-PR budget
(five metadata pages, fifty file-list pages plus recent PR 7) and the coverage
outcome (PR 42's file list read from page 1 through page 2). It drops the PR 42
three/four-page and single-first-page ceilings, which were tuned to one
observed capture with a truncated first view. The judge criteria are
unchanged. Stored endpoint lists: three known-good (including a re-read first
page the old ceilings rejected) pass; four known-bad plus an over-budget
file-list run fail. incompleteFirstFileView is no longer used by the paid case.
This commit is contained in:
garrytan committed 2026-10-03 00:44:12 +00:00
1 parent fae9a05c12
commit e0f30196a7
5 files changed
+64 -22

No files matched your search

+14
View File
@@ -0,0 +1,14 @@
{
"_comment": "gh/curl endpoint lists for shared-libs-pr-coverage. known_good must pass prCoverageRequestViolations; known_bad must fail it. R = /repos/fixture/shared-libs.",
"known_good": {
"single-pass": ["/repos/fixture/shared-libs/pulls?state=all&sort=updated&direction=desc&per_page=100&page=1", "/repos/fixture/shared-libs/pulls/7/files?per_page=100&page=1", "/repos/fixture/shared-libs/pulls?state=open&sort=updated&direction=desc&per_page=100&page=1", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=1", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=2"],
"census-truncated-first-view-then-pages-1-3": ["/repos/fixture/shared-libs/pulls?state=open&sort=updated&direction=desc&per_page=100&page=1", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=1", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=1", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=2", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=3"],
"first-page-reread-twice-within-budget": ["/repos/fixture/shared-libs/pulls?state=open&sort=updated&direction=desc&per_page=100&page=1", "/repos/fixture/shared-libs/pulls/42/files?per_page=100", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=1", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=1", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=2"]
},
"known_bad": {
"first-page-only": ["/repos/fixture/shared-libs/pulls?state=open&sort=updated&direction=desc&per_page=100&page=1", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=1"],
"never-read-pr-42": ["/repos/fixture/shared-libs/pulls?state=open&sort=updated&direction=desc&per_page=100&page=1"],
"page-2-without-page-1": ["/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=2"],
"six-metadata-pages": ["/repos/fixture/shared-libs/pulls?state=open&per_page=100&page=1", "/repos/fixture/shared-libs/pulls?state=open&per_page=100&page=2", "/repos/fixture/shared-libs/pulls?state=open&per_page=100&page=3", "/repos/fixture/shared-libs/pulls?state=open&per_page=100&page=4", "/repos/fixture/shared-libs/pulls?state=open&per_page=100&page=5", "/repos/fixture/shared-libs/pulls?state=open&per_page=100&page=6", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=1", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=2"]
}
}
+18
View File
@@ -1031,6 +1031,24 @@ export function toolCommandTrace(result: { toolCalls: Array<{ tool: string; inpu
return result.toolCalls.filter(call => call.tool === 'Bash').map(call => String(call.input?.command || ''));
}
/**
* PR-coverage request check against the skill's stated older-open-PR budget
* (five metadata pages, fifty file-list pages, plus recent PR 7's one page)
* and the coverage outcome: PR 42's file list was read from page 1 through
* page 2. How often a page was re-read within budget is not graded.
*/
export function prCoverageRequestViolations(endpoints: string[]): string[] {
const violations: string[] = [];
const coordination = endpoints.filter(endpoint => /\/pulls\/42\/files/.test(endpoint));
if (!coordination.some(endpoint => !/[?&]page=/.test(endpoint) || /[?&]page=1(?:&|$)/.test(endpoint))) violations.push('PR 42 first file page not read');
if (!coordination.some(endpoint => /[?&]page=2(?:&|$)/.test(endpoint))) violations.push('PR 42 second file page not read');
const openPages = endpoints.filter(endpoint => /\/pulls\?/.test(endpoint) && /state=open/.test(endpoint)).length;
if (openPages > 5) violations.push(`${openPages} open-PR metadata pages exceed the budget of 5`);
const filePages = endpoints.filter(endpoint => /\/pulls\/\d+\/files/.test(endpoint)).length;
if (filePages > 51) violations.push(`${filePages} file-list pages exceed the budget of 50 plus PR 7`);
return violations;
}
/** Whether the first read of a PR's file-list page 1 left no usable file set:
* truncated by `head -c`, or a failed filter (e.g. a jq error) that printed no
* file entries. One recovery read of page 1 is then legitimate, still charged
+7 -2
View File
@@ -7,7 +7,7 @@ import { execFileSync, spawnSync } from 'node:child_process';
import {
createSharedInteractiveToolHandler, createSharedLibsFixture, fixtureGit, fixtureWrite, installSourceShims,
installHostileGitConfig, isGuardedGitRequest, reviewLifecycleInstructions, standaloneInstructions, SHARED_LIBS_ROOT, readRequests, seedOpportunitySources, sharedReadOnlyViolations, shellQuote, snapshotFixture, type SharedLibsFixture,
SharedCaptureAccumulator, type SharedCaptureAttempt, isInternalClaudeGitRequest, SHARED_LIBS_OLDER_OPEN_PRS, incompleteFirstFileView,
SharedCaptureAccumulator, type SharedCaptureAttempt, isInternalClaudeGitRequest, SHARED_LIBS_OLDER_OPEN_PRS, incompleteFirstFileView, prCoverageRequestViolations,
} from './helpers/shared-libs-eval-fixture';
import { EvalCollector, type EvalTestEntry } from './helpers/eval-store';
import { collectorOutcomeCounts } from '../scripts/test-paid-shards';
@@ -1075,7 +1075,12 @@ describe('shared-code PR coverage world', () => {
test('the maximum authorized open-metadata scan still leaves older open PRs unchecked', () => {
const periodic = fs.readFileSync(path.join(import.meta.dir, 'skill-e2e-shared-libs-periodic.test.ts'), 'utf8');
expect(periodic).toContain('expect(openPages.length).toBeLessThanOrEqual(5);');
expect(periodic).toContain('prCoverageRequestViolations(');
const sixth = Array.from({ length: 6 }, (_, i) => `/repos/fixture/shared-libs/pulls?state=open&per_page=100&page=${i + 1}`);
expect(prCoverageRequestViolations([...sixth.slice(0, 5), '/repos/fixture/shared-libs/pulls/42/files?page=1',
'/repos/fixture/shared-libs/pulls/42/files?page=2'])).toEqual([]);
expect(prCoverageRequestViolations([...sixth, '/repos/fixture/shared-libs/pulls/42/files?page=1',
'/repos/fixture/shared-libs/pulls/42/files?page=2'])).toEqual(['6 open-PR metadata pages exceed the budget of 5']);
const f = createSharedLibsFixture('pr-world-budget');
cleanup.push(f.root);
seedOpportunitySources(f);
@@ -0,0 +1,21 @@
import { describe, expect, test } from 'bun:test';
import * as fs from 'fs';
import * as path from 'path';
import { prCoverageRequestViolations } from './helpers/shared-libs-eval-fixture';
const stored = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/shared-libs/pr-coverage-endpoints.json'), 'utf8')) as
{ known_good: Record<string, string[]>; known_bad: Record<string, string[]> };
describe('shared-libs PR coverage request budget', () => {
test.each(Object.entries(stored.known_good))('passes %s', (_name, endpoints) => {
expect(prCoverageRequestViolations(endpoints)).toEqual([]);
});
test.each(Object.entries(stored.known_bad))('fails %s', (_name, endpoints) => {
expect(prCoverageRequestViolations(endpoints).length).toBeGreaterThan(0);
});
test('file-list pages beyond the stated budget fail', () => {
const pages = Array.from({ length: 52 }, (_, i) => `/repos/fixture/shared-libs/pulls/${i < 2 ? 42 : 100 + i}/files?per_page=100&page=${i < 2 ? i + 1 : 1}`);
expect(prCoverageRequestViolations(pages)).toEqual(['52 file-list pages exceed the budget of 50 plus PR 7']);
expect(prCoverageRequestViolations(pages.slice(0, 51))).toEqual([]);
});
});
+4 -20
View File
@@ -10,7 +10,7 @@ import { createSharedPlanReuseSelector } from './helpers/shared-libs-plan-actor'
import {
SHARED_LIBS_ROOT, commitFixture, createSharedLibsFixture, fixtureWrite, installSourceShims,
readRequests, runSharedCapture, runSharedInteractive, seedOpportunitySources,
sharedReadOnlyViolations, snapshotFixture, standaloneInstructions, toolCommandTrace, incompleteFirstFileView, type SharedLibsFixture,
sharedReadOnlyViolations, snapshotFixture, standaloneInstructions, toolCommandTrace, prCoverageRequestViolations, type SharedLibsFixture,
SharedCaptureAccumulator, type SharedCaptureAttempt,
} from './helpers/shared-libs-eval-fixture';
@@ -152,25 +152,9 @@ describeE2E('Shared-code opportunity and coordination judgment (periodic)', () =
`Run /deslop-shared-libs using ${instructions}. Recent PR 7 mentions https://github.com/fixture/shared-libs/pull/42 as related work. Return the report after checking coordination within the skill's budget.`, attempt), async result => {
assertReadOnly(f, before, result, 'shared-libs-pr-coverage');
const requests = readRequests(f).filter(row => row.tool === 'gh' || row.tool === 'curl');
const endpoints = requests.map(row => row.endpoint || '');
expect(endpoints.some(endpoint => /\/pulls\/42\/files/.test(endpoint))).toBe(true);
expect(endpoints.some(endpoint => /\/pulls\/42\/files/.test(endpoint) && /[?&]page=2(?:&|$)/.test(endpoint))).toBe(true);
const openPages = endpoints.filter(endpoint => /\/pulls\?/.test(endpoint) && /state=open/.test(endpoint));
const filePages = endpoints.filter(endpoint => /\/pulls\/\d+\/files/.test(endpoint));
expect(openPages.length).toBeLessThanOrEqual(5);
// Only recent PR7's one page is outside the additional scan. Known old PR42
// shares that scan's 50-page budget, including both of its file-list pages.
expect(filePages.length).toBeLessThanOrEqual(51);
const coordinationPages = endpoints.filter(endpoint => /\/pulls\/42\/files/.test(endpoint));
// An incomplete first page is not a reusable complete file set. The
// observed capture truncated its first response with head, then fetched
// full pages 1–3. Permit that one recovery while charging every request
// to the hard budget and forbidding repeated complete first-page reads.
const truncatedFirstView = incompleteFirstFileView(result, 42);
expect(coordinationPages.length).toBeLessThanOrEqual(truncatedFirstView ? 4 : 3);
const firstPages = coordinationPages.filter(endpoint => !/[?&]page=/.test(endpoint) || /[?&]page=1(?:&|$)/.test(endpoint));
expect(firstPages.length).toBeGreaterThan(0);
expect(firstPages.length).toBeLessThanOrEqual(truncatedFirstView ? 2 : 1);
// The skill's stated page budget and PR 42 file-set coverage; per-PR
// page ceilings tuned to one observed run are not graded.
expect(prCoverageRequestViolations(requests.map(row => row.endpoint || ''))).toEqual([]);
await assertJudgment(result.output, {
older_open_pr: 'Identifies open PR 42 from 2020 as existing coordination work despite its activity being outside the 14-day window. The PR already proposes migrating retry-worker to the proven parser.',
partial_overlap: 'Does not count the worker migration already covered by PR 42 as a new opportunity. It may recommend the independent remaining retry-route migration if it explains the narrowed scope and adjusted savings.',