mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-02 06:54:50 +01:00
test: shared-libs PR coverage checks the stated page budget, not per-PR ceilings from one run (W7d shared-libs per-endpoint page caps)
prCoverageRequestViolations keeps the skill's stated older-open-PR budget (five metadata pages, fifty file-list pages plus recent PR 7) and the coverage outcome (PR 42's file list read from page 1 through page 2). It drops the PR 42 three/four-page and single-first-page ceilings, which were tuned to one observed capture with a truncated first view. The judge criteria are unchanged. Stored endpoint lists: three known-good (including a re-read first page the old ceilings rejected) pass; four known-bad plus an over-budget file-list run fail. incompleteFirstFileView is no longer used by the paid case.
This commit is contained in:
1 parent
fae9a05c12
commit
e0f30196a7
5 files changed
+64
-22
No files matched your search
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"_comment": "gh/curl endpoint lists for shared-libs-pr-coverage. known_good must pass prCoverageRequestViolations; known_bad must fail it. R = /repos/fixture/shared-libs.",
|
||||
"known_good": {
|
||||
"single-pass": ["/repos/fixture/shared-libs/pulls?state=all&sort=updated&direction=desc&per_page=100&page=1", "/repos/fixture/shared-libs/pulls/7/files?per_page=100&page=1", "/repos/fixture/shared-libs/pulls?state=open&sort=updated&direction=desc&per_page=100&page=1", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=1", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=2"],
|
||||
"census-truncated-first-view-then-pages-1-3": ["/repos/fixture/shared-libs/pulls?state=open&sort=updated&direction=desc&per_page=100&page=1", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=1", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=1", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=2", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=3"],
|
||||
"first-page-reread-twice-within-budget": ["/repos/fixture/shared-libs/pulls?state=open&sort=updated&direction=desc&per_page=100&page=1", "/repos/fixture/shared-libs/pulls/42/files?per_page=100", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=1", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=1", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=2"]
|
||||
},
|
||||
"known_bad": {
|
||||
"first-page-only": ["/repos/fixture/shared-libs/pulls?state=open&sort=updated&direction=desc&per_page=100&page=1", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=1"],
|
||||
"never-read-pr-42": ["/repos/fixture/shared-libs/pulls?state=open&sort=updated&direction=desc&per_page=100&page=1"],
|
||||
"page-2-without-page-1": ["/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=2"],
|
||||
"six-metadata-pages": ["/repos/fixture/shared-libs/pulls?state=open&per_page=100&page=1", "/repos/fixture/shared-libs/pulls?state=open&per_page=100&page=2", "/repos/fixture/shared-libs/pulls?state=open&per_page=100&page=3", "/repos/fixture/shared-libs/pulls?state=open&per_page=100&page=4", "/repos/fixture/shared-libs/pulls?state=open&per_page=100&page=5", "/repos/fixture/shared-libs/pulls?state=open&per_page=100&page=6", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=1", "/repos/fixture/shared-libs/pulls/42/files?per_page=100&page=2"]
|
||||
}
|
||||
}
|
||||
@@ -1031,6 +1031,24 @@ export function toolCommandTrace(result: { toolCalls: Array<{ tool: string; inpu
|
||||
return result.toolCalls.filter(call => call.tool === 'Bash').map(call => String(call.input?.command || ''));
|
||||
}
|
||||
|
||||
/**
|
||||
* PR-coverage request check against the skill's stated older-open-PR budget
|
||||
* (five metadata pages, fifty file-list pages, plus recent PR 7's one page)
|
||||
* and the coverage outcome: PR 42's file list was read from page 1 through
|
||||
* page 2. How often a page was re-read within budget is not graded.
|
||||
*/
|
||||
export function prCoverageRequestViolations(endpoints: string[]): string[] {
|
||||
const violations: string[] = [];
|
||||
const coordination = endpoints.filter(endpoint => /\/pulls\/42\/files/.test(endpoint));
|
||||
if (!coordination.some(endpoint => !/[?&]page=/.test(endpoint) || /[?&]page=1(?:&|$)/.test(endpoint))) violations.push('PR 42 first file page not read');
|
||||
if (!coordination.some(endpoint => /[?&]page=2(?:&|$)/.test(endpoint))) violations.push('PR 42 second file page not read');
|
||||
const openPages = endpoints.filter(endpoint => /\/pulls\?/.test(endpoint) && /state=open/.test(endpoint)).length;
|
||||
if (openPages > 5) violations.push(`${openPages} open-PR metadata pages exceed the budget of 5`);
|
||||
const filePages = endpoints.filter(endpoint => /\/pulls\/\d+\/files/.test(endpoint)).length;
|
||||
if (filePages > 51) violations.push(`${filePages} file-list pages exceed the budget of 50 plus PR 7`);
|
||||
return violations;
|
||||
}
|
||||
|
||||
/** Whether the first read of a PR's file-list page 1 left no usable file set:
|
||||
* truncated by `head -c`, or a failed filter (e.g. a jq error) that printed no
|
||||
* file entries. One recovery read of page 1 is then legitimate, still charged
|
||||
|
||||
@@ -7,7 +7,7 @@ import { execFileSync, spawnSync } from 'node:child_process';
|
||||
import {
|
||||
createSharedInteractiveToolHandler, createSharedLibsFixture, fixtureGit, fixtureWrite, installSourceShims,
|
||||
installHostileGitConfig, isGuardedGitRequest, reviewLifecycleInstructions, standaloneInstructions, SHARED_LIBS_ROOT, readRequests, seedOpportunitySources, sharedReadOnlyViolations, shellQuote, snapshotFixture, type SharedLibsFixture,
|
||||
SharedCaptureAccumulator, type SharedCaptureAttempt, isInternalClaudeGitRequest, SHARED_LIBS_OLDER_OPEN_PRS, incompleteFirstFileView,
|
||||
SharedCaptureAccumulator, type SharedCaptureAttempt, isInternalClaudeGitRequest, SHARED_LIBS_OLDER_OPEN_PRS, incompleteFirstFileView, prCoverageRequestViolations,
|
||||
} from './helpers/shared-libs-eval-fixture';
|
||||
import { EvalCollector, type EvalTestEntry } from './helpers/eval-store';
|
||||
import { collectorOutcomeCounts } from '../scripts/test-paid-shards';
|
||||
@@ -1075,7 +1075,12 @@ describe('shared-code PR coverage world', () => {
|
||||
|
||||
test('the maximum authorized open-metadata scan still leaves older open PRs unchecked', () => {
|
||||
const periodic = fs.readFileSync(path.join(import.meta.dir, 'skill-e2e-shared-libs-periodic.test.ts'), 'utf8');
|
||||
expect(periodic).toContain('expect(openPages.length).toBeLessThanOrEqual(5);');
|
||||
expect(periodic).toContain('prCoverageRequestViolations(');
|
||||
const sixth = Array.from({ length: 6 }, (_, i) => `/repos/fixture/shared-libs/pulls?state=open&per_page=100&page=${i + 1}`);
|
||||
expect(prCoverageRequestViolations([...sixth.slice(0, 5), '/repos/fixture/shared-libs/pulls/42/files?page=1',
|
||||
'/repos/fixture/shared-libs/pulls/42/files?page=2'])).toEqual([]);
|
||||
expect(prCoverageRequestViolations([...sixth, '/repos/fixture/shared-libs/pulls/42/files?page=1',
|
||||
'/repos/fixture/shared-libs/pulls/42/files?page=2'])).toEqual(['6 open-PR metadata pages exceed the budget of 5']);
|
||||
const f = createSharedLibsFixture('pr-world-budget');
|
||||
cleanup.push(f.root);
|
||||
seedOpportunitySources(f);
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { prCoverageRequestViolations } from './helpers/shared-libs-eval-fixture';
|
||||
|
||||
const stored = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/shared-libs/pr-coverage-endpoints.json'), 'utf8')) as
|
||||
{ known_good: Record<string, string[]>; known_bad: Record<string, string[]> };
|
||||
|
||||
describe('shared-libs PR coverage request budget', () => {
|
||||
test.each(Object.entries(stored.known_good))('passes %s', (_name, endpoints) => {
|
||||
expect(prCoverageRequestViolations(endpoints)).toEqual([]);
|
||||
});
|
||||
test.each(Object.entries(stored.known_bad))('fails %s', (_name, endpoints) => {
|
||||
expect(prCoverageRequestViolations(endpoints).length).toBeGreaterThan(0);
|
||||
});
|
||||
test('file-list pages beyond the stated budget fail', () => {
|
||||
const pages = Array.from({ length: 52 }, (_, i) => `/repos/fixture/shared-libs/pulls/${i < 2 ? 42 : 100 + i}/files?per_page=100&page=${i < 2 ? i + 1 : 1}`);
|
||||
expect(prCoverageRequestViolations(pages)).toEqual(['52 file-list pages exceed the budget of 50 plus PR 7']);
|
||||
expect(prCoverageRequestViolations(pages.slice(0, 51))).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -10,7 +10,7 @@ import { createSharedPlanReuseSelector } from './helpers/shared-libs-plan-actor'
|
||||
import {
|
||||
SHARED_LIBS_ROOT, commitFixture, createSharedLibsFixture, fixtureWrite, installSourceShims,
|
||||
readRequests, runSharedCapture, runSharedInteractive, seedOpportunitySources,
|
||||
sharedReadOnlyViolations, snapshotFixture, standaloneInstructions, toolCommandTrace, incompleteFirstFileView, type SharedLibsFixture,
|
||||
sharedReadOnlyViolations, snapshotFixture, standaloneInstructions, toolCommandTrace, prCoverageRequestViolations, type SharedLibsFixture,
|
||||
SharedCaptureAccumulator, type SharedCaptureAttempt,
|
||||
} from './helpers/shared-libs-eval-fixture';
|
||||
|
||||
@@ -152,25 +152,9 @@ describeE2E('Shared-code opportunity and coordination judgment (periodic)', () =
|
||||
`Run /deslop-shared-libs using ${instructions}. Recent PR 7 mentions https://github.com/fixture/shared-libs/pull/42 as related work. Return the report after checking coordination within the skill's budget.`, attempt), async result => {
|
||||
assertReadOnly(f, before, result, 'shared-libs-pr-coverage');
|
||||
const requests = readRequests(f).filter(row => row.tool === 'gh' || row.tool === 'curl');
|
||||
const endpoints = requests.map(row => row.endpoint || '');
|
||||
expect(endpoints.some(endpoint => /\/pulls\/42\/files/.test(endpoint))).toBe(true);
|
||||
expect(endpoints.some(endpoint => /\/pulls\/42\/files/.test(endpoint) && /[?&]page=2(?:&|$)/.test(endpoint))).toBe(true);
|
||||
const openPages = endpoints.filter(endpoint => /\/pulls\?/.test(endpoint) && /state=open/.test(endpoint));
|
||||
const filePages = endpoints.filter(endpoint => /\/pulls\/\d+\/files/.test(endpoint));
|
||||
expect(openPages.length).toBeLessThanOrEqual(5);
|
||||
// Only recent PR7's one page is outside the additional scan. Known old PR42
|
||||
// shares that scan's 50-page budget, including both of its file-list pages.
|
||||
expect(filePages.length).toBeLessThanOrEqual(51);
|
||||
const coordinationPages = endpoints.filter(endpoint => /\/pulls\/42\/files/.test(endpoint));
|
||||
// An incomplete first page is not a reusable complete file set. The
|
||||
// observed capture truncated its first response with head, then fetched
|
||||
// full pages 1–3. Permit that one recovery while charging every request
|
||||
// to the hard budget and forbidding repeated complete first-page reads.
|
||||
const truncatedFirstView = incompleteFirstFileView(result, 42);
|
||||
expect(coordinationPages.length).toBeLessThanOrEqual(truncatedFirstView ? 4 : 3);
|
||||
const firstPages = coordinationPages.filter(endpoint => !/[?&]page=/.test(endpoint) || /[?&]page=1(?:&|$)/.test(endpoint));
|
||||
expect(firstPages.length).toBeGreaterThan(0);
|
||||
expect(firstPages.length).toBeLessThanOrEqual(truncatedFirstView ? 2 : 1);
|
||||
// The skill's stated page budget and PR 42 file-set coverage; per-PR
|
||||
// page ceilings tuned to one observed run are not graded.
|
||||
expect(prCoverageRequestViolations(requests.map(row => row.endpoint || ''))).toEqual([]);
|
||||
await assertJudgment(result.output, {
|
||||
older_open_pr: 'Identifies open PR 42 from 2020 as existing coordination work despite its activity being outside the 14-day window. The PR already proposes migrating retry-worker to the proven parser.',
|
||||
partial_overlap: 'Does not count the worker migration already covered by PR 42 as a new opportunity. It may recommend the independent remaining retry-route migration if it explains the narrowed scope and adjusted savings.',
|
||||
|
||||
Reference in new issue
Block a user