test(evals): record the read-only and detector-row invariants as contracts

shared-libs-opportunity-judgment and review-design-lite are behavior
cases: their recommendation and checklist judgments may vary, but the
read-only invariant (commands, provider requests, fixture bytes, hooks,
state) and the deterministic fake-engine detector rows are contracts.
Both now go through expectContract, so any failure vetoes the panel.
This commit is contained in:
garrytan committed 2026-09-29 19:51:35 +00:00
1 parent b2ca207cf0
commit 2fb55e4c7a
2 files changed
+19 -10

No files matched your search

+3 -1
View File
@@ -8,6 +8,7 @@ import {
createEvalCollector, finalizeEvalCollector,
} from './helpers/e2e-helpers';
import { extractSkillSections, REVIEW_E2E_SECTIONS } from './helpers/skill-fixture';
import { expectContract } from './helpers/eval-store';
import { spawnSync } from 'child_process';
import * as fs from 'fs';
import * as path from 'path';
@@ -291,7 +292,8 @@ Important: The design checklist should catch issues like blacklisted fonts, smal
console.log(`Design review detected ${detected}/7 planted checklist signals; detector rows surfaced: ${detectorSeen}`);
expect(detected).toBeGreaterThanOrEqual(4); // the LLM-checklist bar, unchanged by the detector
expect(detectorSeen).toBe(true); // the fake engine's rows are deterministic; the review must carry them
// The fake engine's rows are deterministic; carrying them is the contract.
expectContract(detectorSeen, 'review-design-lite: the review omitted the mechanical detector rows', { collector: evalCollector, name: '/review design lite' });
}
}, CAPTURE_MS + REVIEW_FINALIZE_MS);
});
+16 -9
View File
@@ -4,7 +4,7 @@ import * as fs from 'node:fs';
import * as path from 'node:path';
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier, e2eTierEnabled } from './helpers/e2e-gate';
import { EvalCollector } from './helpers/eval-store';
import { EvalCollector, expectContract } from './helpers/eval-store';
import { sharedLibsPlanExcerpt } from './helpers/shared-libs-plan-excerpt';
import { createSharedPlanReuseSelector } from './helpers/shared-libs-plan-actor';
import {
@@ -51,17 +51,24 @@ async function assertJudgment(report: string, criteria: Record<string, string>)
for (const key of Object.keys(criteria)) expect(judgment.checks[key], `${key}: ${judgment.reasoning}`).toBe(true);
}
function assertReadOnly(f: SharedLibsFixture, before: Record<string, string>, result: any) {
function assertReadOnly(f: SharedLibsFixture, before: Record<string, string>, result: any, name: string) {
// Read-only is the contract of every audit, even where the recommendation
// itself is a tolerated judgment call.
const record = { collector, name };
result.providerRequests = readRequests(f);
expect(sharedReadOnlyViolations(result.toolCalls, result.providerRequests)).toEqual([]);
const violations = sharedReadOnlyViolations(result.toolCalls, result.providerRequests);
expectContract(violations.length === 0, `read-only: disallowed commands or provider requests ${JSON.stringify(violations)}`, record);
const expected = { ...before }, after = snapshotFixture(f.root);
// Only the source-provider instrumentation can change. Snapshot the outer
// fixture as well as the repository; inspect commands for writes beyond it.
delete expected[path.relative(f.root, f.trace)];
delete after[path.relative(f.root, f.trace)];
expect(after).toEqual(expected);
expect(fs.existsSync(f.hookTrace) ? fs.readFileSync(f.hookTrace, 'utf8') : '').toBe('');
expect(fs.readdirSync(f.state)).toEqual([]);
const changed = [...new Set([...Object.keys(expected), ...Object.keys(after)])].filter(file => expected[file] !== after[file]);
expectContract(changed.length === 0, `read-only: fixture files changed: ${changed.join(', ')}`, record);
const hookTrace = fs.existsSync(f.hookTrace) ? fs.readFileSync(f.hookTrace, 'utf8') : '';
expectContract(hookTrace === '', `read-only: a configured hook ran: ${hookTrace.slice(0, 500)}`, record);
const state = fs.readdirSync(f.state);
expectContract(state.length === 0, `read-only: gstack state written: ${state.join(', ')}`, record);
}
describeE2E('Shared-code opportunity and coordination judgment (periodic)', () => {
@@ -77,7 +84,7 @@ describeE2E('Shared-code opportunity and coordination judgment (periodic)', () =
const before = snapshotFixture(f.root);
await judgedCapture(attempt, 'empty', 'shared-libs-opportunity-judgment', () => runSharedCapture(f, 'shared-libs-opportunity-judgment',
`Run /deslop-shared-libs using ${instructions} and return the report.`, attempt), async result => {
assertReadOnly(f, before, result);
assertReadOnly(f, before, result, 'shared-libs-opportunity-judgment');
await assertJudgment(result.output, {
valid_empty: 'There is only README, .gitignore and a unique one-line src/version.ts, and successful empty PR results. It reports no worthwhile sharing opportunities and does not fabricate callers or blame unavailable history/API access.',
});
@@ -110,7 +117,7 @@ describeE2E('Shared-code opportunity and coordination judgment (periodic)', () =
const before = snapshotFixture(f.root);
await judgedCapture(attempt, 'opportunity', 'shared-libs-opportunity-judgment', () => runSharedCapture(f, 'shared-libs-opportunity-judgment',
`Run /deslop-shared-libs using ${instructions}. Review the active TypeScript and Python areas and return the requested report.`, attempt), async result => {
assertReadOnly(f, before, result);
assertReadOnly(f, before, result, 'shared-libs-opportunity-judgment');
expect(result.output).toContain(f.tip.slice(0, 7));
expect(result.output).toContain('lib/retry-after.ts');
expect(JSON.stringify(result.transcript ?? result.toolCalls)).toMatch(/inventory\.py|search\.py|negative inventory|src\/\*\.py/);
@@ -143,7 +150,7 @@ describeE2E('Shared-code opportunity and coordination judgment (periodic)', () =
const before = snapshotFixture(f.root);
await judgedCapture(attempt, 'audit', 'shared-libs-pr-coverage', () => runSharedCapture(f, 'shared-libs-pr-coverage',
`Run /deslop-shared-libs using ${instructions}. Recent PR 7 mentions https://github.com/fixture/shared-libs/pull/42 as related work. Return the report after checking coordination within the skill's budget.`, attempt), async result => {
assertReadOnly(f, before, result);
assertReadOnly(f, before, result, 'shared-libs-pr-coverage');
const requests = readRequests(f).filter(row => row.tool === 'gh' || row.tool === 'curl');
const endpoints = requests.map(row => row.endpoint || '');
expect(endpoints.some(endpoint => /\/pulls\/42\/files/.test(endpoint))).toBe(true);