mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-02 17:40:02 +02:00
test(evals): record the read-only and detector-row invariants as contracts
shared-libs-opportunity-judgment and review-design-lite are behavior cases: their recommendation and checklist judgments may vary, but the read-only invariant (commands, provider requests, fixture bytes, hooks, state) and the deterministic fake-engine detector rows are contracts. Both now go through expectContract, so any failure vetoes the panel.
This commit is contained in:
1 parent
b2ca207cf0
commit
2fb55e4c7a
2 files changed
+19
-10
No files matched your search
@@ -8,6 +8,7 @@ import {
|
||||
createEvalCollector, finalizeEvalCollector,
|
||||
} from './helpers/e2e-helpers';
|
||||
import { extractSkillSections, REVIEW_E2E_SECTIONS } from './helpers/skill-fixture';
|
||||
import { expectContract } from './helpers/eval-store';
|
||||
import { spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
@@ -291,7 +292,8 @@ Important: The design checklist should catch issues like blacklisted fonts, smal
|
||||
|
||||
console.log(`Design review detected ${detected}/7 planted checklist signals; detector rows surfaced: ${detectorSeen}`);
|
||||
expect(detected).toBeGreaterThanOrEqual(4); // the LLM-checklist bar, unchanged by the detector
|
||||
expect(detectorSeen).toBe(true); // the fake engine's rows are deterministic; the review must carry them
|
||||
// The fake engine's rows are deterministic; carrying them is the contract.
|
||||
expectContract(detectorSeen, 'review-design-lite: the review omitted the mechanical detector rows', { collector: evalCollector, name: '/review design lite' });
|
||||
}
|
||||
}, CAPTURE_MS + REVIEW_FINALIZE_MS);
|
||||
});
|
||||
|
||||
@@ -4,7 +4,7 @@ import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier, e2eTierEnabled } from './helpers/e2e-gate';
|
||||
import { EvalCollector } from './helpers/eval-store';
|
||||
import { EvalCollector, expectContract } from './helpers/eval-store';
|
||||
import { sharedLibsPlanExcerpt } from './helpers/shared-libs-plan-excerpt';
|
||||
import { createSharedPlanReuseSelector } from './helpers/shared-libs-plan-actor';
|
||||
import {
|
||||
@@ -51,17 +51,24 @@ async function assertJudgment(report: string, criteria: Record<string, string>)
|
||||
for (const key of Object.keys(criteria)) expect(judgment.checks[key], `${key}: ${judgment.reasoning}`).toBe(true);
|
||||
}
|
||||
|
||||
function assertReadOnly(f: SharedLibsFixture, before: Record<string, string>, result: any) {
|
||||
function assertReadOnly(f: SharedLibsFixture, before: Record<string, string>, result: any, name: string) {
|
||||
// Read-only is the contract of every audit, even where the recommendation
|
||||
// itself is a tolerated judgment call.
|
||||
const record = { collector, name };
|
||||
result.providerRequests = readRequests(f);
|
||||
expect(sharedReadOnlyViolations(result.toolCalls, result.providerRequests)).toEqual([]);
|
||||
const violations = sharedReadOnlyViolations(result.toolCalls, result.providerRequests);
|
||||
expectContract(violations.length === 0, `read-only: disallowed commands or provider requests ${JSON.stringify(violations)}`, record);
|
||||
const expected = { ...before }, after = snapshotFixture(f.root);
|
||||
// Only the source-provider instrumentation can change. Snapshot the outer
|
||||
// fixture as well as the repository; inspect commands for writes beyond it.
|
||||
delete expected[path.relative(f.root, f.trace)];
|
||||
delete after[path.relative(f.root, f.trace)];
|
||||
expect(after).toEqual(expected);
|
||||
expect(fs.existsSync(f.hookTrace) ? fs.readFileSync(f.hookTrace, 'utf8') : '').toBe('');
|
||||
expect(fs.readdirSync(f.state)).toEqual([]);
|
||||
const changed = [...new Set([...Object.keys(expected), ...Object.keys(after)])].filter(file => expected[file] !== after[file]);
|
||||
expectContract(changed.length === 0, `read-only: fixture files changed: ${changed.join(', ')}`, record);
|
||||
const hookTrace = fs.existsSync(f.hookTrace) ? fs.readFileSync(f.hookTrace, 'utf8') : '';
|
||||
expectContract(hookTrace === '', `read-only: a configured hook ran: ${hookTrace.slice(0, 500)}`, record);
|
||||
const state = fs.readdirSync(f.state);
|
||||
expectContract(state.length === 0, `read-only: gstack state written: ${state.join(', ')}`, record);
|
||||
}
|
||||
|
||||
describeE2E('Shared-code opportunity and coordination judgment (periodic)', () => {
|
||||
@@ -77,7 +84,7 @@ describeE2E('Shared-code opportunity and coordination judgment (periodic)', () =
|
||||
const before = snapshotFixture(f.root);
|
||||
await judgedCapture(attempt, 'empty', 'shared-libs-opportunity-judgment', () => runSharedCapture(f, 'shared-libs-opportunity-judgment',
|
||||
`Run /deslop-shared-libs using ${instructions} and return the report.`, attempt), async result => {
|
||||
assertReadOnly(f, before, result);
|
||||
assertReadOnly(f, before, result, 'shared-libs-opportunity-judgment');
|
||||
await assertJudgment(result.output, {
|
||||
valid_empty: 'There is only README, .gitignore and a unique one-line src/version.ts, and successful empty PR results. It reports no worthwhile sharing opportunities and does not fabricate callers or blame unavailable history/API access.',
|
||||
});
|
||||
@@ -110,7 +117,7 @@ describeE2E('Shared-code opportunity and coordination judgment (periodic)', () =
|
||||
const before = snapshotFixture(f.root);
|
||||
await judgedCapture(attempt, 'opportunity', 'shared-libs-opportunity-judgment', () => runSharedCapture(f, 'shared-libs-opportunity-judgment',
|
||||
`Run /deslop-shared-libs using ${instructions}. Review the active TypeScript and Python areas and return the requested report.`, attempt), async result => {
|
||||
assertReadOnly(f, before, result);
|
||||
assertReadOnly(f, before, result, 'shared-libs-opportunity-judgment');
|
||||
expect(result.output).toContain(f.tip.slice(0, 7));
|
||||
expect(result.output).toContain('lib/retry-after.ts');
|
||||
expect(JSON.stringify(result.transcript ?? result.toolCalls)).toMatch(/inventory\.py|search\.py|negative inventory|src\/\*\.py/);
|
||||
@@ -143,7 +150,7 @@ describeE2E('Shared-code opportunity and coordination judgment (periodic)', () =
|
||||
const before = snapshotFixture(f.root);
|
||||
await judgedCapture(attempt, 'audit', 'shared-libs-pr-coverage', () => runSharedCapture(f, 'shared-libs-pr-coverage',
|
||||
`Run /deslop-shared-libs using ${instructions}. Recent PR 7 mentions https://github.com/fixture/shared-libs/pull/42 as related work. Return the report after checking coordination within the skill's budget.`, attempt), async result => {
|
||||
assertReadOnly(f, before, result);
|
||||
assertReadOnly(f, before, result, 'shared-libs-pr-coverage');
|
||||
const requests = readRequests(f).filter(row => row.tool === 'gh' || row.tool === 'curl');
|
||||
const endpoints = requests.map(row => row.endpoint || '');
|
||||
expect(endpoints.some(endpoint => /\/pulls\/42\/files/.test(endpoint))).toBe(true);
|
||||
|
||||
Reference in new issue
Block a user