v1.89.0.0 feat: add shared-code extraction audit (#2925)

* feat: bind shared-code review advice to source and branch * feat: add shared-code extraction audit and scoped review checks * test: recognize complete source reads and explicit coverage legends * chore: bump version and changelog (v1.88.0.0) Co-Authored-By: OpenAI Codex <noreply@openai.com> * test: capture native review questions and retain public evidence Capture the actual first public native question with strict ownership and display matching. Preserve terminal failures and raw evidence, and retain SDK completion checks. * test: recognize verified review evidence and complete fixtures Recognize complete source and diagram evidence, concrete design and developer-experience decisions, and the complete planted scenario contracts. Preserve negative controls and grading thresholds. * fix: preserve decision brief structure in native questions Keep the required pros-and-cons heading and final Net field in native question text. Regenerate host outputs and document the release and evaluation repairs. Co-Authored-By: OpenAI Codex <noreply@openai.com> * docs: update project documentation for v1.88.0.0 Co-Authored-By: OpenAI Codex <noreply@openai.com> * fix: correct eval retry accounting and ship workflow gates * fix: capture native eval evidence and stabilize CI fixtures * fix: keep shared-code eval skips read-only Choose explicit no-change answers instead of mixed fix/preservation options. Reuse the bounded revalidation prompt for path fixtures so required review metadata is available without repeated discovery. Preserve source checks, retry limits, and failed native terminal outcomes. Add captured-question and callback regressions, plus evaluation selection coverage for the affected fixtures. --------- Co-authored-by: OpenAI Codex <noreply@openai.com>
2026-09-27 23:21:53 +02:00 · 2026-09-24 01:53:58 -04:00
parent b9706f3635
commit 06ed920a97
177 changed files with 13244 additions and 2477 deletions
@@ -0,0 +1,130 @@
+/** Gate: real filesystem boundaries must invalidate reuse of a skipped extraction. */
+import { afterAll, expect, test } from 'bun:test';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
+import { describeE2ETier, e2eTierEnabled } from './helpers/e2e-gate';
+import { EvalCollector } from './helpers/eval-store';
+import {
+  fixtureWorkingTree, reviewLifecycleInstructions, reviewRevalidationPrompt, reviewRecords,
+  runSharedInteractive, toolCommandTrace, readRequests, SharedCaptureAccumulator, type SharedLibsFixture,
+} from './helpers/shared-libs-eval-fixture';
+import { preparePathEligibilityFixture, type PathEligibilityCase } from './helpers/shared-libs-path-fixture';
+
+const describeE2E = describeE2ETier('gate');
+const collector = e2eTierEnabled('gate') ? new EvalCollector('e2e') : null;
+const captures = new SharedCaptureAccumulator();
+afterAll(async () => { await captures.finalize(collector); });
+
+function sourceReadTrace(result: any): string {
+  return result.toolCalls.flatMap((call: any) => {
+    if (call.tool === 'Read') return [String(call.input?.file_path || '')];
+    const command = String(call.input?.command || '');
+    if (call.tool === 'Bash' && /\b(?:cat|sed|head|tail|nl)\b|readFile|Bun\.file/.test(command)) return [command];
+    return [];
+  }).join('\n');
+}
+
+async function exerciseEligibility(testId: string, kinds: PathEligibilityCase[]) {
+  return captures.runAttempt(testId, kinds, CAPTURE_LONG_MS, async attempt => {
+    // One wave fits the SDK's default semaphore of three. Serial outer tests keep
+    // --concurrent from queueing later waves inside another test's 600-second wall.
+    if (kinds.length > 3) throw new Error('Path eligibility groups must fit one SDK capture wave');
+    const exercise = async (kind: PathEligibilityCase) => {
+      let f: SharedLibsFixture | undefined;
+      let result: any, passed = false;
+      let scenarioError: string | undefined;
+      let captureDiagnostic: string | undefined;
+      try {
+        const prepared = preparePathEligibilityFixture(kind);
+        f = prepared.fixture;
+        const sourceBefore = new Map(prepared.current.evidence_paths.map((source: string) =>
+          [source, fs.readFileSync(path.join(prepared.fixture.repo, source), 'utf8')]));
+        const instructions = reviewLifecycleInstructions(f);
+        const supplied = path.join(f.root, 'current-advisory.jsonl');
+        fs.writeFileSync(supplied, JSON.stringify({ ...prepared.current, specialist: 'maintainability' }) + '\n');
+        const prompt = reviewRevalidationPrompt(f, instructions, supplied)
+          + '\nAll named caller sources are first-party authored runtime code. Inspect them directly, including any Git/path boundary, before deciding whether the previous review decision can be reused. The fixture contains no generated caller sources.';
+        const capture = await runSharedInteractive(f, testId, prompt, 'skip');
+        result = capture.result;
+        expect(result.exitReason, `${kind}: ${result.output}`).toBe('success');
+        expect(result.toolCalls.length).toBeGreaterThan(0);
+        expect(capture.questions.length, `${kind}: old decision must be revalidated and presented again`).toBeGreaterThan(0);
+        const trace = toolCommandTrace(result).join('\n');
+        expect(trace).toContain('gstack-review-read');
+        expect(trace).toContain('sharedLibsFingerprint');
+        expect(trace).toContain('gstack-review-log');
+        expect(trace).toContain('--start');
+        expect(trace).toContain('--finish');
+        const reads = sourceReadTrace(result);
+        expect(reads).toContain('src/retry-worker.ts');
+        expect(reads).toContain('lib/retry-after.ts');
+        for (const source of prepared.sourcePaths) {
+          expect(reads, `${kind}: reread ${source}`).toContain(source);
+        }
+        if (kind === 'symlinks') expect(trace).toMatch(/readlink|lstat|stat\b|test\s+-L|\[\s+-L|ls-files[^\n]*(?:--stage|-s\b)/);
+        if (kind === 'submodule') expect(trace + '\n' + result.output).toMatch(/submodule|160000/i);
+        if (kind === 'ignored') expect(trace + '\n' + result.output).toMatch(/check-ignore|ignored|exclude-standard/i);
+        expect(fixtureWorkingTree(f), `${kind}: Skip must not refactor any source`).toBe(prepared.beforeTree);
+        for (const [source, before] of sourceBefore) {
+          expect(fs.readFileSync(path.join(f.repo, source as string), 'utf8'), `${kind}: preserve raw ${source}`).toBe(before);
+        }
+        const rows = reviewRecords(f).filter(row => row.skill === 'review');
+        expect(rows.length).toBeGreaterThanOrEqual(2);
+        const last = rows.at(-1);
+        expect(last).toMatchObject({ status: 'clean', issues_found: 0, completed: true, converged: true });
+        expect(last.review_binding.state).toBe('verified');
+        const skipped = (last.findings || []).filter((finding: any) => finding.advisory === true && finding.action === 'skipped');
+        expect(skipped.length, `${kind}: the new explicit decision must be saved`).toBeGreaterThan(0);
+        expect(skipped.some((finding: any) => finding.helper_target?.path === 'lib/retry-after.ts'
+          && finding.helper_target?.symbol === 'retrySeconds')).toBe(true);
+        for (const finding of skipped) {
+          expect(finding.fingerprint).toMatch(/^shared-libs:[0-9a-f]{64}$/);
+          expect(finding.evidence_paths).toContain('src/retry-worker.ts');
+          expect(Array.isArray(finding.snapshot_covered_paths)).toBe(true);
+          if (kind !== 'legacy' && kind !== 'removed-filter') {
+            for (const source of prepared.sourcePaths) {
+              expect(finding.snapshot_covered_paths, `${kind}: unsafe raw source must not receive coverage proof`).not.toContain(source);
+            }
+          }
+        }
+        passed = true;
+      } catch (error) {
+        const partial = (error as any)?.sharedCapture;
+        result ??= partial?.result;
+        captureDiagnostic = partial?.diagnostic;
+        scenarioError = String(error);
+        throw error;
+      } finally {
+        try {
+          attempt.add(kind, { name: testId, suite: 'shared-libs', tier: 'e2e', passed,
+            duration_ms: result?.durationMs ?? 0, cost_usd: result?.costUsd ?? 0,
+            model: result?.model, turns_used: result?.turnsUsed ?? 0,
+            transcript: [{ scenario: kind, error: scenarioError, diagnostic: captureDiagnostic,
+              cost_known: result?.costKnown, provider_requests: f ? readRequests(f) : [] }, ...(result?.events ?? [])],
+            output: `[${kind}]${scenarioError ? ` ${scenarioError}` : ''}\n${result?.output ?? ''}`,
+            error: scenarioError,
+            exit_reason: result?.exitReason ?? 'capture_threw' });
+        } finally { if (f) fs.rmSync(f.root, { recursive: true, force: true }); }
+      }
+    };
+    const results = await Promise.allSettled(kinds.map(exercise));
+    const failures = results.flatMap((result, index) => result.status === 'rejected'
+      ? [`${kinds[index]}: ${String(result.reason)}`] : []);
+    expect(failures).toEqual([]);
+  });
+}
+
+describeE2E('Shared-code skipped advice across real source boundaries (gate)', () => {
+  test.serial('shared-libs-review-path-eligibility', () => exerciseEligibility(
+    'shared-libs-review-path-eligibility', ['symlinks', 'submodule', 'ignored'],
+  ), CAPTURE_LONG_MS);
+
+  test.serial('shared-libs-review-index-flags', () => exerciseEligibility(
+    'shared-libs-review-index-flags', ['assume-unchanged', 'skip-worktree'],
+  ), CAPTURE_LONG_MS);
+
+  test.serial('shared-libs-review-prior-coverage', () => exerciseEligibility(
+    'shared-libs-review-prior-coverage', ['legacy', 'removed-filter'],
+  ), CAPTURE_LONG_MS);
+});