mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-27 07:01:54 +02:00
v1.89.0.0 feat: add shared-code extraction audit (#2925)
* feat: bind shared-code review advice to source and branch * feat: add shared-code extraction audit and scoped review checks * test: recognize complete source reads and explicit coverage legends * chore: bump version and changelog (v1.88.0.0) Co-Authored-By: OpenAI Codex <noreply@openai.com> * test: capture native review questions and retain public evidence Capture the actual first public native question with strict ownership and display matching. Preserve terminal failures and raw evidence, and retain SDK completion checks. * test: recognize verified review evidence and complete fixtures Recognize complete source and diagram evidence, concrete design and developer-experience decisions, and the complete planted scenario contracts. Preserve negative controls and grading thresholds. * fix: preserve decision brief structure in native questions Keep the required pros-and-cons heading and final Net field in native question text. Regenerate host outputs and document the release and evaluation repairs. Co-Authored-By: OpenAI Codex <noreply@openai.com> * docs: update project documentation for v1.88.0.0 Co-Authored-By: OpenAI Codex <noreply@openai.com> * fix: correct eval retry accounting and ship workflow gates * fix: capture native eval evidence and stabilize CI fixtures * fix: keep shared-code eval skips read-only Choose explicit no-change answers instead of mixed fix/preservation options. Reuse the bounded revalidation prompt for path fixtures so required review metadata is available without repeated discovery. Preserve source checks, retry limits, and failed native terminal outcomes. Add captured-question and callback regressions, plus evaluation selection coverage for the affected fixtures. --------- Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Garry Tan
co-authored by
OpenAI Codex
OpenAI Codex
parent
b9706f3635
commit
06ed920a97
@@ -7,7 +7,7 @@ import {
|
||||
ROOT, runId, describeIfSelected, testConcurrentIfSelected,
|
||||
logCost, recordE2E, createEvalCollector, finalizeEvalCollector,
|
||||
} from './helpers/e2e-helpers';
|
||||
import { extractSkillSections, REVIEW_ARMY_E2E_SECTIONS } from './helpers/skill-fixture';
|
||||
import { extractSkillSections, REVIEW_ARMY_E2E_SECTIONS, sliceBetween } from './helpers/skill-fixture';
|
||||
import { spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
@@ -328,38 +328,51 @@ describeIfSelected('Review Army: Quality Score', ['review-army-quality-score'],
|
||||
repo.run('git', ['commit', '-m', 'initial']);
|
||||
|
||||
repo.run('git', ['checkout', '-b', 'feature/add-controller']);
|
||||
// Code with obvious issues for quality score computation
|
||||
// Real source for one defect and one legitimate optional simplification.
|
||||
fs.writeFileSync(path.join(dir, 'user_controller.rb'), `class UserController
|
||||
def create
|
||||
# SQL injection
|
||||
User.where("name = '#{params[:name]}'")
|
||||
# Magic number
|
||||
if users.count > 42
|
||||
raise "too many"
|
||||
end
|
||||
end
|
||||
|
||||
def lookup
|
||||
User.find(NumberParser.parse(params[:id]))
|
||||
end
|
||||
end
|
||||
`);
|
||||
fs.writeFileSync(path.join(dir, 'number_parser.rb'), `class NumberParser
|
||||
def self.parse(value)
|
||||
Integer(value)
|
||||
end
|
||||
end
|
||||
`);
|
||||
repo.run('git', ['add', '.']);
|
||||
repo.run('git', ['commit', '-m', 'add user controller']);
|
||||
|
||||
copyReviewFiles(dir);
|
||||
// Use the actual merge stage without copying its classification or score
|
||||
// logic into the prompt. The supplied malformed metadata is the regression.
|
||||
fs.writeFileSync(path.join(dir, 'review-merge.md'), sliceBetween(
|
||||
readReviewSection('review-army.md'), '### Step 4.6: Collect and merge findings', '### Red Team dispatch',
|
||||
));
|
||||
fs.writeFileSync(path.join(dir, 'specialist-findings.jsonl'), [
|
||||
{ severity: 'CRITICAL', advisory: true, confidence: 9, path: 'user_controller.rb', line: 3,
|
||||
category: 'injection', summary: 'Interpolating params[:name] into SQL allows injection.',
|
||||
fix: 'Use a parameterized query.', fingerprint: 'user_controller.rb:3:injection', specialist: 'security' },
|
||||
{ severity: 'INFORMATIONAL', advisory: true, confidence: 8, path: 'number_parser.rb', line: 1,
|
||||
category: 'stdlib-wrapper', summary: 'The one-method NumberParser wrapper only forwards to Integer.',
|
||||
fix: 'Optionally call Integer directly in lookup and remove the wrapper.',
|
||||
fingerprint: 'number_parser.rb:1:stdlib-wrapper', specialist: 'simplification', lines_removable: 5 },
|
||||
].map(finding => JSON.stringify(finding)).join('\n') + '\n');
|
||||
});
|
||||
|
||||
afterAll(() => { try { fs.rmSync(dir, { recursive: true, force: true }); } catch {} });
|
||||
|
||||
testConcurrentIfSelected('review-army-quality-score', async () => {
|
||||
const before = ['user_controller.rb', 'number_parser.rb'].map(file => fs.readFileSync(path.join(dir, file), 'utf8'));
|
||||
const result = await runSkillTest({
|
||||
prompt: `You are in a git repo with a vulnerable user controller.
|
||||
Read review-SKILL.md and review-checklist.md.
|
||||
Skip preamble, lake intro, telemetry.
|
||||
|
||||
Run the Critical pass (Step 4) against the diff (git diff main...HEAD).
|
||||
Then compute the PR Quality Score as described in the Review Army merge step:
|
||||
quality_score = max(0, 10 - (critical_count * 2 + informational_count * 0.5))
|
||||
|
||||
Write your findings AND the computed quality score to ${dir}/review-output.md
|
||||
Include the line: "PR Quality Score: X/10" where X is the computed score.`,
|
||||
prompt: `Replay the completed specialist results in specialist-findings.jsonl through the actual Collect and merge instructions in review-merge.md. Read user_controller.rb and number_parser.rb to verify these findings against the source.
|
||||
This capture covers only the merge, classification, and scoring stage: do not dispatch additional reviewers, discover unrelated findings, or enter Fix-First. Do not edit application source.
|
||||
Write the standard merged findings report to ${dir}/review-output.md.
|
||||
Also write ${dir}/merged-review.json as one JSON object with findings (all final merged finding records, including optional advice), critical_count, informational_count, issues_found (defect count), and quality_score. Preserve each finding's final severity, category, and advisory classification in that artifact.`,
|
||||
workingDirectory: dir,
|
||||
maxTurns: 15,
|
||||
timeout: JUDGE_MS,
|
||||
@@ -368,17 +381,34 @@ Include the line: "PR Quality Score: X/10" where X is the computed score.`,
|
||||
});
|
||||
|
||||
logCost('/review army quality', result);
|
||||
recordE2E(evalCollector, '/review army quality score', 'Review Army', result);
|
||||
expect(result.exitReason).toBe('success');
|
||||
|
||||
const outputPath = path.join(dir, 'review-output.md');
|
||||
if (fs.existsSync(outputPath)) {
|
||||
let passed = false, failure: unknown;
|
||||
try {
|
||||
expect(result.exitReason).toBe('success');
|
||||
expect(result.toolCalls.length).toBeGreaterThan(0);
|
||||
const outputPath = path.join(dir, 'review-output.md');
|
||||
expect(fs.existsSync(outputPath)).toBe(true);
|
||||
const content = fs.readFileSync(outputPath, 'utf-8');
|
||||
// Should contain a quality score
|
||||
const hasScore =
|
||||
content.toLowerCase().includes('quality score') ||
|
||||
content.match(/\d+\/10/);
|
||||
expect(hasScore).toBeTruthy();
|
||||
const merged = JSON.parse(fs.readFileSync(path.join(dir, 'merged-review.json'), 'utf8'));
|
||||
expect(merged).toMatchObject({ critical_count: 1, informational_count: 0, issues_found: 1, quality_score: 8 });
|
||||
expect(merged.findings).toHaveLength(2);
|
||||
const defect = merged.findings.find((finding: any) => finding.category === 'injection');
|
||||
expect(defect).toMatchObject({ severity: 'CRITICAL', specialist: 'security' });
|
||||
expect(defect.advisory).not.toBe(true);
|
||||
const advice = merged.findings.find((finding: any) => finding.category === 'stdlib-wrapper');
|
||||
expect(advice).toMatchObject({ severity: 'INFORMATIONAL', advisory: true, specialist: 'simplification' });
|
||||
expect(content).toMatch(/SPECIALIST REVIEW:\s*1 findings?\s*\(1 critical, 0 informational\)/i);
|
||||
expect(content).toMatch(/PR Quality Score:\s*8(?:\.0)?\/10/i);
|
||||
expect(content).toContain('[ADVISORY]');
|
||||
expect(['user_controller.rb', 'number_parser.rb'].map(file => fs.readFileSync(path.join(dir, file), 'utf8'))).toEqual(before);
|
||||
passed = true;
|
||||
} catch (cause) {
|
||||
failure = cause;
|
||||
throw cause;
|
||||
} finally {
|
||||
recordE2E(evalCollector, 'review-army-quality-score', 'Review Army', result, {
|
||||
passed, error: failure ? String(failure) : undefined,
|
||||
exit_reason: passed ? 'success' : result.exitReason === 'success' ? 'assertion_failed' : result.exitReason,
|
||||
});
|
||||
}
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user