mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-27 15:11:47 +02:00
v1.89.0.0 feat: add shared-code extraction audit (#2925)
* feat: bind shared-code review advice to source and branch * feat: add shared-code extraction audit and scoped review checks * test: recognize complete source reads and explicit coverage legends * chore: bump version and changelog (v1.88.0.0) Co-Authored-By: OpenAI Codex <noreply@openai.com> * test: capture native review questions and retain public evidence Capture the actual first public native question with strict ownership and display matching. Preserve terminal failures and raw evidence, and retain SDK completion checks. * test: recognize verified review evidence and complete fixtures Recognize complete source and diagram evidence, concrete design and developer-experience decisions, and the complete planted scenario contracts. Preserve negative controls and grading thresholds. * fix: preserve decision brief structure in native questions Keep the required pros-and-cons heading and final Net field in native question text. Regenerate host outputs and document the release and evaluation repairs. Co-Authored-By: OpenAI Codex <noreply@openai.com> * docs: update project documentation for v1.88.0.0 Co-Authored-By: OpenAI Codex <noreply@openai.com> * fix: correct eval retry accounting and ship workflow gates * fix: capture native eval evidence and stabilize CI fixtures * fix: keep shared-code eval skips read-only Choose explicit no-change answers instead of mixed fix/preservation options. Reuse the bounded revalidation prompt for path fixtures so required review metadata is available without repeated discovery. Preserve source checks, retry limits, and failed native terminal outcomes. Add captured-question and callback regressions, plus evaluation selection coverage for the affected fixtures. --------- Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Garry Tan
co-authored by
OpenAI Codex
OpenAI Codex
parent
b9706f3635
commit
06ed920a97
@@ -28,6 +28,55 @@ const verdict = (s:ReturnType<typeof synthetic>) => coverageAuditVerdict(s.resul
|
||||
const block = (s:ReturnType<typeof synthetic>,i:number) => s.result.transcript[i].message.content[0];
|
||||
|
||||
describe('coverage audit native evidence',()=>{
|
||||
test('literal cat operands preserve quoted whitespace for single and multiple owned paths', () => {
|
||||
for (const quote of ["'", '"']) for (const flags of ['', '-n ', '-n -- ']) {
|
||||
const s = synthetic();
|
||||
s.files.cwd = '/repo with space';
|
||||
s.files.source.path = s.files.cwd + '/src/billing source.ts';
|
||||
s.files.tests.path = s.files.cwd + '/test/billing test.ts';
|
||||
s.result.transcript[0].cwd = s.files.cwd;
|
||||
for (const [i, file] of [[1, s.files.source], [3, s.files.tests]] as const) {
|
||||
Object.assign(block(s, i), { name: 'Bash', input: { command: `cat ${flags}${quote}${file.path}${quote}` } });
|
||||
}
|
||||
expect(verdict(s)).toMatchObject({ sourceRead: true, testsRead: true });
|
||||
block(s, 1).input.command = `cat ${flags}${quote}${s.files.source.path}${quote} ${quote}${s.files.tests.path}${quote}`;
|
||||
block(s, 2).content = s.files.source.content + '\n' + s.files.tests.content;
|
||||
s.result.transcript.splice(3);
|
||||
expect(verdict(s)).toMatchObject({ sourceRead: true, testsRead: true });
|
||||
for (const operands of [
|
||||
`${quote}${s.files.source.path}${quote}suffix`,
|
||||
`${quote}${s.files.source.path}${quote}${quote}${s.files.tests.path}${quote}`,
|
||||
`${quote}${s.files.source.path}`, '"$SOURCE_FILE"', '`cat path`',
|
||||
]) {
|
||||
block(s, 1).input.command = `cat ${flags}${operands}`;
|
||||
expect(verdict(s), operands).toMatchObject({ sourceRead: false, testsRead: false });
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('single word legend entries consume complete unqualified coverage and quality clauses', () => {
|
||||
const s = synthetic();
|
||||
const output = (legend: string) => '```text\nprocessPayment()\n└─ happy path [OK]\nrefundPayment()\n└─ happy path [GAP]\n' + legend + '\n```';
|
||||
for (const legend of [
|
||||
'Legend: [OK] covered\nLegend: [GAP] no test',
|
||||
'Legend: ★★★ edges + errors ★★ happy path only ★ smoke [OK] tested\nLegend: [GAP] no test [→E2E] recommend integration test',
|
||||
]) {
|
||||
s.result.output = output(legend);
|
||||
expect(verdict(s).diagram, legend).toBe(true);
|
||||
for (const qualified of [
|
||||
legend.replace('[OK] covered', '[OK] covered only if approved').replace('[OK] tested', '[OK] tested only if approved'),
|
||||
legend.replace('[GAP] no test', '[GAP] no test except refunds'),
|
||||
legend.replace('[GAP] no test', '[GAP] no test unless approved'),
|
||||
legend.replace('[GAP] no test', 'hypothetical [GAP] no test'),
|
||||
legend.replace('[OK]', 'not [OK]'),
|
||||
legend + ' unknown qualifier',
|
||||
]) {
|
||||
s.result.output = output(qualified);
|
||||
expect(verdict(s).diagram, qualified).toBe(false);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('all four exact completed public attempts delivered both files and the seeded diagram',()=>{
|
||||
expect(fixture.provenance.actualPassedCases).toBe(0);
|
||||
for(const row of fixture.rows){
|
||||
@@ -44,6 +93,110 @@ describe('coverage audit native evidence',()=>{
|
||||
expect(verdict(s)).toEqual({ sourceRead: true, testsRead: true, diagram: true, passed: true, failures: [] });
|
||||
}
|
||||
});
|
||||
const displayLegend = (legend: string, covered = '#', gap = ' ') => '```text\n' + legend + '\n'
|
||||
+ 'processPayment(amount, currency)\n└─ valid return success [' + covered + ']\n'
|
||||
+ 'refundPayment(paymentId, reason)\n└─ valid return refunded [' + gap + '] GAP\n```';
|
||||
test('paid coverage diagrams accept a branch line without an arrowhead and a declared hash checkbox', () => {
|
||||
for (const output of [
|
||||
displayLegend('Legend: [✓] tested [✗] GAP (no test) ── branch', '✓', '✗'),
|
||||
displayLegend('src/billing.ts — coverage map [#] tested [ ] GAP'),
|
||||
displayLegend('Legend: [#] tested [ ] no test'),
|
||||
displayLegend('src/billing.ts — coverage map [x] tested [ ] GAP', 'x'),
|
||||
]) {
|
||||
const s = synthetic(); s.result.output = output;
|
||||
expect(verdict(s)).toEqual({ sourceRead: true, testsRead: true, diagram: true, passed: true, failures: [] });
|
||||
}
|
||||
});
|
||||
// Exact public diagram from the completed, failed September 21 /review run.
|
||||
// Its footer declares both symbol meanings without punctuation after Legend.
|
||||
const symbolFooterDiagram = `\`\`\`
|
||||
src/billing.ts test/billing.test.ts
|
||||
======================================================================================
|
||||
|
||||
processPayment(amount, currency) describe('processPayment')
|
||||
│
|
||||
├── [✓] amount > 0 && currency in {USD, EUR} processes valid payment (L6-9)
|
||||
│ → return { status: 'success', ... } (L5) processPayment(100, 'USD')
|
||||
│
|
||||
├── [✗] amount <= 0 ── NO TEST ──
|
||||
│ → throw 'Invalid amount' (L3) gap: 0, negative values untested
|
||||
│
|
||||
└── [✗] currency not USD/EUR ── NO TEST ──
|
||||
→ throw 'Unsupported currency' (L4) gap: 'GBP', '', lowercase 'usd'
|
||||
|
||||
|
||||
refundPayment(paymentId, reason) (no describe block; not imported)
|
||||
│
|
||||
├── [✗] paymentId && reason truthy ── NO TEST ──
|
||||
│ → return { status: 'refunded', ... } (L11) gap: happy path never exercised
|
||||
│
|
||||
├── [✗] !paymentId ── NO TEST ──
|
||||
│ → throw 'Payment ID required' (L9) gap: '' / undefined untested
|
||||
│
|
||||
└── [✗] !reason ── NO TEST ──
|
||||
→ throw 'Reason required' (L10) gap: '' / undefined untested
|
||||
|
||||
======================================================================================
|
||||
Legend [✓] covered [✗] gap
|
||||
|
||||
Branches: 1 / 6 covered (17%)
|
||||
Functions: 1 / 2 covered (50%)
|
||||
Guard clauses tested: 0 / 4
|
||||
\`\`\``;
|
||||
test('the exact paid symbol footer may omit its colon', () => {
|
||||
const s = synthetic(); s.result.output = symbolFooterDiagram;
|
||||
expect(verdict(s)).toEqual({ sourceRead: true, testsRead: true, diagram: true, passed: true, failures: [] });
|
||||
});
|
||||
test('a colonless symbol footer still requires a current, affirmative, owned key', () => {
|
||||
const key = 'Legend [✓] covered [✗] gap';
|
||||
for (const replacement of ['', '> ' + key, '"' + key + '"', 'Example: ' + key,
|
||||
'If approved: ' + key, key.replace('covered [✗] gap', 'gap [✗] covered'),
|
||||
key.replace('[✗] gap', '[✗] covered'), key.replace('[✗]', '[✓]'),
|
||||
key + ' except refunds', key + '\nLegend [✓] gap [✗] covered',
|
||||
key + '\nThis legend is withdrawn.', key + '\nThis legend applies only if approved.',
|
||||
]) {
|
||||
const s = synthetic(); s.result.output = symbolFooterDiagram.replace(key, replacement);
|
||||
expect(verdict(s).diagram, replacement).toBe(false);
|
||||
}
|
||||
for (const output of [
|
||||
'\`\`\`text\n' + key + '\n\`\`\`\n' + symbolFooterDiagram.replace(key, ''),
|
||||
symbolFooterDiagram.replaceAll('refundPayment', 'otherRefund'),
|
||||
symbolFooterDiagram.replace('├── [✓] amount', '├── [✗] amount'),
|
||||
'Example:\n' + symbolFooterDiagram, '\`\`\`\`markdown\n' + symbolFooterDiagram + '\n\`\`\`\`',
|
||||
]) {
|
||||
const s = synthetic(); s.result.output = output; expect(verdict(s).diagram).toBe(false);
|
||||
}
|
||||
});
|
||||
test('hash checkbox and branch-line legends retain explicit local meanings and ownership', () => {
|
||||
const caption = 'src/billing.ts — coverage map [#] tested [ ] GAP';
|
||||
for (const legend of ['', '> ' + caption, '"' + caption + '"', 'Example: ' + caption,
|
||||
'If approved: ' + caption, caption.replace('[#] tested [ ] GAP', '[#] GAP [ ] tested'),
|
||||
caption.replace('[ ] GAP', '[ ] tested'), caption.replace('[ ] GAP', '[#] GAP'),
|
||||
caption + ' except refunds', caption + '\nLegend: [#] untested [ ] covered',
|
||||
caption + '\nLegend:[#] untested [ ] covered',
|
||||
caption + '\nsrc/billing.ts — coverage map[#] untested [ ] covered',
|
||||
caption + '\nThis legend is withdrawn.', caption + '\nThis legend applies only if approved.',
|
||||
]) {
|
||||
const s = synthetic(); s.result.output = displayLegend(legend); expect(verdict(s).diagram).toBe(false);
|
||||
}
|
||||
const valid = displayLegend(caption);
|
||||
for (const output of [
|
||||
'```text\n' + caption + '\n```\n' + displayLegend(''),
|
||||
valid.replace('processPayment', 'otherPayment'), valid.replace('refundPayment', 'otherRefund'),
|
||||
valid.replace('return success [#]', 'return success not [#]'),
|
||||
valid.replace('return success [#]', 'return success [#] -> [ ]'),
|
||||
valid.replace('return refunded [ ]', 'return refunded [ ] -> [#]'),
|
||||
valid.replace('return refunded [ ]', 'return refunded [ ] [#]'),
|
||||
valid.replace('return success [#]', 'return success ├─ [#]'),
|
||||
'````markdown\n' + valid + '\n````', 'Example:\n' + valid,
|
||||
displayLegend('Legend: [✓] tested [✗] GAP ── covered', '✓', '✗'),
|
||||
displayLegend('Legend: [✓] tested [✗] GAP ── branch except refunds', '✓', '✗'),
|
||||
]) {
|
||||
const s = synthetic(); s.result.output = output; expect(verdict(s).diagram).toBe(false);
|
||||
}
|
||||
const s = synthetic(); s.result.output = valid; s.result.transcript = [];
|
||||
expect(verdict(s).diagram).toBe(true); expect(verdict(s).passed).toBe(false);
|
||||
});
|
||||
test('CI symbol legends remain current, unambiguous and owned by their diagram', () => {
|
||||
for (const row of ciDiagrams.diagrams) {
|
||||
const text = row.text, key = text.split('\n').find(line => line.startsWith('Legend:'))!;
|
||||
@@ -95,6 +248,117 @@ describe('coverage audit native evidence',()=>{
|
||||
const s=synthetic();block(s,2).content=[{type:'text',text:fixture.files.source.split('\n').map((line,i)=>`${i+1}→${line}`).join('\n')}];
|
||||
expect(verdict(s).passed).toBe(true);
|
||||
});
|
||||
function mixedDisplay(context: boolean) {
|
||||
const s = synthetic();
|
||||
const command = context
|
||||
? 'cat review/specialists/testing.md && echo ==== SRC ==== && cat -n src/billing.ts && echo ==== TEST ==== && cat -n test/billing.test.ts && echo ==== GIT ==== && git log --oneline main..HEAD; git diff main --stat'
|
||||
: 'cat -n test/billing.test.ts && git log --oneline main..feature/billing 2>/dev/null; git diff main...feature/billing --stat 2>/dev/null';
|
||||
const numbered = (body: string) => body.replace(/\n$/, '').split('\n').map((line, index) => `${index + 1}\t${line}`).join('\n');
|
||||
if (context) s.result.transcript.splice(1, 2);
|
||||
const use = s.result.transcript.at(-2).message.content[0];
|
||||
const result = s.result.transcript.at(-1).message.content[0];
|
||||
Object.assign(use, {name: 'Bash', input: {command}});
|
||||
result.content = context
|
||||
? '# Testing Specialist Review Checklist\n\nCoverage Gaps\n==== SRC ====\n' + numbered(s.files.source.content)
|
||||
+ '\n==== TEST ====\n' + numbered(s.files.tests.content) + '\n==== GIT ===='
|
||||
: numbered(s.files.tests.content);
|
||||
return {s, use, result};
|
||||
}
|
||||
test('mixed Git display tails retain separately delivered files and numbered reads after context', () => {
|
||||
// Shell forms from the two failed 2026-09-20 paid /review captures.
|
||||
for (const context of [false, true]) expect(verdict(mixedDisplay(context).s).passed).toBe(true);
|
||||
});
|
||||
test('mixed display reads retain ordered bodies and successful parent ownership', () => {
|
||||
for (const context of [false, true]) for (const mutate of [
|
||||
(x: ReturnType<typeof mixedDisplay>) => { x.result.is_error = true; },
|
||||
(x: ReturnType<typeof mixedDisplay>) => { x.result.content = 'test/billing.test.ts was read'; },
|
||||
(x: ReturnType<typeof mixedDisplay>) => { x.result.content = x.result.content.replace(/.*import \{ describe.*\n/, ''); },
|
||||
(x: ReturnType<typeof mixedDisplay>) => { x.s.result.transcript.at(-1).session_id = 'foreign'; },
|
||||
(x: ReturnType<typeof mixedDisplay>) => { x.s.result.transcript.at(-1).parent_tool_use_id = 'child'; },
|
||||
(x: ReturnType<typeof mixedDisplay>) => { x.result.tool_use_id = 'unpaired'; },
|
||||
(x: ReturnType<typeof mixedDisplay>) => { x.s.result.transcript.push(clone(x.s.result.transcript.at(-1))); },
|
||||
]) {
|
||||
const x = mixedDisplay(context); mutate(x); expect(verdict(x.s).testsRead).toBe(false);
|
||||
}
|
||||
for (const context of [false, true]) for (const suffix of [
|
||||
'git diff main --output=src/billing.ts --stat', 'git diff main --ext-diff --stat',
|
||||
'git diff main --stat > output.txt', 'git diff main --stat || echo ok',
|
||||
]) {
|
||||
const x = mixedDisplay(context); x.use.input.command = x.use.input.command.replace(/git diff[^;]+$/, suffix);
|
||||
expect(verdict(x.s).testsRead).toBe(false);
|
||||
}
|
||||
for (const prefix of ['cat ../foreign.md', 'cat --help.md', 'cat /foreign.md', 'cat "$CONTEXT"', 'cat review/specialists/testing.md | head -2',
|
||||
'false', 'python3 -c "pass"', 'echo -e "replacement"', 'cat review/specialists/testing.md; false']) {
|
||||
const x = mixedDisplay(true); x.use.input.command = x.use.input.command.replace('cat review/specialists/testing.md', prefix);
|
||||
expect(verdict(x.s).sourceRead).toBe(false); expect(verdict(x.s).testsRead).toBe(false);
|
||||
}
|
||||
const x = mixedDisplay(true); x.result.content = x.result.content.replace('==== SRC ====', '==== OTHER ====');
|
||||
expect(verdict(x.s).sourceRead).toBe(false); expect(verdict(x.s).testsRead).toBe(false);
|
||||
const repeated = mixedDisplay(true); repeated.result.content += '\n==== SRC ====';
|
||||
expect(verdict(repeated.s).sourceRead).toBe(false); expect(verdict(repeated.s).testsRead).toBe(false);
|
||||
const missing = mixedDisplay(true); missing.result.content = missing.result.content.slice(missing.result.content.indexOf('==== SRC ===='));
|
||||
expect(verdict(missing.s).sourceRead).toBe(false); expect(verdict(missing.s).testsRead).toBe(false);
|
||||
});
|
||||
function boundDisplay(kind: 'and-log' | 'quoted-grep') {
|
||||
const s = synthetic();
|
||||
const numbered = (body: string) => body.replace(/\n$/, '').split('\n').map((line, index) => `${index + 1}\t${line}`).join('\n');
|
||||
// Exact commands from the two completed, failed 2026-09-20 bound reruns.
|
||||
const command = kind === 'and-log'
|
||||
? 'cat -n src/billing.ts && echo ==== && cat -n test/billing.test.ts && echo ==== && git log --oneline main..HEAD && git diff main --stat'
|
||||
: "grep -n -i 'diagram\\|coverage\\|tested\\|gap' review/SKILL.md | head -60; echo ======SRC; cat -n src/billing.ts; echo ======TEST; cat -n test/billing.test.ts; echo ======DIFF; git diff main...HEAD --stat";
|
||||
s.result.transcript.splice(3, 2);
|
||||
const use = block(s, 1), result = block(s, 2);
|
||||
Object.assign(use, {name: 'Bash', input: {command}});
|
||||
result.content = kind === 'and-log'
|
||||
? numbered(s.files.source.content) + '\n====\n' + numbered(s.files.tests.content) + '\n===='
|
||||
: '119: Test coverage gaps for stated requirements\n======SRC\n' + numbered(s.files.source.content)
|
||||
+ '\n======TEST\n' + numbered(s.files.tests.content) + '\n======DIFF';
|
||||
return {s, use, result};
|
||||
}
|
||||
test.each(['and-log', 'quoted-grep'] as const)('complete parent reads survive closed neighboring displays: %s', kind => {
|
||||
expect(verdict(boundDisplay(kind).s)).toEqual({sourceRead:true, testsRead:true, diagram:true, passed:true, failures:[]});
|
||||
});
|
||||
test('neighboring log and quoted grep displays cannot replace complete owned delivery', () => {
|
||||
for (const kind of ['and-log', 'quoted-grep'] as const) for (const mutate of [
|
||||
(x: ReturnType<typeof boundDisplay>) => { x.result.is_error = true; },
|
||||
(x: ReturnType<typeof boundDisplay>) => { x.result.content = 'src/billing.ts and test/billing.test.ts were read'; },
|
||||
(x: ReturnType<typeof boundDisplay>) => { x.s.result.transcript[2].session_id = 'foreign'; },
|
||||
(x: ReturnType<typeof boundDisplay>) => { x.s.result.transcript[2].parent_tool_use_id = 'child'; },
|
||||
(x: ReturnType<typeof boundDisplay>) => { x.s.result.transcript[1].parent_tool_use_id = 'child'; },
|
||||
(x: ReturnType<typeof boundDisplay>) => { x.result.tool_use_id = 'unpaired'; },
|
||||
(x: ReturnType<typeof boundDisplay>) => { x.s.result.transcript.push(clone(x.s.result.transcript[2])); },
|
||||
]) {
|
||||
const x = boundDisplay(kind); mutate(x);
|
||||
expect(verdict(x.s).sourceRead).toBe(false); expect(verdict(x.s).testsRead).toBe(false);
|
||||
}
|
||||
for (const kind of ['and-log', 'quoted-grep'] as const) for (const [key, line] of [
|
||||
['sourceRead', /.*export function processPayment.*\n/], ['testsRead', /.*import \{ describe.*\n/],
|
||||
] as const) {
|
||||
const x = boundDisplay(kind); x.result.content = x.result.content.replace(line, '');
|
||||
expect(verdict(x.s)[key]).toBe(false); expect(verdict(x.s).passed).toBe(false);
|
||||
}
|
||||
});
|
||||
test('closed neighboring log and grep grammars reject unsafe lookalikes', () => {
|
||||
for (const display of [
|
||||
'git log --oneline main..HEAD --output=src/billing.ts', 'git log --oneline main..HEAD --format=%B',
|
||||
'git log --oneline main..HEAD --ext-diff', 'git log --oneline main..HEAD > output.txt',
|
||||
'git log --oneline "main..HEAD"', 'git log --oneline main..HEAD || echo ok',
|
||||
]) {
|
||||
const x = boundDisplay('and-log'); x.use.input.command = x.use.input.command.replace('git log --oneline main..HEAD', display);
|
||||
expect(verdict(x.s).sourceRead).toBe(false); expect(verdict(x.s).testsRead).toBe(false);
|
||||
}
|
||||
for (const display of [
|
||||
'grep -n -i "$(touch sentinel)" review/SKILL.md | head -60',
|
||||
'grep -n -i "`touch sentinel`" review/SKILL.md | head -60',
|
||||
"grep -n -i 'diagram\\|coverage' --help | head -60",
|
||||
"grep -n -i 'diagram\\|coverage' review/SKILL.md > output.txt",
|
||||
"grep -n -i 'diagram\\|coverage' review/SKILL.md | python3 -c 'pass'",
|
||||
"grep -n -i 'diagram\\ncoverage' review/SKILL.md | head -60",
|
||||
]) {
|
||||
const x = boundDisplay('quoted-grep'); x.use.input.command = x.use.input.command.replace(/^[^;]+/, display);
|
||||
expect(verdict(x.s).sourceRead).toBe(false); expect(verdict(x.s).testsRead).toBe(false);
|
||||
}
|
||||
});
|
||||
test('each exact source and test file must be successfully delivered',()=>{
|
||||
for(const mutate of [
|
||||
(s:any)=>{block(s,2).content='src/billing.ts was read';},
|
||||
|
||||
Reference in New Issue
Block a user