From c9e7a422be47b69d855f48cfce9024457e0dd4fc Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Thu, 10 Sep 2026 03:04:49 +0000 Subject: [PATCH] test: repair native workflow observation and fixture isolation --- test/autoplan-edit-header-ag.test.ts | 109 +++ test/ceo-contract-assertions-ag.test.ts | 181 +++++ test/ceo-hold-posture-ag.test.ts | 129 ++++ test/eng-regression-pinning-ag.test.ts | 49 ++ test/fixtures/autoplan-edit-header-ag.json | 727 ++++++++++++++++++ .../ceo-contract-assertions-ag-retry.json | 175 +++++ test/fixtures/ceo-contract-assertions-ag.json | 131 ++++ test/fixtures/ceo-hold-posture-ag.json | 58 ++ test/fixtures/design-plan-scope-ag.json | 126 +++ test/fixtures/eng-regression-pinning-ag.json | 382 +++++++++ test/fixtures/native-auto-decide-ag.json | 456 +++++++++++ test/fixtures/sdk-order-b-ag.json | 21 + test/helpers/autoplan-artifact-permission.ts | 33 +- test/helpers/ceo-mode-option.ts | 34 +- test/helpers/ceo-section-loading-fixture.ts | 123 +++ test/helpers/claude-pty-runner.ts | 70 +- test/helpers/eng-seeded-coverage.ts | 6 +- test/helpers/native-auto-decide.ts | 96 +++ test/helpers/plan-scope-selection.ts | 12 +- test/helpers/touchfiles-data.ts | 30 +- test/native-auto-decide-pty.test.ts | 49 ++ test/native-auto-decide.test.ts | 106 +++ test/office-posture-recording.test.ts | 110 +++ test/plan-scope-selection.test.ts | 88 ++- test/plan-tune-cathedral-fixture.test.ts | 18 +- test/sdk-order-b-ag.test.ts | 145 ++++ test/skill-e2e-office-hours.test.ts | 76 +- test/skill-e2e-plan-tune-cathedral.test.ts | 23 +- ...2e-setup-gbrain-path4-local-pglite.test.ts | 4 +- 29 files changed, 3498 insertions(+), 69 deletions(-) create mode 100644 test/autoplan-edit-header-ag.test.ts create mode 100644 test/ceo-contract-assertions-ag.test.ts create mode 100644 test/ceo-hold-posture-ag.test.ts create mode 100644 test/eng-regression-pinning-ag.test.ts create mode 100644 test/fixtures/autoplan-edit-header-ag.json create mode 100644 test/fixtures/ceo-contract-assertions-ag-retry.json create mode 100644 test/fixtures/ceo-contract-assertions-ag.json create mode 100644 test/fixtures/ceo-hold-posture-ag.json create mode 100644 test/fixtures/design-plan-scope-ag.json create mode 100644 test/fixtures/eng-regression-pinning-ag.json create mode 100644 test/fixtures/native-auto-decide-ag.json create mode 100644 test/fixtures/sdk-order-b-ag.json create mode 100644 test/helpers/native-auto-decide.ts create mode 100644 test/native-auto-decide-pty.test.ts create mode 100644 test/native-auto-decide.test.ts create mode 100644 test/office-posture-recording.test.ts create mode 100644 test/sdk-order-b-ag.test.ts diff --git a/test/autoplan-edit-header-ag.test.ts b/test/autoplan-edit-header-ag.test.ts new file mode 100644 index 000000000..1894a31d0 --- /dev/null +++ b/test/autoplan-edit-header-ag.test.ts @@ -0,0 +1,109 @@ +import { afterEach, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { autoplanArtifactPermissionInput, pendingAutoplanArtifactPermissionInput, autoplanArtifactMenuKey } from './helpers/autoplan-artifact-permission'; +import type { NativePublicToolEvent } from './helpers/plan-count-transcript'; +import { E2E_TOUCHFILES, selectTests } from './helpers/touchfiles'; +import captured from './fixtures/autoplan-edit-header-ag.json'; + +const roots: string[] = []; +afterEach(() => { for (const root of roots.splice(0)) fs.rmSync(root, {recursive:true,force:true}); }); +function replay() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autoplan-edit-header-')); roots.push(root); + const cwd = path.join(root,path.basename(captured.cwd)); + const ownedStateRoot = path.join(root,'home','.gstack'); + const file = captured.pending.file.replace(captured.ownedStateRoot,ownedStateRoot); + fs.mkdirSync(cwd,{recursive:true}); fs.mkdirSync(path.dirname(file),{recursive:true}); + fs.writeFileSync(file,captured.before); + const beforeTime = new Date(Date.parse(captured.pending.timestamp)-1000); + fs.utimesSync(file,beforeTime,beforeTime); + const publicTools = structuredClone(captured.events) as NativePublicToolEvent[]; + for (const event of publicTools) if (event.input?.file_path === captured.pending.file) event.input.file_path = file; + const pending = {...captured.pending,file,source:'pre_tool_use' as const,tool:'Edit' as const}; + const context = {cwd,ownedStateRoot,commandStartedAt:Date.parse(publicTools[0]!.timestamp)-1, + now:Date.parse(captured.viewportCapturedAt),viewportCapturedAt:Date.parse(captured.viewportCapturedAt), + transcriptStatus:'ready',publicTools,pending}; + const viewport = captured.viewport.replace(/^ (…[^\n]+)$/m,' …'+file.slice(root.length+1)); + return {root,file,context,viewport}; +} +const pick = (r:ReturnType, seen = new Set()) => + pendingAutoplanArtifactPermissionInput(r.viewport,r.context,seen); + +test('the captured native edit header preserves the current owned hook and diff', () => { + const r = replay(); + expect(r.context.publicTools).toHaveLength(88); + expect(autoplanArtifactPermissionInput(r.viewport,r.context,new Set())).toBeNull(); + expect(pick(r)).toEqual({input:'1\r',signature:captured.pending.sessionId+':'+captured.pending.toolUseId,file:r.file}); +}); + +test('exact absolute paths, full owned suffixes and launcher-owned aliases bind the same one-time request', () => { + for (const absoluteTitle of [false,true]) for (const displayedPath of ['cropped','absolute','relative']) { + const r = replay(); + if (absoluteTitle) r.viewport = r.viewport.replace(/^([●⏺] Update\()[^\n]+(?=\)$)/m,'$1'+r.file); + if (displayedPath === 'absolute') r.viewport = r.viewport.replace(/^ …[^\n]+$/m,' '+r.file); + if (displayedPath === 'relative') r.viewport = r.viewport.replace(/^ …[^\n]+$/m,' …'+path.relative(r.context.ownedStateRoot,r.file)); + const result = pick(r); expect(result?.input).toBe('1\r'); + expect(pick(r,new Set([result!.signature]))).toBeNull(); + expect(pick(r,new Set([autoplanArtifactMenuKey(r.viewport)]))).toBeNull(); + } +}); + +test('a retained header does not permit unrelated, ambiguous or quoted prefix rows', () => { + const changes = [ + (s:string) => s.replace('● Update(', '● Write('), + (s:string) => s.replace(/^● Update\([^\n]+\)/, '● Update(/tmp/foreign.md)'), + (s:string) => s.replace('~/.gstack/projects/', '~/.gstack/../projects/'), + (s:string) => s.replace(/(^ …[^\n]+)dashboard.md/m, '$1other.md'), + (s:string) => s.replace(/^ …[^\n]+$/m, ' …2026-09-10-user-dashboard.md'), + (s:string) => s.replace(/^ …[^\n]+$/m, ' …projects/sibling/ceo-plans/2026-09-10-user-dashboard.md'), + (s:string) => s.replace(' Edit file', ' Read file'), + (s:string) => s.replace(' Edit file', ' Run this first\n Edit file'), + (s:string) => s.replace(' Edit file', ' Edit file\n Edit file'), + (s:string) => 'Example:\n'+s, + (s:string) => '> '+s.replaceAll('\n','\n> '), + (s:string) => '```text\n'+s+'\n```', + (s:string) => s+'\nRun another action.', + (s:string) => s.replace(' ❯ 1. Yes',' ❯ 1. Yes, always allow'), + (s:string) => s.replace('to 2026-09-10-user-dashboard.md?','to sibling.md?'), + ]; + for (const change of changes) { const r=replay(); r.viewport=change(r.viewport); expect(pick(r),change.toString()).toBeNull(); } +}); + +test('framed edits retain stale, wrong-tool, foreign-path and success-history gates', () => { + const changes: Array<(r:ReturnType)=>void> = [ + r=>{r.context.pending.tool='Write' as 'Edit';}, + r=>{r.context.pending.sessionId='foreign';}, + r=>{r.context.pending.file=r.file+'.sibling';}, + r=>{r.context.viewportCapturedAt=Date.parse(r.context.pending.timestamp)-1;}, + r=>{r.context.pending.timestamp=new Date(r.context.now+1000).toISOString();}, + r=>{r.context.publicTools.push({kind:'result',sessionId:r.context.pending.sessionId,toolUseId:r.context.pending.toolUseId,timestamp:new Date(r.context.now).toISOString(),isError:false});}, + r=>{r.context.publicTools.push({kind:'use',sessionId:r.context.pending.sessionId,toolUseId:'unresolved-other',name:'Write',timestamp:new Date(r.context.now).toISOString(),input:{file_path:r.file}});}, + r=>{for(const event of r.context.publicTools) if(event.kind==='result') event.isError=true;}, + r=>{fs.writeFileSync(r.file,'Unrelated replacement content');}, + r=>{fs.utimesSync(r.file,new Date(r.context.now+1000),new Date(r.context.now+1000));}, + ]; + for(const change of changes) { const r=replay();change(r);expect(pick(r),change.toString()).toBeNull(); } +}); + +test('the same header works for fully published synthetic Edit inputs without replacing their comparison', () => { + const r = replay(); + const oldString = captured.before.split('\n')[0]!; + const newString = oldString+' (revised)'; + const lines = r.viewport.split('\n'); + const menu = r.viewport.slice(r.viewport.indexOf(' Do you want')); + r.viewport = lines.slice(0,6).join('\n')+'\n 1 -'+oldString+'\n 1 +'+newString+'\n────────\n'+menu; + r.context.publicTools.push({kind:'use',sessionId:r.context.pending.sessionId,toolUseId:r.context.pending.toolUseId, + name:'Edit',timestamp:r.context.pending.timestamp,input:{file_path:r.file,old_string:oldString,new_string:newString}}); + expect(pendingAutoplanArtifactPermissionInput(r.viewport,r.context,new Set())).toBeNull(); + expect(autoplanArtifactPermissionInput(r.viewport,r.context,new Set())?.input).toBe('1\r'); + r.context.publicTools.at(-1)!.input!.new_string='Different unpublished replacement'; + expect(autoplanArtifactPermissionInput(r.viewport,r.context,new Set())).toBeNull(); +}); + +test('the new native header evidence selects only the existing Autoplan paid case', () => { + for(const file of ['test/autoplan-edit-header-ag.test.ts','test/fixtures/autoplan-edit-header-ag.json']) { + expect(Object.entries(E2E_TOUCHFILES).filter(([,files])=>files.includes(file)).map(([owner])=>owner)).toEqual(['autoplan-chain-pty']); + expect(selectTests([file],E2E_TOUCHFILES,[]).selected).toEqual(['autoplan-chain-pty']); + } +}); diff --git a/test/ceo-contract-assertions-ag.test.ts b/test/ceo-contract-assertions-ag.test.ts new file mode 100644 index 000000000..0fc5185eb --- /dev/null +++ b/test/ceo-contract-assertions-ag.test.ts @@ -0,0 +1,181 @@ +import { expect, test } from 'bun:test'; +import captured from './fixtures/ceo-contract-assertions-ag.json'; +import retry from './fixtures/ceo-contract-assertions-ag-retry.json'; +import { ceoFirstReviewAUQ, ceoStep0Boundary, nativePlanCallFingerprint, planCountQuestionPhase } from './helpers/claude-pty-runner'; +import type { NativePlanQuestionCall } from './helpers/plan-count-transcript'; +import { E2E_TOUCHFILES, selectTests } from './helpers/touchfiles'; + +const calls = () => structuredClone(captured.calls) as NativePlanQuestionCall[]; +const fp = (call: NativePlanQuestionCall) => nativePlanCallFingerprint(call, 0, true); +function reanswer(call: NativePlanQuestionCall) { + call.answers = { [call.questions[0]!.question]: call.questions[0]!.options[0]!.label }; + return call; +} + +test('actual declarative assertion defects start review after routing and approach', () => { + let started = false; + const counts = { setup: 0, review: 0 }; + for (const call of calls()) { + const phase = planCountQuestionPhase(fp(call), started, ceoStep0Boundary, ceoFirstReviewAUQ); + started = phase.reviewStarted; + counts[phase.preReview ? 'setup' : 'review']++; + } + expect(counts).toEqual({ setup: 2, review: 2 }); + for (const call of calls().slice(2)) expect(ceoFirstReviewAUQ(fp(call))).toBe(true); + // Correct classification cannot retroactively complete the original paid run. + expect(captured.observedOutcome).toBe('no_review_questions'); + expect(captured.observedReviewCount).toBe(0); +}); + +test('assertion briefs still require completed native identity and their actual remedy', () => { + for (const original of calls().slice(2)) { + for (const mutate of [ + (c: NativePlanQuestionCall) => { c.answered = false; }, + (c: NativePlanQuestionCall) => { c.failed = true; }, + (c: NativePlanQuestionCall) => { c.answers = {}; }, + (c: NativePlanQuestionCall) => { c.unansweredQuestionIndices = [0]; }, + (c: NativePlanQuestionCall) => { c.questions.push(structuredClone(c.questions[0]!)); }, + (c: NativePlanQuestionCall) => { c.questions[0]!.header = 'Issue 99'; }, + (c: NativePlanQuestionCall) => { c.questions[0]!.header = 'Approach'; }, + (c: NativePlanQuestionCall) => { c.questions[0]!.question = c.questions[0]!.question.replace(/Recommendation: \d[A-Z]/, 'Recommendation: 99Z'); }, + (c: NativePlanQuestionCall) => { c.questions[0]!.options.forEach((o, i) => { o.label = `${i + 1}A) Keep`; o.description = 'Keep the saved report.'; }); }, + ]) { + const call = structuredClone(original); mutate(call); + if (call.answers && Object.keys(call.answers).length) reanswer(call); + expect(ceoFirstReviewAUQ(fp(call))).toBe(false); + } + expect(ceoFirstReviewAUQ({ ...fp(original), signature: 'foreign:call' })).toBe(false); + expect(ceoFirstReviewAUQ({ ...fp(original), options: [] })).toBe(false); + } +}); + +test('historical, hypothetical, quoted and withdrawn assertion problems are not current findings', () => { + for (const original of calls().slice(2)) { + for (const prefix of ['If ', 'Example: ', 'Whether ', 'Unless ']) { + const call = structuredClone(original); + call.questions[0]!.question = call.questions[0]!.question.replace(/(Issue \d+: )/, `$1${prefix}`); + expect(ceoFirstReviewAUQ(fp(reanswer(call)))).toBe(false); + } + for (const replacement of [ + 'test 2 can detect all retry or backoff regressions', + 'test 2 previously could not detect retry or backoff regressions', + 'test 1 does not accept any truthy value as a correct receipt', + '"test 2 cannot detect retry or backoff regressions"', + ]) { + const call = structuredClone(original); + call.questions[0]!.question = call.questions[0]!.question.replace(/(Issue \d+: )[^\n]+/, `$1${replacement}`); + expect(ceoFirstReviewAUQ(fp(reanswer(call)))).toBe(false); + } + const withdrawn = structuredClone(original); + withdrawn.questions[0]!.question = withdrawn.questions[0]!.question.replace(/(ELI10:[^\n]+)/, '$1 No current defect exists.'); + expect(ceoFirstReviewAUQ(fp(reanswer(withdrawn)))).toBe(false); + } +}); + +test('the captured assertion regression selects the existing CEO count eval', () => { + for (const file of ['test/ceo-contract-assertions-ag.test.ts', 'test/fixtures/ceo-contract-assertions-ag.json']) { + expect(selectTests([file], E2E_TOUCHFILES).selected).toContain('plan-ceo-finding-count'); + } +}); + + +test('actual retry contract wording recognizes its first repair and counts three review decisions', () => { + let started = false; + const counts = { setup: 0, review: 0 }; + for (const call of structuredClone(retry.calls) as NativePlanQuestionCall[]) { + const phase = planCountQuestionPhase(fp(call), started, ceoStep0Boundary, ceoFirstReviewAUQ); + started = phase.reviewStarted; + counts[phase.preReview ? 'setup' : 'review']++; + } + expect(counts).toEqual({ setup: 2, review: 3 }); + for (const original of retry.calls.slice(2, 4)) { + const call = structuredClone(original) as NativePlanQuestionCall; + expect(ceoFirstReviewAUQ(fp(call))).toBe(true); + call.answers = { [call.questions[0]!.question]: call.questions[0]!.options.at(-1)!.label }; + expect(ceoFirstReviewAUQ(fp(call))).toBe(true); + } + expect(retry.observedOutcome).toBe('no_review_questions'); + expect(retry.observedReviewCount).toBe(0); + expect(selectTests(['test/fixtures/ceo-contract-assertions-ag-retry.json'], E2E_TOUCHFILES).selected).toContain('plan-ceo-finding-count'); +}); + +test('already complete assertions and layout-only choices do not invent a defect', () => { + const cases = [ + [2, 'D2 — Issue 1: test 2 cannot detect retry regressions (historical assessment)', 'The assertion gap was fixed yesterday. The current test pins the retry count and delay; this choice only arranges the already complete tests.'], + [3, 'D3 — Issue 2: test 1 accepts any truthy value as specified by its success contract', 'The contract intentionally accepts every truthy success marker. The current assertion covers the contract completely; this choice only arranges the existing test.'], + ] as const; + for (const [index, title, explanation] of cases) { + const call = calls()[index]!; + const q = call.questions[0]!; + q.question = `${title}\nELI10: ${explanation}\nRecommendation: A`; + q.options = [{ label: 'A) Use a table-driven layout', description: 'Use a table-driven layout for the existing assertions.' }, { label: 'B) Keep the existing layout', description: 'Keep the existing assertions in place.' }]; + expect(ceoFirstReviewAUQ(fp(reanswer(call)))).toBe(false); + } +}); + +test('retry assertion brief keeps native identity, exact contract and repair requirements', () => { + for (const original of retry.calls.slice(2, 4)) { + for (const mutate of [ + (c: NativePlanQuestionCall) => { c.answered = false; }, + (c: NativePlanQuestionCall) => { c.failed = true; }, + (c: NativePlanQuestionCall) => { c.questions[0]!.header = 'Finding 99'; }, + (c: NativePlanQuestionCall) => { c.questions[0]!.question = 'Example: ' + c.questions[0]!.question; }, + (c: NativePlanQuestionCall) => { c.questions[0]!.question = c.questions[0]!.question.replace('but the contract is', 'but there is no contract for'); }, + (c: NativePlanQuestionCall) => { c.questions[0]!.question = c.questions[0]!.question.replace(/^ELI10:.*$/m, 'ELI10: The current assertion covers the contract completely.'); }, + (c: NativePlanQuestionCall) => { c.questions[0]!.question = c.questions[0]!.question.replace('Fix the assertion?', 'Save the report?'); }, + (c: NativePlanQuestionCall) => { c.questions[0]!.options.forEach(o => { o.label = o.label.replace(/\).*/, ') Use the existing layout'); o.description = 'Use the existing layout.'; }); }, + ]) { + const call = structuredClone(original) as NativePlanQuestionCall; + mutate(call); + expect(ceoFirstReviewAUQ(fp(reanswer(call)))).toBe(false); + } + } +}); + + +test('one offered option must repair the assertion rather than borrow report and layout actions', () => { + for (const original of [...captured.calls.slice(2), ...retry.calls.slice(2, 4)]) { + for (const administrative of ['Verify the saved report', 'Assert the full report', 'Pin the exact saved plan', 'Verify the expected layout']) { + const call = structuredClone(original) as NativePlanQuestionCall; + const q = call.questions[0]!; + const prefix = /^([1-9]\d*)?[A-Z]/.exec(q.options[0]!.label)![1] ?? ''; + q.options = [ + { label: `${prefix}A) ${administrative}`, description: administrative + '.' }, + { label: `${prefix}B) Use a table-driven layout`, description: 'Use a table-driven layout for the existing assertions.' }, + ]; + expect(ceoFirstReviewAUQ(fp(reanswer(call)))).toBe(false); + } + } +}); + + +test('administrative report qualifiers cannot strengthen the unchanged assertion clause', () => { + for (const suffix of [' and include a full report.', '; write an exact report.', '. Save the complete plan.']) { + const call = calls()[2]!; + const q = call.questions[0]!; + q.options = [ + { label: '1A) Assert the error class only', description: 'Assert the error class only' + suffix }, + { label: '1B) Keep the current test', description: 'Leave the current rejection-only assertion unchanged.' }, + ]; + expect(ceoFirstReviewAUQ(fp(reanswer(call)))).toBe(false); + } +}); + + +test('each assertion clause owns its strong qualifier and actual assertion target', () => { + for (const suffix of [' and verify the full report.', ' and check the full report.', ' with a full report.', ' with a complete saved plan.']) { + const call = calls()[2]!; + call.questions[0]!.options = [ + { label: '1A) Assert the error class only', description: 'Assert the error class only' + suffix }, + { label: '1B) Keep the current test', description: 'Leave the current rejection-only assertion unchanged.' }, + ]; + expect(ceoFirstReviewAUQ(fp(reanswer(call)))).toBe(false); + } + for (const description of ['Assert the rejection class and exactly two Stripe attempts.', 'Assert the error class only and assert exactly two Stripe attempts.']) { + const call = calls()[2]!; + call.questions[0]!.options[0]!.label = '1A) Strengthen the assertions'; + call.questions[0]!.options[0]!.description = description; + call.questions[0]!.options = [call.questions[0]!.options[0]!, { label: '1B) Keep the current test', description: 'Leave the rejection-only assertion unchanged.' }]; + expect(ceoFirstReviewAUQ(fp(reanswer(call)))).toBe(true); + } +}); diff --git a/test/ceo-hold-posture-ag.test.ts b/test/ceo-hold-posture-ag.test.ts new file mode 100644 index 000000000..92eddc63f --- /dev/null +++ b/test/ceo-hold-posture-ag.test.ts @@ -0,0 +1,129 @@ +import { expect, test } from 'bun:test'; +import { hasNativePostAnswerCeoPosture, nativeCeoModeAnswer } from './helpers/ceo-mode-option'; +import type { PlanCountTranscript } from './helpers/plan-count-transcript'; +import captured from './fixtures/ceo-hold-posture-ag.json'; +import { E2E_TOUCHFILES, selectTests } from './helpers/touchfiles'; + +const posture = /\b(rigor|bulletproof|hold\s*scope|maximum\s+rigor)\b/i; +const original = captured.transcript.assistantMessages[0]!.text; +const replay = () => structuredClone(captured.transcript) as PlanCountTranscript; +const matches = (transcript = replay()) => hasNativePostAnswerCeoPosture( + transcript, 'HOLD SCOPE', posture, captured.selectionStartedAt, +); + +test('the captured selected HOLD scope lock and hardening establish posture without a keyword', () => { + const transcript = replay(); + expect(captured.provenance.actualState).toBe('failed'); + expect(nativeCeoModeAnswer(transcript, 'HOLD SCOPE', captured.selectionStartedAt)?.toolUseId) + .toBe('toolu_011bt3yabPDSEsPNm97EhqV4'); + expect(posture.test(original)).toBe(false); + expect(matches(transcript)).toBe(true); +}); + +test('ordinary current scope declarations preserve the same three obligations', () => { + for (const text of [ + original.replace("I'm locking", 'We are keeping').replace('the four PLAN.md bullets from approach B', 'the agreed plan') + .replace('flagging anything beyond', 'treating everything outside').replace('hunting', 'checking'), + original.replace("I'm locking", 'I am holding').replace('four PLAN.md bullets from approach B', 'PLAN.md requirements') + .replace('flagging', 'marking').replace('hunting', 'looking'), + original.replace("I'm", 'I’m').replace('PLAN.md', '**PLAN.md**'), + ]) { + const transcript = replay(); transcript.assistantMessages[0]!.text = text; + expect(matches(transcript)).toBe(true); + } +}); + +test('promises, conditions and quotation cannot establish the current posture', () => { + for (const text of [ + original.replace("I'm locking", 'I will lock'), + original.replace("I'm locking", 'I would lock'), + 'If you approve, ' + original, + 'Later, ' + original, + 'Example only: ' + original, + 'An unproven hypothesis: ' + original, + 'Example only. ' + original, + '"' + original + '"', + '> ' + original, + '```text\n' + original + '\n```', + '~~~~\n' + original + '\n~~~~', + 'Read(file)\n' + original, + 'The user said: ' + original, + original.replace('and hunting', 'and not hunting'), + ]) { + const transcript = replay(); transcript.assistantMessages[0]!.text = text; + expect(matches(transcript), text).toBe(false); + } +}); + +test('all three obligations refer to the selected current scope', () => { + for (const text of [ + original.replace('PLAN.md', 'OTHER.md'), + original.replace('PLAN.md', 'archive/PLAN.md'), + original.replace('the four PLAN.md bullets from approach B', 'the future expanded plan'), + original.replace('the four PLAN.md bullets from approach B', 'the two imagined requirements'), + original.replace('out of scope', 'in scope'), + original.replace('as out of scope', 'as not out of scope'), + original.replace('flagging anything beyond that (defaults, sharing, deep links) as out of scope, and ', ''), + original.replace(/, and hunting[^.]+\./, '.'), + original.replace('constraints, error handling, UI edge cases, access-rule leaks', 'word choice and formatting'), + original + ' I am expanding scope to include a new feature.', + original + ' I am adding extra features to scope.', + ]) { + const transcript = replay(); transcript.assistantMessages[0]!.text = text; + expect(matches(transcript), text).toBe(false); + } + const ambiguous = replay(); + const question = ambiguous.calls[0]!.questions[0]!; + const oldQuestion = question.question; + question.question = question.question.replace('reviewing PLAN.md', 'reviewing PLAN.md and OTHER.md'); + ambiguous.calls[0]!.answers = { [question.question]: ambiguous.calls[0]!.answers![oldQuestion]! }; + expect(matches(ambiguous)).toBe(false); +}); + +test('explicit later corrections withdraw scope locking, while quoted examples do not', () => { + const corrections = [ + 'Correction: the previously excluded defaults, sharing, and deep links are now in scope.', + 'Correction: I am no longer locking scope to those requirements.', + 'I am not keeping scope to those requirements.', + 'The formerly excluded additions are in scope.', + ]; + for (const correction of corrections) { + const transcript = replay(); + transcript.assistantMessages[0]!.text = original + '\n\n' + correction; + expect(matches(transcript), correction).toBe(false); + for (const quote of ['> ' + correction, '```text\n' + correction + '\n```', + '~~~text\n' + correction + '\n~~~', 'An example of withdrawn wording is: "' + correction + '"']) { + transcript.assistantMessages[0]!.text = original + '\n\n' + quote; + expect(matches(transcript), quote).toBe(true); + } + } +}); + +test('only a real selected HOLD answer followed by its own public statement supplies evidence', () => { + for (const change of [ + (t: PlanCountTranscript) => { t.status = 'missing'; }, + (t: PlanCountTranscript) => { t.calls[0]!.answered = false; }, + (t: PlanCountTranscript) => { t.calls[0]!.failed = true; }, + (t: PlanCountTranscript) => { t.calls[0]!.answeredAt = new Date(captured.selectionStartedAt - 1).toISOString(); }, + (t: PlanCountTranscript) => { t.calls[0]!.answers![t.calls[0]!.questions[0]!.question] = 'Scope Expansion'; }, + (t: PlanCountTranscript) => { t.calls[0]!.answers![t.calls[0]!.questions[0]!.question] = 'Unknown'; }, + (t: PlanCountTranscript) => { t.assistantMessages[0]!.sessionId = 'foreign'; }, + (t: PlanCountTranscript) => { t.assistantMessages[0]!.timestamp = t.calls[0]!.answeredAt!; }, + (t: PlanCountTranscript) => { t.assistantMessages[0]!.timestamp = 'invalid'; }, + (t: PlanCountTranscript) => { t.assistantMessages[0]!.timestamp = new Date(Date.now() + 60_000).toISOString(); }, + (t: PlanCountTranscript) => { t.assistantMessages = []; }, + ]) { + const transcript = replay(); change(transcript); expect(matches(transcript)).toBe(false); + } + const expansion = replay(); + expansion.calls[0]!.answers![expansion.calls[0]!.questions[0]!.question] = 'Scope Expansion'; + expect(hasNativePostAnswerCeoPosture(expansion, 'SCOPE EXPANSION', posture, captured.selectionStartedAt)).toBe(false); +}); + +test('new evidence controls select only the existing mode paid owner', () => { + for (const file of ['test/ceo-hold-posture-ag.test.ts', 'test/fixtures/ceo-hold-posture-ag.json']) { + expect(Object.entries(E2E_TOUCHFILES).filter(([, files]) => files.includes(file)).map(([owner]) => owner)) + .toEqual(['plan-ceo-mode-routing']); + expect(selectTests([file], E2E_TOUCHFILES, []).selected).toEqual(['plan-ceo-mode-routing']); + } +}); diff --git a/test/eng-regression-pinning-ag.test.ts b/test/eng-regression-pinning-ag.test.ts new file mode 100644 index 000000000..fae54b86a --- /dev/null +++ b/test/eng-regression-pinning-ag.test.ts @@ -0,0 +1,49 @@ +import { expect, test } from 'bun:test'; +import fixture from './fixtures/eng-regression-pinning-ag.json'; +import { evaluateEngSeedCoverage } from './helpers/eng-seeded-coverage'; +import type { PlanCountTranscript } from './helpers/plan-count-transcript'; +import { E2E_TOUCHFILES, selectTests } from './helpers/touchfiles'; +const transcript = fixture.transcript as PlanCountTranscript; +const started = Math.min(...transcript.calls.map(c => Date.parse(c.answeredAt!))) - 1; +const finished = Date.parse(fixture.finishedAt); +const evaluate = (task: string) => evaluateEngSeedCoverage(transcript, + task + '\n\n## GSTACK REVIEW REPORT\nCoverage fixture.\n', started, finished); + +test('actual required characterization task pins existing legacy behavior before other work', () => { + expect(evaluate(fixture.requiredTask).ok).toBe(true); + expect(evaluate(fixture.requiredTask).regression).toBe('plan'); + expect(fixture.requiredTask).toContain('before any other task lands'); + expect(fixture.observedOutcome).toBe('plan_ready'); + expect(fixture.observedFailure).toBe('mandatory legacy regression coverage absent'); +}); + +test('pinning instruction must target current legacy behavior, not borrow names from notes', () => { + const task = fixture.requiredTask.split('\n')[0]!; + for (const text of [ + task.replace('legacyAuthFlow()', 'newAuthFlow()') + '; legacyAuthFlow is mentioned in notes.', + task.replace('pinning', 'describing'), + task.replace('current behavior', 'future behavior'), + task.replace('Write characterization tests', 'Write a report about characterization tests'), + task.replace('Write characterization tests', 'Do not write characterization tests'), + task.replace('Write characterization tests', 'Maybe write characterization tests'), + '> ' + task, + '\"' + task + '\"', + '```\n' + task + '\n```', + task.replace('Write characterization tests', 'If approved, write characterization tests'), + ]) expect(evaluate(text).regression, text).toBeUndefined(); + for (const tense of ['current', 'existing', 'prior']) { + expect(evaluate(task.replace('current behavior', tense + ' behavior')).regression).toBe('plan'); + } +}); + +test('regression task cannot replace absent distinct decisions or a final report', () => { + const missing = structuredClone(transcript); missing.calls = []; + expect(evaluateEngSeedCoverage(missing, fixture.requiredTask, started, finished).ok).toBe(false); + expect(evaluateEngSeedCoverage(transcript, fixture.requiredTask, started, finished).problems).toContain('final review report absent or empty'); +}); + +test('the new regression evidence selects the affected engineering count owners', () => { + for (const file of ['test/eng-regression-pinning-ag.test.ts', 'test/fixtures/eng-regression-pinning-ag.json']) { + expect(selectTests([file], E2E_TOUCHFILES, []).selected.sort()).toEqual(['plan-eng-finding-count', 'plan-eng-multi-finding-batching']); + } +}); diff --git a/test/fixtures/autoplan-edit-header-ag.json b/test/fixtures/autoplan-edit-header-ag.json new file mode 100644 index 000000000..6898ffea4 --- /dev/null +++ b/test/fixtures/autoplan-edit-header-ag.json @@ -0,0 +1,727 @@ +{ + "provenance": { + "sourceCommit": "8d8537e5d341cc9f3d186822f06efb245ec7b8fd", + "evidenceDir": ".context/ship-source-ag-delta-paid-20260910-v1/autoplan-edit-pending-0209-v1", + "observationSha256": "003ad8731d98f55ba02c894ed4d5c7a17fb0aa1f0dfd42848278e6b671905083", + "screenSha256": "bca8523bdfd3681f59cb6f0fbd011e63a68b8793ad7f9642a2664b6653784680", + "hookSha256": "60a61f738f4522833d7c362cffce4752d18ca2fa5b64e42f9b1ecb967c760186", + "projection": "Exact current pending hook, viewport and owned file bytes. Public tool history retains only IDs/names/file paths/timestamps/error flags; edit bodies omitted.", + "replayLimitation": "Free controls rebase owned filesystem paths only; any published current Edit input is explicitly synthetic because the actual input was unpublished." + }, + "cwd": "/tmp/gstack-paid-shard-zmS3XD/tmp/gstack-autoplan-chain-vsRLi1", + "ownedStateRoot": "/tmp/gstack-paid-shard-zmS3XD/tmp/gstack-hermetic-4169247-CCA4dg/skill-home-tlU9KV/.gstack", + "pending": { + "source": "pre_tool_use", + "sessionId": "49915381-304b-4c53-a36f-b421aa5f153c", + "toolUseId": "toolu_011MQGGjAj1xmUbr8Q1cUQ9T", + "tool": "Edit", + "file": "/tmp/gstack-paid-shard-zmS3XD/tmp/gstack-hermetic-4169247-CCA4dg/skill-home-tlU9KV/.gstack/projects/gstack-autoplan-chain-vsRLi1/ceo-plans/2026-09-10-user-dashboard.md", + "timestamp": "2026-09-10T02:07:26.812Z" + }, + "viewportCapturedAt": "2026-09-10T02:09:10.779Z", + "commandStartedAtMeaning": "Reconstructed 1ms before first retained public tool; every retained event included.", + "viewport": "\u25cf Update(~/.gstack/projects/gstack-autoplan-chain-vsRLi1/ceo-plans/2026-09-10-user-dashboard.md)\n \n\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n Edit file\n \u20264169247-CCA4dg/skill-home-tlU9KV/.gstack/projects/gstack-autoplan-chain-vsRLi1/ceo-plans/2026-09-10-user-dashboard.md\n\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\n 23 \n 24 | # | Proposal | Effort (human / CC) | Decision | Reasoning |\n 25 |---|----------|--------|----------|-----------|\n 26 -| E1 | Refetch on window focus/visibility, at most once per 60s, stale-while-revalidate; a 503, network or other re\n -quest-level error during a background refetch keeps the stale render and shows an error toast, never a blank page; \n -the refetch is suppressed while the mark-all-read dialog is open or its request is in flight, and the mutation's co\n -mpletion triggers the next refetch, so stale pre-mutation data can never repaint over a completed mutation | S (~2h\n - / ~10min) | ACCEPTED | In blast radius; removes the stale-dashboard-after-lunch failure | \n 26 +| E1 | Refetch on window focus/visibility, at most once per 60s, stale-while-revalidate; a 503, network or other re\n +quest-level error during a background refetch keeps the stale render and shows an error toast, never a blank page; \n +the refetch is suppressed while the mark-all-read dialog is open or its request is in flight, and the mutation's co\n +mpletion triggers the next refetch, so stale pre-mutation data can never repaint over a completed mutation. The 60s\n + throttle applies to focus/visibility triggers only, measured from completion of the most recent fetch of any kind;\n + a mutation-completion refetch bypasses the throttle and resets the window. On a background refetch, panels whose e\n +nvelope is `ok` update; panels whose envelope is `error` keep stale data and contribute to one deduplicated error t\n +oast; panel-level error states render only when there is no data to show (initial load or Retry from error) | S (~2\n +h / ~10min) | ACCEPTED | In blast radius; removes the stale-dashboard-after-lunch failure | \n 27 | E2 | Relative timestamps (\"4 min ago\") with the absolute value in `