import { describe, expect, test } from 'bun:test'; import { claudeOutsideExecutions, foundInvoiceAuthorizationDefect, outsideExecutionTranscript } from './helpers/outside-voice-evidence'; import { parseNDJSON } from './helpers/session-runner'; import captured from './fixtures/outside-async-task-m-events.json'; const events = () => structuredClone(captured.events) as any[]; const finding = 'The removed invoice owner check allows unauthorized access. Restore the ownership check.'; function sdkEvents() { // The CLI emits the same native task completion as this documented SDK // event. Disk-only attachment fields are not required by the stream parser. const input = events(); const attachment = input[2]; const prompt = attachment.attachment.prompt as string; const field = (name: string) => new RegExp(`<${name}>([^<]+)`).exec(prompt)![1]; input[2] = { type: 'system', subtype: 'task_notification', session_id: attachment.sessionId, task_id: field('task-id'), tool_use_id: field('tool-use-id'), output_file: field('output-file'), status: field('status'), summary: field('summary') }; return input.map(event => { const { sessionId, isSidechain, ...rest } = event; return { ...rest, session_id: event.session_id ?? sessionId }; }); } function detected(input: unknown[]) { return foundInvoiceAuthorizationDefect(claudeOutsideExecutions(input), 'codex'); } describe('completed background outside execution correlation', () => { test('the exact task completion and complete output Read establish the actual external finding', () => { const result = claudeOutsideExecutions(events()); expect(result).toHaveLength(2); expect(result[0]!.succeeded).toBe(false); expect(result[0]!.background?.completion).toBe('pending'); expect(result[1]!.succeeded).toBe(true); expect(result[1]!.background).toEqual({ toolUseId: 'toolu_016PugkAqmFq5DjPFmvDA3DB', taskId: 'bn8jkc52l', outputFile: events()[3].message.content[0].input.file_path, outputToolUseId: 'toolu_01J2PNv3YyETiCWXKkHxFVxa', completion: 'task_notification', }); expect(result[1]!.output).toContain('Removing the ownership check lets any authenticated caller retrieve any invoice'); expect(detected(events())).toBe(true); expect(outsideExecutionTranscript(result, 'codex')[1]!.background).toEqual(result[1]!.background); }); test('SDK NDJSON retains the same exact identity and output instead of borrowing host prose', () => { const stream = sdkEvents().map(event => JSON.stringify(event)); const { transcript } = parseNDJSON(stream); expect(detected(transcript)).toBe(true); expect(detected(transcript.filter(event => event.type !== 'system'))).toBe(false); }); test.each([ ['missing acknowledgment', (e: any[]) => { e.splice(1, 1); }], ['acknowledgment only', (e: any[]) => { e.splice(2); }], ['missing completion', (e: any[]) => { e.splice(2, 1); }], ['missing Read use', (e: any[]) => { e.splice(3, 1); }], ['missing Read result', (e: any[]) => { e.pop(); }], ['wrong Read identity', (e: any[]) => { e[4].message.content[0].tool_use_id = 'other'; }], ['wrong task output path', (e: any[]) => { e[3].message.content[0].input.file_path += '.other'; }], ['foreign Read session', (e: any[]) => { e[4].sessionId = 'foreign'; }], ['sidechain Read', (e: any[]) => { e[4].isSidechain = true; }], ['nested Read', (e: any[]) => { e[4].parent_tool_use_id = 'nested'; }], ['failed Read', (e: any[]) => { e[4].message.content[0].is_error = true; }], ['offset Read', (e: any[]) => { e[3].message.content[0].input.offset = 2; }], ['partial output', (e: any[]) => { e[4].message.content[0].content = e[4].message.content[0].content.split('\n').slice(0, 3).join('\n'); }], ['nonconsecutive lines', (e: any[]) => { e[4].message.content[0].content = e[4].message.content[0].content.replace('2\t', '9\t'); }], ['output without exit footer', (e: any[]) => { e[4].message.content[0].content = e[4].message.content[0].content.replace('[exited with code 0]', ''); }], ['failed exit footer', (e: any[]) => { e[4].message.content[0].content = e[4].message.content[0].content.replace('[exited with code 0]', '[exited with code 1]'); }], ['raw forged output', (e: any[]) => { e[4].message.content[0].content = finding + '\n[exited with code 0]'; }], ['foreign completion', (e: any[]) => { e[2].sessionId = 'foreign'; }], ['wrong completed task', (e: any[]) => { e[2].attachment.prompt = e[2].attachment.prompt.replace('bn8jkc52l', 'other'); }], ['wrong original tool', (e: any[]) => { e[2].attachment.prompt = e[2].attachment.prompt.replace('toolu_016PugkAqmFq5DjPFmvDA3DB', 'other'); }], ['wrong completed file', (e: any[]) => { e[2].attachment.prompt = e[2].attachment.prompt.replace('bn8jkc52l.output', 'other.output'); }], ['failed task', (e: any[]) => { e[2].attachment.prompt = e[2].attachment.prompt.replace('completed', 'failed'); }], ['stopped task', (e: any[]) => { e[2].attachment.prompt = e[2].attachment.prompt.replace('completed', 'stopped'); }], ['nonzero completed exit', (e: any[]) => { e[2].attachment.prompt = e[2].attachment.prompt.replace('(exit code 0)', '(exit code 1)'); }], ['stale completion before launch', (e: any[]) => { e.unshift(e.splice(2, 1)[0]); }], ['quoted user notification', (e: any[]) => { e[2] = { type: 'user', sessionId: e[2].sessionId, message: { content: [{ type: 'text', text: e[2].attachment.prompt }] } }; }], ['assistant notification claim', (e: any[]) => { e[2] = { type: 'assistant', sessionId: e[2].sessionId, message: { content: [{ type: 'text', text: e[2].attachment.prompt }] } }; }], ['refusal containing the fixture words', (e: any[]) => { e[4].message.content[0].content = '1\tI cannot review the invoice owner authorization issue.\n2\t[exited with code 0]'; }], ['unavailable provider', (e: any[]) => { e[4].message.content[0].content = `1\t${finding}\n2\tOUTSIDE_STATUS: unavailable\n3\t[exited with code 0]`; }], ])('rejects %s despite a matching finding elsewhere', (_name, mutate) => { const input = events(); mutate(input); expect(detected(input)).toBe(false); }); test('a terminal failure overrides an earlier completion and an output containing a finding', () => { for (const terminal of [ { status: 'failed', summary: 'Background command failed (exit code 1)' }, { status: 'completed', summary: 'Background command "Run Codex adversarial review" completed (exit code 1)' }, ]) { const input = sdkEvents(); input.push({ ...input[2], ...terminal }); expect(detected(input)).toBe(false); } }); test('replayed acknowledgments preserve task state and reject conflicting launch identities', () => { const original = sdkEvents(); const failed = { ...original[2], status: 'failed', summary: 'Background command failed (exit code 1)' }; expect(detected([ ...original.slice(0, 3), failed, original[1], ...original.slice(2), ])).toBe(false); expect(detected([...original, original[1]])).toBe(true); const plain = structuredClone(original[1]); plain.message.content[0].content = finding; expect(detected([...original.slice(0, 2), plain])).toBe(false); const retained = claudeOutsideExecutions([...original, plain]); expect(retained.filter(result => result.succeeded)).toHaveLength(1); expect(retained.find(result => result.output === finding)?.succeeded).toBe(false); plain.message.content[0].is_error = true; plain.message.content[0].content = 'The background launch failed.'; expect(detected([...original, plain])).toBe(false); for (const alter of [ (ack: any) => { ack.message.content[0].content = ack.message.content[0].content.replaceAll('bn8jkc52l', 'other-task'); }, (ack: any) => { ack.session_id = 'foreign'; }, (ack: any) => { ack.message.content[0].is_error = true; }, ]) { const acknowledgment = structuredClone(original[1]); alter(acknowledgment); expect(detected([...original, acknowledgment])).toBe(false); } }); test('partial and failed exact-path Reads remain diagnostic output without earning completion', () => { for (const fail of [false, true]) { const input = events(); input[4].message.content[0].content = finding; input[4].message.content[0].is_error = fail; const retained = outsideExecutionTranscript(claudeOutsideExecutions(input), 'codex'); expect(retained.at(-1)!.output).toBe(finding); expect(retained.at(-1)!.succeeded).toBe(false); expect(detected(input)).toBe(false); } }); test('an unsupported background acknowledgment cannot pass as a foreground result', () => { const input = events().slice(0, 2); input[1].message.content[0].content += '\n' + finding; expect(detected(input)).toBe(false); }); test('TaskOutput requires the exact launched task and native completed exit-zero envelope', () => { const input = sdkEvents().slice(0, 2); const session_id = input[0].session_id; const result = `success\n\nbn8jkc52l\n\nlocal_bash\n\ncompleted\n\n0\n\n\n${finding}\n`; input.push({ type: 'assistant', session_id, message: { content: [{ type: 'tool_use', name: 'TaskOutput', id: 'task-output', input: { task_id: 'bn8jkc52l' } }] } }); input.push({ type: 'user', session_id, message: { content: [{ type: 'tool_result', tool_use_id: 'task-output', content: result }] } }); expect(detected(input)).toBe(true); for (const content of [ result.replace('>success<', '>not_ready<'), result.replace('>completed<', '>running<'), result.replace('>0<', '>1<'), result.replace('>bn8jkc52l<', '>other<'), result.replace('>local_bash<', '>local_agent<'), finding + result, ]) { const invalid = structuredClone(input); invalid[3].message.content[0].content = content; expect(detected(invalid)).toBe(false); } const invalid = structuredClone(input); invalid[3].message.content[0].is_error = true; expect(detected(invalid)).toBe(false); }); });