/** Private evaluator inputs. Never copy this module or its JSON output into producer snapshots. */ import { createHash } from 'node:crypto'; import { CORPUS_VERSION, type EvalFamily, type EvalStack } from '../fixtures/cso-eval/materialize'; export interface OracleRequest { method: 'GET' | 'POST'; path: string; body?: string; headers?: Record } export interface OracleResponse { status: number; body: string; headers?: Record } export interface PrivateOracle { version: string; controls: Array<{ request: OracleRequest; response: (value: OracleResponse) => boolean }>; attack: OracleRequest; vulnerable: (value: OracleResponse) => boolean; fixed: (value: OracleResponse) => boolean; /** More than the exposed training assertion: alternative attack payloads. */ heldOut: Array<{ request: OracleRequest; fixed: (value: OracleResponse) => boolean }>; } const get = (params: Record): OracleRequest => ({ method: 'GET', path: `/action?${new URLSearchParams(params)}` }); const post = (value: object): OracleRequest => ({ method: 'POST', path: '/action', body: JSON.stringify(value), headers: { 'content-type': 'application/json' } }); const contains = (status: number, expected: string) => (response: OracleResponse) => response.status === status && response.body.includes(expected); const rejects = (status: number) => (response: OracleResponse) => response.status === status; const jsonValue = (value: OracleResponse): any => { try { return JSON.parse(value.body); } catch { return null; } }; export function oracleFor(family: EvalFamily): PrivateOracle { const input: Record> = { 'sql-injection': { controls: [{ request: get({ name: 'alice' }), response: contains(200, 'HELLO_ALICE') }], attack: get({ name: "nobody' OR 1=1 --" }), vulnerable: contains(200, 'SECRET_ADMIN_NOTE'), fixed: value => value.status === 200 && Array.isArray(jsonValue(value)) && jsonValue(value).length === 0, heldOut: [{ request: get({ name: "' UNION SELECT 'ORACLE_SQL_MARKER' --" }), fixed: value => value.status === 200 && !value.body.includes('ORACLE_SQL_MARKER') }], }, 'command-injection': { controls: [{ request: get({ name: 'Ada' }), response: value => value.status === 200 && jsonValue(value)?.output === 'Ada' }], attack: get({ name: '"; printf CSO_COMMAND_EXECUTED; #' }), vulnerable: value => value.status === 200 && jsonValue(value)?.output === 'CSO_COMMAND_EXECUTED', fixed: rejects(400), heldOut: [{ request: get({ name: '$(printf ORACLE_COMMAND_MARKER)' }), fixed: rejects(400) }], }, 'path-traversal': { controls: [{ request: get({ file: 'hello.txt' }), response: contains(200, 'PUBLIC_HELLO') }], attack: get({ file: '../private.txt' }), vulnerable: contains(200, 'PRIVATE_DOCUMENT_SECRET'), fixed: rejects(403), heldOut: [{ request: get({ file: 'subdir/../../private.txt' }), fixed: rejects(403) }], }, ssrf: { controls: [{ request: get({ url: 'http://127.0.0.1:8001/public' }), response: contains(200, 'PUBLIC_STATUS') }], attack: get({ url: 'http://127.0.0.1:8001/private' }), vulnerable: contains(200, 'INTERNAL_SERVICE_SECRET'), fixed: rejects(403), heldOut: [{ request: get({ url: 'http://localhost:8001/private' }), fixed: rejects(403) }], }, 'object-authorization': { controls: [{ request: get({ id: 'invoice-1' }), response: value => value.status === 200 && jsonValue(value)?.total === 12 }], attack: get({ id: 'invoice-2' }), vulnerable: value => value.status === 200 && jsonValue(value)?.total === 9800, fixed: rejects(403), heldOut: [{ request: get({ id: 'invoice-2', owner: 'member-1' }), fixed: rejects(403) }], }, 'tenant-isolation': { controls: [{ request: get({ tenant: 'tenant-a' }), response: contains(200, 'OWN_TENANT_DOCUMENT') }], attack: get({ tenant: 'tenant-b' }), vulnerable: contains(200, 'OTHER_TENANT_SECRET'), fixed: value => value.status === 200 && value.body.includes('OWN_TENANT_DOCUMENT') && !value.body.includes('OTHER_TENANT_SECRET'), heldOut: [{ request: get({ tenant: 'tenant-b', actor_tenant: 'tenant-b' }), fixed: value => value.status === 200 && !value.body.includes('OTHER_TENANT_SECRET') }], }, 'html-injection': { controls: [{ request: get({ name: 'Alice' }), response: contains(200, '
Welcome Alice
') }], attack: get({ name: '' }), vulnerable: contains(200, '' }), fixed: value => value.status === 200 && !value.body.includes(' value.status === 302 && jsonValue(value)?.next === '/account' && value.headers?.location === '/account' }], attack: get({ next: 'https://attacker.invalid/collect' }), vulnerable: value => value.status === 302 && value.headers?.location === 'https://attacker.invalid/collect', fixed: rejects(400), heldOut: [{ request: get({ next: '//attacker.invalid/collect' }), fixed: rejects(400) }, { request: get({ next: '/\\attacker.invalid/collect' }), fixed: rejects(400) }], }, 'mass-assignment': { controls: [{ request: post({ display_name: 'Ada' }), response: value => value.status === 200 && jsonValue(value)?.display_name === 'Ada' }], attack: post({ role: 'admin' }), vulnerable: value => value.status === 200 && jsonValue(value)?.role === 'admin', fixed: value => value.status === 200 && jsonValue(value)?.role === 'member', heldOut: [{ request: post({ id: 'member-2', role: 'owner', display_name: 'Grace' }), fixed: value => value.status === 200 && jsonValue(value)?.id === 'member-1' && jsonValue(value)?.role === 'member' && jsonValue(value)?.display_name === 'Grace' }], }, 'resource-exhaustion': { controls: [{ request: get({ count: '5' }), response: value => value.status === 200 && jsonValue(value)?.count === 5 }], // Bounded proof of missing admission, not an attempt to exhaust the runner. attack: get({ count: '250' }), vulnerable: value => value.status === 200 && jsonValue(value)?.count === 250, fixed: rejects(400), heldOut: [{ request: get({ count: '101' }), fixed: rejects(400) }, { request: get({ count: '0' }), fixed: rejects(400) }], }, }; return { version: CORPUS_VERSION, ...input[family] }; } export interface Observation { request: OracleRequest; response: OracleResponse } export interface PrivateEvidence { original: { booted: boolean; controls: Observation[]; attack: Observation }; patched: { booted: boolean; controls: Observation[]; attack: Observation; heldOut: Observation[]; existingTestsPassed: boolean }; immutableVerifier: boolean; independentRootCauseReview: boolean; featurePreserved: boolean; boundaryMocks: boolean; } const sameRequest = (left: OracleRequest, right: OracleRequest) => JSON.stringify(left) === JSON.stringify(right); /** Only trusted runner observations may enter this function; producer claims are not observations. */ export function judgeRepair(family: EvalFamily, evidence: PrivateEvidence): { reproduced: boolean; correctRepair: boolean; evidenceHash: string } { const oracle = oracleFor(family); const controls = (observations: Observation[]) => oracle.controls.every(control => observations.some(observation => sameRequest(control.request, observation.request) && control.response(observation.response))); const original = evidence.original, patched = evidence.patched; const reproduced = original.booted && controls(original.controls) && sameRequest(original.attack.request, oracle.attack) && oracle.vulnerable(original.attack.response); const correctRepair = reproduced && patched.booted && controls(patched.controls) && sameRequest(patched.attack.request, oracle.attack) && oracle.fixed(patched.attack.response) && oracle.heldOut.every(assertion => patched.heldOut.some(observation => sameRequest(assertion.request, observation.request) && assertion.fixed(observation.response))) && patched.existingTestsPassed && evidence.immutableVerifier && evidence.independentRootCauseReview && evidence.featurePreserved && !evidence.boundaryMocks; return { reproduced, correctRepair, evidenceHash: createHash('sha256').update(JSON.stringify({ version: oracle.version, family, evidence })).digest('hex') }; } export function runtimeStart(stack: EvalStack): { executable: string; args: string[]; port: 8000; environment: Record } { return stack === 'node' ? { executable: '/usr/local/bin/node', args: ['app.mjs'], port: 8000, environment: {} } : stack === 'bun' ? { executable: '/usr/local/bin/bun', args: ['--no-install', 'app.ts'], port: 8000, environment: {} } : stack === 'python' ? { executable: '/usr/local/bin/python', args: ['-I', 'app.py'], port: 8000, environment: {} } : { executable: '/usr/local/bin/ruby', args: ['bin/rails', 'server', '-e', 'test', '-b', '127.0.0.1', '-p', '8000'], port: 8000, environment: { RAILS_ENV: 'test', RACK_ENV: 'test' } }; }