From fe87890edb47b5e57fa89cf2cacaa857c05f2245 Mon Sep 17 00:00:00 2001 From: garrytan Date: Mon, 5 Oct 2026 01:31:51 +0000 Subject: [PATCH] fix(cso): opaque producer inputs and receipts; requalification triggers independent of cwd Producer inputs are {schemaVersion: 2, cellRef, skill, source, execution} and receipts bind cellRef + inputHash; collection rebuilds the cell from the matrix, so a cell VM never learns its case or variant. requalificationTriggers resolves Bun's cwd-relative metafile inputs against the repository root. --- scripts/cso-eval-producer.ts | 61 ++++++++++++++++++-------------- scripts/cso-eval-protocol.ts | 39 +++++++++++++++++--- scripts/cso-eval.ts | 29 ++++++++------- scripts/cso-requalification.ts | 3 +- test/cso-eval.test.ts | 31 ++++++++++++++-- test/cso-requalification.test.ts | 13 ++++++- test/fixtures/cso-eval/README.md | 8 +++-- 7 files changed, 133 insertions(+), 51 deletions(-) diff --git a/scripts/cso-eval-producer.ts b/scripts/cso-eval-producer.ts index d16c68dc2..32ea5592e 100644 --- a/scripts/cso-eval-producer.ts +++ b/scripts/cso-eval-producer.ts @@ -25,7 +25,7 @@ import { producerProviderIdentityHash, producerReceiptHash, sha256, - type ProducerCell, + type ProducerExecution, type ProducerHost, type ProducerInput, type ProducerArtifactInventory, @@ -199,9 +199,18 @@ function repositoryIdentity(root: string): string { return sha256(JSON.stringify({ config: sha256(config), head: read(['rev-parse', '--verify', 'HEAD']).trim(), branch: read(['symbolic-ref', '--short', 'HEAD']).trim(), status: read(['status', '--porcelain=v2', '--untracked-files=all']) })); } -function validateCell(cell: ProducerCell): void { - if (!cell || !HEX.test(cell.id) || !cell.caseId || !['node', 'bun', 'python', 'rails'].includes(cell.stack) || !['vulnerable', 'fixed'].includes(cell.variant) || !['v2', 'v3'].includes(cell.version) || !['daily', 'comprehensive'].includes(cell.mode) || ![1, 2, 3].includes(cell.repetition) || !cell.model || !['claude', 'codex', 'gemini'].includes(cell.host) || !Number.isInteger(cell.budgetSeconds) || cell.budgetSeconds <= 60 || cell.budgetSeconds > 3600 || !HEX.test(cell.sourceHash) || !HEX.test(cell.skillHash) || (cell.platform !== undefined && !PRODUCER_PLATFORMS.includes(cell.platform))) throw new Error('INVALID_PRODUCER_INPUT'); - if (cell.platform !== undefined && cell.platform !== producerHostPlatform()) throw new Error('UNMATCHED_PRODUCER_PLATFORM'); +const INPUT_FIELDS = 'cellRef,execution,schemaVersion,skill,source'; +const EXECUTION_FIELDS = 'budgetSeconds,host,mode,model,skillHash,sourceHash'; +/** The input has exactly the opaque fields: no case, variant, version, or repetition can ride along. */ +function validateInput(input: ProducerInput): void { + const execution = input?.execution; + if (!input || typeof input !== 'object' || Object.keys(input).sort().join(',') !== INPUT_FIELDS || input.schemaVersion !== 2 || !HEX.test(input.cellRef) || + typeof input.skill !== 'string' || Buffer.byteLength(input.skill) > INPUT_LIMIT || !Array.isArray(input.source) || + !execution || typeof execution !== 'object' || Object.keys(execution).filter(key => key !== 'platform').sort().join(',') !== EXECUTION_FIELDS || + !['daily', 'comprehensive'].includes(execution.mode) || typeof execution.model !== 'string' || !execution.model || !['claude', 'codex', 'gemini'].includes(execution.host) || + !Number.isInteger(execution.budgetSeconds) || execution.budgetSeconds <= 60 || execution.budgetSeconds > 3600 || !HEX.test(execution.sourceHash) || !HEX.test(execution.skillHash) || + ('platform' in execution && !PRODUCER_PLATFORMS.includes(execution.platform!))) throw new Error('INVALID_PRODUCER_INPUT'); + if (execution.platform !== undefined && execution.platform !== producerHostPlatform()) throw new Error('UNMATCHED_PRODUCER_PLATFORM'); } export interface ProducerHelperBinding { @@ -420,7 +429,7 @@ export function resolveProducerProviderIdentity(host: ProducerHost, stateRoot: s }; } -function promptFor(cell: ProducerCell, skill: string, sourceRoot: string, helper: ProducerHelperBinding): string { +function promptFor(cell: ProducerExecution, skill: string, sourceRoot: string, helper: ProducerHelperBinding): string { const invocation = `/cso${cell.mode === 'comprehensive' ? ' --comprehensive' : ''} --budget ${cell.budgetSeconds}`; const sourceAccess = cell.host === 'codex' ? 'The Codex permission profile grants read-only access to exactly that immutable snapshot so the trusted helper can read it. Use only the trusted helper to inspect or act on source; direct provider file or shell access is outside this evaluation policy.' @@ -487,20 +496,20 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op const bytes = readBoundedStable(controlPath, INPUT_LIMIT, 'Producer input'); let input: ProducerInput; try { input = JSON.parse(bytes.toString('utf8')); } catch { throw new Error('INVALID_PRODUCER_INPUT'); } - if (input.schemaVersion !== 1 || typeof input.skill !== 'string' || Buffer.byteLength(input.skill) > INPUT_LIMIT || !Array.isArray(input.source)) throw new Error('INVALID_PRODUCER_INPUT'); - validateCell(input.cell); - if (basename(receipt) !== `${input.cell.id}.json`) throw new Error('UNMATCHED_RECEIPT_DESTINATION'); - if (sha256(input.skill) !== input.cell.skillHash) throw new Error('INVALID_PRODUCER_SKILL'); - validateSource(sourceRoot, input.source, input.cell.sourceHash); + validateInput(input); + const cell = input.execution; + if (basename(receipt) !== `${input.cellRef}.json`) throw new Error('UNMATCHED_RECEIPT_DESTINATION'); + if (sha256(input.skill) !== cell.skillHash) throw new Error('INVALID_PRODUCER_SKILL'); + validateSource(sourceRoot, input.source, cell.sourceHash); sealProducerSource(sourceRoot); - validateSource(sourceRoot, input.source, input.cell.sourceHash); + validateSource(sourceRoot, input.source, cell.sourceHash); const originalRepositoryIdentity = repositoryIdentity(sourceRoot); const stateRoot = join(jobRoot, 'state'); const helperHome = join(stateRoot, 'cso-home'); const helper = resolveProducerHelperBinding(sourceRoot, stateRoot, options.testHelperLauncherPath); const installationIdentity = producerInstallationIdentity(helper); - const adapter = options.adapter ?? adapterFor(input.cell.host); - if ((input.cell.host === 'codex' ? 'gpt' : input.cell.host) !== adapter.family) throw new Error('UNMATCHED_PRODUCER_ADAPTER'); + const adapter = options.adapter ?? adapterFor(cell.host); + if ((cell.host === 'codex' ? 'gpt' : cell.host) !== adapter.family) throw new Error('UNMATCHED_PRODUCER_ADAPTER'); fs.mkdirSync(stateRoot, { recursive: false, mode: 0o700 }); fs.mkdirSync(helperHome, { recursive: false, mode: 0o700 }); const provider = options.testProviderIdentity @@ -508,13 +517,13 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op identity: validateProviderIdentity(options.testProviderIdentity, adapter.family), command: options.testProviderCommand ?? { executable: process.execPath, argsPrefix: [] }, } - : resolveProducerProviderIdentity(input.cell.host, stateRoot); + : resolveProducerProviderIdentity(cell.host, stateRoot); const providerIdentity = provider.identity; const runOptions = { - prompt: promptFor(input.cell, input.skill, sourceRoot, helper), + prompt: promptFor(cell, input.skill, sourceRoot, helper), workdir: stateRoot, - timeoutMs: input.cell.budgetSeconds * 1000, - model: input.cell.model, + timeoutMs: cell.budgetSeconds * 1000, + model: cell.model, csoProducer: { stateDirectory: stateRoot, sourceDirectory: sourceRoot, @@ -524,7 +533,7 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op }, } satisfies RunOpts; const availability = await adapter.available(runOptions); - if (!availability.ok) throw new Error(`PRODUCER_UNAVAILABLE: ${availability.reason ?? input.cell.host}`); + if (!availability.ok) throw new Error(`PRODUCER_UNAVAILABLE: ${availability.reason ?? cell.host}`); const inputHash = producerInputHash(input); // Load the opaque metadata into memory, then remove it before starting the @@ -539,11 +548,11 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op const startedAt = new Date().toISOString(); let run: RunResult; try { - if (input.cell.host === 'gemini') prepareGeminiProducerState(stateRoot, helper.launcher); + if (cell.host === 'gemini') prepareGeminiProducerState(stateRoot, helper.launcher); run = await adapter.run(runOptions); } finally { try { - if (input.cell.host === 'gemini') removeGeminiProducerState(stateRoot); + if (cell.host === 'gemini') removeGeminiProducerState(stateRoot); } finally { if (previousHome === undefined) delete process.env.GSTACK_HOME; else process.env.GSTACK_HOME = previousHome; if (previousSessionKind === undefined) delete process.env.GSTACK_SESSION_KIND; else process.env.GSTACK_SESSION_KIND = previousSessionKind; @@ -560,7 +569,7 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op throw new Error('PRODUCER_PROVIDER_INSTALLATION_RACE'); } assertProducerSourceSealed(sourceRoot); - validateSource(sourceRoot, input.source, input.cell.sourceHash); + validateSource(sourceRoot, input.source, cell.sourceHash); if (repositoryIdentity(sourceRoot) !== originalRepositoryIdentity) throw new Error('PRODUCER_CHANGED_SOURCE'); const artifacts = inventoryProducerArtifacts(helperHome); run = sanitizeProducerRun(run); @@ -569,8 +578,8 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op const tokensReported = run.tokens.input > 0 || run.tokens.output > 0 || (run.tokens.cached ?? 0) > 0; const estimatedCostUSD = tokensReported && PRICING[run.modelUsed] ? adapter.estimateCost(run.tokens, run.modelUsed) : null; const withoutHash: Omit = { - schemaVersion: 1, - cell: input.cell, + schemaVersion: 2, + cellRef: input.cellRef, inputHash, installationIdentity, providerIdentity, @@ -578,9 +587,9 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op startedAt, finishedAt, status: run.error ? 'failed' : 'succeeded', - requestedModel: input.cell.model, + requestedModel: cell.model, modelUsed: run.modelUsed, - modelIdentitySource: run.modelUsed === input.cell.model ? 'requested_pin' : 'provider_reported', + modelIdentitySource: run.modelUsed === cell.model ? 'requested_pin' : 'provider_reported', durationMs: run.durationMs, firstUsefulResultMs: null, toolCalls: run.toolCalls, @@ -603,7 +612,7 @@ async function cli(args: string[]): Promise { if (args.length !== 4 || args[0] !== 'run' || args[3] !== '--execute-paid') throw new Error('Usage: cso-eval-producer run --execute-paid'); if (process.env.CSO_EVAL_PAID !== '1') throw new Error('PAID_EXECUTION_NOT_AUTHORIZED: also set CSO_EVAL_PAID=1 on the isolated producer host'); const receipt = await runProducerCell(args[1], args[2], { paidExecutionAuthorized: true }); - console.log(JSON.stringify({ cellId: receipt.cell.id, status: receipt.status, durationMs: receipt.durationMs, receiptHash: receipt.receiptHash })); + console.log(JSON.stringify({ cellRef: receipt.cellRef, status: receipt.status, durationMs: receipt.durationMs, receiptHash: receipt.receiptHash })); } if (import.meta.main) { diff --git a/scripts/cso-eval-protocol.ts b/scripts/cso-eval-protocol.ts index 827db2e75..88174ecff 100644 --- a/scripts/cso-eval-protocol.ts +++ b/scripts/cso-eval-protocol.ts @@ -31,16 +31,40 @@ export interface ProducerSourceEntry { bytes: number; } +/** + * Only what a producer needs to run one cell. It never names the corpus case, + * variant, version, or repetition: an agent that can read its own control file + * or receipt still cannot tell which answer it is being graded against. + */ +export interface ProducerExecution { + mode: ProducerMode; + model: string; + host: ProducerHost; + budgetSeconds: number; + sourceHash: string; + skillHash: string; + platform?: ProducerPlatform; +} + /** * This control file is consumed before a producing agent starts. It must never * be placed inside the application repository or retained on the producer. */ export interface ProducerInput { - schemaVersion: 1; - cell: ProducerCell; + schemaVersion: 2; + /** The opaque matrix cell id; only the trusted evaluator maps it back to a cell. */ + cellRef: string; /** Exact canonical portable payload: root SKILL.md plus its manifest-listed sections. */ skill: string; source: ProducerSourceEntry[]; + execution: ProducerExecution; +} + +export function producerExecution(cell: ProducerCell): ProducerExecution { + return { + mode: cell.mode, model: cell.model, host: cell.host, budgetSeconds: cell.budgetSeconds, + sourceHash: cell.sourceHash, skillHash: cell.skillHash, ...(cell.platform === undefined ? {} : { platform: cell.platform }), + }; } export interface ProducerArtifactIdentity { @@ -89,8 +113,9 @@ export interface ProducerArtifactInventory { } export interface ProducerReceipt { - schemaVersion: 1; - cell: ProducerCell; + schemaVersion: 2; + /** The opaque cell id from the consumed input; bound together with inputHash. */ + cellRef: string; inputHash: string; installationIdentity: ProducerInstallationIdentity; providerIdentity: ProducerProviderIdentity; @@ -119,8 +144,12 @@ export interface ProducerReceipt { receiptHash: string; } -/** Compact trusted index; raw output remains in the separately retained receipt. */ +/** + * Compact trusted index; raw output remains in the separately retained receipt. + * Collection adds the matrix cell that the receipt's cellRef names. + */ export type ProducerReceiptIndex = Omit & { + cell: ProducerCell; error?: { code: string }; }; diff --git a/scripts/cso-eval.ts b/scripts/cso-eval.ts index d5aa1516a..69e475a61 100644 --- a/scripts/cso-eval.ts +++ b/scripts/cso-eval.ts @@ -11,6 +11,7 @@ import { loadCorpusManifest, materializeCase, publicCorpus, STACKS, validateCorp import { PRODUCER_PLATFORMS, producerArtifactInventoryHash, + producerExecution, producerInputHash, producerInstallationIdentityHash, producerProviderIdentityHash, @@ -358,7 +359,7 @@ export function prepareEvalJobs(matrix: EvalMatrix, payloads: Partial receipt.usage[field as keyof typeof receipt.usage] === null || validNumber(receipt.usage[field as keyof typeof receipt.usage])) || ((receipt.status === 'failed') !== !!receipt.error) || (receipt.error && !receipt.error.code)) throw new Error('INVALID_PRODUCER_BATCH'); validateInstallationIdentity(receipt.installationIdentity, 'INVALID_PRODUCER_BATCH'); @@ -473,24 +474,26 @@ export function collectProducerReceipts(matrix: EvalMatrix, schedule: PreparedEv const jobs = new Map(schedule.jobs.map(job => [job.cellId, job])); const seen = new Set(); for (const receipt of receipts) { - const cell = cells.get(receipt?.cell?.id), job = jobs.get(receipt?.cell?.id); + const cell = cells.get(receipt?.cellRef), job = jobs.get(receipt?.cellRef); if (!cell || !job || seen.has(cell.id)) throw new Error('UNKNOWN_OR_DUPLICATE_PRODUCER_RECEIPT'); validateReceipt(receipt, cell, job.inputHash); seen.add(cell.id); } - assertPlatformIdentities(receipts); - const pairs = new Map>>(); - if (profileOf(matrix) === 'full') for (const receipt of receipts) { + // Receipts name only the opaque cellRef; the trusted matrix supplies the cell. + const bound = receipts.map(receipt => ({ ...receipt, cell: cells.get(receipt.cellRef)! })); + assertPlatformIdentities(bound); + const pairs = new Map>>(); + if (profileOf(matrix) === 'full') for (const receipt of bound) { const key = matchedPairKey(receipt.cell as EvalCell); const pair = pairs.get(key) ?? {}; pair[receipt.cell.version] = receipt; pairs.set(key, pair); } - else if (new Set(receipts.map(receipt => receipt.modelUsed)).size > 1) throw new Error('UNMATCHED_EFFECTIVE_MODELS: single-version cells used different normalized model identities'); - const completePairs = [...pairs.entries()].filter(([, pair]) => pair.v2 && pair.v3) as Array<[string, { v2: ProducerReceipt; v3: ProducerReceipt }]>; + else if (new Set(bound.map(receipt => receipt.modelUsed)).size > 1) throw new Error('UNMATCHED_EFFECTIVE_MODELS: single-version cells used different normalized model identities'); + const completePairs = [...pairs.entries()].filter(([, pair]) => pair.v2 && pair.v3) as Array<[string, { v2: (typeof bound)[number]; v3: (typeof bound)[number] }]>; const modelMismatches = completePairs.filter(([, pair]) => pair.v2.modelUsed !== pair.v3.modelUsed).map(([pair, value]) => ({ pair: digest(pair), v2: value.v2.modelUsed, v3: value.v3.modelUsed })); if (modelMismatches.length) throw new Error(`UNMATCHED_EFFECTIVE_MODELS: ${modelMismatches.length} matched v2/v3 pair(s) used different normalized model identities`); const groups: ProducerGroupSummary[] = []; for (const [version, mode] of PROFILE_GROUPS[profileOf(matrix)]) { const expected = matrix.cells.filter(cell => cell.version === version && cell.mode === mode); - const submitted = receipts.filter(receipt => receipt.cell.version === version && receipt.cell.mode === mode); + const submitted = bound.filter(receipt => receipt.cell.version === version && receipt.cell.mode === mode); const tokens = submitted.flatMap(receipt => receipt.usage.inputTokens === null || receipt.usage.outputTokens === null ? [] : [receipt.usage.inputTokens + receipt.usage.outputTokens]); const costs = submitted.flatMap(receipt => receipt.usage.estimatedCostUSD === null ? [] : [receipt.usage.estimatedCostUSD]); groups.push({ version, mode, scheduled: expected.length, submitted: submitted.length, missing: expected.length - submitted.length, succeeded: submitted.filter(receipt => receipt.status === 'succeeded').length, failed: submitted.filter(receipt => receipt.status === 'failed').length, @@ -500,9 +503,9 @@ export function collectProducerReceipts(matrix: EvalMatrix, schedule: PreparedEv estimatedCost: { measured: costs.length, denominator: expected.length, totalUSD: costs.length ? costs.reduce((sum, value) => sum + value, 0) : null, source: 'pricing-table-estimate' }, }); } - const indexes: ProducerReceiptIndex[] = receipts.map(({ output: _output, error, ...receipt }) => ({ ...receipt, ...(error ? { error: { code: error.code } } : {}) })); + const indexes: ProducerReceiptIndex[] = bound.map(({ output: _output, error, ...receipt }) => ({ ...receipt, ...(error ? { error: { code: error.code } } : {}) })); const base = { schemaVersion: 1 as const, matrixHash: matrixHash(matrix), scheduleHash: digest(JSON.stringify(schedule)), receipts: indexes.sort((left, right) => left.cell.id.localeCompare(right.cell.id)), summary: { - scheduled: matrix.cells.length, prepared: schedule.preparedCells, submitted: receipts.length, missing: matrix.cells.length - receipts.length, + scheduled: matrix.cells.length, prepared: schedule.preparedCells, submitted: bound.length, missing: matrix.cells.length - bound.length, matchedPairsExpected: expectedMatchedPairs(matrix), matchedPairsSubmitted: completePairs.length, modelMismatches, groups, note: 'Costs are pricing-table estimates. First-useful timing is unmeasured because the reused provider adapters return completed runs. Trusted findings and runtime outcomes require separate oracle adjudication.', } }; @@ -785,7 +788,7 @@ function validateCollectedBatch(matrix: EvalMatrix, batch: ProducerBatch, observ const cells = new Map(matrix.cells.map(cell => [cell.id, cell])); const receipts = new Map(); for (const receipt of batch.receipts) { - const cell = cells.get(receipt?.cell?.id); + const cell = cells.get(receipt?.cellRef); if (!cell || receipts.has(cell.id)) throw new Error('INVALID_PRODUCER_BATCH'); validateReceiptIndex(receipt, cell); receipts.set(cell.id, receipt); } diff --git a/scripts/cso-requalification.ts b/scripts/cso-requalification.ts index 5dbf13c42..d64110687 100644 --- a/scripts/cso-requalification.ts +++ b/scripts/cso-requalification.ts @@ -22,7 +22,8 @@ const IMAGE_CONTEXT_EXCLUDED = new Set(['README.md', 'qualification.json', 'buil export async function imageSourceFiles(root = ROOT): Promise { const build = await Bun.build({ entrypoints: IMAGE_HELPER_ENTRYPOINTS.map(entry => path.join(root, entry)), target: 'bun', metafile: true }); if (!build.success || !build.metafile) throw new Error('REQUALIFICATION_SOURCE_CLOSURE_FAILED'); - const closure = Object.keys(build.metafile.inputs).map(input => path.relative(root, path.resolve(root, input)).split(path.sep).join('/')); + // Bun reports metafile inputs relative to the process working directory, not to the entrypoints. + const closure = Object.keys(build.metafile.inputs).map(input => path.relative(root, path.resolve(process.cwd(), input)).split(path.sep).join('/')); if (closure.some(file => file.startsWith('..') || file.includes('node_modules/'))) throw new Error('REQUALIFICATION_SOURCE_OUTSIDE_REPOSITORY'); const context = fs.readdirSync(path.join(root, IMAGE_CONTEXT), { withFileTypes: true }) .filter(entry => entry.isFile() && !IMAGE_CONTEXT_EXCLUDED.has(entry.name)) diff --git a/test/cso-eval.test.ts b/test/cso-eval.test.ts index 58c389bbc..444b43422 100644 --- a/test/cso-eval.test.ts +++ b/test/cso-eval.test.ts @@ -84,7 +84,7 @@ function syntheticArtifactInventory(): ProducerArtifactInventory { /** Platform-pinned cells get one native installation and provider binary per platform, sharing catalogs and provider policy. */ function syntheticReceipt(cell: EvalCell): ProducerReceipt { const seed = cell.platform ?? '1'; - const withoutHash: Omit = { schemaVersion: 1, cell, inputHash: 'e'.repeat(64), installationIdentity: syntheticInstallationIdentity(seed, '1'), providerIdentity: syntheticProviderIdentity(cell.host === 'codex' ? 'gpt' : cell.host, seed), artifacts: syntheticArtifactInventory(), startedAt: '2026-01-01T00:00:00.000Z', finishedAt: '2026-01-01T00:00:01.000Z', status: 'succeeded', requestedModel: cell.model, modelUsed: `resolved-${cell.model}`, modelIdentitySource: 'provider_reported', durationMs: 1000, firstUsefulResultMs: null, toolCalls: 1, output: 'synthetic producer transcript', outputHash: sha256('synthetic producer transcript'), usage: { inputTokens: 10, outputTokens: 5, cachedTokens: 0, estimatedCostUSD: 0.001 } }; + const withoutHash: Omit = { schemaVersion: 2, cellRef: cell.id, inputHash: 'e'.repeat(64), installationIdentity: syntheticInstallationIdentity(seed, '1'), providerIdentity: syntheticProviderIdentity(cell.host === 'codex' ? 'gpt' : cell.host, seed), artifacts: syntheticArtifactInventory(), startedAt: '2026-01-01T00:00:00.000Z', finishedAt: '2026-01-01T00:00:01.000Z', status: 'succeeded', requestedModel: cell.model, modelUsed: `resolved-${cell.model}`, modelIdentitySource: 'provider_reported', durationMs: 1000, firstUsefulResultMs: null, toolCalls: 1, output: 'synthetic producer transcript', outputHash: sha256('synthetic producer transcript'), usage: { inputTokens: 10, outputTokens: 5, cachedTokens: 0, estimatedCostUSD: 0.001 } }; return { ...withoutHash, receiptHash: producerReceiptHash(withoutHash) }; } function syntheticSchedule(target: EvalMatrix = matrix): PreparedEvalSchedule { @@ -243,6 +243,20 @@ describe('CSO matched evaluation accounting', () => { mixedProvider.providerIdentity = syntheticProviderIdentity('gpt', 'different-provider'); expect(() => collectProducerReceipts(matrix, syntheticSchedule(), [first, rehashReceipt(mixedProvider)])).toThrow('UNMATCHED_PRODUCER_PROVIDERS'); }); + test('receipts name only an opaque cellRef that the trusted matrix resolves', () => { + const cell = matrix.cells.find(item => item.version === 'v3')!, receipt = syntheticReceipt(cell); + expect(Object.keys(receipt)).not.toContain('cell'); expect(JSON.stringify(receipt)).not.toMatch(/caseId|variant|vulnerable|"fixed"/); + expect(JSON.stringify(receipt)).not.toContain(cell.caseId); + const unknown = structuredClone(receipt); unknown.cellRef = 'f'.repeat(64); + expect(() => collectProducerReceipts(matrix, syntheticSchedule(), [rehashReceipt(unknown)])).toThrow('UNKNOWN_OR_DUPLICATE_PRODUCER_RECEIPT'); + const smuggled = rehashReceipt({ ...structuredClone(receipt), cell } as ProducerReceipt); + expect(() => collectProducerReceipts(matrix, syntheticSchedule(), [smuggled])).toThrow('INVALID_PRODUCER_RECEIPT'); + const batch = syntheticBatch(); + expect(batch.receipts.every(index => index.cellRef === index.cell.id && JSON.stringify(index.cell) === JSON.stringify(matrix.cells.find(item => item.id === index.cellRef)))).toBe(true); + const relabeled = structuredClone(batch); relabeled.receipts[0].cellRef = relabeled.receipts[1].cellRef; + const { batchHash: _hash, ...rest } = relabeled; relabeled.batchHash = createHash('sha256').update(JSON.stringify(rest)).digest('hex'); + expect(() => scoreCollectedEval(matrix, relabeled, [], qualification)).toThrow('INVALID_PRODUCER_BATCH'); + }); test('high/critical recall includes critical cases and detects a v3 regression',()=>{ const criticalCorpus=structuredClone(corpus);criticalCorpus.cases.find(item=>item.severity==='medium')!.severity='critical'; const criticalMatrix=createEvalMatrix({model:'matched-test-model',host:'codex',skillHashes:{v2:'a'.repeat(64),v3:'b'.repeat(64)}},criticalCorpus),complete=criticalMatrix.cells.map(syntheticResult); @@ -550,7 +564,8 @@ describe('CSO matched producer orchestration', () => { expect(existsSync(join(destination, 'jobs', cell.id, 'producer-input.json'))).toBe(true); const input = JSON.parse(readFileSync(join(destination, 'jobs', cell.id, 'producer-input.json'), 'utf8')); expect(input.skill).toBe(skills[cell.version]); - expect(input.cell.skillHash).toBe(sha256(input.skill)); + expect(input.execution.skillHash).toBe(sha256(input.skill)); + expect(input.cellRef).toBe(cell.id); expect(input.skill).toContain(`${cell.version.toUpperCase()}_SECTION_ONLY`); expect(input.skill).not.toContain(cell.version === 'v2' ? 'V3_SECTION_ONLY' : 'V2_SECTION_ONLY'); expect(validatePortableSkillPayload(input.skill, cell.version).files.map(file => file.path)).toEqual(['SKILL.md', 'sections/manifest.json', 'sections/audit-phases.md']); @@ -953,12 +968,22 @@ describe('CSO matched producer orchestration', () => { prepareEvalJobs(release, { v3: skills.v3 }, destination, [other.id, ...(native ? [native.id] : [])], privateCorpus); const receipts = join(root(), 'receipts'); mkdirSync(receipts); const isolatedOther = isolate(destination, other); + const opaque = readFileSync(isolatedOther.input, 'utf8'), parsed = JSON.parse(opaque); + expect(Object.keys(parsed).sort()).toEqual(['cellRef', 'execution', 'schemaVersion', 'skill', 'source']); + expect(parsed).toMatchObject({ schemaVersion: 2, cellRef: other.id, execution: { mode: 'comprehensive', platform: other.platform, sourceHash: other.sourceHash } }); + expect(opaque).not.toMatch(/caseId|variant|vulnerable|"fixed"|"version"|repetition/); expect(opaque).not.toContain(other.caseId); + const smuggled = isolate(destination, other); + writeFileSync(smuggled.input, JSON.stringify({ ...parsed, caseId: other.caseId })); + await expect(runProducerCell(smuggled.input, join(receipts, `${other.id}.json`), withHelper({ adapter: new FakeAdapter(() => {}), paidExecutionAuthorized: true }))).rejects.toThrow('INVALID_PRODUCER_INPUT'); expect(readFileSync(join(isolatedOther.source, 'src/app.txt'), 'utf8')).toBe(`synthetic ${other.caseId} ${other.variant}\n`); await expect(runProducerCell(isolatedOther.input, join(receipts, `${other.id}.json`), withHelper({ adapter: new FakeAdapter(() => {}), paidExecutionAuthorized: true }))).rejects.toThrow('UNMATCHED_PRODUCER_PLATFORM'); if (!native) return; const isolatedNative = isolate(destination, native); const receipt = await runProducerCell(isolatedNative.input, join(receipts, `${native.id}.json`), withHelper({ adapter: new FakeAdapter(() => {}), paidExecutionAuthorized: true })); - expect(receipt.cell).toEqual(native); + expect(receipt.cellRef).toBe(native.id); + const written = readFileSync(join(receipts, `${native.id}.json`), 'utf8'); + expect(JSON.parse(written)).not.toHaveProperty('cell'); + expect(written).not.toMatch(/caseId|variant|vulnerable|"fixed"/); expect(written).not.toContain(native.caseId); }); test('seals source read-only and withholds a receipt after a mode/content mutation', async () => { diff --git a/test/cso-requalification.test.ts b/test/cso-requalification.test.ts index 4bd5e3176..03cc5c85b 100644 --- a/test/cso-requalification.test.ts +++ b/test/cso-requalification.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, test } from 'bun:test'; -import { cpSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { cpSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import { spawnSync } from 'node:child_process'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; @@ -35,6 +35,17 @@ describe('CSO runtime requalification triggers', () => { expect((await requalificationTriggers(copy)).preparationSha256).not.toBe(current.preparationSha256); }); + test.skipIf(process.platform==='win32')('triggers resolve against the repository root from any working directory', async () => { + const elsewhere = mkdtempSync(join(tmpdir(), 'cso-requalification-cwd-')); temps.push(elsewhere); + const nested = join(elsewhere, 'a', 'b', 'c'); mkdirSync(nested, { recursive: true }); + const cli = spawnSync(process.execPath, [join(ROOT, 'scripts/cso-requalification.ts'), 'triggers'], { cwd: nested, encoding: 'utf8', timeout: 30_000 }); + expect(cli.status, cli.stderr).toBe(0); expect(JSON.parse(cli.stdout)).toEqual(current); + const copy = join(elsewhere, 'deeper', 'checkout'); mkdirSync(copy, { recursive: true }); + cpSync(join(ROOT, 'lib'), join(copy, 'lib'), { recursive: true }); + expect(await requalificationTriggers(copy)).toEqual(current); + expect(await imageSourceFiles(copy)).toEqual(await imageSourceFiles()); + }); + test('a promoted catalog names each trigger that changed after qualification', () => { const catalog = completeRuntimeCatalogFixture(); const recorded = catalog.promotion!.requalification; diff --git a/test/fixtures/cso-eval/README.md b/test/fixtures/cso-eval/README.md index eca16982b..c05915c84 100644 --- a/test/fixtures/cso-eval/README.md +++ b/test/fixtures/cso-eval/README.md @@ -127,8 +127,12 @@ gstack helper/runtime files. Do not install or copy any CSO `SKILL.md`, carved section, generated skill tree, schedule, this repository checkout, sibling jobs, fixed alternatives, or evaluator code to that filesystem. The producer runner reads its opaque control file into memory and deletes it before -the agent process starts, so the case identifier and vulnerable/fixed label are -not agent inputs. Copy the selected hash-named directory as the literal path +the agent process starts. That file never names the case, variant, version, or +repetition: it is `{schemaVersion: 2, cellRef, skill, source, execution}`, where +`cellRef` is the matrix cell id and `execution` holds only the mode, model, host, +budget, source and skill hashes, and platform. The receipt binds `cellRef` and +`inputHash`, and `collect` rebuilds each cell from the matrix, so neither file +tells an agent with root on its cell VM which answer it is graded against. Copy the selected hash-named directory as the literal path `/producer/job`; `/producer` must contain only `job`. Put the runner in a system tool directory and use a separate receipt mount. The runner rejects the full prepared batch layout, extra files beside `job`, a reused job with prior state,