fix(cso): opaque producer inputs and receipts; requalification triggers independent of cwd

Producer inputs are {schemaVersion: 2, cellRef, skill, source, execution} and
receipts bind cellRef + inputHash; collection rebuilds the cell from the
matrix, so a cell VM never learns its case or variant. requalificationTriggers
resolves Bun's cwd-relative metafile inputs against the repository root.
This commit is contained in:
garrytan committed 2026-10-05 01:31:51 +00:00
1 parent fa564ba771
commit fe87890edb
7 files changed
+133 -51

No files matched your search

+35 -26
View File
@@ -25,7 +25,7 @@ import {
producerProviderIdentityHash,
producerReceiptHash,
sha256,
type ProducerCell,
type ProducerExecution,
type ProducerHost,
type ProducerInput,
type ProducerArtifactInventory,
@@ -199,9 +199,18 @@ function repositoryIdentity(root: string): string {
return sha256(JSON.stringify({ config: sha256(config), head: read(['rev-parse', '--verify', 'HEAD']).trim(), branch: read(['symbolic-ref', '--short', 'HEAD']).trim(), status: read(['status', '--porcelain=v2', '--untracked-files=all']) }));
}
function validateCell(cell: ProducerCell): void {
if (!cell || !HEX.test(cell.id) || !cell.caseId || !['node', 'bun', 'python', 'rails'].includes(cell.stack) || !['vulnerable', 'fixed'].includes(cell.variant) || !['v2', 'v3'].includes(cell.version) || !['daily', 'comprehensive'].includes(cell.mode) || ![1, 2, 3].includes(cell.repetition) || !cell.model || !['claude', 'codex', 'gemini'].includes(cell.host) || !Number.isInteger(cell.budgetSeconds) || cell.budgetSeconds <= 60 || cell.budgetSeconds > 3600 || !HEX.test(cell.sourceHash) || !HEX.test(cell.skillHash) || (cell.platform !== undefined && !PRODUCER_PLATFORMS.includes(cell.platform))) throw new Error('INVALID_PRODUCER_INPUT');
if (cell.platform !== undefined && cell.platform !== producerHostPlatform()) throw new Error('UNMATCHED_PRODUCER_PLATFORM');
const INPUT_FIELDS = 'cellRef,execution,schemaVersion,skill,source';
const EXECUTION_FIELDS = 'budgetSeconds,host,mode,model,skillHash,sourceHash';
/** The input has exactly the opaque fields: no case, variant, version, or repetition can ride along. */
function validateInput(input: ProducerInput): void {
const execution = input?.execution;
if (!input || typeof input !== 'object' || Object.keys(input).sort().join(',') !== INPUT_FIELDS || input.schemaVersion !== 2 || !HEX.test(input.cellRef) ||
typeof input.skill !== 'string' || Buffer.byteLength(input.skill) > INPUT_LIMIT || !Array.isArray(input.source) ||
!execution || typeof execution !== 'object' || Object.keys(execution).filter(key => key !== 'platform').sort().join(',') !== EXECUTION_FIELDS ||
!['daily', 'comprehensive'].includes(execution.mode) || typeof execution.model !== 'string' || !execution.model || !['claude', 'codex', 'gemini'].includes(execution.host) ||
!Number.isInteger(execution.budgetSeconds) || execution.budgetSeconds <= 60 || execution.budgetSeconds > 3600 || !HEX.test(execution.sourceHash) || !HEX.test(execution.skillHash) ||
('platform' in execution && !PRODUCER_PLATFORMS.includes(execution.platform!))) throw new Error('INVALID_PRODUCER_INPUT');
if (execution.platform !== undefined && execution.platform !== producerHostPlatform()) throw new Error('UNMATCHED_PRODUCER_PLATFORM');
}
export interface ProducerHelperBinding {
@@ -420,7 +429,7 @@ export function resolveProducerProviderIdentity(host: ProducerHost, stateRoot: s
};
}
function promptFor(cell: ProducerCell, skill: string, sourceRoot: string, helper: ProducerHelperBinding): string {
function promptFor(cell: ProducerExecution, skill: string, sourceRoot: string, helper: ProducerHelperBinding): string {
const invocation = `/cso${cell.mode === 'comprehensive' ? ' --comprehensive' : ''} --budget ${cell.budgetSeconds}`;
const sourceAccess = cell.host === 'codex'
? 'The Codex permission profile grants read-only access to exactly that immutable snapshot so the trusted helper can read it. Use only the trusted helper to inspect or act on source; direct provider file or shell access is outside this evaluation policy.'
@@ -487,20 +496,20 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op
const bytes = readBoundedStable(controlPath, INPUT_LIMIT, 'Producer input');
let input: ProducerInput;
try { input = JSON.parse(bytes.toString('utf8')); } catch { throw new Error('INVALID_PRODUCER_INPUT'); }
if (input.schemaVersion !== 1 || typeof input.skill !== 'string' || Buffer.byteLength(input.skill) > INPUT_LIMIT || !Array.isArray(input.source)) throw new Error('INVALID_PRODUCER_INPUT');
validateCell(input.cell);
if (basename(receipt) !== `${input.cell.id}.json`) throw new Error('UNMATCHED_RECEIPT_DESTINATION');
if (sha256(input.skill) !== input.cell.skillHash) throw new Error('INVALID_PRODUCER_SKILL');
validateSource(sourceRoot, input.source, input.cell.sourceHash);
validateInput(input);
const cell = input.execution;
if (basename(receipt) !== `${input.cellRef}.json`) throw new Error('UNMATCHED_RECEIPT_DESTINATION');
if (sha256(input.skill) !== cell.skillHash) throw new Error('INVALID_PRODUCER_SKILL');
validateSource(sourceRoot, input.source, cell.sourceHash);
sealProducerSource(sourceRoot);
validateSource(sourceRoot, input.source, input.cell.sourceHash);
validateSource(sourceRoot, input.source, cell.sourceHash);
const originalRepositoryIdentity = repositoryIdentity(sourceRoot);
const stateRoot = join(jobRoot, 'state');
const helperHome = join(stateRoot, 'cso-home');
const helper = resolveProducerHelperBinding(sourceRoot, stateRoot, options.testHelperLauncherPath);
const installationIdentity = producerInstallationIdentity(helper);
const adapter = options.adapter ?? adapterFor(input.cell.host);
if ((input.cell.host === 'codex' ? 'gpt' : input.cell.host) !== adapter.family) throw new Error('UNMATCHED_PRODUCER_ADAPTER');
const adapter = options.adapter ?? adapterFor(cell.host);
if ((cell.host === 'codex' ? 'gpt' : cell.host) !== adapter.family) throw new Error('UNMATCHED_PRODUCER_ADAPTER');
fs.mkdirSync(stateRoot, { recursive: false, mode: 0o700 });
fs.mkdirSync(helperHome, { recursive: false, mode: 0o700 });
const provider = options.testProviderIdentity
@@ -508,13 +517,13 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op
identity: validateProviderIdentity(options.testProviderIdentity, adapter.family),
command: options.testProviderCommand ?? { executable: process.execPath, argsPrefix: [] },
}
: resolveProducerProviderIdentity(input.cell.host, stateRoot);
: resolveProducerProviderIdentity(cell.host, stateRoot);
const providerIdentity = provider.identity;
const runOptions = {
prompt: promptFor(input.cell, input.skill, sourceRoot, helper),
prompt: promptFor(cell, input.skill, sourceRoot, helper),
workdir: stateRoot,
timeoutMs: input.cell.budgetSeconds * 1000,
model: input.cell.model,
timeoutMs: cell.budgetSeconds * 1000,
model: cell.model,
csoProducer: {
stateDirectory: stateRoot,
sourceDirectory: sourceRoot,
@@ -524,7 +533,7 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op
},
} satisfies RunOpts;
const availability = await adapter.available(runOptions);
if (!availability.ok) throw new Error(`PRODUCER_UNAVAILABLE: ${availability.reason ?? input.cell.host}`);
if (!availability.ok) throw new Error(`PRODUCER_UNAVAILABLE: ${availability.reason ?? cell.host}`);
const inputHash = producerInputHash(input);
// Load the opaque metadata into memory, then remove it before starting the
@@ -539,11 +548,11 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op
const startedAt = new Date().toISOString();
let run: RunResult;
try {
if (input.cell.host === 'gemini') prepareGeminiProducerState(stateRoot, helper.launcher);
if (cell.host === 'gemini') prepareGeminiProducerState(stateRoot, helper.launcher);
run = await adapter.run(runOptions);
} finally {
try {
if (input.cell.host === 'gemini') removeGeminiProducerState(stateRoot);
if (cell.host === 'gemini') removeGeminiProducerState(stateRoot);
} finally {
if (previousHome === undefined) delete process.env.GSTACK_HOME; else process.env.GSTACK_HOME = previousHome;
if (previousSessionKind === undefined) delete process.env.GSTACK_SESSION_KIND; else process.env.GSTACK_SESSION_KIND = previousSessionKind;
@@ -560,7 +569,7 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op
throw new Error('PRODUCER_PROVIDER_INSTALLATION_RACE');
}
assertProducerSourceSealed(sourceRoot);
validateSource(sourceRoot, input.source, input.cell.sourceHash);
validateSource(sourceRoot, input.source, cell.sourceHash);
if (repositoryIdentity(sourceRoot) !== originalRepositoryIdentity) throw new Error('PRODUCER_CHANGED_SOURCE');
const artifacts = inventoryProducerArtifacts(helperHome);
run = sanitizeProducerRun(run);
@@ -569,8 +578,8 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op
const tokensReported = run.tokens.input > 0 || run.tokens.output > 0 || (run.tokens.cached ?? 0) > 0;
const estimatedCostUSD = tokensReported && PRICING[run.modelUsed] ? adapter.estimateCost(run.tokens, run.modelUsed) : null;
const withoutHash: Omit<ProducerReceipt, 'receiptHash'> = {
schemaVersion: 1,
cell: input.cell,
schemaVersion: 2,
cellRef: input.cellRef,
inputHash,
installationIdentity,
providerIdentity,
@@ -578,9 +587,9 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op
startedAt,
finishedAt,
status: run.error ? 'failed' : 'succeeded',
requestedModel: input.cell.model,
requestedModel: cell.model,
modelUsed: run.modelUsed,
modelIdentitySource: run.modelUsed === input.cell.model ? 'requested_pin' : 'provider_reported',
modelIdentitySource: run.modelUsed === cell.model ? 'requested_pin' : 'provider_reported',
durationMs: run.durationMs,
firstUsefulResultMs: null,
toolCalls: run.toolCalls,
@@ -603,7 +612,7 @@ async function cli(args: string[]): Promise<void> {
if (args.length !== 4 || args[0] !== 'run' || args[3] !== '--execute-paid') throw new Error('Usage: cso-eval-producer run <consumable-input.json> <new-receipt.json> --execute-paid');
if (process.env.CSO_EVAL_PAID !== '1') throw new Error('PAID_EXECUTION_NOT_AUTHORIZED: also set CSO_EVAL_PAID=1 on the isolated producer host');
const receipt = await runProducerCell(args[1], args[2], { paidExecutionAuthorized: true });
console.log(JSON.stringify({ cellId: receipt.cell.id, status: receipt.status, durationMs: receipt.durationMs, receiptHash: receipt.receiptHash }));
console.log(JSON.stringify({ cellRef: receipt.cellRef, status: receipt.status, durationMs: receipt.durationMs, receiptHash: receipt.receiptHash }));
}
if (import.meta.main) {
+34 -5
View File
@@ -31,16 +31,40 @@ export interface ProducerSourceEntry {
bytes: number;
}
/**
* Only what a producer needs to run one cell. It never names the corpus case,
* variant, version, or repetition: an agent that can read its own control file
* or receipt still cannot tell which answer it is being graded against.
*/
export interface ProducerExecution {
mode: ProducerMode;
model: string;
host: ProducerHost;
budgetSeconds: number;
sourceHash: string;
skillHash: string;
platform?: ProducerPlatform;
}
/**
* This control file is consumed before a producing agent starts. It must never
* be placed inside the application repository or retained on the producer.
*/
export interface ProducerInput {
schemaVersion: 1;
cell: ProducerCell;
schemaVersion: 2;
/** The opaque matrix cell id; only the trusted evaluator maps it back to a cell. */
cellRef: string;
/** Exact canonical portable payload: root SKILL.md plus its manifest-listed sections. */
skill: string;
source: ProducerSourceEntry[];
execution: ProducerExecution;
}
export function producerExecution(cell: ProducerCell): ProducerExecution {
return {
mode: cell.mode, model: cell.model, host: cell.host, budgetSeconds: cell.budgetSeconds,
sourceHash: cell.sourceHash, skillHash: cell.skillHash, ...(cell.platform === undefined ? {} : { platform: cell.platform }),
};
}
export interface ProducerArtifactIdentity {
@@ -89,8 +113,9 @@ export interface ProducerArtifactInventory {
}
export interface ProducerReceipt {
schemaVersion: 1;
cell: ProducerCell;
schemaVersion: 2;
/** The opaque cell id from the consumed input; bound together with inputHash. */
cellRef: string;
inputHash: string;
installationIdentity: ProducerInstallationIdentity;
providerIdentity: ProducerProviderIdentity;
@@ -119,8 +144,12 @@ export interface ProducerReceipt {
receiptHash: string;
}
/** Compact trusted index; raw output remains in the separately retained receipt. */
/**
* Compact trusted index; raw output remains in the separately retained receipt.
* Collection adds the matrix cell that the receipt's cellRef names.
*/
export type ProducerReceiptIndex = Omit<ProducerReceipt, 'output' | 'error'> & {
cell: ProducerCell;
error?: { code: string };
};
+16 -13
View File
@@ -11,6 +11,7 @@ import { loadCorpusManifest, materializeCase, publicCorpus, STACKS, validateCorp
import {
PRODUCER_PLATFORMS,
producerArtifactInventoryHash,
producerExecution,
producerInputHash,
producerInstallationIdentityHash,
producerProviderIdentityHash,
@@ -358,7 +359,7 @@ export function prepareEvalJobs(matrix: EvalMatrix, payloads: Partial<Record<Eva
const source = materializeCase(cell.caseId, cell.variant, join(jobRoot, 'source'), corpus);
if (source.sourceHash !== cell.sourceHash) throw new Error('CORPUS_INTEGRITY_MISMATCH');
initializeFixtureRepository(source.path);
const input: ProducerInput = { schemaVersion: 1, cell, skill: payloads[cell.version]!, source: sourceEntries(corpus, cell.caseId, cell.variant) };
const input: ProducerInput = { schemaVersion: 2, cellRef: cell.id, skill: payloads[cell.version]!, source: sourceEntries(corpus, cell.caseId, cell.variant), execution: producerExecution(cell) };
const inputHash = producerInputHash(input);
safeWriteNew(join(jobRoot, 'producer-input.json'), input);
jobs.push({ cellId: cell.id, relativePath: `jobs/${cell.id}`, inputHash });
@@ -379,7 +380,7 @@ function validateSchedule(matrix: EvalMatrix, schedule: PreparedEvalSchedule): v
}
function validateReceipt(receipt: ProducerReceipt, cell: EvalCell, inputHash: string): void {
if (receipt?.schemaVersion !== 1 || JSON.stringify(receipt.cell) !== JSON.stringify(cell) || receipt.inputHash !== inputHash || receipt.requestedModel !== cell.model || !['succeeded', 'failed'].includes(receipt.status) || typeof receipt.modelUsed !== 'string' || !receipt.modelUsed || !['provider_reported', 'requested_pin'].includes(receipt.modelIdentitySource) || (receipt.modelIdentitySource === 'requested_pin' && receipt.modelUsed !== receipt.requestedModel) || (receipt.modelIdentitySource === 'provider_reported' && receipt.modelUsed === receipt.requestedModel) || !validNumber(receipt.durationMs) || receipt.firstUsefulResultMs !== null || !Number.isSafeInteger(receipt.toolCalls) || receipt.toolCalls < 0 || typeof receipt.output !== 'string' || receipt.outputHash !== sha256(receipt.output) || !HEX.test(receipt.receiptHash)) throw new Error('INVALID_PRODUCER_RECEIPT');
if (receipt?.schemaVersion !== 2 || receipt.cellRef !== cell.id || 'cell' in receipt || receipt.inputHash !== inputHash || receipt.requestedModel !== cell.model || !['succeeded', 'failed'].includes(receipt.status) || typeof receipt.modelUsed !== 'string' || !receipt.modelUsed || !['provider_reported', 'requested_pin'].includes(receipt.modelIdentitySource) || (receipt.modelIdentitySource === 'requested_pin' && receipt.modelUsed !== receipt.requestedModel) || (receipt.modelIdentitySource === 'provider_reported' && receipt.modelUsed === receipt.requestedModel) || !validNumber(receipt.durationMs) || receipt.firstUsefulResultMs !== null || !Number.isSafeInteger(receipt.toolCalls) || receipt.toolCalls < 0 || typeof receipt.output !== 'string' || receipt.outputHash !== sha256(receipt.output) || !HEX.test(receipt.receiptHash)) throw new Error('INVALID_PRODUCER_RECEIPT');
const { receiptHash, ...withoutHash } = receipt;
if (producerReceiptHash(withoutHash) !== receiptHash) throw new Error('INVALID_PRODUCER_RECEIPT');
const started = Date.parse(receipt.startedAt), finished = Date.parse(receipt.finishedAt);
@@ -424,7 +425,7 @@ function validateArtifactInventory(inventory: ProducerArtifactInventory, error:
}
function validateReceiptIndex(receipt: ProducerReceiptIndex, cell: EvalCell): void {
if (!receipt || receipt.schemaVersion !== 1 || JSON.stringify(receipt.cell) !== JSON.stringify(cell) || !HEX.test(receipt.inputHash) || receipt.requestedModel !== cell.model || !receipt.modelUsed || !['provider_reported', 'requested_pin'].includes(receipt.modelIdentitySource) || (receipt.modelIdentitySource === 'requested_pin' && receipt.modelUsed !== receipt.requestedModel) || (receipt.modelIdentitySource === 'provider_reported' && receipt.modelUsed === receipt.requestedModel) || !['succeeded', 'failed'].includes(receipt.status) || !HEX.test(receipt.outputHash) || !HEX.test(receipt.receiptHash) || !validNumber(receipt.durationMs) || receipt.firstUsefulResultMs !== null || !Number.isSafeInteger(receipt.toolCalls) || receipt.toolCalls < 0 || !receipt.usage) throw new Error('INVALID_PRODUCER_BATCH');
if (!receipt || receipt.schemaVersion !== 2 || receipt.cellRef !== cell.id || JSON.stringify(receipt.cell) !== JSON.stringify(cell) || !HEX.test(receipt.inputHash) || receipt.requestedModel !== cell.model || !receipt.modelUsed || !['provider_reported', 'requested_pin'].includes(receipt.modelIdentitySource) || (receipt.modelIdentitySource === 'requested_pin' && receipt.modelUsed !== receipt.requestedModel) || (receipt.modelIdentitySource === 'provider_reported' && receipt.modelUsed === receipt.requestedModel) || !['succeeded', 'failed'].includes(receipt.status) || !HEX.test(receipt.outputHash) || !HEX.test(receipt.receiptHash) || !validNumber(receipt.durationMs) || receipt.firstUsefulResultMs !== null || !Number.isSafeInteger(receipt.toolCalls) || receipt.toolCalls < 0 || !receipt.usage) throw new Error('INVALID_PRODUCER_BATCH');
const started = Date.parse(receipt.startedAt), finished = Date.parse(receipt.finishedAt);
if (!Number.isFinite(started) || !Number.isFinite(finished) || finished < started || !['inputTokens', 'outputTokens', 'cachedTokens', 'estimatedCostUSD'].every(field => receipt.usage[field as keyof typeof receipt.usage] === null || validNumber(receipt.usage[field as keyof typeof receipt.usage])) || ((receipt.status === 'failed') !== !!receipt.error) || (receipt.error && !receipt.error.code)) throw new Error('INVALID_PRODUCER_BATCH');
validateInstallationIdentity(receipt.installationIdentity, 'INVALID_PRODUCER_BATCH');
@@ -473,24 +474,26 @@ export function collectProducerReceipts(matrix: EvalMatrix, schedule: PreparedEv
const jobs = new Map(schedule.jobs.map(job => [job.cellId, job]));
const seen = new Set<string>();
for (const receipt of receipts) {
const cell = cells.get(receipt?.cell?.id), job = jobs.get(receipt?.cell?.id);
const cell = cells.get(receipt?.cellRef), job = jobs.get(receipt?.cellRef);
if (!cell || !job || seen.has(cell.id)) throw new Error('UNKNOWN_OR_DUPLICATE_PRODUCER_RECEIPT');
validateReceipt(receipt, cell, job.inputHash); seen.add(cell.id);
}
assertPlatformIdentities(receipts);
const pairs = new Map<string, Partial<Record<EvalVersion, ProducerReceipt>>>();
if (profileOf(matrix) === 'full') for (const receipt of receipts) {
// Receipts name only the opaque cellRef; the trusted matrix supplies the cell.
const bound = receipts.map(receipt => ({ ...receipt, cell: cells.get(receipt.cellRef)! }));
assertPlatformIdentities(bound);
const pairs = new Map<string, Partial<Record<EvalVersion, (typeof bound)[number]>>>();
if (profileOf(matrix) === 'full') for (const receipt of bound) {
const key = matchedPairKey(receipt.cell as EvalCell);
const pair = pairs.get(key) ?? {}; pair[receipt.cell.version] = receipt; pairs.set(key, pair);
}
else if (new Set(receipts.map(receipt => receipt.modelUsed)).size > 1) throw new Error('UNMATCHED_EFFECTIVE_MODELS: single-version cells used different normalized model identities');
const completePairs = [...pairs.entries()].filter(([, pair]) => pair.v2 && pair.v3) as Array<[string, { v2: ProducerReceipt; v3: ProducerReceipt }]>;
else if (new Set(bound.map(receipt => receipt.modelUsed)).size > 1) throw new Error('UNMATCHED_EFFECTIVE_MODELS: single-version cells used different normalized model identities');
const completePairs = [...pairs.entries()].filter(([, pair]) => pair.v2 && pair.v3) as Array<[string, { v2: (typeof bound)[number]; v3: (typeof bound)[number] }]>;
const modelMismatches = completePairs.filter(([, pair]) => pair.v2.modelUsed !== pair.v3.modelUsed).map(([pair, value]) => ({ pair: digest(pair), v2: value.v2.modelUsed, v3: value.v3.modelUsed }));
if (modelMismatches.length) throw new Error(`UNMATCHED_EFFECTIVE_MODELS: ${modelMismatches.length} matched v2/v3 pair(s) used different normalized model identities`);
const groups: ProducerGroupSummary[] = [];
for (const [version, mode] of PROFILE_GROUPS[profileOf(matrix)]) {
const expected = matrix.cells.filter(cell => cell.version === version && cell.mode === mode);
const submitted = receipts.filter(receipt => receipt.cell.version === version && receipt.cell.mode === mode);
const submitted = bound.filter(receipt => receipt.cell.version === version && receipt.cell.mode === mode);
const tokens = submitted.flatMap(receipt => receipt.usage.inputTokens === null || receipt.usage.outputTokens === null ? [] : [receipt.usage.inputTokens + receipt.usage.outputTokens]);
const costs = submitted.flatMap(receipt => receipt.usage.estimatedCostUSD === null ? [] : [receipt.usage.estimatedCostUSD]);
groups.push({ version, mode, scheduled: expected.length, submitted: submitted.length, missing: expected.length - submitted.length, succeeded: submitted.filter(receipt => receipt.status === 'succeeded').length, failed: submitted.filter(receipt => receipt.status === 'failed').length,
@@ -500,9 +503,9 @@ export function collectProducerReceipts(matrix: EvalMatrix, schedule: PreparedEv
estimatedCost: { measured: costs.length, denominator: expected.length, totalUSD: costs.length ? costs.reduce((sum, value) => sum + value, 0) : null, source: 'pricing-table-estimate' },
});
}
const indexes: ProducerReceiptIndex[] = receipts.map(({ output: _output, error, ...receipt }) => ({ ...receipt, ...(error ? { error: { code: error.code } } : {}) }));
const indexes: ProducerReceiptIndex[] = bound.map(({ output: _output, error, ...receipt }) => ({ ...receipt, ...(error ? { error: { code: error.code } } : {}) }));
const base = { schemaVersion: 1 as const, matrixHash: matrixHash(matrix), scheduleHash: digest(JSON.stringify(schedule)), receipts: indexes.sort((left, right) => left.cell.id.localeCompare(right.cell.id)), summary: {
scheduled: matrix.cells.length, prepared: schedule.preparedCells, submitted: receipts.length, missing: matrix.cells.length - receipts.length,
scheduled: matrix.cells.length, prepared: schedule.preparedCells, submitted: bound.length, missing: matrix.cells.length - bound.length,
matchedPairsExpected: expectedMatchedPairs(matrix), matchedPairsSubmitted: completePairs.length, modelMismatches, groups,
note: 'Costs are pricing-table estimates. First-useful timing is unmeasured because the reused provider adapters return completed runs. Trusted findings and runtime outcomes require separate oracle adjudication.',
} };
@@ -785,7 +788,7 @@ function validateCollectedBatch(matrix: EvalMatrix, batch: ProducerBatch, observ
const cells = new Map(matrix.cells.map(cell => [cell.id, cell]));
const receipts = new Map<string, ProducerReceiptIndex>();
for (const receipt of batch.receipts) {
const cell = cells.get(receipt?.cell?.id);
const cell = cells.get(receipt?.cellRef);
if (!cell || receipts.has(cell.id)) throw new Error('INVALID_PRODUCER_BATCH');
validateReceiptIndex(receipt, cell); receipts.set(cell.id, receipt);
}
+2 -1
View File
@@ -22,7 +22,8 @@ const IMAGE_CONTEXT_EXCLUDED = new Set(['README.md', 'qualification.json', 'buil
export async function imageSourceFiles(root = ROOT): Promise<string[]> {
const build = await Bun.build({ entrypoints: IMAGE_HELPER_ENTRYPOINTS.map(entry => path.join(root, entry)), target: 'bun', metafile: true });
if (!build.success || !build.metafile) throw new Error('REQUALIFICATION_SOURCE_CLOSURE_FAILED');
const closure = Object.keys(build.metafile.inputs).map(input => path.relative(root, path.resolve(root, input)).split(path.sep).join('/'));
// Bun reports metafile inputs relative to the process working directory, not to the entrypoints.
const closure = Object.keys(build.metafile.inputs).map(input => path.relative(root, path.resolve(process.cwd(), input)).split(path.sep).join('/'));
if (closure.some(file => file.startsWith('..') || file.includes('node_modules/'))) throw new Error('REQUALIFICATION_SOURCE_OUTSIDE_REPOSITORY');
const context = fs.readdirSync(path.join(root, IMAGE_CONTEXT), { withFileTypes: true })
.filter(entry => entry.isFile() && !IMAGE_CONTEXT_EXCLUDED.has(entry.name))
+28 -3
View File
@@ -84,7 +84,7 @@ function syntheticArtifactInventory(): ProducerArtifactInventory {
/** Platform-pinned cells get one native installation and provider binary per platform, sharing catalogs and provider policy. */
function syntheticReceipt(cell: EvalCell): ProducerReceipt {
const seed = cell.platform ?? '1';
const withoutHash: Omit<ProducerReceipt, 'receiptHash'> = { schemaVersion: 1, cell, inputHash: 'e'.repeat(64), installationIdentity: syntheticInstallationIdentity(seed, '1'), providerIdentity: syntheticProviderIdentity(cell.host === 'codex' ? 'gpt' : cell.host, seed), artifacts: syntheticArtifactInventory(), startedAt: '2026-01-01T00:00:00.000Z', finishedAt: '2026-01-01T00:00:01.000Z', status: 'succeeded', requestedModel: cell.model, modelUsed: `resolved-${cell.model}`, modelIdentitySource: 'provider_reported', durationMs: 1000, firstUsefulResultMs: null, toolCalls: 1, output: 'synthetic producer transcript', outputHash: sha256('synthetic producer transcript'), usage: { inputTokens: 10, outputTokens: 5, cachedTokens: 0, estimatedCostUSD: 0.001 } };
const withoutHash: Omit<ProducerReceipt, 'receiptHash'> = { schemaVersion: 2, cellRef: cell.id, inputHash: 'e'.repeat(64), installationIdentity: syntheticInstallationIdentity(seed, '1'), providerIdentity: syntheticProviderIdentity(cell.host === 'codex' ? 'gpt' : cell.host, seed), artifacts: syntheticArtifactInventory(), startedAt: '2026-01-01T00:00:00.000Z', finishedAt: '2026-01-01T00:00:01.000Z', status: 'succeeded', requestedModel: cell.model, modelUsed: `resolved-${cell.model}`, modelIdentitySource: 'provider_reported', durationMs: 1000, firstUsefulResultMs: null, toolCalls: 1, output: 'synthetic producer transcript', outputHash: sha256('synthetic producer transcript'), usage: { inputTokens: 10, outputTokens: 5, cachedTokens: 0, estimatedCostUSD: 0.001 } };
return { ...withoutHash, receiptHash: producerReceiptHash(withoutHash) };
}
function syntheticSchedule(target: EvalMatrix = matrix): PreparedEvalSchedule {
@@ -243,6 +243,20 @@ describe('CSO matched evaluation accounting', () => {
mixedProvider.providerIdentity = syntheticProviderIdentity('gpt', 'different-provider');
expect(() => collectProducerReceipts(matrix, syntheticSchedule(), [first, rehashReceipt(mixedProvider)])).toThrow('UNMATCHED_PRODUCER_PROVIDERS');
});
test('receipts name only an opaque cellRef that the trusted matrix resolves', () => {
const cell = matrix.cells.find(item => item.version === 'v3')!, receipt = syntheticReceipt(cell);
expect(Object.keys(receipt)).not.toContain('cell'); expect(JSON.stringify(receipt)).not.toMatch(/caseId|variant|vulnerable|"fixed"/);
expect(JSON.stringify(receipt)).not.toContain(cell.caseId);
const unknown = structuredClone(receipt); unknown.cellRef = 'f'.repeat(64);
expect(() => collectProducerReceipts(matrix, syntheticSchedule(), [rehashReceipt(unknown)])).toThrow('UNKNOWN_OR_DUPLICATE_PRODUCER_RECEIPT');
const smuggled = rehashReceipt({ ...structuredClone(receipt), cell } as ProducerReceipt);
expect(() => collectProducerReceipts(matrix, syntheticSchedule(), [smuggled])).toThrow('INVALID_PRODUCER_RECEIPT');
const batch = syntheticBatch();
expect(batch.receipts.every(index => index.cellRef === index.cell.id && JSON.stringify(index.cell) === JSON.stringify(matrix.cells.find(item => item.id === index.cellRef)))).toBe(true);
const relabeled = structuredClone(batch); relabeled.receipts[0].cellRef = relabeled.receipts[1].cellRef;
const { batchHash: _hash, ...rest } = relabeled; relabeled.batchHash = createHash('sha256').update(JSON.stringify(rest)).digest('hex');
expect(() => scoreCollectedEval(matrix, relabeled, [], qualification)).toThrow('INVALID_PRODUCER_BATCH');
});
test('high/critical recall includes critical cases and detects a v3 regression',()=>{
const criticalCorpus=structuredClone(corpus);criticalCorpus.cases.find(item=>item.severity==='medium')!.severity='critical';
const criticalMatrix=createEvalMatrix({model:'matched-test-model',host:'codex',skillHashes:{v2:'a'.repeat(64),v3:'b'.repeat(64)}},criticalCorpus),complete=criticalMatrix.cells.map(syntheticResult);
@@ -550,7 +564,8 @@ describe('CSO matched producer orchestration', () => {
expect(existsSync(join(destination, 'jobs', cell.id, 'producer-input.json'))).toBe(true);
const input = JSON.parse(readFileSync(join(destination, 'jobs', cell.id, 'producer-input.json'), 'utf8'));
expect(input.skill).toBe(skills[cell.version]);
expect(input.cell.skillHash).toBe(sha256(input.skill));
expect(input.execution.skillHash).toBe(sha256(input.skill));
expect(input.cellRef).toBe(cell.id);
expect(input.skill).toContain(`${cell.version.toUpperCase()}_SECTION_ONLY`);
expect(input.skill).not.toContain(cell.version === 'v2' ? 'V3_SECTION_ONLY' : 'V2_SECTION_ONLY');
expect(validatePortableSkillPayload(input.skill, cell.version).files.map(file => file.path)).toEqual(['SKILL.md', 'sections/manifest.json', 'sections/audit-phases.md']);
@@ -953,12 +968,22 @@ describe('CSO matched producer orchestration', () => {
prepareEvalJobs(release, { v3: skills.v3 }, destination, [other.id, ...(native ? [native.id] : [])], privateCorpus);
const receipts = join(root(), 'receipts'); mkdirSync(receipts);
const isolatedOther = isolate(destination, other);
const opaque = readFileSync(isolatedOther.input, 'utf8'), parsed = JSON.parse(opaque);
expect(Object.keys(parsed).sort()).toEqual(['cellRef', 'execution', 'schemaVersion', 'skill', 'source']);
expect(parsed).toMatchObject({ schemaVersion: 2, cellRef: other.id, execution: { mode: 'comprehensive', platform: other.platform, sourceHash: other.sourceHash } });
expect(opaque).not.toMatch(/caseId|variant|vulnerable|"fixed"|"version"|repetition/); expect(opaque).not.toContain(other.caseId);
const smuggled = isolate(destination, other);
writeFileSync(smuggled.input, JSON.stringify({ ...parsed, caseId: other.caseId }));
await expect(runProducerCell(smuggled.input, join(receipts, `${other.id}.json`), withHelper({ adapter: new FakeAdapter(() => {}), paidExecutionAuthorized: true }))).rejects.toThrow('INVALID_PRODUCER_INPUT');
expect(readFileSync(join(isolatedOther.source, 'src/app.txt'), 'utf8')).toBe(`synthetic ${other.caseId} ${other.variant}\n`);
await expect(runProducerCell(isolatedOther.input, join(receipts, `${other.id}.json`), withHelper({ adapter: new FakeAdapter(() => {}), paidExecutionAuthorized: true }))).rejects.toThrow('UNMATCHED_PRODUCER_PLATFORM');
if (!native) return;
const isolatedNative = isolate(destination, native);
const receipt = await runProducerCell(isolatedNative.input, join(receipts, `${native.id}.json`), withHelper({ adapter: new FakeAdapter(() => {}), paidExecutionAuthorized: true }));
expect(receipt.cell).toEqual(native);
expect(receipt.cellRef).toBe(native.id);
const written = readFileSync(join(receipts, `${native.id}.json`), 'utf8');
expect(JSON.parse(written)).not.toHaveProperty('cell');
expect(written).not.toMatch(/caseId|variant|vulnerable|"fixed"/); expect(written).not.toContain(native.caseId);
});
test('seals source read-only and withholds a receipt after a mode/content mutation', async () => {
+12 -1
View File
@@ -1,5 +1,5 @@
import { afterEach, describe, expect, test } from 'bun:test';
import { cpSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { cpSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { spawnSync } from 'node:child_process';
import { tmpdir } from 'node:os';
import { join, resolve } from 'node:path';
@@ -35,6 +35,17 @@ describe('CSO runtime requalification triggers', () => {
expect((await requalificationTriggers(copy)).preparationSha256).not.toBe(current.preparationSha256);
});
test.skipIf(process.platform==='win32')('triggers resolve against the repository root from any working directory', async () => {
const elsewhere = mkdtempSync(join(tmpdir(), 'cso-requalification-cwd-')); temps.push(elsewhere);
const nested = join(elsewhere, 'a', 'b', 'c'); mkdirSync(nested, { recursive: true });
const cli = spawnSync(process.execPath, [join(ROOT, 'scripts/cso-requalification.ts'), 'triggers'], { cwd: nested, encoding: 'utf8', timeout: 30_000 });
expect(cli.status, cli.stderr).toBe(0); expect(JSON.parse(cli.stdout)).toEqual(current);
const copy = join(elsewhere, 'deeper', 'checkout'); mkdirSync(copy, { recursive: true });
cpSync(join(ROOT, 'lib'), join(copy, 'lib'), { recursive: true });
expect(await requalificationTriggers(copy)).toEqual(current);
expect(await imageSourceFiles(copy)).toEqual(await imageSourceFiles());
});
test('a promoted catalog names each trigger that changed after qualification', () => {
const catalog = completeRuntimeCatalogFixture();
const recorded = catalog.promotion!.requalification;
+6 -2
View File
@@ -127,8 +127,12 @@ gstack helper/runtime files. Do not install or copy any CSO `SKILL.md`, carved
section, generated skill tree, schedule, this repository checkout,
sibling jobs, fixed alternatives, or evaluator code to that filesystem. The
producer runner reads its opaque control file into memory and deletes it before
the agent process starts, so the case identifier and vulnerable/fixed label are
not agent inputs. Copy the selected hash-named directory as the literal path
the agent process starts. That file never names the case, variant, version, or
repetition: it is `{schemaVersion: 2, cellRef, skill, source, execution}`, where
`cellRef` is the matrix cell id and `execution` holds only the mode, model, host,
budget, source and skill hashes, and platform. The receipt binds `cellRef` and
`inputHash`, and `collect` rebuilds each cell from the matrix, so neither file
tells an agent with root on its cell VM which answer it is graded against. Copy the selected hash-named directory as the literal path
`/producer/job`; `/producer` must contain only `job`. Put the runner in a system
tool directory and use a separate receipt mount. The runner rejects the full
prepared batch layout, extra files beside `job`, a reused job with prior state,