fix(cso): opaque producer inputs and receipts; requalification triggers independent of cwd

Producer inputs are {schemaVersion: 2, cellRef, skill, source, execution} and
receipts bind cellRef + inputHash; collection rebuilds the cell from the
matrix, so a cell VM never learns its case or variant. requalificationTriggers
resolves Bun's cwd-relative metafile inputs against the repository root.
This commit is contained in:
garrytan committed 2026-10-05 01:31:51 +00:00
1 parent fa564ba771
commit fe87890edb
7 files changed
+133 -51

No files matched your search

+35 -26
View File
@@ -25,7 +25,7 @@ import {
producerProviderIdentityHash,
producerReceiptHash,
sha256,
type ProducerCell,
type ProducerExecution,
type ProducerHost,
type ProducerInput,
type ProducerArtifactInventory,
@@ -199,9 +199,18 @@ function repositoryIdentity(root: string): string {
return sha256(JSON.stringify({ config: sha256(config), head: read(['rev-parse', '--verify', 'HEAD']).trim(), branch: read(['symbolic-ref', '--short', 'HEAD']).trim(), status: read(['status', '--porcelain=v2', '--untracked-files=all']) }));
}
function validateCell(cell: ProducerCell): void {
if (!cell || !HEX.test(cell.id) || !cell.caseId || !['node', 'bun', 'python', 'rails'].includes(cell.stack) || !['vulnerable', 'fixed'].includes(cell.variant) || !['v2', 'v3'].includes(cell.version) || !['daily', 'comprehensive'].includes(cell.mode) || ![1, 2, 3].includes(cell.repetition) || !cell.model || !['claude', 'codex', 'gemini'].includes(cell.host) || !Number.isInteger(cell.budgetSeconds) || cell.budgetSeconds <= 60 || cell.budgetSeconds > 3600 || !HEX.test(cell.sourceHash) || !HEX.test(cell.skillHash) || (cell.platform !== undefined && !PRODUCER_PLATFORMS.includes(cell.platform))) throw new Error('INVALID_PRODUCER_INPUT');
if (cell.platform !== undefined && cell.platform !== producerHostPlatform()) throw new Error('UNMATCHED_PRODUCER_PLATFORM');
const INPUT_FIELDS = 'cellRef,execution,schemaVersion,skill,source';
const EXECUTION_FIELDS = 'budgetSeconds,host,mode,model,skillHash,sourceHash';
/** The input has exactly the opaque fields: no case, variant, version, or repetition can ride along. */
function validateInput(input: ProducerInput): void {
const execution = input?.execution;
if (!input || typeof input !== 'object' || Object.keys(input).sort().join(',') !== INPUT_FIELDS || input.schemaVersion !== 2 || !HEX.test(input.cellRef) ||
typeof input.skill !== 'string' || Buffer.byteLength(input.skill) > INPUT_LIMIT || !Array.isArray(input.source) ||
!execution || typeof execution !== 'object' || Object.keys(execution).filter(key => key !== 'platform').sort().join(',') !== EXECUTION_FIELDS ||
!['daily', 'comprehensive'].includes(execution.mode) || typeof execution.model !== 'string' || !execution.model || !['claude', 'codex', 'gemini'].includes(execution.host) ||
!Number.isInteger(execution.budgetSeconds) || execution.budgetSeconds <= 60 || execution.budgetSeconds > 3600 || !HEX.test(execution.sourceHash) || !HEX.test(execution.skillHash) ||
('platform' in execution && !PRODUCER_PLATFORMS.includes(execution.platform!))) throw new Error('INVALID_PRODUCER_INPUT');
if (execution.platform !== undefined && execution.platform !== producerHostPlatform()) throw new Error('UNMATCHED_PRODUCER_PLATFORM');
}
export interface ProducerHelperBinding {
@@ -420,7 +429,7 @@ export function resolveProducerProviderIdentity(host: ProducerHost, stateRoot: s
};
}
function promptFor(cell: ProducerCell, skill: string, sourceRoot: string, helper: ProducerHelperBinding): string {
function promptFor(cell: ProducerExecution, skill: string, sourceRoot: string, helper: ProducerHelperBinding): string {
const invocation = `/cso${cell.mode === 'comprehensive' ? ' --comprehensive' : ''} --budget ${cell.budgetSeconds}`;
const sourceAccess = cell.host === 'codex'
? 'The Codex permission profile grants read-only access to exactly that immutable snapshot so the trusted helper can read it. Use only the trusted helper to inspect or act on source; direct provider file or shell access is outside this evaluation policy.'
@@ -487,20 +496,20 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op
const bytes = readBoundedStable(controlPath, INPUT_LIMIT, 'Producer input');
let input: ProducerInput;
try { input = JSON.parse(bytes.toString('utf8')); } catch { throw new Error('INVALID_PRODUCER_INPUT'); }
if (input.schemaVersion !== 1 || typeof input.skill !== 'string' || Buffer.byteLength(input.skill) > INPUT_LIMIT || !Array.isArray(input.source)) throw new Error('INVALID_PRODUCER_INPUT');
validateCell(input.cell);
if (basename(receipt) !== `${input.cell.id}.json`) throw new Error('UNMATCHED_RECEIPT_DESTINATION');
if (sha256(input.skill) !== input.cell.skillHash) throw new Error('INVALID_PRODUCER_SKILL');
validateSource(sourceRoot, input.source, input.cell.sourceHash);
validateInput(input);
const cell = input.execution;
if (basename(receipt) !== `${input.cellRef}.json`) throw new Error('UNMATCHED_RECEIPT_DESTINATION');
if (sha256(input.skill) !== cell.skillHash) throw new Error('INVALID_PRODUCER_SKILL');
validateSource(sourceRoot, input.source, cell.sourceHash);
sealProducerSource(sourceRoot);
validateSource(sourceRoot, input.source, input.cell.sourceHash);
validateSource(sourceRoot, input.source, cell.sourceHash);
const originalRepositoryIdentity = repositoryIdentity(sourceRoot);
const stateRoot = join(jobRoot, 'state');
const helperHome = join(stateRoot, 'cso-home');
const helper = resolveProducerHelperBinding(sourceRoot, stateRoot, options.testHelperLauncherPath);
const installationIdentity = producerInstallationIdentity(helper);
const adapter = options.adapter ?? adapterFor(input.cell.host);
if ((input.cell.host === 'codex' ? 'gpt' : input.cell.host) !== adapter.family) throw new Error('UNMATCHED_PRODUCER_ADAPTER');
const adapter = options.adapter ?? adapterFor(cell.host);
if ((cell.host === 'codex' ? 'gpt' : cell.host) !== adapter.family) throw new Error('UNMATCHED_PRODUCER_ADAPTER');
fs.mkdirSync(stateRoot, { recursive: false, mode: 0o700 });
fs.mkdirSync(helperHome, { recursive: false, mode: 0o700 });
const provider = options.testProviderIdentity
@@ -508,13 +517,13 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op
identity: validateProviderIdentity(options.testProviderIdentity, adapter.family),
command: options.testProviderCommand ?? { executable: process.execPath, argsPrefix: [] },
}
: resolveProducerProviderIdentity(input.cell.host, stateRoot);
: resolveProducerProviderIdentity(cell.host, stateRoot);
const providerIdentity = provider.identity;
const runOptions = {
prompt: promptFor(input.cell, input.skill, sourceRoot, helper),
prompt: promptFor(cell, input.skill, sourceRoot, helper),
workdir: stateRoot,
timeoutMs: input.cell.budgetSeconds * 1000,
model: input.cell.model,
timeoutMs: cell.budgetSeconds * 1000,
model: cell.model,
csoProducer: {
stateDirectory: stateRoot,
sourceDirectory: sourceRoot,
@@ -524,7 +533,7 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op
},
} satisfies RunOpts;
const availability = await adapter.available(runOptions);
if (!availability.ok) throw new Error(`PRODUCER_UNAVAILABLE: ${availability.reason ?? input.cell.host}`);
if (!availability.ok) throw new Error(`PRODUCER_UNAVAILABLE: ${availability.reason ?? cell.host}`);
const inputHash = producerInputHash(input);
// Load the opaque metadata into memory, then remove it before starting the
@@ -539,11 +548,11 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op
const startedAt = new Date().toISOString();
let run: RunResult;
try {
if (input.cell.host === 'gemini') prepareGeminiProducerState(stateRoot, helper.launcher);
if (cell.host === 'gemini') prepareGeminiProducerState(stateRoot, helper.launcher);
run = await adapter.run(runOptions);
} finally {
try {
if (input.cell.host === 'gemini') removeGeminiProducerState(stateRoot);
if (cell.host === 'gemini') removeGeminiProducerState(stateRoot);
} finally {
if (previousHome === undefined) delete process.env.GSTACK_HOME; else process.env.GSTACK_HOME = previousHome;
if (previousSessionKind === undefined) delete process.env.GSTACK_SESSION_KIND; else process.env.GSTACK_SESSION_KIND = previousSessionKind;
@@ -560,7 +569,7 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op
throw new Error('PRODUCER_PROVIDER_INSTALLATION_RACE');
}
assertProducerSourceSealed(sourceRoot);
validateSource(sourceRoot, input.source, input.cell.sourceHash);
validateSource(sourceRoot, input.source, cell.sourceHash);
if (repositoryIdentity(sourceRoot) !== originalRepositoryIdentity) throw new Error('PRODUCER_CHANGED_SOURCE');
const artifacts = inventoryProducerArtifacts(helperHome);
run = sanitizeProducerRun(run);
@@ -569,8 +578,8 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op
const tokensReported = run.tokens.input > 0 || run.tokens.output > 0 || (run.tokens.cached ?? 0) > 0;
const estimatedCostUSD = tokensReported && PRICING[run.modelUsed] ? adapter.estimateCost(run.tokens, run.modelUsed) : null;
const withoutHash: Omit<ProducerReceipt, 'receiptHash'> = {
schemaVersion: 1,
cell: input.cell,
schemaVersion: 2,
cellRef: input.cellRef,
inputHash,
installationIdentity,
providerIdentity,
@@ -578,9 +587,9 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op
startedAt,
finishedAt,
status: run.error ? 'failed' : 'succeeded',
requestedModel: input.cell.model,
requestedModel: cell.model,
modelUsed: run.modelUsed,
modelIdentitySource: run.modelUsed === input.cell.model ? 'requested_pin' : 'provider_reported',
modelIdentitySource: run.modelUsed === cell.model ? 'requested_pin' : 'provider_reported',
durationMs: run.durationMs,
firstUsefulResultMs: null,
toolCalls: run.toolCalls,
@@ -603,7 +612,7 @@ async function cli(args: string[]): Promise<void> {
if (args.length !== 4 || args[0] !== 'run' || args[3] !== '--execute-paid') throw new Error('Usage: cso-eval-producer run <consumable-input.json> <new-receipt.json> --execute-paid');
if (process.env.CSO_EVAL_PAID !== '1') throw new Error('PAID_EXECUTION_NOT_AUTHORIZED: also set CSO_EVAL_PAID=1 on the isolated producer host');
const receipt = await runProducerCell(args[1], args[2], { paidExecutionAuthorized: true });
console.log(JSON.stringify({ cellId: receipt.cell.id, status: receipt.status, durationMs: receipt.durationMs, receiptHash: receipt.receiptHash }));
console.log(JSON.stringify({ cellRef: receipt.cellRef, status: receipt.status, durationMs: receipt.durationMs, receiptHash: receipt.receiptHash }));
}
if (import.meta.main) {
+34 -5
View File
@@ -31,16 +31,40 @@ export interface ProducerSourceEntry {
bytes: number;
}
/**
* Only what a producer needs to run one cell. It never names the corpus case,
* variant, version, or repetition: an agent that can read its own control file
* or receipt still cannot tell which answer it is being graded against.
*/
export interface ProducerExecution {
mode: ProducerMode;
model: string;
host: ProducerHost;
budgetSeconds: number;
sourceHash: string;
skillHash: string;
platform?: ProducerPlatform;
}
/**
* This control file is consumed before a producing agent starts. It must never
* be placed inside the application repository or retained on the producer.
*/
export interface ProducerInput {
schemaVersion: 1;
cell: ProducerCell;
schemaVersion: 2;
/** The opaque matrix cell id; only the trusted evaluator maps it back to a cell. */
cellRef: string;
/** Exact canonical portable payload: root SKILL.md plus its manifest-listed sections. */
skill: string;
source: ProducerSourceEntry[];
execution: ProducerExecution;
}
export function producerExecution(cell: ProducerCell): ProducerExecution {
return {
mode: cell.mode, model: cell.model, host: cell.host, budgetSeconds: cell.budgetSeconds,
sourceHash: cell.sourceHash, skillHash: cell.skillHash, ...(cell.platform === undefined ? {} : { platform: cell.platform }),
};
}
export interface ProducerArtifactIdentity {
@@ -89,8 +113,9 @@ export interface ProducerArtifactInventory {
}
export interface ProducerReceipt {
schemaVersion: 1;
cell: ProducerCell;
schemaVersion: 2;
/** The opaque cell id from the consumed input; bound together with inputHash. */
cellRef: string;
inputHash: string;
installationIdentity: ProducerInstallationIdentity;
providerIdentity: ProducerProviderIdentity;
@@ -119,8 +144,12 @@ export interface ProducerReceipt {
receiptHash: string;
}
/** Compact trusted index; raw output remains in the separately retained receipt. */
/**
* Compact trusted index; raw output remains in the separately retained receipt.
* Collection adds the matrix cell that the receipt's cellRef names.
*/
export type ProducerReceiptIndex = Omit<ProducerReceipt, 'output' | 'error'> & {
cell: ProducerCell;
error?: { code: string };
};
+16 -13
View File
@@ -11,6 +11,7 @@ import { loadCorpusManifest, materializeCase, publicCorpus, STACKS, validateCorp
import {
PRODUCER_PLATFORMS,
producerArtifactInventoryHash,
producerExecution,
producerInputHash,
producerInstallationIdentityHash,
producerProviderIdentityHash,
@@ -358,7 +359,7 @@ export function prepareEvalJobs(matrix: EvalMatrix, payloads: Partial<Record<Eva
const source = materializeCase(cell.caseId, cell.variant, join(jobRoot, 'source'), corpus);
if (source.sourceHash !== cell.sourceHash) throw new Error('CORPUS_INTEGRITY_MISMATCH');
initializeFixtureRepository(source.path);
const input: ProducerInput = { schemaVersion: 1, cell, skill: payloads[cell.version]!, source: sourceEntries(corpus, cell.caseId, cell.variant) };
const input: ProducerInput = { schemaVersion: 2, cellRef: cell.id, skill: payloads[cell.version]!, source: sourceEntries(corpus, cell.caseId, cell.variant), execution: producerExecution(cell) };
const inputHash = producerInputHash(input);
safeWriteNew(join(jobRoot, 'producer-input.json'), input);
jobs.push({ cellId: cell.id, relativePath: `jobs/${cell.id}`, inputHash });
@@ -379,7 +380,7 @@ function validateSchedule(matrix: EvalMatrix, schedule: PreparedEvalSchedule): v
}
function validateReceipt(receipt: ProducerReceipt, cell: EvalCell, inputHash: string): void {
if (receipt?.schemaVersion !== 1 || JSON.stringify(receipt.cell) !== JSON.stringify(cell) || receipt.inputHash !== inputHash || receipt.requestedModel !== cell.model || !['succeeded', 'failed'].includes(receipt.status) || typeof receipt.modelUsed !== 'string' || !receipt.modelUsed || !['provider_reported', 'requested_pin'].includes(receipt.modelIdentitySource) || (receipt.modelIdentitySource === 'requested_pin' && receipt.modelUsed !== receipt.requestedModel) || (receipt.modelIdentitySource === 'provider_reported' && receipt.modelUsed === receipt.requestedModel) || !validNumber(receipt.durationMs) || receipt.firstUsefulResultMs !== null || !Number.isSafeInteger(receipt.toolCalls) || receipt.toolCalls < 0 || typeof receipt.output !== 'string' || receipt.outputHash !== sha256(receipt.output) || !HEX.test(receipt.receiptHash)) throw new Error('INVALID_PRODUCER_RECEIPT');
if (receipt?.schemaVersion !== 2 || receipt.cellRef !== cell.id || 'cell' in receipt || receipt.inputHash !== inputHash || receipt.requestedModel !== cell.model || !['succeeded', 'failed'].includes(receipt.status) || typeof receipt.modelUsed !== 'string' || !receipt.modelUsed || !['provider_reported', 'requested_pin'].includes(receipt.modelIdentitySource) || (receipt.modelIdentitySource === 'requested_pin' && receipt.modelUsed !== receipt.requestedModel) || (receipt.modelIdentitySource === 'provider_reported' && receipt.modelUsed === receipt.requestedModel) || !validNumber(receipt.durationMs) || receipt.firstUsefulResultMs !== null || !Number.isSafeInteger(receipt.toolCalls) || receipt.toolCalls < 0 || typeof receipt.output !== 'string' || receipt.outputHash !== sha256(receipt.output) || !HEX.test(receipt.receiptHash)) throw new Error('INVALID_PRODUCER_RECEIPT');
const { receiptHash, ...withoutHash } = receipt;
if (producerReceiptHash(withoutHash) !== receiptHash) throw new Error('INVALID_PRODUCER_RECEIPT');
const started = Date.parse(receipt.startedAt), finished = Date.parse(receipt.finishedAt);
@@ -424,7 +425,7 @@ function validateArtifactInventory(inventory: ProducerArtifactInventory, error:
}
function validateReceiptIndex(receipt: ProducerReceiptIndex, cell: EvalCell): void {
if (!receipt || receipt.schemaVersion !== 1 || JSON.stringify(receipt.cell) !== JSON.stringify(cell) || !HEX.test(receipt.inputHash) || receipt.requestedModel !== cell.model || !receipt.modelUsed || !['provider_reported', 'requested_pin'].includes(receipt.modelIdentitySource) || (receipt.modelIdentitySource === 'requested_pin' && receipt.modelUsed !== receipt.requestedModel) || (receipt.modelIdentitySource === 'provider_reported' && receipt.modelUsed === receipt.requestedModel) || !['succeeded', 'failed'].includes(receipt.status) || !HEX.test(receipt.outputHash) || !HEX.test(receipt.receiptHash) || !validNumber(receipt.durationMs) || receipt.firstUsefulResultMs !== null || !Number.isSafeInteger(receipt.toolCalls) || receipt.toolCalls < 0 || !receipt.usage) throw new Error('INVALID_PRODUCER_BATCH');
if (!receipt || receipt.schemaVersion !== 2 || receipt.cellRef !== cell.id || JSON.stringify(receipt.cell) !== JSON.stringify(cell) || !HEX.test(receipt.inputHash) || receipt.requestedModel !== cell.model || !receipt.modelUsed || !['provider_reported', 'requested_pin'].includes(receipt.modelIdentitySource) || (receipt.modelIdentitySource === 'requested_pin' && receipt.modelUsed !== receipt.requestedModel) || (receipt.modelIdentitySource === 'provider_reported' && receipt.modelUsed === receipt.requestedModel) || !['succeeded', 'failed'].includes(receipt.status) || !HEX.test(receipt.outputHash) || !HEX.test(receipt.receiptHash) || !validNumber(receipt.durationMs) || receipt.firstUsefulResultMs !== null || !Number.isSafeInteger(receipt.toolCalls) || receipt.toolCalls < 0 || !receipt.usage) throw new Error('INVALID_PRODUCER_BATCH');
const started = Date.parse(receipt.startedAt), finished = Date.parse(receipt.finishedAt);
if (!Number.isFinite(started) || !Number.isFinite(finished) || finished < started || !['inputTokens', 'outputTokens', 'cachedTokens', 'estimatedCostUSD'].every(field => receipt.usage[field as keyof typeof receipt.usage] === null || validNumber(receipt.usage[field as keyof typeof receipt.usage])) || ((receipt.status === 'failed') !== !!receipt.error) || (receipt.error && !receipt.error.code)) throw new Error('INVALID_PRODUCER_BATCH');
validateInstallationIdentity(receipt.installationIdentity, 'INVALID_PRODUCER_BATCH');
@@ -473,24 +474,26 @@ export function collectProducerReceipts(matrix: EvalMatrix, schedule: PreparedEv
const jobs = new Map(schedule.jobs.map(job => [job.cellId, job]));
const seen = new Set<string>();
for (const receipt of receipts) {
const cell = cells.get(receipt?.cell?.id), job = jobs.get(receipt?.cell?.id);
const cell = cells.get(receipt?.cellRef), job = jobs.get(receipt?.cellRef);
if (!cell || !job || seen.has(cell.id)) throw new Error('UNKNOWN_OR_DUPLICATE_PRODUCER_RECEIPT');
validateReceipt(receipt, cell, job.inputHash); seen.add(cell.id);
}
assertPlatformIdentities(receipts);
const pairs = new Map<string, Partial<Record<EvalVersion, ProducerReceipt>>>();
if (profileOf(matrix) === 'full') for (const receipt of receipts) {
// Receipts name only the opaque cellRef; the trusted matrix supplies the cell.
const bound = receipts.map(receipt => ({ ...receipt, cell: cells.get(receipt.cellRef)! }));
assertPlatformIdentities(bound);
const pairs = new Map<string, Partial<Record<EvalVersion, (typeof bound)[number]>>>();
if (profileOf(matrix) === 'full') for (const receipt of bound) {
const key = matchedPairKey(receipt.cell as EvalCell);
const pair = pairs.get(key) ?? {}; pair[receipt.cell.version] = receipt; pairs.set(key, pair);
}
else if (new Set(receipts.map(receipt => receipt.modelUsed)).size > 1) throw new Error('UNMATCHED_EFFECTIVE_MODELS: single-version cells used different normalized model identities');
const completePairs = [...pairs.entries()].filter(([, pair]) => pair.v2 && pair.v3) as Array<[string, { v2: ProducerReceipt; v3: ProducerReceipt }]>;
else if (new Set(bound.map(receipt => receipt.modelUsed)).size > 1) throw new Error('UNMATCHED_EFFECTIVE_MODELS: single-version cells used different normalized model identities');
const completePairs = [...pairs.entries()].filter(([, pair]) => pair.v2 && pair.v3) as Array<[string, { v2: (typeof bound)[number]; v3: (typeof bound)[number] }]>;
const modelMismatches = completePairs.filter(([, pair]) => pair.v2.modelUsed !== pair.v3.modelUsed).map(([pair, value]) => ({ pair: digest(pair), v2: value.v2.modelUsed, v3: value.v3.modelUsed }));
if (modelMismatches.length) throw new Error(`UNMATCHED_EFFECTIVE_MODELS: ${modelMismatches.length} matched v2/v3 pair(s) used different normalized model identities`);
const groups: ProducerGroupSummary[] = [];
for (const [version, mode] of PROFILE_GROUPS[profileOf(matrix)]) {
const expected = matrix.cells.filter(cell => cell.version === version && cell.mode === mode);
const submitted = receipts.filter(receipt => receipt.cell.version === version && receipt.cell.mode === mode);
const submitted = bound.filter(receipt => receipt.cell.version === version && receipt.cell.mode === mode);
const tokens = submitted.flatMap(receipt => receipt.usage.inputTokens === null || receipt.usage.outputTokens === null ? [] : [receipt.usage.inputTokens + receipt.usage.outputTokens]);
const costs = submitted.flatMap(receipt => receipt.usage.estimatedCostUSD === null ? [] : [receipt.usage.estimatedCostUSD]);
groups.push({ version, mode, scheduled: expected.length, submitted: submitted.length, missing: expected.length - submitted.length, succeeded: submitted.filter(receipt => receipt.status === 'succeeded').length, failed: submitted.filter(receipt => receipt.status === 'failed').length,
@@ -500,9 +503,9 @@ export function collectProducerReceipts(matrix: EvalMatrix, schedule: PreparedEv
estimatedCost: { measured: costs.length, denominator: expected.length, totalUSD: costs.length ? costs.reduce((sum, value) => sum + value, 0) : null, source: 'pricing-table-estimate' },
});
}
const indexes: ProducerReceiptIndex[] = receipts.map(({ output: _output, error, ...receipt }) => ({ ...receipt, ...(error ? { error: { code: error.code } } : {}) }));
const indexes: ProducerReceiptIndex[] = bound.map(({ output: _output, error, ...receipt }) => ({ ...receipt, ...(error ? { error: { code: error.code } } : {}) }));
const base = { schemaVersion: 1 as const, matrixHash: matrixHash(matrix), scheduleHash: digest(JSON.stringify(schedule)), receipts: indexes.sort((left, right) => left.cell.id.localeCompare(right.cell.id)), summary: {
scheduled: matrix.cells.length, prepared: schedule.preparedCells, submitted: receipts.length, missing: matrix.cells.length - receipts.length,
scheduled: matrix.cells.length, prepared: schedule.preparedCells, submitted: bound.length, missing: matrix.cells.length - bound.length,
matchedPairsExpected: expectedMatchedPairs(matrix), matchedPairsSubmitted: completePairs.length, modelMismatches, groups,
note: 'Costs are pricing-table estimates. First-useful timing is unmeasured because the reused provider adapters return completed runs. Trusted findings and runtime outcomes require separate oracle adjudication.',
} };
@@ -785,7 +788,7 @@ function validateCollectedBatch(matrix: EvalMatrix, batch: ProducerBatch, observ
const cells = new Map(matrix.cells.map(cell => [cell.id, cell]));
const receipts = new Map<string, ProducerReceiptIndex>();
for (const receipt of batch.receipts) {
const cell = cells.get(receipt?.cell?.id);
const cell = cells.get(receipt?.cellRef);
if (!cell || receipts.has(cell.id)) throw new Error('INVALID_PRODUCER_BATCH');
validateReceiptIndex(receipt, cell); receipts.set(cell.id, receipt);
}
+2 -1
View File
@@ -22,7 +22,8 @@ const IMAGE_CONTEXT_EXCLUDED = new Set(['README.md', 'qualification.json', 'buil
export async function imageSourceFiles(root = ROOT): Promise<string[]> {
const build = await Bun.build({ entrypoints: IMAGE_HELPER_ENTRYPOINTS.map(entry => path.join(root, entry)), target: 'bun', metafile: true });
if (!build.success || !build.metafile) throw new Error('REQUALIFICATION_SOURCE_CLOSURE_FAILED');
const closure = Object.keys(build.metafile.inputs).map(input => path.relative(root, path.resolve(root, input)).split(path.sep).join('/'));
// Bun reports metafile inputs relative to the process working directory, not to the entrypoints.
const closure = Object.keys(build.metafile.inputs).map(input => path.relative(root, path.resolve(process.cwd(), input)).split(path.sep).join('/'));
if (closure.some(file => file.startsWith('..') || file.includes('node_modules/'))) throw new Error('REQUALIFICATION_SOURCE_OUTSIDE_REPOSITORY');
const context = fs.readdirSync(path.join(root, IMAGE_CONTEXT), { withFileTypes: true })
.filter(entry => entry.isFile() && !IMAGE_CONTEXT_EXCLUDED.has(entry.name))