mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-04 07:47:27 +01:00
fix(cso): opaque producer inputs and receipts; requalification triggers independent of cwd
Producer inputs are {schemaVersion: 2, cellRef, skill, source, execution} and
receipts bind cellRef + inputHash; collection rebuilds the cell from the
matrix, so a cell VM never learns its case or variant. requalificationTriggers
resolves Bun's cwd-relative metafile inputs against the repository root.
This commit is contained in:
1 parent
fa564ba771
commit
fe87890edb
7 files changed
+133
-51
No files matched your search
@@ -25,7 +25,7 @@ import {
|
||||
producerProviderIdentityHash,
|
||||
producerReceiptHash,
|
||||
sha256,
|
||||
type ProducerCell,
|
||||
type ProducerExecution,
|
||||
type ProducerHost,
|
||||
type ProducerInput,
|
||||
type ProducerArtifactInventory,
|
||||
@@ -199,9 +199,18 @@ function repositoryIdentity(root: string): string {
|
||||
return sha256(JSON.stringify({ config: sha256(config), head: read(['rev-parse', '--verify', 'HEAD']).trim(), branch: read(['symbolic-ref', '--short', 'HEAD']).trim(), status: read(['status', '--porcelain=v2', '--untracked-files=all']) }));
|
||||
}
|
||||
|
||||
function validateCell(cell: ProducerCell): void {
|
||||
if (!cell || !HEX.test(cell.id) || !cell.caseId || !['node', 'bun', 'python', 'rails'].includes(cell.stack) || !['vulnerable', 'fixed'].includes(cell.variant) || !['v2', 'v3'].includes(cell.version) || !['daily', 'comprehensive'].includes(cell.mode) || ![1, 2, 3].includes(cell.repetition) || !cell.model || !['claude', 'codex', 'gemini'].includes(cell.host) || !Number.isInteger(cell.budgetSeconds) || cell.budgetSeconds <= 60 || cell.budgetSeconds > 3600 || !HEX.test(cell.sourceHash) || !HEX.test(cell.skillHash) || (cell.platform !== undefined && !PRODUCER_PLATFORMS.includes(cell.platform))) throw new Error('INVALID_PRODUCER_INPUT');
|
||||
if (cell.platform !== undefined && cell.platform !== producerHostPlatform()) throw new Error('UNMATCHED_PRODUCER_PLATFORM');
|
||||
const INPUT_FIELDS = 'cellRef,execution,schemaVersion,skill,source';
|
||||
const EXECUTION_FIELDS = 'budgetSeconds,host,mode,model,skillHash,sourceHash';
|
||||
/** The input has exactly the opaque fields: no case, variant, version, or repetition can ride along. */
|
||||
function validateInput(input: ProducerInput): void {
|
||||
const execution = input?.execution;
|
||||
if (!input || typeof input !== 'object' || Object.keys(input).sort().join(',') !== INPUT_FIELDS || input.schemaVersion !== 2 || !HEX.test(input.cellRef) ||
|
||||
typeof input.skill !== 'string' || Buffer.byteLength(input.skill) > INPUT_LIMIT || !Array.isArray(input.source) ||
|
||||
!execution || typeof execution !== 'object' || Object.keys(execution).filter(key => key !== 'platform').sort().join(',') !== EXECUTION_FIELDS ||
|
||||
!['daily', 'comprehensive'].includes(execution.mode) || typeof execution.model !== 'string' || !execution.model || !['claude', 'codex', 'gemini'].includes(execution.host) ||
|
||||
!Number.isInteger(execution.budgetSeconds) || execution.budgetSeconds <= 60 || execution.budgetSeconds > 3600 || !HEX.test(execution.sourceHash) || !HEX.test(execution.skillHash) ||
|
||||
('platform' in execution && !PRODUCER_PLATFORMS.includes(execution.platform!))) throw new Error('INVALID_PRODUCER_INPUT');
|
||||
if (execution.platform !== undefined && execution.platform !== producerHostPlatform()) throw new Error('UNMATCHED_PRODUCER_PLATFORM');
|
||||
}
|
||||
|
||||
export interface ProducerHelperBinding {
|
||||
@@ -420,7 +429,7 @@ export function resolveProducerProviderIdentity(host: ProducerHost, stateRoot: s
|
||||
};
|
||||
}
|
||||
|
||||
function promptFor(cell: ProducerCell, skill: string, sourceRoot: string, helper: ProducerHelperBinding): string {
|
||||
function promptFor(cell: ProducerExecution, skill: string, sourceRoot: string, helper: ProducerHelperBinding): string {
|
||||
const invocation = `/cso${cell.mode === 'comprehensive' ? ' --comprehensive' : ''} --budget ${cell.budgetSeconds}`;
|
||||
const sourceAccess = cell.host === 'codex'
|
||||
? 'The Codex permission profile grants read-only access to exactly that immutable snapshot so the trusted helper can read it. Use only the trusted helper to inspect or act on source; direct provider file or shell access is outside this evaluation policy.'
|
||||
@@ -487,20 +496,20 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op
|
||||
const bytes = readBoundedStable(controlPath, INPUT_LIMIT, 'Producer input');
|
||||
let input: ProducerInput;
|
||||
try { input = JSON.parse(bytes.toString('utf8')); } catch { throw new Error('INVALID_PRODUCER_INPUT'); }
|
||||
if (input.schemaVersion !== 1 || typeof input.skill !== 'string' || Buffer.byteLength(input.skill) > INPUT_LIMIT || !Array.isArray(input.source)) throw new Error('INVALID_PRODUCER_INPUT');
|
||||
validateCell(input.cell);
|
||||
if (basename(receipt) !== `${input.cell.id}.json`) throw new Error('UNMATCHED_RECEIPT_DESTINATION');
|
||||
if (sha256(input.skill) !== input.cell.skillHash) throw new Error('INVALID_PRODUCER_SKILL');
|
||||
validateSource(sourceRoot, input.source, input.cell.sourceHash);
|
||||
validateInput(input);
|
||||
const cell = input.execution;
|
||||
if (basename(receipt) !== `${input.cellRef}.json`) throw new Error('UNMATCHED_RECEIPT_DESTINATION');
|
||||
if (sha256(input.skill) !== cell.skillHash) throw new Error('INVALID_PRODUCER_SKILL');
|
||||
validateSource(sourceRoot, input.source, cell.sourceHash);
|
||||
sealProducerSource(sourceRoot);
|
||||
validateSource(sourceRoot, input.source, input.cell.sourceHash);
|
||||
validateSource(sourceRoot, input.source, cell.sourceHash);
|
||||
const originalRepositoryIdentity = repositoryIdentity(sourceRoot);
|
||||
const stateRoot = join(jobRoot, 'state');
|
||||
const helperHome = join(stateRoot, 'cso-home');
|
||||
const helper = resolveProducerHelperBinding(sourceRoot, stateRoot, options.testHelperLauncherPath);
|
||||
const installationIdentity = producerInstallationIdentity(helper);
|
||||
const adapter = options.adapter ?? adapterFor(input.cell.host);
|
||||
if ((input.cell.host === 'codex' ? 'gpt' : input.cell.host) !== adapter.family) throw new Error('UNMATCHED_PRODUCER_ADAPTER');
|
||||
const adapter = options.adapter ?? adapterFor(cell.host);
|
||||
if ((cell.host === 'codex' ? 'gpt' : cell.host) !== adapter.family) throw new Error('UNMATCHED_PRODUCER_ADAPTER');
|
||||
fs.mkdirSync(stateRoot, { recursive: false, mode: 0o700 });
|
||||
fs.mkdirSync(helperHome, { recursive: false, mode: 0o700 });
|
||||
const provider = options.testProviderIdentity
|
||||
@@ -508,13 +517,13 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op
|
||||
identity: validateProviderIdentity(options.testProviderIdentity, adapter.family),
|
||||
command: options.testProviderCommand ?? { executable: process.execPath, argsPrefix: [] },
|
||||
}
|
||||
: resolveProducerProviderIdentity(input.cell.host, stateRoot);
|
||||
: resolveProducerProviderIdentity(cell.host, stateRoot);
|
||||
const providerIdentity = provider.identity;
|
||||
const runOptions = {
|
||||
prompt: promptFor(input.cell, input.skill, sourceRoot, helper),
|
||||
prompt: promptFor(cell, input.skill, sourceRoot, helper),
|
||||
workdir: stateRoot,
|
||||
timeoutMs: input.cell.budgetSeconds * 1000,
|
||||
model: input.cell.model,
|
||||
timeoutMs: cell.budgetSeconds * 1000,
|
||||
model: cell.model,
|
||||
csoProducer: {
|
||||
stateDirectory: stateRoot,
|
||||
sourceDirectory: sourceRoot,
|
||||
@@ -524,7 +533,7 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op
|
||||
},
|
||||
} satisfies RunOpts;
|
||||
const availability = await adapter.available(runOptions);
|
||||
if (!availability.ok) throw new Error(`PRODUCER_UNAVAILABLE: ${availability.reason ?? input.cell.host}`);
|
||||
if (!availability.ok) throw new Error(`PRODUCER_UNAVAILABLE: ${availability.reason ?? cell.host}`);
|
||||
const inputHash = producerInputHash(input);
|
||||
|
||||
// Load the opaque metadata into memory, then remove it before starting the
|
||||
@@ -539,11 +548,11 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op
|
||||
const startedAt = new Date().toISOString();
|
||||
let run: RunResult;
|
||||
try {
|
||||
if (input.cell.host === 'gemini') prepareGeminiProducerState(stateRoot, helper.launcher);
|
||||
if (cell.host === 'gemini') prepareGeminiProducerState(stateRoot, helper.launcher);
|
||||
run = await adapter.run(runOptions);
|
||||
} finally {
|
||||
try {
|
||||
if (input.cell.host === 'gemini') removeGeminiProducerState(stateRoot);
|
||||
if (cell.host === 'gemini') removeGeminiProducerState(stateRoot);
|
||||
} finally {
|
||||
if (previousHome === undefined) delete process.env.GSTACK_HOME; else process.env.GSTACK_HOME = previousHome;
|
||||
if (previousSessionKind === undefined) delete process.env.GSTACK_SESSION_KIND; else process.env.GSTACK_SESSION_KIND = previousSessionKind;
|
||||
@@ -560,7 +569,7 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op
|
||||
throw new Error('PRODUCER_PROVIDER_INSTALLATION_RACE');
|
||||
}
|
||||
assertProducerSourceSealed(sourceRoot);
|
||||
validateSource(sourceRoot, input.source, input.cell.sourceHash);
|
||||
validateSource(sourceRoot, input.source, cell.sourceHash);
|
||||
if (repositoryIdentity(sourceRoot) !== originalRepositoryIdentity) throw new Error('PRODUCER_CHANGED_SOURCE');
|
||||
const artifacts = inventoryProducerArtifacts(helperHome);
|
||||
run = sanitizeProducerRun(run);
|
||||
@@ -569,8 +578,8 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op
|
||||
const tokensReported = run.tokens.input > 0 || run.tokens.output > 0 || (run.tokens.cached ?? 0) > 0;
|
||||
const estimatedCostUSD = tokensReported && PRICING[run.modelUsed] ? adapter.estimateCost(run.tokens, run.modelUsed) : null;
|
||||
const withoutHash: Omit<ProducerReceipt, 'receiptHash'> = {
|
||||
schemaVersion: 1,
|
||||
cell: input.cell,
|
||||
schemaVersion: 2,
|
||||
cellRef: input.cellRef,
|
||||
inputHash,
|
||||
installationIdentity,
|
||||
providerIdentity,
|
||||
@@ -578,9 +587,9 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op
|
||||
startedAt,
|
||||
finishedAt,
|
||||
status: run.error ? 'failed' : 'succeeded',
|
||||
requestedModel: input.cell.model,
|
||||
requestedModel: cell.model,
|
||||
modelUsed: run.modelUsed,
|
||||
modelIdentitySource: run.modelUsed === input.cell.model ? 'requested_pin' : 'provider_reported',
|
||||
modelIdentitySource: run.modelUsed === cell.model ? 'requested_pin' : 'provider_reported',
|
||||
durationMs: run.durationMs,
|
||||
firstUsefulResultMs: null,
|
||||
toolCalls: run.toolCalls,
|
||||
@@ -603,7 +612,7 @@ async function cli(args: string[]): Promise<void> {
|
||||
if (args.length !== 4 || args[0] !== 'run' || args[3] !== '--execute-paid') throw new Error('Usage: cso-eval-producer run <consumable-input.json> <new-receipt.json> --execute-paid');
|
||||
if (process.env.CSO_EVAL_PAID !== '1') throw new Error('PAID_EXECUTION_NOT_AUTHORIZED: also set CSO_EVAL_PAID=1 on the isolated producer host');
|
||||
const receipt = await runProducerCell(args[1], args[2], { paidExecutionAuthorized: true });
|
||||
console.log(JSON.stringify({ cellId: receipt.cell.id, status: receipt.status, durationMs: receipt.durationMs, receiptHash: receipt.receiptHash }));
|
||||
console.log(JSON.stringify({ cellRef: receipt.cellRef, status: receipt.status, durationMs: receipt.durationMs, receiptHash: receipt.receiptHash }));
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
|
||||
@@ -31,16 +31,40 @@ export interface ProducerSourceEntry {
|
||||
bytes: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Only what a producer needs to run one cell. It never names the corpus case,
|
||||
* variant, version, or repetition: an agent that can read its own control file
|
||||
* or receipt still cannot tell which answer it is being graded against.
|
||||
*/
|
||||
export interface ProducerExecution {
|
||||
mode: ProducerMode;
|
||||
model: string;
|
||||
host: ProducerHost;
|
||||
budgetSeconds: number;
|
||||
sourceHash: string;
|
||||
skillHash: string;
|
||||
platform?: ProducerPlatform;
|
||||
}
|
||||
|
||||
/**
|
||||
* This control file is consumed before a producing agent starts. It must never
|
||||
* be placed inside the application repository or retained on the producer.
|
||||
*/
|
||||
export interface ProducerInput {
|
||||
schemaVersion: 1;
|
||||
cell: ProducerCell;
|
||||
schemaVersion: 2;
|
||||
/** The opaque matrix cell id; only the trusted evaluator maps it back to a cell. */
|
||||
cellRef: string;
|
||||
/** Exact canonical portable payload: root SKILL.md plus its manifest-listed sections. */
|
||||
skill: string;
|
||||
source: ProducerSourceEntry[];
|
||||
execution: ProducerExecution;
|
||||
}
|
||||
|
||||
export function producerExecution(cell: ProducerCell): ProducerExecution {
|
||||
return {
|
||||
mode: cell.mode, model: cell.model, host: cell.host, budgetSeconds: cell.budgetSeconds,
|
||||
sourceHash: cell.sourceHash, skillHash: cell.skillHash, ...(cell.platform === undefined ? {} : { platform: cell.platform }),
|
||||
};
|
||||
}
|
||||
|
||||
export interface ProducerArtifactIdentity {
|
||||
@@ -89,8 +113,9 @@ export interface ProducerArtifactInventory {
|
||||
}
|
||||
|
||||
export interface ProducerReceipt {
|
||||
schemaVersion: 1;
|
||||
cell: ProducerCell;
|
||||
schemaVersion: 2;
|
||||
/** The opaque cell id from the consumed input; bound together with inputHash. */
|
||||
cellRef: string;
|
||||
inputHash: string;
|
||||
installationIdentity: ProducerInstallationIdentity;
|
||||
providerIdentity: ProducerProviderIdentity;
|
||||
@@ -119,8 +144,12 @@ export interface ProducerReceipt {
|
||||
receiptHash: string;
|
||||
}
|
||||
|
||||
/** Compact trusted index; raw output remains in the separately retained receipt. */
|
||||
/**
|
||||
* Compact trusted index; raw output remains in the separately retained receipt.
|
||||
* Collection adds the matrix cell that the receipt's cellRef names.
|
||||
*/
|
||||
export type ProducerReceiptIndex = Omit<ProducerReceipt, 'output' | 'error'> & {
|
||||
cell: ProducerCell;
|
||||
error?: { code: string };
|
||||
};
|
||||
|
||||
|
||||
+16
-13
@@ -11,6 +11,7 @@ import { loadCorpusManifest, materializeCase, publicCorpus, STACKS, validateCorp
|
||||
import {
|
||||
PRODUCER_PLATFORMS,
|
||||
producerArtifactInventoryHash,
|
||||
producerExecution,
|
||||
producerInputHash,
|
||||
producerInstallationIdentityHash,
|
||||
producerProviderIdentityHash,
|
||||
@@ -358,7 +359,7 @@ export function prepareEvalJobs(matrix: EvalMatrix, payloads: Partial<Record<Eva
|
||||
const source = materializeCase(cell.caseId, cell.variant, join(jobRoot, 'source'), corpus);
|
||||
if (source.sourceHash !== cell.sourceHash) throw new Error('CORPUS_INTEGRITY_MISMATCH');
|
||||
initializeFixtureRepository(source.path);
|
||||
const input: ProducerInput = { schemaVersion: 1, cell, skill: payloads[cell.version]!, source: sourceEntries(corpus, cell.caseId, cell.variant) };
|
||||
const input: ProducerInput = { schemaVersion: 2, cellRef: cell.id, skill: payloads[cell.version]!, source: sourceEntries(corpus, cell.caseId, cell.variant), execution: producerExecution(cell) };
|
||||
const inputHash = producerInputHash(input);
|
||||
safeWriteNew(join(jobRoot, 'producer-input.json'), input);
|
||||
jobs.push({ cellId: cell.id, relativePath: `jobs/${cell.id}`, inputHash });
|
||||
@@ -379,7 +380,7 @@ function validateSchedule(matrix: EvalMatrix, schedule: PreparedEvalSchedule): v
|
||||
}
|
||||
|
||||
function validateReceipt(receipt: ProducerReceipt, cell: EvalCell, inputHash: string): void {
|
||||
if (receipt?.schemaVersion !== 1 || JSON.stringify(receipt.cell) !== JSON.stringify(cell) || receipt.inputHash !== inputHash || receipt.requestedModel !== cell.model || !['succeeded', 'failed'].includes(receipt.status) || typeof receipt.modelUsed !== 'string' || !receipt.modelUsed || !['provider_reported', 'requested_pin'].includes(receipt.modelIdentitySource) || (receipt.modelIdentitySource === 'requested_pin' && receipt.modelUsed !== receipt.requestedModel) || (receipt.modelIdentitySource === 'provider_reported' && receipt.modelUsed === receipt.requestedModel) || !validNumber(receipt.durationMs) || receipt.firstUsefulResultMs !== null || !Number.isSafeInteger(receipt.toolCalls) || receipt.toolCalls < 0 || typeof receipt.output !== 'string' || receipt.outputHash !== sha256(receipt.output) || !HEX.test(receipt.receiptHash)) throw new Error('INVALID_PRODUCER_RECEIPT');
|
||||
if (receipt?.schemaVersion !== 2 || receipt.cellRef !== cell.id || 'cell' in receipt || receipt.inputHash !== inputHash || receipt.requestedModel !== cell.model || !['succeeded', 'failed'].includes(receipt.status) || typeof receipt.modelUsed !== 'string' || !receipt.modelUsed || !['provider_reported', 'requested_pin'].includes(receipt.modelIdentitySource) || (receipt.modelIdentitySource === 'requested_pin' && receipt.modelUsed !== receipt.requestedModel) || (receipt.modelIdentitySource === 'provider_reported' && receipt.modelUsed === receipt.requestedModel) || !validNumber(receipt.durationMs) || receipt.firstUsefulResultMs !== null || !Number.isSafeInteger(receipt.toolCalls) || receipt.toolCalls < 0 || typeof receipt.output !== 'string' || receipt.outputHash !== sha256(receipt.output) || !HEX.test(receipt.receiptHash)) throw new Error('INVALID_PRODUCER_RECEIPT');
|
||||
const { receiptHash, ...withoutHash } = receipt;
|
||||
if (producerReceiptHash(withoutHash) !== receiptHash) throw new Error('INVALID_PRODUCER_RECEIPT');
|
||||
const started = Date.parse(receipt.startedAt), finished = Date.parse(receipt.finishedAt);
|
||||
@@ -424,7 +425,7 @@ function validateArtifactInventory(inventory: ProducerArtifactInventory, error:
|
||||
}
|
||||
|
||||
function validateReceiptIndex(receipt: ProducerReceiptIndex, cell: EvalCell): void {
|
||||
if (!receipt || receipt.schemaVersion !== 1 || JSON.stringify(receipt.cell) !== JSON.stringify(cell) || !HEX.test(receipt.inputHash) || receipt.requestedModel !== cell.model || !receipt.modelUsed || !['provider_reported', 'requested_pin'].includes(receipt.modelIdentitySource) || (receipt.modelIdentitySource === 'requested_pin' && receipt.modelUsed !== receipt.requestedModel) || (receipt.modelIdentitySource === 'provider_reported' && receipt.modelUsed === receipt.requestedModel) || !['succeeded', 'failed'].includes(receipt.status) || !HEX.test(receipt.outputHash) || !HEX.test(receipt.receiptHash) || !validNumber(receipt.durationMs) || receipt.firstUsefulResultMs !== null || !Number.isSafeInteger(receipt.toolCalls) || receipt.toolCalls < 0 || !receipt.usage) throw new Error('INVALID_PRODUCER_BATCH');
|
||||
if (!receipt || receipt.schemaVersion !== 2 || receipt.cellRef !== cell.id || JSON.stringify(receipt.cell) !== JSON.stringify(cell) || !HEX.test(receipt.inputHash) || receipt.requestedModel !== cell.model || !receipt.modelUsed || !['provider_reported', 'requested_pin'].includes(receipt.modelIdentitySource) || (receipt.modelIdentitySource === 'requested_pin' && receipt.modelUsed !== receipt.requestedModel) || (receipt.modelIdentitySource === 'provider_reported' && receipt.modelUsed === receipt.requestedModel) || !['succeeded', 'failed'].includes(receipt.status) || !HEX.test(receipt.outputHash) || !HEX.test(receipt.receiptHash) || !validNumber(receipt.durationMs) || receipt.firstUsefulResultMs !== null || !Number.isSafeInteger(receipt.toolCalls) || receipt.toolCalls < 0 || !receipt.usage) throw new Error('INVALID_PRODUCER_BATCH');
|
||||
const started = Date.parse(receipt.startedAt), finished = Date.parse(receipt.finishedAt);
|
||||
if (!Number.isFinite(started) || !Number.isFinite(finished) || finished < started || !['inputTokens', 'outputTokens', 'cachedTokens', 'estimatedCostUSD'].every(field => receipt.usage[field as keyof typeof receipt.usage] === null || validNumber(receipt.usage[field as keyof typeof receipt.usage])) || ((receipt.status === 'failed') !== !!receipt.error) || (receipt.error && !receipt.error.code)) throw new Error('INVALID_PRODUCER_BATCH');
|
||||
validateInstallationIdentity(receipt.installationIdentity, 'INVALID_PRODUCER_BATCH');
|
||||
@@ -473,24 +474,26 @@ export function collectProducerReceipts(matrix: EvalMatrix, schedule: PreparedEv
|
||||
const jobs = new Map(schedule.jobs.map(job => [job.cellId, job]));
|
||||
const seen = new Set<string>();
|
||||
for (const receipt of receipts) {
|
||||
const cell = cells.get(receipt?.cell?.id), job = jobs.get(receipt?.cell?.id);
|
||||
const cell = cells.get(receipt?.cellRef), job = jobs.get(receipt?.cellRef);
|
||||
if (!cell || !job || seen.has(cell.id)) throw new Error('UNKNOWN_OR_DUPLICATE_PRODUCER_RECEIPT');
|
||||
validateReceipt(receipt, cell, job.inputHash); seen.add(cell.id);
|
||||
}
|
||||
assertPlatformIdentities(receipts);
|
||||
const pairs = new Map<string, Partial<Record<EvalVersion, ProducerReceipt>>>();
|
||||
if (profileOf(matrix) === 'full') for (const receipt of receipts) {
|
||||
// Receipts name only the opaque cellRef; the trusted matrix supplies the cell.
|
||||
const bound = receipts.map(receipt => ({ ...receipt, cell: cells.get(receipt.cellRef)! }));
|
||||
assertPlatformIdentities(bound);
|
||||
const pairs = new Map<string, Partial<Record<EvalVersion, (typeof bound)[number]>>>();
|
||||
if (profileOf(matrix) === 'full') for (const receipt of bound) {
|
||||
const key = matchedPairKey(receipt.cell as EvalCell);
|
||||
const pair = pairs.get(key) ?? {}; pair[receipt.cell.version] = receipt; pairs.set(key, pair);
|
||||
}
|
||||
else if (new Set(receipts.map(receipt => receipt.modelUsed)).size > 1) throw new Error('UNMATCHED_EFFECTIVE_MODELS: single-version cells used different normalized model identities');
|
||||
const completePairs = [...pairs.entries()].filter(([, pair]) => pair.v2 && pair.v3) as Array<[string, { v2: ProducerReceipt; v3: ProducerReceipt }]>;
|
||||
else if (new Set(bound.map(receipt => receipt.modelUsed)).size > 1) throw new Error('UNMATCHED_EFFECTIVE_MODELS: single-version cells used different normalized model identities');
|
||||
const completePairs = [...pairs.entries()].filter(([, pair]) => pair.v2 && pair.v3) as Array<[string, { v2: (typeof bound)[number]; v3: (typeof bound)[number] }]>;
|
||||
const modelMismatches = completePairs.filter(([, pair]) => pair.v2.modelUsed !== pair.v3.modelUsed).map(([pair, value]) => ({ pair: digest(pair), v2: value.v2.modelUsed, v3: value.v3.modelUsed }));
|
||||
if (modelMismatches.length) throw new Error(`UNMATCHED_EFFECTIVE_MODELS: ${modelMismatches.length} matched v2/v3 pair(s) used different normalized model identities`);
|
||||
const groups: ProducerGroupSummary[] = [];
|
||||
for (const [version, mode] of PROFILE_GROUPS[profileOf(matrix)]) {
|
||||
const expected = matrix.cells.filter(cell => cell.version === version && cell.mode === mode);
|
||||
const submitted = receipts.filter(receipt => receipt.cell.version === version && receipt.cell.mode === mode);
|
||||
const submitted = bound.filter(receipt => receipt.cell.version === version && receipt.cell.mode === mode);
|
||||
const tokens = submitted.flatMap(receipt => receipt.usage.inputTokens === null || receipt.usage.outputTokens === null ? [] : [receipt.usage.inputTokens + receipt.usage.outputTokens]);
|
||||
const costs = submitted.flatMap(receipt => receipt.usage.estimatedCostUSD === null ? [] : [receipt.usage.estimatedCostUSD]);
|
||||
groups.push({ version, mode, scheduled: expected.length, submitted: submitted.length, missing: expected.length - submitted.length, succeeded: submitted.filter(receipt => receipt.status === 'succeeded').length, failed: submitted.filter(receipt => receipt.status === 'failed').length,
|
||||
@@ -500,9 +503,9 @@ export function collectProducerReceipts(matrix: EvalMatrix, schedule: PreparedEv
|
||||
estimatedCost: { measured: costs.length, denominator: expected.length, totalUSD: costs.length ? costs.reduce((sum, value) => sum + value, 0) : null, source: 'pricing-table-estimate' },
|
||||
});
|
||||
}
|
||||
const indexes: ProducerReceiptIndex[] = receipts.map(({ output: _output, error, ...receipt }) => ({ ...receipt, ...(error ? { error: { code: error.code } } : {}) }));
|
||||
const indexes: ProducerReceiptIndex[] = bound.map(({ output: _output, error, ...receipt }) => ({ ...receipt, ...(error ? { error: { code: error.code } } : {}) }));
|
||||
const base = { schemaVersion: 1 as const, matrixHash: matrixHash(matrix), scheduleHash: digest(JSON.stringify(schedule)), receipts: indexes.sort((left, right) => left.cell.id.localeCompare(right.cell.id)), summary: {
|
||||
scheduled: matrix.cells.length, prepared: schedule.preparedCells, submitted: receipts.length, missing: matrix.cells.length - receipts.length,
|
||||
scheduled: matrix.cells.length, prepared: schedule.preparedCells, submitted: bound.length, missing: matrix.cells.length - bound.length,
|
||||
matchedPairsExpected: expectedMatchedPairs(matrix), matchedPairsSubmitted: completePairs.length, modelMismatches, groups,
|
||||
note: 'Costs are pricing-table estimates. First-useful timing is unmeasured because the reused provider adapters return completed runs. Trusted findings and runtime outcomes require separate oracle adjudication.',
|
||||
} };
|
||||
@@ -785,7 +788,7 @@ function validateCollectedBatch(matrix: EvalMatrix, batch: ProducerBatch, observ
|
||||
const cells = new Map(matrix.cells.map(cell => [cell.id, cell]));
|
||||
const receipts = new Map<string, ProducerReceiptIndex>();
|
||||
for (const receipt of batch.receipts) {
|
||||
const cell = cells.get(receipt?.cell?.id);
|
||||
const cell = cells.get(receipt?.cellRef);
|
||||
if (!cell || receipts.has(cell.id)) throw new Error('INVALID_PRODUCER_BATCH');
|
||||
validateReceiptIndex(receipt, cell); receipts.set(cell.id, receipt);
|
||||
}
|
||||
|
||||
@@ -22,7 +22,8 @@ const IMAGE_CONTEXT_EXCLUDED = new Set(['README.md', 'qualification.json', 'buil
|
||||
export async function imageSourceFiles(root = ROOT): Promise<string[]> {
|
||||
const build = await Bun.build({ entrypoints: IMAGE_HELPER_ENTRYPOINTS.map(entry => path.join(root, entry)), target: 'bun', metafile: true });
|
||||
if (!build.success || !build.metafile) throw new Error('REQUALIFICATION_SOURCE_CLOSURE_FAILED');
|
||||
const closure = Object.keys(build.metafile.inputs).map(input => path.relative(root, path.resolve(root, input)).split(path.sep).join('/'));
|
||||
// Bun reports metafile inputs relative to the process working directory, not to the entrypoints.
|
||||
const closure = Object.keys(build.metafile.inputs).map(input => path.relative(root, path.resolve(process.cwd(), input)).split(path.sep).join('/'));
|
||||
if (closure.some(file => file.startsWith('..') || file.includes('node_modules/'))) throw new Error('REQUALIFICATION_SOURCE_OUTSIDE_REPOSITORY');
|
||||
const context = fs.readdirSync(path.join(root, IMAGE_CONTEXT), { withFileTypes: true })
|
||||
.filter(entry => entry.isFile() && !IMAGE_CONTEXT_EXCLUDED.has(entry.name))
|
||||
|
||||
+28
-3
@@ -84,7 +84,7 @@ function syntheticArtifactInventory(): ProducerArtifactInventory {
|
||||
/** Platform-pinned cells get one native installation and provider binary per platform, sharing catalogs and provider policy. */
|
||||
function syntheticReceipt(cell: EvalCell): ProducerReceipt {
|
||||
const seed = cell.platform ?? '1';
|
||||
const withoutHash: Omit<ProducerReceipt, 'receiptHash'> = { schemaVersion: 1, cell, inputHash: 'e'.repeat(64), installationIdentity: syntheticInstallationIdentity(seed, '1'), providerIdentity: syntheticProviderIdentity(cell.host === 'codex' ? 'gpt' : cell.host, seed), artifacts: syntheticArtifactInventory(), startedAt: '2026-01-01T00:00:00.000Z', finishedAt: '2026-01-01T00:00:01.000Z', status: 'succeeded', requestedModel: cell.model, modelUsed: `resolved-${cell.model}`, modelIdentitySource: 'provider_reported', durationMs: 1000, firstUsefulResultMs: null, toolCalls: 1, output: 'synthetic producer transcript', outputHash: sha256('synthetic producer transcript'), usage: { inputTokens: 10, outputTokens: 5, cachedTokens: 0, estimatedCostUSD: 0.001 } };
|
||||
const withoutHash: Omit<ProducerReceipt, 'receiptHash'> = { schemaVersion: 2, cellRef: cell.id, inputHash: 'e'.repeat(64), installationIdentity: syntheticInstallationIdentity(seed, '1'), providerIdentity: syntheticProviderIdentity(cell.host === 'codex' ? 'gpt' : cell.host, seed), artifacts: syntheticArtifactInventory(), startedAt: '2026-01-01T00:00:00.000Z', finishedAt: '2026-01-01T00:00:01.000Z', status: 'succeeded', requestedModel: cell.model, modelUsed: `resolved-${cell.model}`, modelIdentitySource: 'provider_reported', durationMs: 1000, firstUsefulResultMs: null, toolCalls: 1, output: 'synthetic producer transcript', outputHash: sha256('synthetic producer transcript'), usage: { inputTokens: 10, outputTokens: 5, cachedTokens: 0, estimatedCostUSD: 0.001 } };
|
||||
return { ...withoutHash, receiptHash: producerReceiptHash(withoutHash) };
|
||||
}
|
||||
function syntheticSchedule(target: EvalMatrix = matrix): PreparedEvalSchedule {
|
||||
@@ -243,6 +243,20 @@ describe('CSO matched evaluation accounting', () => {
|
||||
mixedProvider.providerIdentity = syntheticProviderIdentity('gpt', 'different-provider');
|
||||
expect(() => collectProducerReceipts(matrix, syntheticSchedule(), [first, rehashReceipt(mixedProvider)])).toThrow('UNMATCHED_PRODUCER_PROVIDERS');
|
||||
});
|
||||
test('receipts name only an opaque cellRef that the trusted matrix resolves', () => {
|
||||
const cell = matrix.cells.find(item => item.version === 'v3')!, receipt = syntheticReceipt(cell);
|
||||
expect(Object.keys(receipt)).not.toContain('cell'); expect(JSON.stringify(receipt)).not.toMatch(/caseId|variant|vulnerable|"fixed"/);
|
||||
expect(JSON.stringify(receipt)).not.toContain(cell.caseId);
|
||||
const unknown = structuredClone(receipt); unknown.cellRef = 'f'.repeat(64);
|
||||
expect(() => collectProducerReceipts(matrix, syntheticSchedule(), [rehashReceipt(unknown)])).toThrow('UNKNOWN_OR_DUPLICATE_PRODUCER_RECEIPT');
|
||||
const smuggled = rehashReceipt({ ...structuredClone(receipt), cell } as ProducerReceipt);
|
||||
expect(() => collectProducerReceipts(matrix, syntheticSchedule(), [smuggled])).toThrow('INVALID_PRODUCER_RECEIPT');
|
||||
const batch = syntheticBatch();
|
||||
expect(batch.receipts.every(index => index.cellRef === index.cell.id && JSON.stringify(index.cell) === JSON.stringify(matrix.cells.find(item => item.id === index.cellRef)))).toBe(true);
|
||||
const relabeled = structuredClone(batch); relabeled.receipts[0].cellRef = relabeled.receipts[1].cellRef;
|
||||
const { batchHash: _hash, ...rest } = relabeled; relabeled.batchHash = createHash('sha256').update(JSON.stringify(rest)).digest('hex');
|
||||
expect(() => scoreCollectedEval(matrix, relabeled, [], qualification)).toThrow('INVALID_PRODUCER_BATCH');
|
||||
});
|
||||
test('high/critical recall includes critical cases and detects a v3 regression',()=>{
|
||||
const criticalCorpus=structuredClone(corpus);criticalCorpus.cases.find(item=>item.severity==='medium')!.severity='critical';
|
||||
const criticalMatrix=createEvalMatrix({model:'matched-test-model',host:'codex',skillHashes:{v2:'a'.repeat(64),v3:'b'.repeat(64)}},criticalCorpus),complete=criticalMatrix.cells.map(syntheticResult);
|
||||
@@ -550,7 +564,8 @@ describe('CSO matched producer orchestration', () => {
|
||||
expect(existsSync(join(destination, 'jobs', cell.id, 'producer-input.json'))).toBe(true);
|
||||
const input = JSON.parse(readFileSync(join(destination, 'jobs', cell.id, 'producer-input.json'), 'utf8'));
|
||||
expect(input.skill).toBe(skills[cell.version]);
|
||||
expect(input.cell.skillHash).toBe(sha256(input.skill));
|
||||
expect(input.execution.skillHash).toBe(sha256(input.skill));
|
||||
expect(input.cellRef).toBe(cell.id);
|
||||
expect(input.skill).toContain(`${cell.version.toUpperCase()}_SECTION_ONLY`);
|
||||
expect(input.skill).not.toContain(cell.version === 'v2' ? 'V3_SECTION_ONLY' : 'V2_SECTION_ONLY');
|
||||
expect(validatePortableSkillPayload(input.skill, cell.version).files.map(file => file.path)).toEqual(['SKILL.md', 'sections/manifest.json', 'sections/audit-phases.md']);
|
||||
@@ -953,12 +968,22 @@ describe('CSO matched producer orchestration', () => {
|
||||
prepareEvalJobs(release, { v3: skills.v3 }, destination, [other.id, ...(native ? [native.id] : [])], privateCorpus);
|
||||
const receipts = join(root(), 'receipts'); mkdirSync(receipts);
|
||||
const isolatedOther = isolate(destination, other);
|
||||
const opaque = readFileSync(isolatedOther.input, 'utf8'), parsed = JSON.parse(opaque);
|
||||
expect(Object.keys(parsed).sort()).toEqual(['cellRef', 'execution', 'schemaVersion', 'skill', 'source']);
|
||||
expect(parsed).toMatchObject({ schemaVersion: 2, cellRef: other.id, execution: { mode: 'comprehensive', platform: other.platform, sourceHash: other.sourceHash } });
|
||||
expect(opaque).not.toMatch(/caseId|variant|vulnerable|"fixed"|"version"|repetition/); expect(opaque).not.toContain(other.caseId);
|
||||
const smuggled = isolate(destination, other);
|
||||
writeFileSync(smuggled.input, JSON.stringify({ ...parsed, caseId: other.caseId }));
|
||||
await expect(runProducerCell(smuggled.input, join(receipts, `${other.id}.json`), withHelper({ adapter: new FakeAdapter(() => {}), paidExecutionAuthorized: true }))).rejects.toThrow('INVALID_PRODUCER_INPUT');
|
||||
expect(readFileSync(join(isolatedOther.source, 'src/app.txt'), 'utf8')).toBe(`synthetic ${other.caseId} ${other.variant}\n`);
|
||||
await expect(runProducerCell(isolatedOther.input, join(receipts, `${other.id}.json`), withHelper({ adapter: new FakeAdapter(() => {}), paidExecutionAuthorized: true }))).rejects.toThrow('UNMATCHED_PRODUCER_PLATFORM');
|
||||
if (!native) return;
|
||||
const isolatedNative = isolate(destination, native);
|
||||
const receipt = await runProducerCell(isolatedNative.input, join(receipts, `${native.id}.json`), withHelper({ adapter: new FakeAdapter(() => {}), paidExecutionAuthorized: true }));
|
||||
expect(receipt.cell).toEqual(native);
|
||||
expect(receipt.cellRef).toBe(native.id);
|
||||
const written = readFileSync(join(receipts, `${native.id}.json`), 'utf8');
|
||||
expect(JSON.parse(written)).not.toHaveProperty('cell');
|
||||
expect(written).not.toMatch(/caseId|variant|vulnerable|"fixed"/); expect(written).not.toContain(native.caseId);
|
||||
});
|
||||
|
||||
test('seals source read-only and withholds a receipt after a mode/content mutation', async () => {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { afterEach, describe, expect, test } from 'bun:test';
|
||||
import { cpSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { cpSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join, resolve } from 'node:path';
|
||||
@@ -35,6 +35,17 @@ describe('CSO runtime requalification triggers', () => {
|
||||
expect((await requalificationTriggers(copy)).preparationSha256).not.toBe(current.preparationSha256);
|
||||
});
|
||||
|
||||
test.skipIf(process.platform==='win32')('triggers resolve against the repository root from any working directory', async () => {
|
||||
const elsewhere = mkdtempSync(join(tmpdir(), 'cso-requalification-cwd-')); temps.push(elsewhere);
|
||||
const nested = join(elsewhere, 'a', 'b', 'c'); mkdirSync(nested, { recursive: true });
|
||||
const cli = spawnSync(process.execPath, [join(ROOT, 'scripts/cso-requalification.ts'), 'triggers'], { cwd: nested, encoding: 'utf8', timeout: 30_000 });
|
||||
expect(cli.status, cli.stderr).toBe(0); expect(JSON.parse(cli.stdout)).toEqual(current);
|
||||
const copy = join(elsewhere, 'deeper', 'checkout'); mkdirSync(copy, { recursive: true });
|
||||
cpSync(join(ROOT, 'lib'), join(copy, 'lib'), { recursive: true });
|
||||
expect(await requalificationTriggers(copy)).toEqual(current);
|
||||
expect(await imageSourceFiles(copy)).toEqual(await imageSourceFiles());
|
||||
});
|
||||
|
||||
test('a promoted catalog names each trigger that changed after qualification', () => {
|
||||
const catalog = completeRuntimeCatalogFixture();
|
||||
const recorded = catalog.promotion!.requalification;
|
||||
|
||||
Vendored
+6
-2
@@ -127,8 +127,12 @@ gstack helper/runtime files. Do not install or copy any CSO `SKILL.md`, carved
|
||||
section, generated skill tree, schedule, this repository checkout,
|
||||
sibling jobs, fixed alternatives, or evaluator code to that filesystem. The
|
||||
producer runner reads its opaque control file into memory and deletes it before
|
||||
the agent process starts, so the case identifier and vulnerable/fixed label are
|
||||
not agent inputs. Copy the selected hash-named directory as the literal path
|
||||
the agent process starts. That file never names the case, variant, version, or
|
||||
repetition: it is `{schemaVersion: 2, cellRef, skill, source, execution}`, where
|
||||
`cellRef` is the matrix cell id and `execution` holds only the mode, model, host,
|
||||
budget, source and skill hashes, and platform. The receipt binds `cellRef` and
|
||||
`inputHash`, and `collect` rebuilds each cell from the matrix, so neither file
|
||||
tells an agent with root on its cell VM which answer it is graded against. Copy the selected hash-named directory as the literal path
|
||||
`/producer/job`; `/producer` must contain only `job`. Put the runner in a system
|
||||
tool directory and use a separate receipt mount. The runner rejects the full
|
||||
prepared batch layout, extra files beside `job`, a reused job with prior state,
|
||||
|
||||
Reference in new issue
Block a user