fix(cso): opaque producer inputs and receipts; requalification triggers independent of cwd

Producer inputs are {schemaVersion: 2, cellRef, skill, source, execution} and
receipts bind cellRef + inputHash; collection rebuilds the cell from the
matrix, so a cell VM never learns its case or variant. requalificationTriggers
resolves Bun's cwd-relative metafile inputs against the repository root.
This commit is contained in:
garrytan committed 2026-10-05 01:31:51 +00:00
1 parent fa564ba771
commit fe87890edb
7 files changed
+133 -51

No files matched your search

+28 -3
View File
@@ -84,7 +84,7 @@ function syntheticArtifactInventory(): ProducerArtifactInventory {
/** Platform-pinned cells get one native installation and provider binary per platform, sharing catalogs and provider policy. */
function syntheticReceipt(cell: EvalCell): ProducerReceipt {
const seed = cell.platform ?? '1';
const withoutHash: Omit<ProducerReceipt, 'receiptHash'> = { schemaVersion: 1, cell, inputHash: 'e'.repeat(64), installationIdentity: syntheticInstallationIdentity(seed, '1'), providerIdentity: syntheticProviderIdentity(cell.host === 'codex' ? 'gpt' : cell.host, seed), artifacts: syntheticArtifactInventory(), startedAt: '2026-01-01T00:00:00.000Z', finishedAt: '2026-01-01T00:00:01.000Z', status: 'succeeded', requestedModel: cell.model, modelUsed: `resolved-${cell.model}`, modelIdentitySource: 'provider_reported', durationMs: 1000, firstUsefulResultMs: null, toolCalls: 1, output: 'synthetic producer transcript', outputHash: sha256('synthetic producer transcript'), usage: { inputTokens: 10, outputTokens: 5, cachedTokens: 0, estimatedCostUSD: 0.001 } };
const withoutHash: Omit<ProducerReceipt, 'receiptHash'> = { schemaVersion: 2, cellRef: cell.id, inputHash: 'e'.repeat(64), installationIdentity: syntheticInstallationIdentity(seed, '1'), providerIdentity: syntheticProviderIdentity(cell.host === 'codex' ? 'gpt' : cell.host, seed), artifacts: syntheticArtifactInventory(), startedAt: '2026-01-01T00:00:00.000Z', finishedAt: '2026-01-01T00:00:01.000Z', status: 'succeeded', requestedModel: cell.model, modelUsed: `resolved-${cell.model}`, modelIdentitySource: 'provider_reported', durationMs: 1000, firstUsefulResultMs: null, toolCalls: 1, output: 'synthetic producer transcript', outputHash: sha256('synthetic producer transcript'), usage: { inputTokens: 10, outputTokens: 5, cachedTokens: 0, estimatedCostUSD: 0.001 } };
return { ...withoutHash, receiptHash: producerReceiptHash(withoutHash) };
}
function syntheticSchedule(target: EvalMatrix = matrix): PreparedEvalSchedule {
@@ -243,6 +243,20 @@ describe('CSO matched evaluation accounting', () => {
mixedProvider.providerIdentity = syntheticProviderIdentity('gpt', 'different-provider');
expect(() => collectProducerReceipts(matrix, syntheticSchedule(), [first, rehashReceipt(mixedProvider)])).toThrow('UNMATCHED_PRODUCER_PROVIDERS');
});
test('receipts name only an opaque cellRef that the trusted matrix resolves', () => {
const cell = matrix.cells.find(item => item.version === 'v3')!, receipt = syntheticReceipt(cell);
expect(Object.keys(receipt)).not.toContain('cell'); expect(JSON.stringify(receipt)).not.toMatch(/caseId|variant|vulnerable|"fixed"/);
expect(JSON.stringify(receipt)).not.toContain(cell.caseId);
const unknown = structuredClone(receipt); unknown.cellRef = 'f'.repeat(64);
expect(() => collectProducerReceipts(matrix, syntheticSchedule(), [rehashReceipt(unknown)])).toThrow('UNKNOWN_OR_DUPLICATE_PRODUCER_RECEIPT');
const smuggled = rehashReceipt({ ...structuredClone(receipt), cell } as ProducerReceipt);
expect(() => collectProducerReceipts(matrix, syntheticSchedule(), [smuggled])).toThrow('INVALID_PRODUCER_RECEIPT');
const batch = syntheticBatch();
expect(batch.receipts.every(index => index.cellRef === index.cell.id && JSON.stringify(index.cell) === JSON.stringify(matrix.cells.find(item => item.id === index.cellRef)))).toBe(true);
const relabeled = structuredClone(batch); relabeled.receipts[0].cellRef = relabeled.receipts[1].cellRef;
const { batchHash: _hash, ...rest } = relabeled; relabeled.batchHash = createHash('sha256').update(JSON.stringify(rest)).digest('hex');
expect(() => scoreCollectedEval(matrix, relabeled, [], qualification)).toThrow('INVALID_PRODUCER_BATCH');
});
test('high/critical recall includes critical cases and detects a v3 regression',()=>{
const criticalCorpus=structuredClone(corpus);criticalCorpus.cases.find(item=>item.severity==='medium')!.severity='critical';
const criticalMatrix=createEvalMatrix({model:'matched-test-model',host:'codex',skillHashes:{v2:'a'.repeat(64),v3:'b'.repeat(64)}},criticalCorpus),complete=criticalMatrix.cells.map(syntheticResult);
@@ -550,7 +564,8 @@ describe('CSO matched producer orchestration', () => {
expect(existsSync(join(destination, 'jobs', cell.id, 'producer-input.json'))).toBe(true);
const input = JSON.parse(readFileSync(join(destination, 'jobs', cell.id, 'producer-input.json'), 'utf8'));
expect(input.skill).toBe(skills[cell.version]);
expect(input.cell.skillHash).toBe(sha256(input.skill));
expect(input.execution.skillHash).toBe(sha256(input.skill));
expect(input.cellRef).toBe(cell.id);
expect(input.skill).toContain(`${cell.version.toUpperCase()}_SECTION_ONLY`);
expect(input.skill).not.toContain(cell.version === 'v2' ? 'V3_SECTION_ONLY' : 'V2_SECTION_ONLY');
expect(validatePortableSkillPayload(input.skill, cell.version).files.map(file => file.path)).toEqual(['SKILL.md', 'sections/manifest.json', 'sections/audit-phases.md']);
@@ -953,12 +968,22 @@ describe('CSO matched producer orchestration', () => {
prepareEvalJobs(release, { v3: skills.v3 }, destination, [other.id, ...(native ? [native.id] : [])], privateCorpus);
const receipts = join(root(), 'receipts'); mkdirSync(receipts);
const isolatedOther = isolate(destination, other);
const opaque = readFileSync(isolatedOther.input, 'utf8'), parsed = JSON.parse(opaque);
expect(Object.keys(parsed).sort()).toEqual(['cellRef', 'execution', 'schemaVersion', 'skill', 'source']);
expect(parsed).toMatchObject({ schemaVersion: 2, cellRef: other.id, execution: { mode: 'comprehensive', platform: other.platform, sourceHash: other.sourceHash } });
expect(opaque).not.toMatch(/caseId|variant|vulnerable|"fixed"|"version"|repetition/); expect(opaque).not.toContain(other.caseId);
const smuggled = isolate(destination, other);
writeFileSync(smuggled.input, JSON.stringify({ ...parsed, caseId: other.caseId }));
await expect(runProducerCell(smuggled.input, join(receipts, `${other.id}.json`), withHelper({ adapter: new FakeAdapter(() => {}), paidExecutionAuthorized: true }))).rejects.toThrow('INVALID_PRODUCER_INPUT');
expect(readFileSync(join(isolatedOther.source, 'src/app.txt'), 'utf8')).toBe(`synthetic ${other.caseId} ${other.variant}\n`);
await expect(runProducerCell(isolatedOther.input, join(receipts, `${other.id}.json`), withHelper({ adapter: new FakeAdapter(() => {}), paidExecutionAuthorized: true }))).rejects.toThrow('UNMATCHED_PRODUCER_PLATFORM');
if (!native) return;
const isolatedNative = isolate(destination, native);
const receipt = await runProducerCell(isolatedNative.input, join(receipts, `${native.id}.json`), withHelper({ adapter: new FakeAdapter(() => {}), paidExecutionAuthorized: true }));
expect(receipt.cell).toEqual(native);
expect(receipt.cellRef).toBe(native.id);
const written = readFileSync(join(receipts, `${native.id}.json`), 'utf8');
expect(JSON.parse(written)).not.toHaveProperty('cell');
expect(written).not.toMatch(/caseId|variant|vulnerable|"fixed"/); expect(written).not.toContain(native.caseId);
});
test('seals source read-only and withholds a receipt after a mode/content mutation', async () => {
+12 -1
View File
@@ -1,5 +1,5 @@
import { afterEach, describe, expect, test } from 'bun:test';
import { cpSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { cpSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { spawnSync } from 'node:child_process';
import { tmpdir } from 'node:os';
import { join, resolve } from 'node:path';
@@ -35,6 +35,17 @@ describe('CSO runtime requalification triggers', () => {
expect((await requalificationTriggers(copy)).preparationSha256).not.toBe(current.preparationSha256);
});
test.skipIf(process.platform==='win32')('triggers resolve against the repository root from any working directory', async () => {
const elsewhere = mkdtempSync(join(tmpdir(), 'cso-requalification-cwd-')); temps.push(elsewhere);
const nested = join(elsewhere, 'a', 'b', 'c'); mkdirSync(nested, { recursive: true });
const cli = spawnSync(process.execPath, [join(ROOT, 'scripts/cso-requalification.ts'), 'triggers'], { cwd: nested, encoding: 'utf8', timeout: 30_000 });
expect(cli.status, cli.stderr).toBe(0); expect(JSON.parse(cli.stdout)).toEqual(current);
const copy = join(elsewhere, 'deeper', 'checkout'); mkdirSync(copy, { recursive: true });
cpSync(join(ROOT, 'lib'), join(copy, 'lib'), { recursive: true });
expect(await requalificationTriggers(copy)).toEqual(current);
expect(await imageSourceFiles(copy)).toEqual(await imageSourceFiles());
});
test('a promoted catalog names each trigger that changed after qualification', () => {
const catalog = completeRuntimeCatalogFixture();
const recorded = catalog.promotion!.requalification;
+6 -2
View File
@@ -127,8 +127,12 @@ gstack helper/runtime files. Do not install or copy any CSO `SKILL.md`, carved
section, generated skill tree, schedule, this repository checkout,
sibling jobs, fixed alternatives, or evaluator code to that filesystem. The
producer runner reads its opaque control file into memory and deletes it before
the agent process starts, so the case identifier and vulnerable/fixed label are
not agent inputs. Copy the selected hash-named directory as the literal path
the agent process starts. That file never names the case, variant, version, or
repetition: it is `{schemaVersion: 2, cellRef, skill, source, execution}`, where
`cellRef` is the matrix cell id and `execution` holds only the mode, model, host,
budget, source and skill hashes, and platform. The receipt binds `cellRef` and
`inputHash`, and `collect` rebuilds each cell from the matrix, so neither file
tells an agent with root on its cell VM which answer it is graded against. Copy the selected hash-named directory as the literal path
`/producer/job`; `/producer` must contain only `job`. Put the runner in a system
tool directory and use a separate receipt mount. The runner rejects the full
prepared batch layout, extra files beside `job`, a reused job with prior state,