mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-13 03:48:06 +01:00
fix(cso): opaque producer inputs and receipts; requalification triggers independent of cwd
Producer inputs are {schemaVersion: 2, cellRef, skill, source, execution} and
receipts bind cellRef + inputHash; collection rebuilds the cell from the
matrix, so a cell VM never learns its case or variant. requalificationTriggers
resolves Bun's cwd-relative metafile inputs against the repository root.
This commit is contained in:
1 parent
fa564ba771
commit
fe87890edb
7 files changed
+133
-51
No files matched your search
+28
-3
@@ -84,7 +84,7 @@ function syntheticArtifactInventory(): ProducerArtifactInventory {
|
||||
/** Platform-pinned cells get one native installation and provider binary per platform, sharing catalogs and provider policy. */
|
||||
function syntheticReceipt(cell: EvalCell): ProducerReceipt {
|
||||
const seed = cell.platform ?? '1';
|
||||
const withoutHash: Omit<ProducerReceipt, 'receiptHash'> = { schemaVersion: 1, cell, inputHash: 'e'.repeat(64), installationIdentity: syntheticInstallationIdentity(seed, '1'), providerIdentity: syntheticProviderIdentity(cell.host === 'codex' ? 'gpt' : cell.host, seed), artifacts: syntheticArtifactInventory(), startedAt: '2026-01-01T00:00:00.000Z', finishedAt: '2026-01-01T00:00:01.000Z', status: 'succeeded', requestedModel: cell.model, modelUsed: `resolved-${cell.model}`, modelIdentitySource: 'provider_reported', durationMs: 1000, firstUsefulResultMs: null, toolCalls: 1, output: 'synthetic producer transcript', outputHash: sha256('synthetic producer transcript'), usage: { inputTokens: 10, outputTokens: 5, cachedTokens: 0, estimatedCostUSD: 0.001 } };
|
||||
const withoutHash: Omit<ProducerReceipt, 'receiptHash'> = { schemaVersion: 2, cellRef: cell.id, inputHash: 'e'.repeat(64), installationIdentity: syntheticInstallationIdentity(seed, '1'), providerIdentity: syntheticProviderIdentity(cell.host === 'codex' ? 'gpt' : cell.host, seed), artifacts: syntheticArtifactInventory(), startedAt: '2026-01-01T00:00:00.000Z', finishedAt: '2026-01-01T00:00:01.000Z', status: 'succeeded', requestedModel: cell.model, modelUsed: `resolved-${cell.model}`, modelIdentitySource: 'provider_reported', durationMs: 1000, firstUsefulResultMs: null, toolCalls: 1, output: 'synthetic producer transcript', outputHash: sha256('synthetic producer transcript'), usage: { inputTokens: 10, outputTokens: 5, cachedTokens: 0, estimatedCostUSD: 0.001 } };
|
||||
return { ...withoutHash, receiptHash: producerReceiptHash(withoutHash) };
|
||||
}
|
||||
function syntheticSchedule(target: EvalMatrix = matrix): PreparedEvalSchedule {
|
||||
@@ -243,6 +243,20 @@ describe('CSO matched evaluation accounting', () => {
|
||||
mixedProvider.providerIdentity = syntheticProviderIdentity('gpt', 'different-provider');
|
||||
expect(() => collectProducerReceipts(matrix, syntheticSchedule(), [first, rehashReceipt(mixedProvider)])).toThrow('UNMATCHED_PRODUCER_PROVIDERS');
|
||||
});
|
||||
test('receipts name only an opaque cellRef that the trusted matrix resolves', () => {
|
||||
const cell = matrix.cells.find(item => item.version === 'v3')!, receipt = syntheticReceipt(cell);
|
||||
expect(Object.keys(receipt)).not.toContain('cell'); expect(JSON.stringify(receipt)).not.toMatch(/caseId|variant|vulnerable|"fixed"/);
|
||||
expect(JSON.stringify(receipt)).not.toContain(cell.caseId);
|
||||
const unknown = structuredClone(receipt); unknown.cellRef = 'f'.repeat(64);
|
||||
expect(() => collectProducerReceipts(matrix, syntheticSchedule(), [rehashReceipt(unknown)])).toThrow('UNKNOWN_OR_DUPLICATE_PRODUCER_RECEIPT');
|
||||
const smuggled = rehashReceipt({ ...structuredClone(receipt), cell } as ProducerReceipt);
|
||||
expect(() => collectProducerReceipts(matrix, syntheticSchedule(), [smuggled])).toThrow('INVALID_PRODUCER_RECEIPT');
|
||||
const batch = syntheticBatch();
|
||||
expect(batch.receipts.every(index => index.cellRef === index.cell.id && JSON.stringify(index.cell) === JSON.stringify(matrix.cells.find(item => item.id === index.cellRef)))).toBe(true);
|
||||
const relabeled = structuredClone(batch); relabeled.receipts[0].cellRef = relabeled.receipts[1].cellRef;
|
||||
const { batchHash: _hash, ...rest } = relabeled; relabeled.batchHash = createHash('sha256').update(JSON.stringify(rest)).digest('hex');
|
||||
expect(() => scoreCollectedEval(matrix, relabeled, [], qualification)).toThrow('INVALID_PRODUCER_BATCH');
|
||||
});
|
||||
test('high/critical recall includes critical cases and detects a v3 regression',()=>{
|
||||
const criticalCorpus=structuredClone(corpus);criticalCorpus.cases.find(item=>item.severity==='medium')!.severity='critical';
|
||||
const criticalMatrix=createEvalMatrix({model:'matched-test-model',host:'codex',skillHashes:{v2:'a'.repeat(64),v3:'b'.repeat(64)}},criticalCorpus),complete=criticalMatrix.cells.map(syntheticResult);
|
||||
@@ -550,7 +564,8 @@ describe('CSO matched producer orchestration', () => {
|
||||
expect(existsSync(join(destination, 'jobs', cell.id, 'producer-input.json'))).toBe(true);
|
||||
const input = JSON.parse(readFileSync(join(destination, 'jobs', cell.id, 'producer-input.json'), 'utf8'));
|
||||
expect(input.skill).toBe(skills[cell.version]);
|
||||
expect(input.cell.skillHash).toBe(sha256(input.skill));
|
||||
expect(input.execution.skillHash).toBe(sha256(input.skill));
|
||||
expect(input.cellRef).toBe(cell.id);
|
||||
expect(input.skill).toContain(`${cell.version.toUpperCase()}_SECTION_ONLY`);
|
||||
expect(input.skill).not.toContain(cell.version === 'v2' ? 'V3_SECTION_ONLY' : 'V2_SECTION_ONLY');
|
||||
expect(validatePortableSkillPayload(input.skill, cell.version).files.map(file => file.path)).toEqual(['SKILL.md', 'sections/manifest.json', 'sections/audit-phases.md']);
|
||||
@@ -953,12 +968,22 @@ describe('CSO matched producer orchestration', () => {
|
||||
prepareEvalJobs(release, { v3: skills.v3 }, destination, [other.id, ...(native ? [native.id] : [])], privateCorpus);
|
||||
const receipts = join(root(), 'receipts'); mkdirSync(receipts);
|
||||
const isolatedOther = isolate(destination, other);
|
||||
const opaque = readFileSync(isolatedOther.input, 'utf8'), parsed = JSON.parse(opaque);
|
||||
expect(Object.keys(parsed).sort()).toEqual(['cellRef', 'execution', 'schemaVersion', 'skill', 'source']);
|
||||
expect(parsed).toMatchObject({ schemaVersion: 2, cellRef: other.id, execution: { mode: 'comprehensive', platform: other.platform, sourceHash: other.sourceHash } });
|
||||
expect(opaque).not.toMatch(/caseId|variant|vulnerable|"fixed"|"version"|repetition/); expect(opaque).not.toContain(other.caseId);
|
||||
const smuggled = isolate(destination, other);
|
||||
writeFileSync(smuggled.input, JSON.stringify({ ...parsed, caseId: other.caseId }));
|
||||
await expect(runProducerCell(smuggled.input, join(receipts, `${other.id}.json`), withHelper({ adapter: new FakeAdapter(() => {}), paidExecutionAuthorized: true }))).rejects.toThrow('INVALID_PRODUCER_INPUT');
|
||||
expect(readFileSync(join(isolatedOther.source, 'src/app.txt'), 'utf8')).toBe(`synthetic ${other.caseId} ${other.variant}\n`);
|
||||
await expect(runProducerCell(isolatedOther.input, join(receipts, `${other.id}.json`), withHelper({ adapter: new FakeAdapter(() => {}), paidExecutionAuthorized: true }))).rejects.toThrow('UNMATCHED_PRODUCER_PLATFORM');
|
||||
if (!native) return;
|
||||
const isolatedNative = isolate(destination, native);
|
||||
const receipt = await runProducerCell(isolatedNative.input, join(receipts, `${native.id}.json`), withHelper({ adapter: new FakeAdapter(() => {}), paidExecutionAuthorized: true }));
|
||||
expect(receipt.cell).toEqual(native);
|
||||
expect(receipt.cellRef).toBe(native.id);
|
||||
const written = readFileSync(join(receipts, `${native.id}.json`), 'utf8');
|
||||
expect(JSON.parse(written)).not.toHaveProperty('cell');
|
||||
expect(written).not.toMatch(/caseId|variant|vulnerable|"fixed"/); expect(written).not.toContain(native.caseId);
|
||||
});
|
||||
|
||||
test('seals source read-only and withholds a receipt after a mode/content mutation', async () => {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { afterEach, describe, expect, test } from 'bun:test';
|
||||
import { cpSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { cpSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join, resolve } from 'node:path';
|
||||
@@ -35,6 +35,17 @@ describe('CSO runtime requalification triggers', () => {
|
||||
expect((await requalificationTriggers(copy)).preparationSha256).not.toBe(current.preparationSha256);
|
||||
});
|
||||
|
||||
test.skipIf(process.platform==='win32')('triggers resolve against the repository root from any working directory', async () => {
|
||||
const elsewhere = mkdtempSync(join(tmpdir(), 'cso-requalification-cwd-')); temps.push(elsewhere);
|
||||
const nested = join(elsewhere, 'a', 'b', 'c'); mkdirSync(nested, { recursive: true });
|
||||
const cli = spawnSync(process.execPath, [join(ROOT, 'scripts/cso-requalification.ts'), 'triggers'], { cwd: nested, encoding: 'utf8', timeout: 30_000 });
|
||||
expect(cli.status, cli.stderr).toBe(0); expect(JSON.parse(cli.stdout)).toEqual(current);
|
||||
const copy = join(elsewhere, 'deeper', 'checkout'); mkdirSync(copy, { recursive: true });
|
||||
cpSync(join(ROOT, 'lib'), join(copy, 'lib'), { recursive: true });
|
||||
expect(await requalificationTriggers(copy)).toEqual(current);
|
||||
expect(await imageSourceFiles(copy)).toEqual(await imageSourceFiles());
|
||||
});
|
||||
|
||||
test('a promoted catalog names each trigger that changed after qualification', () => {
|
||||
const catalog = completeRuntimeCatalogFixture();
|
||||
const recorded = catalog.promotion!.requalification;
|
||||
|
||||
Vendored
+6
-2
@@ -127,8 +127,12 @@ gstack helper/runtime files. Do not install or copy any CSO `SKILL.md`, carved
|
||||
section, generated skill tree, schedule, this repository checkout,
|
||||
sibling jobs, fixed alternatives, or evaluator code to that filesystem. The
|
||||
producer runner reads its opaque control file into memory and deletes it before
|
||||
the agent process starts, so the case identifier and vulnerable/fixed label are
|
||||
not agent inputs. Copy the selected hash-named directory as the literal path
|
||||
the agent process starts. That file never names the case, variant, version, or
|
||||
repetition: it is `{schemaVersion: 2, cellRef, skill, source, execution}`, where
|
||||
`cellRef` is the matrix cell id and `execution` holds only the mode, model, host,
|
||||
budget, source and skill hashes, and platform. The receipt binds `cellRef` and
|
||||
`inputHash`, and `collect` rebuilds each cell from the matrix, so neither file
|
||||
tells an agent with root on its cell VM which answer it is graded against. Copy the selected hash-named directory as the literal path
|
||||
`/producer/job`; `/producer` must contain only `job`. Put the runner in a system
|
||||
tool directory and use a separate receipt mount. The runner rejects the full
|
||||
prepared batch layout, extra files beside `job`, a reused job with prior state,
|
||||
|
||||
Reference in new issue
Block a user