v1.91.69.0 fix(cso-eval): producers keep usage on timeout, give the agent reporting time, and accept the helper's own artifact names

- every helper artifact path tripped the secret redactor: the run ID's
  13-digit epoch reads as a phone number, some 32-hex IDs as wallets.
  Validate paths against the helper layout; schema-shaped ID components
  are accepted by shape, every other component still goes through redact
- the provider was killed at exactly --budget after cell start, before
  the helper deadline (start + budget) and before finish; give it the
  budget plus a 300 s reporting grace (evaluator graceSeconds)
- Claude producers stream events (stream-json --verbose
  --include-partial-messages) and add usage per message, so a timed-out
  receipt keeps tokens, cost, turns and the last text
- failed post-run checks name the check in the receipt reason

scripts/cso-eval-producer.ts

@@ -42,6 +42,16 @@ const OUTPUT_LIMIT = 32 * 1024 * 1024;
 const ARTIFACT_FILE_LIMIT = 32 * 1024 * 1024;
 const ARTIFACT_TOTAL_LIMIT = 128 * 1024 * 1024;
 const ARTIFACT_COUNT_LIMIT = 4096;
+/**
+ * `/cso --budget` bounds the helper run: it accepts evidence until its deadline (start + budget) and `finish` writes the
+ * report after it. A user's agent is never killed at the budget, so the provider gets the budget, the agent's time
+ * before `start`, and the reporting time after the deadline. Matches the evaluator's frozen `graceSeconds`.
+ */
+export const PRODUCER_REPORTING_GRACE_SECONDS = 300;
+
+export function producerProviderTimeoutMs(budgetSeconds: number): number {
+  return (budgetSeconds + PRODUCER_REPORTING_GRACE_SECONDS) * 1000;
+}
 
 function inside(parent: string, child: string): boolean {
   const path = relative(parent, child);
@@ -168,6 +178,34 @@ function emptyArtifactInventory(): ProducerArtifactInventory {
   return { ...base, identityHash: producerArtifactInventoryHash(base) };
 }
 
+/** A whole path component that is one of the helper's own identifiers: `<epoch ms>-<hex>` run and replay IDs, retired runs, and hex artifact IDs with an optional lowercase-word prefix (`gitleaks-<hex>-<hex>.json`). */
+const HELPER_ID_COMPONENT = /^(?:\d{13}-[a-f0-9]{16}|\.retired-\d{13}-[a-f0-9]{16}-[a-f0-9]{32}|(?:[a-z]+-)*[a-f0-9]{16,64}(?:-[a-f0-9]{16})*(?:\.[a-z]+)?)$/;
+
+/**
+ * Helper artifact paths are `<repoId>/<runId>/...` plus the helper's public-cache/ and legacy-imports/. Components that
+ * are wholly a helper identifier are checked by shape, because the secret redactor reads them as data (a 13-digit epoch
+ * as a phone number, some 32-hex IDs as wallets). Every other component (fixed helper names, and source paths mirrored
+ * under snapshot/ and readable/) still goes through the redactor.
+ */
+export function assertProducerArtifactPath(relativePath: string): void {
+  const parts = relativePath.split('/');
+  const reject = (rule: string): never => { throw new Error(`INVALID_PRODUCER_ARTIFACTS: ${rule}`); };
+  if (!relativePath || relativePath.length > 1024) reject('an artifact path is empty or longer than 1024 bytes');
+  if (!/^[A-Za-z0-9._/-]+$/.test(relativePath) || parts.some(part => !part || part === '.' || part === '..')) reject('an artifact path has a component outside [A-Za-z0-9._-]');
+  const inRun = /^[a-f0-9]{24}$/.test(parts[0]) && /^(?:\d{13}-[a-f0-9]{16}|\.retired-\d{13}-[a-f0-9]{16}-[a-f0-9]{32})$/.test(parts[1] ?? '');
+  if (parts.length < 2 || !(inRun ? parts.length >= 3 : ['public-cache', 'legacy-imports'].includes(parts[0]))) reject('an artifact lies outside <repoId>/<runId>/, public-cache/ and legacy-imports/');
+  const schemaId = (part: string, index: number) => index === 0 ? /^[a-f0-9]{24}$/.test(part) : HELPER_ID_COMPONENT.test(part);
+  for (const [index, part] of parts.entries()) {
+    if (schemaId(part, index)) continue;
+    let clean: boolean;
+    try { clean = redact(part) === part; } catch { clean = false; }
+    if (!clean) {
+      const parent = parts.slice(0, index).map((item, at) => schemaId(item, at) ? 'ID' : item).join('/');
+      throw new CsoError('REDACTION_FAILED', `artifact path component ${index + 1} under ${JSON.stringify(parent)} looks like a secret; the component is withheld`);
+    }
+  }
+}
+
 export function inventoryProducerArtifacts(helperHome: string): ProducerArtifactInventory {
   const security = join(helperHome, 'security'), artifactRoot = join(security, 'cso');
   for (const directory of [security, artifactRoot]) {
@@ -187,9 +225,7 @@ export function inventoryProducerArtifacts(helperHome: string): ProducerArtifact
       if (entry.isDirectory()) { visit(full, depth + 1); continue; }
       if (!entry.isFile() || entries.length >= ARTIFACT_COUNT_LIMIT) throw new Error('INVALID_PRODUCER_ARTIFACTS');
       const relativePath = relative(artifactRoot, full).split(sep).join('/');
-      if (!relativePath || relativePath.length > 1024 || !/^[A-Za-z0-9._/-]+$/.test(relativePath) || relativePath.split('/').some(part => !part || part === '.' || part === '..') || redact(relativePath) !== relativePath) {
-        throw new CsoError('REDACTION_FAILED', 'Producer artifact path withheld');
-      }
+      assertProducerArtifactPath(relativePath);
       const contents = readBoundedStable(full, ARTIFACT_FILE_LIMIT, 'Producer artifact');
       totalBytes += contents.byteLength;
       if (totalBytes > ARTIFACT_TOTAL_LIMIT) throw new Error('PRODUCER_ARTIFACTS_TOO_LARGE');
@@ -494,7 +530,11 @@ export function producerFailureMessage(error: unknown): string {
 
 /** A post-run check failed: the receipt records it as a failed cell with no output or artifacts, keeping the run's usage. */
 export function producerIntegrityFailure(error: unknown): { code: string; reason: string } {
-  if (error instanceof CsoError) return { code: error.code, reason: producerFailureMessage(error) };
+  if (error instanceof CsoError) {
+    let reason: string;
+    try { reason = redact(error.message); } catch { reason = 'detail withheld'; }
+    return { code: error.code, reason };
+  }
   const message = producerFailureMessage(error);
   const named = error instanceof Error && !(error as NodeJS.ErrnoException).code ? /^([A-Z][A-Z0-9_]+)(?::\s*([\s\S]*))?$/.exec(message) : null;
   return named ? { code: named[1], reason: named[2] || named[1] } : { code: 'PRODUCER_POST_RUN_CHECK_FAILED', reason: message };
@@ -544,7 +584,7 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op
   const runOptions = {
     prompt: promptFor(cell, input.skill, sourceRoot, helper),
     workdir: stateRoot,
-    timeoutMs: cell.budgetSeconds * 1000,
+    timeoutMs: producerProviderTimeoutMs(cell.budgetSeconds),
     model: cell.model,
     csoProducer: {
       stateDirectory: stateRoot,
@@ -584,23 +624,32 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op
   const finishedAt = new Date().toISOString();
   let artifacts = emptyArtifactInventory();
   let integrityFailure: { code: string; reason: string } | undefined;
+  let check = '';
   try {
+    check = 'helper installation';
     const installationAfter = producerInstallationIdentity(helper);
     if (installationAfter.identityHash !== installationIdentity.identityHash) {
       throw new Error('PRODUCER_HELPER_GENERATION_CHANGED: installed helper identity changed during the run');
     }
+    check = 'provider installation';
     const providerAfter = artifactIdentity(provider.command.executable);
     if (providerAfter.sha256 !== providerIdentity.executable.sha256 || providerAfter.bytes !== providerIdentity.executable.bytes) {
       throw new Error('PRODUCER_PROVIDER_INSTALLATION_RACE: provider executable changed during the run');
     }
+    check = 'source modes';
     assertProducerSourceSealed(sourceRoot);
+    check = 'source content';
     validateSource(sourceRoot, input.source, cell.sourceHash);
+    check = 'source Git state';
     if (repositoryIdentity(sourceRoot) !== originalRepositoryIdentity) throw new Error('PRODUCER_CHANGED_SOURCE: Git HEAD, branch, config or status changed during the run');
+    check = 'artifact inventory';
     artifacts = inventoryProducerArtifacts(helperHome);
+    check = 'output size';
     const outputBytes = typeof run.output === 'string' ? Buffer.byteLength(run.output) : 0;
     if (outputBytes > OUTPUT_LIMIT) throw new Error(`PRODUCER_OUTPUT_TOO_LARGE: output is ${outputBytes} bytes; the limit is ${OUTPUT_LIMIT}`);
   } catch (error) {
-    integrityFailure = producerIntegrityFailure(error);
+    const failure = producerIntegrityFailure(error);
+    integrityFailure = { code: failure.code, reason: `${check} check: ${failure.reason}${run.error ? `; the provider run also failed (${run.error.code})` : ''}` };
     artifacts = emptyArtifactInventory();
   }
   if (!integrityFailure) {