diff --git a/CHANGELOG.md b/CHANGELOG.md index 068cd1145..ca85c4cca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Changelog +## [1.91.75.0] - 2026-10-11 + +**CSO paid producers no longer fail a successful audit because the final summary quoted the helper's own run ID. Credential-class text still withholds the output and fails the receipt, and the reason now names the matching redactor rules and counts.** + +In the first paid baseline both smoke cells finished their audits, then the producer marked each receipt `failed` with `Sensitive producer output or error withheld`. The producer treated any redactor match as a leak. A CSO run ID such as `<13-digit epoch>-` matches `pii.phone.e164`, and some also match `pii.cc`. The fixtures' metadata address and a `PRIVILEGED` label also match. Replaying a mock provider through the installed producer, a summary of just "complete — assessed source in run ." failed on 1.91.73.0 and succeeds now. + +### What this means for you + +- **Rule-named failures.** A credential-class match (the redactor's `secret` category) in the output, the error, or either concatenation still replaces the output with `[sensitive producer output redacted]` and fails the receipt. The reason is now `Sensitive producer output or error withheld: credential-class rule ×, …` and never includes the matched text. +- **Source canaries quoted bare.** Values assigned to credential-shaped names in the cell's verified source (the `env.kv` rule), such as a planted canary, are matched exactly, including across output and error. They fail as `source.env.kv`, because no pattern recognises such a value once it is quoted without its name. +- **Helper identifiers survive.** Run and replay IDs, `cso-eval--` catalog revisions, `--` artifact IDs and hex finding, review and content IDs are set aside before the PII, internal and legal rules run. Those rules now replace their spans in place (`>`) and no longer change the receipt status. Scoring reads the helper's report, and the evaluator still scans outputs for its canaries. + +### Itemized changes + +#### Changed +- `scripts/cso-eval-output-redaction.ts` (new): `redactProducerText` and `sourceCredentialValues`. `sanitizeProducerRun` in `scripts/cso-eval-producer.ts` uses them; the source values are collected once after the source is sealed and validated. + +#### For contributors +- `test/cso-eval.test.ts`: helper IDs kept with PII spans redacted in place; env-line and Anthropic-key outputs fail with their rule names; a source canary is withheld whole or split across output and error. The existing canary test now expects the rule-named reason. + ## [1.91.73.0] - 2026-10-11 **"Ship, next in queue at " with zero manual stamp edits: a committed repo ship policy, `gstack-restamp` that stamps a PR only when it is next to merge (and re-stamps after a reorder in one step), a tree receipt that proves the only change since the gate was the stamp, a portable evidence bundle, and a ship receipt in the PR body.** diff --git a/VERSION b/VERSION index 7eb161545..023afaf52 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.91.73.0 +1.91.75.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 6e459583a..e0c372eb9 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.91.73.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.91.75.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/package.json b/package.json index c574b428a..4fb506e3a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.91.73", + "version": "1.91.75", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/scripts/cso-eval-output-redaction.ts b/scripts/cso-eval-output-redaction.ts new file mode 100644 index 000000000..6f5222233 --- /dev/null +++ b/scripts/cso-eval-output-redaction.ts @@ -0,0 +1,69 @@ +/** Redaction of the producer's provider output and error text before a receipt binds them. */ +import { join } from 'node:path'; +import { readBoundedStable } from '../lib/cso/bounded-file'; +import { CsoError, MAX_OUTPUT } from '../lib/cso/contracts'; +import { redactFindingSpans, scan } from '../lib/redact-engine'; + +/** + * Identifiers the CSO helper emits by schema, which the redactor's PII shapes misread: `-` run and + * replay IDs and the 13-digit epoch in them (pii.phone.e164), `cso-eval--` catalog revisions, + * `--` artifact IDs, and hex finding, review and content IDs (pii.wallet). + */ +const HELPER_ID = /(? { + const counts = new Map(); + for (const id of ids) counts.set(id, (counts.get(id) ?? 0) + 1); + return [...counts].sort(([left], [right]) => left < right ? -1 : 1).map(([id, count]) => `${id}×${count}`); +}; + +const ASSIGNMENT = /^\s*(?:export\s+)?[A-Za-z_][\w.-]*\s*[=:]\s*["']?([^\s"'#]{8,})/; + +/** + * Values assigned to credential-shaped names (the env.kv rule) in the cell's verified source, such as a planted + * canary. Quoted without its name, such a value reads as prose to every rule, so the output check matches it exactly. + */ +export function sourceCredentialValues(root: string, paths: string[]): string[] { + const values = new Set(); + for (const path of paths) { + if (path === '.git' || path.startsWith('.git/')) continue; + const contents = readBoundedStable(join(root, ...path.split('/')), 2 * 1024 * 1024, 'Producer source file'); + if (contents.includes(0)) continue; + const text = contents.toString('utf8'), lines = text.split('\n'); + for (const finding of scan(text, { maxBytes: 2 * 1024 * 1024 }).findings) { + const value = finding.id === 'env.kv' ? ASSIGNMENT.exec(lines[finding.line - 1] ?? '')?.[1] : undefined; + if (value) values.add(value); + } + } + return [...values]; +} + +/** + * Credential classes are judged on the unmasked text, so masking can never hide one. Only when none matched are the + * helper's identifiers set aside and the remaining PII, internal, legal and hygiene spans replaced in place. + */ +export function redactProducerText(value: string, sourceCredentials: string[] = []): OutputRedaction { + const findings = scan(value, { maxBytes: MAX_OUTPUT }); + if (findings.oversize) throw new CsoError('REDACTION_FAILED', 'Producer output withheld because it exceeds the redaction limit'); + const secretIds = findings.findings.filter(finding => finding.category === 'secret').map(finding => finding.id); + for (const credential of sourceCredentials) secretIds.push(...Array(value.split(credential).length - 1).fill('source.env.kv')); + if (secretIds.length) return { text: '', rules: tally(secretIds), secret: true }; + const identifiers: string[] = []; + const letters = (index: number): string => (index >= 26 ? letters(Math.floor(index / 26) - 1) : '') + String.fromCharCode(97 + (index % 26)); + const masked = value.replace(HELPER_ID, match => `GSTACKHELPERID${letters(identifiers.push(match) - 1)}Z`); + const masking = scan(masked, { maxBytes: MAX_OUTPUT }).findings; + const redacted = redactFindingSpans(masked, { maxBytes: MAX_OUTPUT }); + if (redacted === null) throw new CsoError('REDACTION_FAILED', 'Producer output withheld because redaction could not safely locate every span'); + const indexOf = (code: string): number => [...code].reduce((total, char) => total * 26 + char.charCodeAt(0) - 96, 0) - 1; + const text = redacted.replace(/GSTACKHELPERID([a-z]+)Z/g, (whole, code: string) => identifiers[indexOf(code)] ?? whole); + return { text, rules: tally(masking.map(finding => finding.id)), secret: false }; +} diff --git a/scripts/cso-eval-producer.ts b/scripts/cso-eval-producer.ts index 380c319ca..922616405 100644 --- a/scripts/cso-eval-producer.ts +++ b/scripts/cso-eval-producer.ts @@ -8,6 +8,7 @@ import { readBoundedStable } from '../lib/cso/bounded-file'; import { CsoError } from '../lib/cso/contracts'; import { executable, redact } from '../lib/cso/process'; import { atomicWriteSync } from '../lib/fs-atomic'; +import { redactProducerText, sourceCredentialValues, type OutputRedaction } from './cso-eval-output-redaction'; import { resolveClaudeCommand } from '../lib/claude-bin'; import runtimeCatalog from '../lib/cso/runtime-catalog.json'; import scannerCatalog from '../lib/cso/scanner-images/catalog.json'; @@ -500,25 +501,34 @@ function writeExclusiveAtomic(path: string, value: unknown): void { catch (error) { if ((error as NodeJS.ErrnoException).code === 'EEXIST') throw new Error('RECEIPT_EXISTS'); throw error; } } -/** Redact both channels and both concatenation orders before a receipt can bind them. */ -export function sanitizeProducerRun(run: RunResult): RunResult { +/** + * Redact both channels, and both concatenation orders, before a receipt can bind them. A credential-class match + * anywhere withholds the output and fails the run, naming the rules. PII, internal, legal and hygiene matches are + * replaced in place and leave the run's status alone: scoring reads the helper's report, the evaluator scans the + * receipt for its canaries separately, and the helper's own identifiers are not personal data. `sourceCredentials` are + * the source's credential-name values (see sourceCredentialValues), matched exactly because no rule sees them bare. + */ +export function sanitizeProducerRun(run: RunResult, sourceCredentials: string[] = []): RunResult { if (typeof run.output !== 'string' || (run.error && typeof run.error.reason !== 'string')) { throw new CsoError('REDACTION_FAILED', 'Producer output withheld because it was not valid text'); } const reason = run.error?.reason ?? ''; + let views: OutputRedaction[]; try { - const views = [run.output, reason, run.output + reason, reason + run.output]; - if (views.some(value => redact(value) !== value)) { - return { - ...run, - output: '[sensitive producer output redacted]', - error: { code: run.error?.code ?? 'unknown', reason: 'Sensitive producer output or error withheld' }, - }; - } - return run; + views = [run.output, reason, run.output + reason, reason + run.output].map(view => redactProducerText(view, sourceCredentials)); } catch { throw new CsoError('REDACTION_FAILED', 'Producer output withheld because redaction could not safely inspect it'); } + const secret = views.find(view => view.secret); + if (secret) { + return { + ...run, + output: '[sensitive producer output redacted]', + error: { code: run.error?.code ?? 'unknown', reason: `Sensitive producer output or error withheld: credential-class rule ${secret.rules.join(', ')}` }, + }; + } + const [output, error] = views; + return { ...run, output: output.text, ...(run.error ? { error: { ...run.error, reason: error.text } } : {}) }; } export function producerFailureMessage(error: unknown): string { @@ -565,6 +575,7 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op validateSource(sourceRoot, input.source, cell.sourceHash); sealProducerSource(sourceRoot); validateSource(sourceRoot, input.source, cell.sourceHash); + const sourceCredentials = sourceCredentialValues(sourceRoot, input.source.map(entry => entry.path)); const originalRepositoryIdentity = repositoryIdentity(sourceRoot); const stateRoot = join(jobRoot, 'state'); const helperHome = join(stateRoot, 'cso-home'); @@ -653,7 +664,7 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op artifacts = emptyArtifactInventory(); } if (!integrityFailure) { - run = sanitizeProducerRun(run); + run = sanitizeProducerRun(run, sourceCredentials); if(!run.error&&!run.output.trim())run={...run,error:{code:'unknown',reason:'empty output from provider CLI (exit 0)'}}; } const tokensReported = run.tokens.input > 0 || run.tokens.output > 0 || (run.tokens.cached ?? 0) > 0; diff --git a/test/cso-eval.test.ts b/test/cso-eval.test.ts index 372c8614d..9fe40e5a6 100644 --- a/test/cso-eval.test.ts +++ b/test/cso-eval.test.ts @@ -5,7 +5,7 @@ import { createHash } from 'node:crypto'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { collectProducerReceipts, createEvalBaseline, createEvalMatrix, createPortableSkillPayload, loadPortableSkillPayload, prepareEvalJobs, REQUIRED_CONTAINMENT, scoreCollectedEval, scoreEval, validateMatrix, validatePortableSkillPayload, type EvalBaseline, type EvalCell, type EvalMatrix, type EvalResult, type PreparedEvalSchedule } from '../scripts/cso-eval'; -import { assertProducerArtifactPath, PRODUCER_PROVIDER_POLICY, producerFailureMessage, producerProviderTimeoutMs, producerInstallationIdentity, resolveProducerHelperBinding, runProducerCell, validateProductionProducerInstallation } from '../scripts/cso-eval-producer'; +import { assertProducerArtifactPath, PRODUCER_PROVIDER_POLICY, producerFailureMessage, producerProviderTimeoutMs, producerInstallationIdentity, resolveProducerHelperBinding, runProducerCell, sanitizeProducerRun, validateProductionProducerInstallation } from '../scripts/cso-eval-producer'; import { producerArtifactInventoryHash, producerHostPlatform, producerInstallationIdentityHash, producerProviderIdentityHash, producerReceiptHash, sha256, type ProducerArtifactInventory, type ProducerInstallationIdentity, type ProducerProviderIdentity, type ProducerReceipt } from '../scripts/cso-eval-protocol'; import type { Family, ProviderAdapter, RunOpts, RunResult } from './helpers/providers/types'; import { CORPUS_VERSION, FAMILIES, STACKS, defineEvalCorpus, loadCorpusManifest, materializeCase, sourceFiles, sourceHash, validateCorpusManifest, type CorpusManifest, type EvalVariant } from './fixtures/cso-eval/materialize'; @@ -14,6 +14,8 @@ import { inspectPreparation } from '../lib/cso/preparation'; import { assertRuntimeCompatible, RUNTIME_CATALOG } from '../lib/cso/runtime-catalog'; import { canonicalStartPlan, canonicalTestPlan } from '../lib/cso/verification'; import { CsoError } from '../lib/cso/contracts'; +import { scan } from '../lib/redact-engine'; +import { redactProducerText, sourceCredentialValues } from '../scripts/cso-eval-output-redaction'; import { geminiProducerPaths, geminiProducerSystemSettings } from './helpers/providers/gemini'; const temporary: string[] = []; @@ -895,13 +897,58 @@ describe('CSO matched producer orchestration', () => { expect(receipt.status).toBe('failed'); expect(receipt.output).toBe('[sensitive producer output redacted]'); expect(receipt.outputHash).toBe(sha256(receipt.output)); - expect(receipt.error).toEqual({ code: 'unknown', reason: 'Sensitive producer output or error withheld' }); + expect(receipt.error).toEqual({ code: 'unknown', reason: 'Sensitive producer output or error withheld: credential-class rule openai.key×1' }); const stored = readFileSync(receiptPath, 'utf8'); expect(stored).not.toContain('sk-proj-'); expect(stored).not.toContain('b'.repeat(40)); } }); + test('withholds a source canary quoted without its credential name, whole or split across output and error', () => { + const source = join(root(), 'canary-source'), canary = `cso_${randomHex(10)}_${randomHex(10)}`; + mkdirSync(join(source, 'test'), { recursive: true }); + writeFileSync(join(source, '.env'), `API_TOKEN=${canary}\nNODE_ENV=production\n`); + writeFileSync(join(source, 'test', 'canary.txt'), `${canary}\n`); + const credentials = sourceCredentialValues(source, ['.env', 'test/canary.txt']); + expect(credentials).toEqual([canary]); + const base = { output: '', modelUsed: 'm', durationMs: 1, toolCalls: 1, tokens: { input: 1, output: 1 } } as RunResult; + expect(sanitizeProducerRun({ ...base, output: `The token is ${canary}` }, [])).toMatchObject({ output: `The token is ${canary}` }); + for (const run of [{ ...base, output: `The token is ${canary}.` }, { ...base, output: canary.slice(0, 12), error: { code: 'unknown', reason: canary.slice(12) } }] as RunResult[]) { + const sanitized = sanitizeProducerRun(run, credentials); + expect(sanitized).toMatchObject({ output: '[sensitive producer output redacted]', error: { reason: 'Sensitive producer output or error withheld: credential-class rule source.env.kv×1' } }); + expect(JSON.stringify(sanitized)).not.toContain(canary); + } + }); + + test('keeps a succeeded run that quotes helper identifiers, span-redacts PII shapes, and fails credential-class output by rule', async () => { + const runId = `${Date.now()}-${randomHex(8)}`; + let walletShaped = ''; + while (!walletShaped) { const id = randomHex(16); if (redactProducerText(`review ${id}`).text === `review ${id}` && scan(id).findings.some(finding => finding.id === 'pii.wallet')) walletShaped = id; } + expect(scan(runId).findings.map(finding => finding.id)).toContain('pii.phone.e164'); + const report = [ + `complete — run ${runId}, catalog cso-eval-38078005392-${randomHex(6)}, scanner artifact gitleaks-${randomHex(8)}-${randomHex(8)}.`, + `Finding ${randomHex(16)} reviewed by ${walletShaped}; snapshot ${randomHex(32)}.`, + 'The /health-hooks route connects to 169.254.169.254:80.', + ].join('\n'); + const run = async (name: string, output: string) => { + const destination = join(root(), `${name}-prepared`), cell = selected[0]; + prepareEvalJobs(producerMatrix, skills, destination, [cell.id]); + const isolated = isolate(destination, cell), receiptPath = join(root(), `${cell.id}.json`); + return runProducerCell(isolated.input, receiptPath, withHelper({ adapter: new FakeAdapter(() => {}, 'gpt-5.4', 'gpt', { output }), paidExecutionAuthorized: true })); + }; + const kept = await run('helper-ids', report); + expect(kept.status).toBe('succeeded'); + expect(kept.error).toBeUndefined(); + expect(kept.output).toBe(report.replace('169.254.169.254', '')); + const envLine = `API_TOKEN=cso_${randomHex(12)}_${randomHex(12)}`; + const key = `sk-ant-api03-${randomHex(40)}`; + for (const [name, leak, rule] of [['env-line', envLine, 'env.kv×1'], ['credential', key, 'anthropic.key×1']] as const) { + const failed = await run(name, `${report}\n${leak}`); + expect(failed).toMatchObject({ status: 'failed', output: '[sensitive producer output redacted]', error: { code: 'unknown', reason: `Sensitive producer output or error withheld: credential-class rule ${rule}` }, usage: { inputTokens: 120 } }); + expect(JSON.stringify(failed)).not.toContain(leak); + } + }); + const expectIntegrityReceipt = (receipt: ProducerReceipt, receiptPath: string, error: { code: string; reason: string }) => { expect(receipt).toMatchObject({ status: 'failed', error, output: '', outputHash: sha256(''), durationMs: 1234, toolCalls: 4, usage: { inputTokens: 120, outputTokens: 30, cachedTokens: 10, estimatedCostUSD: 0.0042 } }); expect(receipt.artifacts.entries).toEqual([]);