diff --git a/apps/cli/src/env.ts b/apps/cli/src/env.ts index 98feb230..822c8e7d 100644 --- a/apps/cli/src/env.ts +++ b/apps/cli/src/env.ts @@ -31,6 +31,10 @@ const COMMON_FORWARD_VARS = [ 'SHANNON_AI_MODEL', 'SHANNON_AI_BASE_URL', 'SHANNON_AI_OPENAI_FORMAT', + // Opt-in debug flag: when set, the worker persists a bounded, sanitized snippet of a failed + // provider turn's raw error message to error.log. Off by default; provider prose stays out of + // durable state unless an operator deliberately enables it for a diagnosis. + 'SHANNON_DEBUG_PROVIDER_ERRORS', GENERIC_API_KEY_ENV, ] as const; diff --git a/apps/worker/src/ai/pi/pi-executor.ts b/apps/worker/src/ai/pi/pi-executor.ts index ffb9bf5d..7c1f149d 100644 --- a/apps/worker/src/ai/pi/pi-executor.ts +++ b/apps/worker/src/ai/pi/pi-executor.ts @@ -49,7 +49,7 @@ import { PI_RETRY_SETTINGS } from './retry-settings.js'; import { createGlobTool, createTodoWriteTool } from './session-tools.js'; import { createTaskTool } from './task-tool.js'; import { TraceEmitter } from './trace-emitter.js'; -import { providerTurnError } from './turn-error.js'; +import { providerTurnError, type SafeProviderTurnDetails, safeProviderTurnDetails } from './turn-error.js'; declare global { var SHANNON_DISABLE_LOADER: boolean | undefined; @@ -163,6 +163,7 @@ async function writeErrorLog( duration: number, turns: number, retryable: boolean, + providerDetails?: SafeProviderTurnDetails, ): Promise { try { const errorLog = { @@ -172,6 +173,7 @@ async function writeErrorLog( duration, turns, retryable, + ...(providerDetails !== undefined && { provider: providerDetails }), }; const logPath = path.join(deliverablesDir(sourceDir), 'error.log'); await fs.appendFile(logPath, `${JSON.stringify(errorLog)}\n`); @@ -310,6 +312,9 @@ export async function runPiPrompt( let turnCount = 0; let pendingError: PentestError | null = null; + // Bounded, non-sensitive facts about the failed turn, captured alongside pendingError so the + // error log can distinguish a safeguard/refusal from a transport or tool-call lifecycle fault. + let pendingProviderDetails: SafeProviderTurnDetails | null = null; // Declared out here so the catch can bill spend accrued before a failure. let session: AgentSession | undefined; @@ -359,6 +364,8 @@ export async function runPiPrompt( } if (msg.role === 'assistant' && msg.stopReason === 'error') { pendingError = pendingError ?? providerTurnError(msg, 'Agent error', selection.model.contextWindow); + pendingProviderDetails = + pendingProviderDetails ?? safeProviderTurnDetails(msg, selection.model.contextWindow); } break; } @@ -438,7 +445,10 @@ export async function runPiPrompt( await traceEmitter?.flush(); progress.stop(); outputLines(formatErrorOutput(safeError, execContext, duration, turnCount, retryable)); - await writeErrorLog(sourceDir, safeError, duration, turnCount, retryable); + if (pendingProviderDetails) { + console.log(` provider-turn: ${JSON.stringify(pendingProviderDetails)}`); + } + await writeErrorLog(sourceDir, safeError, duration, turnCount, retryable, pendingProviderDetails ?? undefined); // A failed agent still spent money — on its own turns and, since Shannon's // prompts delegate the heavy work, mostly on `task` sub-agents. Both count diff --git a/apps/worker/src/ai/pi/turn-error.ts b/apps/worker/src/ai/pi/turn-error.ts index 6a17ca29..9415b6a8 100644 --- a/apps/worker/src/ai/pi/turn-error.ts +++ b/apps/worker/src/ai/pi/turn-error.ts @@ -6,7 +6,7 @@ import type { AssistantMessage } from '@earendil-works/pi-ai'; import { classifyProviderFailure, PentestError } from '../../services/error-handling.js'; -import { ErrorCode } from '../../types/errors.js'; +import { ErrorCode, type ProviderFailureCategory } from '../../types/errors.js'; /** * Wrap a failed assistant turn, taking the retry verdict from pi. @@ -33,3 +33,124 @@ export function providerTurnError(message: AssistantMessage, label: string, cont error.providerCategory = failure.category; return error; } + +/** One diagnostic entry reduced to bounded, non-prose tokens. */ +interface SafeDiagnostic { + readonly type?: string; + readonly errorName?: string; + readonly errorCode?: string | number; +} + +/** + * Bounded, non-sensitive observability facts about a failed assistant turn, safe to persist. + * + * Carries only closed enums (stop reason, provider category), opaque ids (response id), short + * provider/harness tokens (raw stop reason, diagnostic type/name/code), and structural booleans + * (redacted thinking, tool call in flight). It never carries the provider's error prose, the + * model's generated text, prompts, or credentials. `errorMessage` is reduced to its length, so a + * silent empty failure stays distinguishable from one that returned text. + */ +export interface SafeProviderTurnDetails { + readonly provider?: string; + readonly model?: string; + readonly responseModel?: string; + readonly responseId?: string; + readonly stopReason?: string; + readonly rawStopReason?: string; + readonly endTurn?: boolean; + /** True when a thinking block was redacted by the provider's safety filters. */ + readonly thinkingRedacted: boolean; + /** Names of tool calls the model was emitting when the turn errored (own allowlist). */ + readonly toolCallsInFlight: readonly string[]; + readonly errorMessageLength: number; + /** Present only when SHANNON_DEBUG_PROVIDER_ERRORS is set: a bounded, sanitized error snippet. */ + readonly errorMessageSnippet?: string; + readonly diagnostics?: readonly SafeDiagnostic[]; + readonly providerCategory: ProviderFailureCategory; + readonly retryable: boolean; +} + +/** Whether the operator opted into persisting a bounded snippet of raw provider error text. */ +function debugProviderErrorsEnabled(): boolean { + return process.env.SHANNON_DEBUG_PROVIDER_ERRORS === '1' || process.env.SHANNON_DEBUG_PROVIDER_ERRORS === 'true'; +} + +/** Collapse whitespace, strip control characters, and truncate. A debug-only view of error prose. */ +function boundedSnippet(value: unknown, max = 500): string | undefined { + if (typeof value !== 'string') return undefined; + const collapsed = value.replace(/\s+/gu, ' ').trim(); + return boundedToken(collapsed, max); +} + +/** Strip control characters and truncate, so a provider-controlled token can never carry prose or blobs. */ +function boundedToken(value: unknown, max = 120): string | undefined { + if (typeof value !== 'string') return undefined; + let out = ''; + for (let index = 0; index < value.length && out.length < max; index += 1) { + const code = value.charCodeAt(index); + if (code > 31 && code !== 127) out += value[index]; + } + return out.length > 0 ? out : undefined; +} + +function safeDiagnostics(message: AssistantMessage): SafeDiagnostic[] | undefined { + const raw = message.diagnostics; + if (!Array.isArray(raw) || raw.length === 0) return undefined; + const entries: SafeDiagnostic[] = []; + for (const diagnostic of raw.slice(0, 5)) { + const type = boundedToken(diagnostic?.type); + const errorName = boundedToken(diagnostic?.error?.name); + const rawCode = diagnostic?.error?.code; + const errorCode = typeof rawCode === 'number' ? rawCode : boundedToken(rawCode, 60); + entries.push({ + ...(type !== undefined && { type }), + ...(errorName !== undefined && { errorName }), + ...(errorCode !== undefined && { errorCode }), + }); + } + return entries; +} + +/** + * Extract the bounded observability record from a failed assistant turn. Shares the classifier + * with {@link providerTurnError} so the persisted category and retry verdict match the thrown + * error exactly. + */ +export function safeProviderTurnDetails(message: AssistantMessage, contextWindow?: number): SafeProviderTurnDetails { + const failure = classifyProviderFailure(message, contextWindow); + const content = Array.isArray(message.content) ? message.content : []; + const toolCallsInFlight: string[] = []; + for (const block of content) { + if (block?.type === 'toolCall') { + const name = boundedToken(block.name, 60); + if (name !== undefined && toolCallsInFlight.length < 10) toolCallsInFlight.push(name); + } + } + const thinkingRedacted = content.some((block) => block?.type === 'thinking' && block.redacted === true); + const diagnostics = safeDiagnostics(message); + + const provider = boundedToken(message.provider); + const model = boundedToken(message.model); + const responseModel = boundedToken(message.responseModel); + const responseId = boundedToken(message.responseId); + const stopReason = boundedToken(message.stopReason); + const rawStopReason = boundedToken(message.rawStopReason); + const errorMessageSnippet = debugProviderErrorsEnabled() ? boundedSnippet(message.errorMessage) : undefined; + + return { + ...(provider !== undefined && { provider }), + ...(model !== undefined && { model }), + ...(responseModel !== undefined && { responseModel }), + ...(responseId !== undefined && { responseId }), + ...(stopReason !== undefined && { stopReason }), + ...(rawStopReason !== undefined && { rawStopReason }), + ...(typeof message.endTurn === 'boolean' && { endTurn: message.endTurn }), + thinkingRedacted, + toolCallsInFlight, + errorMessageLength: typeof message.errorMessage === 'string' ? message.errorMessage.length : 0, + ...(errorMessageSnippet !== undefined && { errorMessageSnippet }), + ...(diagnostics !== undefined && { diagnostics }), + providerCategory: failure.category, + retryable: failure.retryable, + }; +}