diff --git a/apps/cli/src/commands/start.ts b/apps/cli/src/commands/start.ts index fe0aabd..6a78a5e 100644 --- a/apps/cli/src/commands/start.ts +++ b/apps/cli/src/commands/start.ts @@ -235,6 +235,22 @@ function printInfo( if (args.pipelineTesting) { console.log(' Mode: Pipeline Testing'); } + + // Surface Fable usage: its safety classifiers route cybersecurity tasks to + // Opus 4.8, so those phases run on Opus 4.8 regardless of the tier setting. + const fableTiers = ( + [ + ['small', process.env.ANTHROPIC_SMALL_MODEL], + ['medium', process.env.ANTHROPIC_MEDIUM_MODEL], + ['large', process.env.ANTHROPIC_LARGE_MODEL], + ] as const + ).filter(([, model]) => model && /fable/i.test(model)); + if (fableTiers.length > 0) { + const tierList = fableTiers.map(([tier, model]) => `${tier} (${model})`).join(', '); + console.log(` Note: ${tierList} set to a Fable model. Fable's safety classifiers`); + console.log(' route cybersecurity tasks to Opus 4.8, so those phases run on Opus 4.8.'); + } + console.log(''); console.log(' Monitor:'); if (workflowId) { diff --git a/apps/worker/src/ai/audit-logger.ts b/apps/worker/src/ai/audit-logger.ts index e7d4491..bfe7531 100644 --- a/apps/worker/src/ai/audit-logger.ts +++ b/apps/worker/src/ai/audit-logger.ts @@ -14,6 +14,7 @@ export interface AuditLogger { logToolStart(toolName: string, parameters: unknown): Promise; logToolEnd(result: unknown): Promise; logError(error: Error, duration: number, turns: number): Promise; + logNote(category: string, message: string): Promise; } class RealAuditLogger implements AuditLogger { @@ -56,6 +57,10 @@ class RealAuditLogger implements AuditLogger { timestamp: formatTimestamp(), }); } + + async logNote(category: string, message: string): Promise { + await this.auditSession.logWorkflowNote(category, message); + } } /** Null Object implementation - all methods are safe no-ops */ @@ -67,6 +72,8 @@ class NullAuditLogger implements AuditLogger { async logToolEnd(_result: unknown): Promise {} async logError(_error: Error, _duration: number, _turns: number): Promise {} + + async logNote(_category: string, _message: string): Promise {} } // Returns no-op when auditSession is null diff --git a/apps/worker/src/ai/message-handlers.ts b/apps/worker/src/ai/message-handlers.ts index 1864915..68a87ea 100644 --- a/apps/worker/src/ai/message-handlers.ts +++ b/apps/worker/src/ai/message-handlers.ts @@ -25,6 +25,7 @@ import type { AssistantResult, ContentBlock, ExecutionContext, + ModelRefusalFallbackMessage, ResultData, ResultMessage, SystemInitMessage, @@ -343,6 +344,15 @@ export async function dispatchMessage( } return { type: 'continue', model: initMsg.model }; } + if (message.subtype === 'model_refusal_fallback') { + const fallback = message as ModelRefusalFallbackMessage; + const category = fallback.api_refusal_category ?? 'policy'; + await auditLogger.logNote( + 'model-fallback', + `Model refused (${category}); fell back ${fallback.original_model} → ${fallback.fallback_model}`, + ); + return { type: 'continue' }; + } return { type: 'continue' }; } diff --git a/apps/worker/src/ai/models.ts b/apps/worker/src/ai/models.ts index 117b590..9f2d73d 100644 --- a/apps/worker/src/ai/models.ts +++ b/apps/worker/src/ai/models.ts @@ -40,3 +40,12 @@ export function resolveModel(tier: ModelTier = 'medium'): string { export function supportsAdaptiveThinking(model: string): boolean { return /opus-4-[678]/.test(model); } + +/** + * Whether a model is in the Fable family. Fable's safety classifiers flag + * cybersecurity tasks and route them to Opus 4.8, so a security scan on Fable + * largely runs on Opus 4.8 anyway. + */ +export function isFableModel(model: string): boolean { + return /fable/i.test(model); +} diff --git a/apps/worker/src/ai/types.ts b/apps/worker/src/ai/types.ts index cbaacfb..b6c762e 100644 --- a/apps/worker/src/ai/types.ts +++ b/apps/worker/src/ai/types.ts @@ -98,6 +98,15 @@ export interface SystemInitMessage { permissionMode?: string; } +/** Emitted when a model refuses a request and the SDK falls back to another model (e.g. Fable 5 routing cybersecurity tasks to Opus 4.8). */ +export interface ModelRefusalFallbackMessage { + type: 'system'; + subtype: 'model_refusal_fallback'; + original_model: string; + fallback_model: string; + api_refusal_category?: string | null; +} + export interface UserMessage { type: 'user'; } diff --git a/apps/worker/src/audit/audit-session.ts b/apps/worker/src/audit/audit-session.ts index 73163df..9d364fe 100644 --- a/apps/worker/src/audit/audit-session.ts +++ b/apps/worker/src/audit/audit-session.ts @@ -158,6 +158,14 @@ export class AuditSession { } } + /** + * Write a human-readable note to the unified workflow log (e.g. a model + * refusal fallback). Independent of agent event logging. + */ + async logWorkflowNote(category: string, message: string): Promise { + await this.workflowLogger.logEvent(category, message); + } + /** * End agent execution (mutex-protected) */ diff --git a/apps/worker/src/audit/workflow-logger.ts b/apps/worker/src/audit/workflow-logger.ts index d8bad90..9637695 100644 --- a/apps/worker/src/audit/workflow-logger.ts +++ b/apps/worker/src/audit/workflow-logger.ts @@ -12,6 +12,7 @@ */ import fs from 'node:fs/promises'; +import { isFableModel, resolveModel } from '../ai/models.js'; import { formatDuration, formatTimestamp } from '../utils/formatting.js'; import { LogStream } from './log-stream.js'; import { generateWorkflowLogPath, type SessionMetadata } from './utils.js'; @@ -77,18 +78,31 @@ export class WorkflowLogger { * Write header to log file */ private async writeHeader(): Promise { - const header = [ + const lines = [ `================================================================================`, `Shannon Pentest - Workflow Log`, `================================================================================`, `Workflow ID: ${this.workflowId ?? this.sessionMetadata.id}`, `Target URL: ${this.sessionMetadata.webUrl}`, `Started: ${formatTimestamp()}`, - `================================================================================`, - ``, - ].join('\n'); + ]; - return this.logStream.write(header); + // Surface Fable usage: its safety classifiers route cybersecurity tasks to + // Opus 4.8, so those phases run on Opus 4.8 regardless of the tier setting. + const fableTiers = (['small', 'medium', 'large'] as const) + .map((tier) => ({ tier, model: resolveModel(tier) })) + .filter(({ model }) => isFableModel(model)); + if (fableTiers.length > 0) { + const tierList = fableTiers.map(({ tier, model }) => `${tier} (${model})`).join(', '); + lines.push( + `Note: ${tierList} set to a Fable model. Fable's safety classifiers`, + ` route cybersecurity tasks to Opus 4.8, so those phases run on Opus 4.8.`, + ); + } + + lines.push(`================================================================================`, ``); + + return this.logStream.write(lines.join('\n')); } /** diff --git a/docs/ai-providers.md b/docs/ai-providers.md index d94375e..3a6dbd0 100644 --- a/docs/ai-providers.md +++ b/docs/ai-providers.md @@ -23,6 +23,8 @@ ANTHROPIC_API_KEY=your-api-key CLAUDE_CODE_MAX_OUTPUT_TOKENS=64000 ``` +Each tier can be pointed at any Claude model via `ANTHROPIC_SMALL_MODEL` / `ANTHROPIC_MEDIUM_MODEL` / `ANTHROPIC_LARGE_MODEL` (or the setup wizard). If you set a tier to `claude-fable-5`, note that Fable's safety classifiers route cybersecurity tasks to Opus 4.8, so those phases run on Opus 4.8 regardless. + ## AWS Bedrock Run `npx @keygraph/shannon setup` and select **AWS Bedrock**. The wizard prompts for region, bearer token, and model IDs. diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7b08ccd..974ff5f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -7,8 +7,8 @@ settings: catalogs: default: '@anthropic-ai/claude-agent-sdk': - specifier: ^0.3.163 - version: 0.3.163 + specifier: ^0.3.173 + version: 0.3.173 importers: @@ -50,7 +50,7 @@ importers: dependencies: '@anthropic-ai/claude-agent-sdk': specifier: 'catalog:' - version: 0.3.163(@anthropic-ai/sdk@0.93.0(zod@4.3.6))(@modelcontextprotocol/sdk@1.29.0(zod@4.3.6))(zod@4.3.6) + version: 0.3.173(@anthropic-ai/sdk@0.93.0(zod@4.3.6))(@modelcontextprotocol/sdk@1.29.0(zod@4.3.6))(zod@4.3.6) '@temporalio/activity': specifier: ^1.11.0 version: 1.15.0 @@ -88,52 +88,52 @@ importers: packages: - '@anthropic-ai/claude-agent-sdk-darwin-arm64@0.3.163': - resolution: {integrity: sha512-2veSQriv2OR1msX3C5ThYVwQUOLBzEXvdGzmkt9y47DTLHPqS6wy7MBWcNVHj7GDKrkPXnH7zz7DbKYM5yKXjg==} + '@anthropic-ai/claude-agent-sdk-darwin-arm64@0.3.173': + resolution: {integrity: sha512-McW1toJ4Qdo/i7bHnsiFMm2AtyCiK/5V90WgL7M9ZO9llrJr3riGRdBlRGHvWnS7rKuv5ttj4/SuBkLoL9o75A==} cpu: [arm64] os: [darwin] - '@anthropic-ai/claude-agent-sdk-darwin-x64@0.3.163': - resolution: {integrity: sha512-0n/vjdW12RQuuvJsy6bettU57a7hi7GTGDZxaAWVKpRr+U9A51GPvmUvnTENqogZ3PqpKCYEDv2he4qkA5zCmw==} + '@anthropic-ai/claude-agent-sdk-darwin-x64@0.3.173': + resolution: {integrity: sha512-m2Oh1pQ69IUg6DSH6n1nAAAtRq+G7J2B/CKTbTfDAEmg5t0lzakZV9l28GZrlP21xl+PIg71dkiJ5u94KYgpRw==} cpu: [x64] os: [darwin] - '@anthropic-ai/claude-agent-sdk-linux-arm64-musl@0.3.163': - resolution: {integrity: sha512-kY39TQiXvniq1902IV4H5VSTwkwav6OkXqcr0T7rN04qXzASMpJ9UYlNhwDICBhIxKwcMlz9rjmg9nCDdeBQjQ==} + '@anthropic-ai/claude-agent-sdk-linux-arm64-musl@0.3.173': + resolution: {integrity: sha512-uu8MnPwFBc9ayFg5c94aHaqJVLS51oHNVxHwud4nK27GliP5WoME3F7pmm1N/Jyy2ry2E2CLNnsAm5l3zemfYA==} cpu: [arm64] os: [linux] libc: [musl] - '@anthropic-ai/claude-agent-sdk-linux-arm64@0.3.163': - resolution: {integrity: sha512-bDwKqn8XT5f9JOcOLaGi0XY6Ndzh3dkCzsub67igXCVYf3i19ElsR9p0LF1vWeQWDnHyCVvCnSfQfmtD3MsHJw==} + '@anthropic-ai/claude-agent-sdk-linux-arm64@0.3.173': + resolution: {integrity: sha512-Vb64WJOD2D9tKn/i0ErmLbO6I1187xTwL/kxEANjg4L1FGQnvdYBnZ6J6jU6+x8UgcuIi82imHROQp80gbPW2w==} cpu: [arm64] os: [linux] libc: [glibc] - '@anthropic-ai/claude-agent-sdk-linux-x64-musl@0.3.163': - resolution: {integrity: sha512-OmvQgIX3X5TZ8wXROOHi90iSmGoVNCREYSP5YR+7o8swR59TjBPhlbaCh22yFX5dc2brftCqwF9WklYMrhaDHQ==} + '@anthropic-ai/claude-agent-sdk-linux-x64-musl@0.3.173': + resolution: {integrity: sha512-ofKxyp/N8+LLSrClt+dJo0laCHDzmBgmN8+Q+zabJ54Jqc1KXee68UsziE7kLCqGbOSY7rsIK9d22T1uQyZkUw==} cpu: [x64] os: [linux] libc: [musl] - '@anthropic-ai/claude-agent-sdk-linux-x64@0.3.163': - resolution: {integrity: sha512-vYPKM5Nfm4yh3jFDqb17iwY7Y59unuZu9JSYEM45POT5idgZMSC5E9O9jqGIt9GF2Ug7sNRdFMEkUZgIXsZIBQ==} + '@anthropic-ai/claude-agent-sdk-linux-x64@0.3.173': + resolution: {integrity: sha512-0l9L5O+Uiw8gq9mu2NqGSYcSmX8RJMAh9GkGacTJqJbkfHCiFxqZmWBWODOt6HP7PTFCV+yMzQK/xJQjnag/jw==} cpu: [x64] os: [linux] libc: [glibc] - '@anthropic-ai/claude-agent-sdk-win32-arm64@0.3.163': - resolution: {integrity: sha512-yQFXwSfD1FQfcCVJoxmVNc9KJGtwzwwrQ5sR3xALmVq7N6yjXy6sU6xqowjx+S98Dgiz7fKBrWVZyKLWWxLyRQ==} + '@anthropic-ai/claude-agent-sdk-win32-arm64@0.3.173': + resolution: {integrity: sha512-aulIrBYjFDm+S5kl4CxC8A3KUiTDKLHoKV46Mat64E+skGEyBkC7WzZAGbpGKB2y8KZo1WbrwJTGefgyHMpDhw==} cpu: [arm64] os: [win32] - '@anthropic-ai/claude-agent-sdk-win32-x64@0.3.163': - resolution: {integrity: sha512-Fe/zfbh3PK5jDO/yoU5BtY6U2vfMN8GdcrvWggzfe20OAqQjdfbxo31SiD9Rg5gTItE28ns3mcP8toohPkmUwA==} + '@anthropic-ai/claude-agent-sdk-win32-x64@0.3.173': + resolution: {integrity: sha512-HjGkfDlNLI3Kh9NIUfevJ3SZY8Xv3td4zx5Cz3YE0VPrrjIkRvdEv9K6WTjT94tlS0TUXQS/kFT+NCmoGXuvZQ==} cpu: [x64] os: [win32] - '@anthropic-ai/claude-agent-sdk@0.3.163': - resolution: {integrity: sha512-JGWfcrQhV5EZggvHb1tfet8JimJci7+4vOgKpQATUxKeE+5rVWH85w9IIND2/R/+qq90mDoMHGM2sYzJYvJXTg==} + '@anthropic-ai/claude-agent-sdk@0.3.173': + resolution: {integrity: sha512-BsdL223y7vCUJA9uBW9osSrhufvwIT+J94IBkh83v+wjyjoBIwLXREdacFabair70bGNdtkw6cWCaYNThSQg7A==} engines: {node: '>=18.0.0'} peerDependencies: '@anthropic-ai/sdk': '>=0.93.0' @@ -1670,44 +1670,44 @@ packages: snapshots: - '@anthropic-ai/claude-agent-sdk-darwin-arm64@0.3.163': + '@anthropic-ai/claude-agent-sdk-darwin-arm64@0.3.173': optional: true - '@anthropic-ai/claude-agent-sdk-darwin-x64@0.3.163': + '@anthropic-ai/claude-agent-sdk-darwin-x64@0.3.173': optional: true - '@anthropic-ai/claude-agent-sdk-linux-arm64-musl@0.3.163': + '@anthropic-ai/claude-agent-sdk-linux-arm64-musl@0.3.173': optional: true - '@anthropic-ai/claude-agent-sdk-linux-arm64@0.3.163': + '@anthropic-ai/claude-agent-sdk-linux-arm64@0.3.173': optional: true - '@anthropic-ai/claude-agent-sdk-linux-x64-musl@0.3.163': + '@anthropic-ai/claude-agent-sdk-linux-x64-musl@0.3.173': optional: true - '@anthropic-ai/claude-agent-sdk-linux-x64@0.3.163': + '@anthropic-ai/claude-agent-sdk-linux-x64@0.3.173': optional: true - '@anthropic-ai/claude-agent-sdk-win32-arm64@0.3.163': + '@anthropic-ai/claude-agent-sdk-win32-arm64@0.3.173': optional: true - '@anthropic-ai/claude-agent-sdk-win32-x64@0.3.163': + '@anthropic-ai/claude-agent-sdk-win32-x64@0.3.173': optional: true - '@anthropic-ai/claude-agent-sdk@0.3.163(@anthropic-ai/sdk@0.93.0(zod@4.3.6))(@modelcontextprotocol/sdk@1.29.0(zod@4.3.6))(zod@4.3.6)': + '@anthropic-ai/claude-agent-sdk@0.3.173(@anthropic-ai/sdk@0.93.0(zod@4.3.6))(@modelcontextprotocol/sdk@1.29.0(zod@4.3.6))(zod@4.3.6)': dependencies: '@anthropic-ai/sdk': 0.93.0(zod@4.3.6) '@modelcontextprotocol/sdk': 1.29.0(zod@4.3.6) zod: 4.3.6 optionalDependencies: - '@anthropic-ai/claude-agent-sdk-darwin-arm64': 0.3.163 - '@anthropic-ai/claude-agent-sdk-darwin-x64': 0.3.163 - '@anthropic-ai/claude-agent-sdk-linux-arm64': 0.3.163 - '@anthropic-ai/claude-agent-sdk-linux-arm64-musl': 0.3.163 - '@anthropic-ai/claude-agent-sdk-linux-x64': 0.3.163 - '@anthropic-ai/claude-agent-sdk-linux-x64-musl': 0.3.163 - '@anthropic-ai/claude-agent-sdk-win32-arm64': 0.3.163 - '@anthropic-ai/claude-agent-sdk-win32-x64': 0.3.163 + '@anthropic-ai/claude-agent-sdk-darwin-arm64': 0.3.173 + '@anthropic-ai/claude-agent-sdk-darwin-x64': 0.3.173 + '@anthropic-ai/claude-agent-sdk-linux-arm64': 0.3.173 + '@anthropic-ai/claude-agent-sdk-linux-arm64-musl': 0.3.173 + '@anthropic-ai/claude-agent-sdk-linux-x64': 0.3.173 + '@anthropic-ai/claude-agent-sdk-linux-x64-musl': 0.3.173 + '@anthropic-ai/claude-agent-sdk-win32-arm64': 0.3.173 + '@anthropic-ai/claude-agent-sdk-win32-x64': 0.3.173 '@anthropic-ai/sdk@0.93.0(zod@4.3.6)': dependencies: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index d402b67..5e346dd 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -2,4 +2,4 @@ packages: - "apps/*" catalog: - "@anthropic-ai/claude-agent-sdk": ^0.3.163 + "@anthropic-ai/claude-agent-sdk": ^0.3.173