feat: multi-provider model support, SARIF output, and exploit-mode fixes (#402)

* feat(worker): record token, cache, and turn usage per agent

* feat: replace model tiers with a single SHANNON_AI_MODEL across five providers

* feat(cli): rebuild the setup wizard for provider and model selection

* docs: document single-model selection and supported providers

* feat(worker): use chat completions for OpenAI behind a custom base URL

* feat: add SHANNON_AI_OPENAI_FORMAT to pick the wire API for OpenAI gateways

* refactor(cli): drop endpoint path hints from the gateway format picker

* feat(worker): enable pi in-session provider retry with retry-after backoff

* refactor(worker): hand provider error classification to pi and drop the Anthropic ladders

* refactor: remove the subscription retry preset and pipeline config section

* fix(worker): validate Bedrock credentials with the same live probe as other providers

* feat(worker): render the report from structured findings instead of agent-written markdown

* fix(worker): dispose the credential probe session on every path

* fix(worker): refuse to replace the assembled report with an empty one

* refactor(worker): catch post-processing throws across the whole finalization block

* revert(worker): drop the report zero-findings guard

* docs(worker): correct the retry split and Bedrock credential claims

* docs: regenerate llms-full.txt from current sources

* feat(cli): build and run the npx flow from a clone

* refactor(cli): flatten the setup summary output

* feat(cli): reject runs with more than one provider configured

* fix(worker): say a rejected bash call never ran

* chore(cli): drop grok-4.3 and gpt-5.6-luna from the setup suggestions

* feat(worker): capture structured finding locations for SARIF output

* fix(worker): enumerate queue confidence so the report inherits it verbatim

* feat(worker): give the reporting phase a mode-specific output schema

* feat(worker): emit a SARIF 2.1.0 log for exploitative runs

* fix(worker): correct SARIF locations and defer fingerprinting to the upload action

* fix(worker): drop the confidence suffix from the analysis-mode summary list

* feat(worker): give exploit findings a dedicated code location field

* feat(worker): carry structured code locations from the vuln queue to the report

* fix(worker): join code locations from the vuln queue instead of re-asking agents

* fix(worker): spell out the finding_id to category mapping in the tool schema

* feat: drop Google/Gemini as a supported AI provider

* fix(worker): stop asking the report agent for code locations

* docs: correct the provider list and drop the removed rate-limit settings

* docs: add provider cyber safeguards and suggested models per provider

* docs: document the SARIF output and the report rating thresholds

apps/worker/src/temporal/activities.ts

@@ -25,7 +25,14 @@ import type { ResumeAttempt } from '../audit/metrics-tracker.js';
 import { authStateFile, generateAuditPath, generateSessionJsonPath, type SessionMetadata } from '../audit/utils.js';
 import type { WorkflowSummary } from '../audit/workflow-logger.js';
 import type { CheckpointContext } from '../interfaces/checkpoint-provider.js';
-import { DEFAULT_DELIVERABLES_SUBDIR, deliverablesDir, resolveSessionJsonPath } from '../paths.js';
+import {
+  ASSEMBLED_REPORT_FILENAME,
+  DEFAULT_DELIVERABLES_SUBDIR,
+  deliverablesDir,
+  REPORT_JSON_FILENAME,
+  resolveSessionJsonPath,
+  SARIF_FILENAME,
+} from '../paths.js';
 import { getAgentGitPaths } from '../services/agent-git-paths.js';
 import { getContainer, getOrCreateContainer, removeContainer } from '../services/container.js';
 import { classifyErrorForTemporal, PentestError } from '../services/error-handling.js';
@@ -34,6 +41,7 @@ import { renderFindingsFromQueues } from '../services/findings-renderer.js';
 import { executeGitCommandWithRetry } from '../services/git-manager.js';
 import { runPreflightChecks } from '../services/preflight.js';
 import type { ExploitationDecision, VulnType } from '../services/queue-validation.js';
+import type { ReportData, ReportMeta } from '../services/report-renderer.js';
 import { assembleFinalReport, copyReportToRunRoot, injectModelIntoReport } from '../services/reporting.js';
 import { validateAuthentication } from '../services/validate-authentication.js';
 import { AGENTS } from '../session-manager.js';
@@ -76,6 +84,10 @@ export interface ActivityInput {
   auditDir?: string;
   promptDir?: string;
   sastSarifPath?: string;
+
+  // Vuln classes whose pipeline failed. Set before the report stage on a partial run so the
+  // report marks them "not assessed" instead of asserting no findings were present.
+  failedClasses?: VulnClass[];
 }
 
 /**
@@ -187,6 +199,7 @@ async function runAgentActivity(
         attemptNumber,
         ...(input.promptDir !== undefined && { promptDir: input.promptDir }),
         ...(input.configYAML !== undefined && { configYAML: input.configYAML }),
+        ...(input.failedClasses !== undefined && { failedClasses: input.failedClasses }),
         ...(customTools && { customTools }),
         ...(writeDeliverable && { writeDeliverable }),
         cancellationSignal: Context.current().cancellationSignal,
@@ -198,10 +211,12 @@ async function runAgentActivity(
     // 4. Return metrics
     return {
       durationMs: Date.now() - startTime,
-      inputTokens: null,
-      outputTokens: null,
+      inputTokens: endResult.input_tokens ?? null,
+      outputTokens: endResult.output_tokens ?? null,
+      cacheReadTokens: endResult.cache_read_tokens ?? null,
+      cacheWriteTokens: endResult.cache_write_tokens ?? null,
       costUsd: endResult.cost_usd,
-      numTurns: null,
+      numTurns: endResult.turns ?? null,
       model: endResult.model,
     };
   } catch (error) {
@@ -432,8 +447,95 @@ export async function runAuthzExploitAgent(input: ActivityInput): Promise<AgentM
   return runExploitAgentWithCollector('authz-exploit', 'authz', input);
 }
 
-export async function runReportAgent(input: ActivityInput): Promise<AgentMetrics> {
-  return runAgentActivity('report', input);
+/**
+ * Write report.sarif when the run is exploitative and the operator asked for it.
+ *
+ * Skipped entirely for analysis-only runs: those findings carry no severity, so every
+ * `result.level` would be invented. Failures are logged and swallowed — the SARIF log is a
+ * secondary artifact and must not fail a run whose report is already written.
+ */
+async function writeSarifIfEnabled(
+  input: ActivityInput,
+  exploit: boolean,
+  reportData: ReportData,
+  deliverablesPath: string,
+  logger: ReturnType<typeof createActivityLogger>,
+): Promise<void> {
+  if (!exploit) return;
+
+  const container = getOrCreateContainer(input.workflowId, buildSessionMetadata(input), buildContainerConfig(input));
+  const configResult = await container.configLoader.loadOptional(input.configPath, undefined, input.configYAML);
+  if (isErr(configResult) || configResult.value?.report?.sarif !== true) return;
+
+  try {
+    const { renderSarif } = await import('../services/sarif-renderer.js');
+    const sarif = renderSarif(reportData, { workspaceName: input.sessionId });
+    await atomicWrite(path.join(deliverablesPath, SARIF_FILENAME), sarif);
+    logger.info(`Wrote ${SARIF_FILENAME}`);
+  } catch (error) {
+    logger.warn(`Failed to write ${SARIF_FILENAME}: ${(error as Error).message}`);
+  }
+}
+
+export async function runReportAgent(input: ActivityInput, exploit: boolean): Promise<AgentMetrics> {
+  const { createFindingCollector } = await import('../collectors/finding-collector.js');
+  const { renderReport } = await import('../services/report-renderer.js');
+
+  const collector = createFindingCollector(exploit);
+
+  const writeDeliverable = async (deliverablesPath: string): Promise<void> => {
+    const logger = createActivityLogger();
+    const { attachQueueCodeLocations } = await import('../services/code-location-join.js');
+    const collected = collector.getAll();
+    logger.info(`Collected ${collected.length} finding(s) from report agent`);
+    const findings = await attachQueueCodeLocations(collected, deliverablesPath, logger);
+
+    // report_meta is written by the set-report-meta CLI while the agent runs; read it back so
+    // the two halves of report.json end up in one document.
+    const reportJsonPath = path.join(deliverablesPath, REPORT_JSON_FILENAME);
+    let reportMeta: ReportMeta = {
+      target: input.webUrl,
+      assessment_date: new Date().toISOString().split('T')[0]!,
+      scope: '',
+      executive_summary: '',
+      exploit,
+    };
+    if (await fileExists(reportJsonPath)) {
+      try {
+        const existing = await readJson<{ report_meta?: Record<string, unknown> }>(reportJsonPath);
+        if (existing.report_meta) {
+          reportMeta = {
+            target: String(existing.report_meta.target ?? input.webUrl),
+            assessment_date: String(existing.report_meta.assessment_date ?? reportMeta.assessment_date),
+            scope: String(existing.report_meta.scope ?? ''),
+            executive_summary: String(existing.report_meta.executive_summary ?? ''),
+            // Run scope, not agent output — keeps the rendered report and the schema the agent
+            // was given in agreement.
+            exploit,
+            ...(existing.report_meta.model !== undefined && { model: String(existing.report_meta.model) }),
+          };
+        }
+      } catch {
+        logger.warn('Failed to read report_meta from report.json, using defaults');
+      }
+    }
+
+    const reportData: ReportData = {
+      report_meta: reportMeta,
+      findings,
+      ...(input.failedClasses && input.failedClasses.length > 0 && { not_assessed: input.failedClasses }),
+    };
+
+    await atomicWrite(reportJsonPath, JSON.stringify(reportData, null, 2));
+    logger.info(`Wrote ${REPORT_JSON_FILENAME} with ${findings.length} finding(s)`);
+
+    await atomicWrite(path.join(deliverablesPath, ASSEMBLED_REPORT_FILENAME), renderReport(reportData));
+    logger.info(`Wrote ${ASSEMBLED_REPORT_FILENAME} from structured data`);
+
+    await writeSarifIfEnabled(input, exploit, reportData, deliverablesPath, logger);
+  };
+
+  return runAgentActivity('report', input, collector.tools, writeDeliverable);
 }
 
 /**

apps/worker/src/temporal/shared.ts

@@ -2,7 +2,7 @@ import { defineQuery } from '@temporalio/workflow';
 
 export type { AgentMetrics } from '../types/metrics.js';
 
-import type { DistributedConfig, PipelineConfig, VulnClass } from '../types/config.js';
+import type { DistributedConfig, VulnClass } from '../types/config.js';
 import type { ErrorCode } from '../types/errors.js';
 import type { AgentMetrics } from '../types/metrics.js';
 
@@ -12,7 +12,6 @@ export interface PipelineInput {
   configPath?: string;
   outputPath?: string;
   pipelineTestingMode?: boolean;
-  pipelineConfig?: PipelineConfig;
   workflowId?: string; // Used for audit correlation
   sessionId?: string; // Workspace directory name (distinct from workflowId for named workspaces)
   resumeFromWorkspace?: string; // Workspace name to resume from

apps/worker/src/temporal/worker.ts

@@ -36,7 +36,7 @@ import dotenv from 'dotenv';
 import { sanitizeHostname } from '../audit/utils.js';
 import { parseConfig } from '../config-parser.js';
 import { ASSEMBLED_REPORT_FILENAME, deliverablesDir, FINAL_REPORT_FILENAME, resolveSessionJsonPath } from '../paths.js';
-import type { PipelineConfig, VulnClass } from '../types/config.js';
+import type { VulnClass } from '../types/config.js';
 import { fileExists, readJson } from '../utils/file-io.js';
 import * as activities from './activities.js';
 import type { PipelineInput, PipelineProgress, PipelineState } from './shared.js';
@@ -276,26 +276,16 @@ async function resolveWorkspace(client: Client, args: CliArgs): Promise<Workspac
 // === Pipeline Input Construction ===
 
 interface OrchestrationConfig {
-  pipelineConfig: PipelineConfig;
   vulnClasses?: VulnClass[];
   exploit?: boolean;
 }
 
 async function loadOrchestrationConfig(configPath: string | undefined): Promise<OrchestrationConfig> {
-  if (!configPath) return { pipelineConfig: {} };
+  if (!configPath) return {};
   try {
     const config = await parseConfig(configPath);
 
-    const pipelineConfig: PipelineConfig = {};
-    if (config.pipeline?.retry_preset !== undefined) {
-      pipelineConfig.retry_preset = config.pipeline.retry_preset;
-    }
-    if (config.pipeline?.max_concurrent_pipelines !== undefined) {
-      pipelineConfig.max_concurrent_pipelines = Number(config.pipeline.max_concurrent_pipelines);
-    }
-
     return {
-      pipelineConfig,
       ...(config.vuln_classes && config.vuln_classes.length > 0 && { vulnClasses: [...config.vuln_classes] }),
       ...(config.exploit !== undefined && { exploit: config.exploit === 'true' }),
     };
@@ -322,7 +312,6 @@ function buildPipelineInput(
     ...(args.pipelineTestingMode && { pipelineTestingMode: args.pipelineTestingMode }),
     ...(workspace.isResume && args.resumeFromWorkspace && { resumeFromWorkspace: args.resumeFromWorkspace }),
     ...(workspace.terminatedWorkflows.length > 0 && { terminatedWorkflows: workspace.terminatedWorkflows }),
-    ...(Object.keys(orchestration.pipelineConfig).length > 0 && { pipelineConfig: orchestration.pipelineConfig }),
     ...(orchestration.vulnClasses && { vulnClasses: orchestration.vulnClasses }),
     ...(orchestration.exploit !== undefined && { exploit: orchestration.exploit }),
   };

apps/worker/src/temporal/workflow-errors.ts

@@ -21,8 +21,6 @@ import { ErrorCode } from '../types/errors.js';
  */
 const ERROR_TYPE_TO_CODE: Record<string, ErrorCode> = {
   AuthenticationError: ErrorCode.AUTH_FAILED,
-  BillingError: ErrorCode.BILLING_ERROR,
-  RateLimitError: ErrorCode.API_RATE_LIMITED,
   ConfigurationError: ErrorCode.CONFIG_VALIDATION_FAILED,
   OutputValidationError: ErrorCode.OUTPUT_VALIDATION_FAILED,
   AgentExecutionError: ErrorCode.AGENT_EXECUTION_FAILED,
@@ -44,13 +42,10 @@ export function classifyErrorCode(error: unknown): ErrorCode | undefined {
 
 /** Maps Temporal error type strings to actionable remediation hints. */
 const REMEDIATION_HINTS: Record<string, string> = {
-  AuthenticationError: 'Verify ANTHROPIC_API_KEY or CLAUDE_CODE_OAUTH_TOKEN in .env is valid and not expired.',
+  AuthenticationError: "Verify the selected provider's API key is valid and not expired.",
   ConfigurationError: 'Check your CONFIG file path and contents.',
-  BillingError: 'Check your Anthropic billing dashboard. Add credits or wait for spending cap reset.',
   GitError: 'Check repository path and git state.',
   InvalidTargetError: 'Verify the target URL is correct and accessible.',
-  PermissionError: 'Check file and network permissions.',
-  ExecutionLimitError: 'Agent exceeded maximum turns or budget. Review prompt complexity.',
 };
 
 /**

apps/worker/src/temporal/workflows.ts

@@ -17,7 +17,7 @@
  *
  * Features:
  * - Queryable state via getProgress
- * - Automatic retry with backoff for transient/billing errors
+ * - Automatic retry with backoff for transient errors
  * - Non-retryable classification for permanent errors
  * - Audit correlation via workflowId
  * - Graceful failure handling: pipelines continue if one fails
@@ -64,21 +64,21 @@ function computeExpectedAgents(vulnClasses: readonly VulnClass[], exploit: boole
   return expected;
 }
 
-// Retry configuration for production (long intervals for billing recovery)
+// Retry configuration for production (long intervals so a rate-limit window can clear)
 const PRODUCTION_RETRY = {
   initialInterval: '5 minutes',
   maximumInterval: '30 minutes',
   backoffCoefficient: 2,
   maximumAttempts: 50,
+  // Belt-and-braces: activities already throw non-retryable ApplicationFailures for
+  // these. Only types that are always permanent belong here — GitError and
+  // AgentExecutionError carry a per-error verdict and must not be listed.
   nonRetryableErrorTypes: [
     'AuthenticationError',
-    'PermissionError',
-    'InvalidRequestError',
-    'RequestTooLargeError',
     'ConfigurationError',
     'InvalidTargetError',
-    'ExecutionLimitError',
     'AuthLoginFailedError',
+    'PermanentError',
   ],
 };
 
@@ -105,22 +105,6 @@ const testActs = proxyActivities<typeof activities>({
   retry: TESTING_RETRY,
 });
 
-// Retry configuration for subscription plans (5h+ rolling rate limit windows)
-const SUBSCRIPTION_RETRY = {
-  initialInterval: '5 minutes',
-  maximumInterval: '6 hours',
-  backoffCoefficient: 2,
-  maximumAttempts: 100,
-  nonRetryableErrorTypes: PRODUCTION_RETRY.nonRetryableErrorTypes,
-};
-
-// Activity proxy for subscription plan recovery (extended timeouts)
-const subscriptionActs = proxyActivities<typeof activities>({
-  startToCloseTimeout: '8 hours',
-  heartbeatTimeout: '2 hours',
-  retry: SUBSCRIPTION_RETRY,
-});
-
 // Retry configuration for preflight validation (short timeout, few retries)
 const PREFLIGHT_RETRY = {
   initialInterval: '10 seconds',
@@ -167,6 +151,9 @@ function computeSummary(state: PipelineState): PipelineSummary {
   };
 }
 
+/** One pipeline per vulnerability class, all five in flight together. */
+const MAX_CONCURRENT_PIPELINES = 5;
+
 const MAX_PIPELINE_ERROR_MESSAGE_LENGTH = 2000;
 
 function truncatePipelineErrorMessage(message: string): string {
@@ -200,14 +187,7 @@ export async function pentestPipeline(input: PipelineInput): Promise<PipelineSta
 
   const { workflowId } = workflowInfo();
 
-  // Select activity proxy based on mode: testing (fast), subscription (extended), or default
-  function selectActivityProxy(pipelineInput: PipelineInput) {
-    if (pipelineInput.pipelineTestingMode) return testActs;
-    if (pipelineInput.pipelineConfig?.retry_preset === 'subscription') return subscriptionActs;
-    return acts;
-  }
-
-  const a = selectActivityProxy(input);
+  const a = input.pipelineTestingMode ? testActs : acts;
 
   const state: PipelineState = {
     status: 'running',
@@ -611,8 +591,6 @@ export async function pentestPipeline(input: PipelineInput): Promise<PipelineSta
       }
     }
 
-    const maxConcurrent = input.pipelineConfig?.max_concurrent_pipelines ?? 5;
-
     const pipelineConfigs = buildPipelineConfigs();
     const pipelineThunks: Array<() => Promise<VulnExploitPipelineResult>> = [];
     let alreadyCompletedPipelineCount = 0;
@@ -632,9 +610,15 @@ export async function pentestPipeline(input: PipelineInput): Promise<PipelineSta
       }
     }
 
-    const pipelineResults = await runWithConcurrencyLimit(pipelineThunks, maxConcurrent);
+    const pipelineResults = await runWithConcurrencyLimit(pipelineThunks, MAX_CONCURRENT_PIPELINES);
     aggregatePipelineResults(pipelineResults, alreadyCompletedPipelineCount);
 
+    // Surface the not-assessed classes to the report stage so a failed class renders as
+    // "analysis did not complete" rather than the absence assertion "no findings".
+    if (state.failedPipelines.length > 0) {
+      activityInput.failedClasses = state.failedPipelines.map((f) => f.vulnType);
+    }
+
     state.currentPhase = 'exploitation';
     state.currentAgent = null;
     await a.logPhaseTransition(activityInput, 'vulnerability-exploitation', 'complete');
@@ -649,7 +633,7 @@ export async function pentestPipeline(input: PipelineInput): Promise<PipelineSta
       await a.assembleReportActivity(activityInput, exploit);
 
       // Then run the report agent to add executive summary and clean up
-      state.agentMetrics.report = await a.runReportAgent(activityInput);
+      state.agentMetrics.report = await a.runReportAgent(activityInput, exploit);
       state.completedAgents.push('report');
       if (input.checkpointsEnabled) {
         await a.saveCheckpoint(activityInput, 'report', 'reporting', state);