CalvinBackup/shannon
1
0
Fork 0
mirror of https://github.com/KeygraphHQ/shannon.git synced 2026-08-11 05:50:21 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat: multi-provider model support, SARIF output, and exploit-mode fixes (#402)

Browse Source 
* feat(worker): record token, cache, and turn usage per agent

* feat: replace model tiers with a single SHANNON_AI_MODEL across five providers

* feat(cli): rebuild the setup wizard for provider and model selection

* docs: document single-model selection and supported providers

* feat(worker): use chat completions for OpenAI behind a custom base URL

* feat: add SHANNON_AI_OPENAI_FORMAT to pick the wire API for OpenAI gateways

* refactor(cli): drop endpoint path hints from the gateway format picker

* feat(worker): enable pi in-session provider retry with retry-after backoff

* refactor(worker): hand provider error classification to pi and drop the Anthropic ladders

* refactor: remove the subscription retry preset and pipeline config section

* fix(worker): validate Bedrock credentials with the same live probe as other providers

* feat(worker): render the report from structured findings instead of agent-written markdown

* fix(worker): dispose the credential probe session on every path

* fix(worker): refuse to replace the assembled report with an empty one

* refactor(worker): catch post-processing throws across the whole finalization block

* revert(worker): drop the report zero-findings guard

* docs(worker): correct the retry split and Bedrock credential claims

* docs: regenerate llms-full.txt from current sources

* feat(cli): build and run the npx flow from a clone

* refactor(cli): flatten the setup summary output

* feat(cli): reject runs with more than one provider configured

* fix(worker): say a rejected bash call never ran

* chore(cli): drop grok-4.3 and gpt-5.6-luna from the setup suggestions

* feat(worker): capture structured finding locations for SARIF output

* fix(worker): enumerate queue confidence so the report inherits it verbatim

* feat(worker): give the reporting phase a mode-specific output schema

* feat(worker): emit a SARIF 2.1.0 log for exploitative runs

* fix(worker): correct SARIF locations and defer fingerprinting to the upload action

* fix(worker): drop the confidence suffix from the analysis-mode summary list

* feat(worker): give exploit findings a dedicated code location field

* feat(worker): carry structured code locations from the vuln queue to the report

* fix(worker): join code locations from the vuln queue instead of re-asking agents

* fix(worker): spell out the finding_id to category mapping in the tool schema

* feat: drop Google/Gemini as a supported AI provider

* fix(worker): stop asking the report agent for code locations

* docs: correct the provider list and drop the removed rate-limit settings

* docs: add provider cyber safeguards and suggested models per provider

* docs: document the SARIF output and the report rating thresholds
This commit is contained in:
ezl-keygraph
2026-07-30 19:31:52 +05:30
committed by GitHub
parent 30a12114ae
commit 1ce250d6a5
69 changed files with 3225 additions and 1471 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/cli/src/commands/build.ts
+6 -9
View File
@@ -1,19 +1,16 @@
/**
* `shannon build` command — build the worker Docker image locally.
* Only available in local mode (running from cloned repository).
* `shannon build` command — build the worker Docker image from the repository.
* Requires a clone (Dockerfile in the working directory).
*/
import { buildImage } from '../docker.js';
import { isLocal } from '../mode.js';
import { buildImage, canBuildImage } from '../docker.js';
export function build(noCache: boolean): void {
if (!isLocal()) {
export function build(noCache: boolean, version: string): void {
if (!canBuildImage()) {
console.error('ERROR: Build is only available when running from the Shannon repository');
console.error(' (Dockerfile not found in current directory)');
console.error('');
console.error('For npx usage, run: shannon update');
process.exit(1);
}
buildImage(noCache);
buildImage(noCache, version);
}
apps/cli/src/commands/setup.ts
+182 -155
View File
@@ -1,56 +1,110 @@
/**
* `npx @keygraph/shannon setup` — interactive TUI wizard for one-time credential configuration.
*
* Walks the user through selecting a provider and entering credentials,
* then persists everything to ~/.shannon/config.toml with 0o600 permissions.
* Walks the user through selecting a provider, entering credentials, and naming
* the model that runs the whole scan, then persists everything to
* ~/.shannon/config.toml with 0o600 permissions.
*/
import os from 'node:os';
import path from 'node:path';
import * as p from '@clack/prompts';
import { type ShannonConfig, saveConfig } from '../config/writer.js';
import { type OpenAiFormat, type ProviderId, SUPPORTED_PROVIDERS } from '../model-spec.js';
import { requireInteractive } from '../tty.js';
const SHANNON_HOME = path.join(os.homedir(), '.shannon');
type Provider = 'anthropic' | 'custom_base_url' | 'bedrock';
const CUSTOM_MODEL = '__custom__';
const CUSTOM_BASE_URL = '__custom_base_url__';
/**
* Wire formats reachable through the gateway route. The format picks the provider
* that supplies the credential, and for OpenAI it also picks which of the two
* OpenAI APIs Shannon calls.
*/
const GATEWAY_DIALECTS: readonly {
value: string;
label: string;
provider: 'anthropic' | 'openai';
format?: OpenAiFormat;
}[] = [
{ value: 'anthropic', label: 'Anthropic Messages', provider: 'anthropic' },
{
value: 'openai-chat-completions',
label: 'OpenAI Chat Completions',
provider: 'openai',
format: 'chat-completions',
},
{ value: 'openai-responses', label: 'OpenAI Responses', provider: 'openai', format: 'responses' },
];
/** Suggested models per provider, best-first. Free-text entry accepts any model in the provider's catalogue. */
const MODEL_SUGGESTIONS: Readonly<Record<ProviderId, readonly string[]>> = {
anthropic: ['claude-sonnet-4-6', 'claude-opus-4-8', 'claude-opus-4-7', 'claude-haiku-4-5-20251001'],
openai: ['gpt-5.6-sol', 'gpt-5.5', 'gpt-5.4'],
xai: ['grok-4.5'],
'amazon-bedrock': ['us.anthropic.claude-sonnet-4-6', 'us.anthropic.claude-opus-4-8', 'us.anthropic.claude-opus-4-7'],
};
/** Placeholder shown in the free-text model ID prompt. */
const MODEL_ID_PLACEHOLDER: Readonly<Record<ProviderId, string>> = {
anthropic: 'claude-sonnet-4-6',
openai: 'gpt-5.6-sol',
xai: 'grok-4.5',
'amazon-bedrock': 'us.anthropic.claude-opus-4-8',
};
export async function setup(): Promise<void> {
requireInteractive('setup', 'For non-interactive use, export credentials as env vars (e.g. ANTHROPIC_API_KEY).');
p.intro('Shannon Setup');
// 1. Select provider
const provider = await p.select({
// 1. Select provider. "Custom Base URL" is a route, not a provider — it asks
// which API dialect the gateway speaks and configures that provider.
const selected = await p.select({
message: 'Select your AI provider',
options: [
{ value: 'anthropic' as const, label: 'Claude Direct', hint: 'recommended' },
{ value: 'custom_base_url' as const, label: 'Custom Base URL', hint: 'proxies, gateways' },
{ value: 'bedrock' as const, label: 'Claude via AWS Bedrock' },
{ value: 'anthropic' as const, label: 'Anthropic', hint: 'Claude models - recommended' },
{ value: 'openai' as const, label: 'OpenAI', hint: 'GPT models' },
{ value: 'xai' as const, label: 'xAI', hint: 'Grok models' },
{ value: 'amazon-bedrock' as const, label: 'AWS Bedrock', hint: 'Claude models via AWS' },
{ value: CUSTOM_BASE_URL as typeof CUSTOM_BASE_URL, label: 'Custom Base URL', hint: 'your own proxy or gateway' },
],
});
if (p.isCancel(provider)) return cancelAndExit();
if (p.isCancel(selected)) return cancelAndExit();
const config = await setupProvider(provider as Provider);
// 2. Credentials — and, on the gateway route, the endpoint and its dialect.
const gateway = selected === CUSTOM_BASE_URL ? await setupGateway() : undefined;
const provider = gateway?.provider ?? (selected as ProviderId);
const config = gateway?.config ?? (await setupProvider(provider));
// 2. Adaptive thinking
await maybePromptAdaptiveThinking(config);
// 3. The model that runs every phase.
const modelId = await promptModel(provider);
config.core = { ...config.core, model: `${provider}:${modelId}` };
if (gateway) config.core = { ...config.core, base_url: gateway.baseUrl };
// 3. Save config
saveConfig(config);
const configPath = path.join(SHANNON_HOME, 'config.toml');
const summary = [`Provider ${provider}`, `Model ${modelId}`];
if (gateway) summary.push(`Endpoint ${gateway.baseUrl}`);
if (gateway?.format) summary.push(`API ${gateway.format}`);
p.log.success(`Configuration saved to ${configPath}`);
p.log.info(summary.join('\n'));
p.outro('Run `npx @keygraph/shannon start` to begin a scan.');
}
async function setupProvider(provider: Provider): Promise<ShannonConfig> {
async function setupProvider(provider: ProviderId): Promise<ShannonConfig> {
switch (provider) {
case 'amazon-bedrock':
return setupBedrock();
case 'anthropic':
return setupAnthropic();
case 'custom_base_url':
return setupCustomBaseUrl();
case 'bedrock':
return setupBedrock();
case 'openai':
return { openai: { api_key: await promptSecret('Enter your OpenAI API key') } };
case 'xai':
return { xai: { api_key: await promptSecret('Enter your xAI API key') } };
}
}
@@ -66,112 +120,13 @@ async function setupAnthropic(): Promise<ShannonConfig> {
});
if (p.isCancel(authMethod)) return cancelAndExit();
const config: ShannonConfig = {};
if (authMethod === 'oauth') {
const token = await promptSecret('Enter your OAuth token');
config.anthropic = { oauth_token: token };
} else {
const apiKey = await promptSecret('Enter your Anthropic API key');
config.anthropic = { api_key: apiKey };
return { anthropic: { oauth_token: token } };
}
const customizeModels = await p.confirm({
message:
'Do you want to change the default models?\n' +
' Small - claude-haiku-4-5-20251001\n' +
' Medium - claude-sonnet-4-6\n' +
' Large - claude-opus-4-8',
initialValue: false,
});
if (p.isCancel(customizeModels)) return cancelAndExit();
if (customizeModels) {
const small = await p.text({
message: 'Small model ID',
initialValue: 'claude-haiku-4-5-20251001',
validate: required('Small model ID is required'),
});
if (p.isCancel(small)) return cancelAndExit();
const medium = await p.text({
message: 'Medium model ID',
initialValue: 'claude-sonnet-4-6',
validate: required('Medium model ID is required'),
});
if (p.isCancel(medium)) return cancelAndExit();
const large = await p.text({
message: 'Large model ID',
initialValue: 'claude-opus-4-8',
validate: required('Large model ID is required'),
});
if (p.isCancel(large)) return cancelAndExit();
config.models = { small, medium, large };
}
return config;
}
async function setupCustomBaseUrl(): Promise<ShannonConfig> {
const baseUrl = await p.text({
message: 'Endpoint URL',
placeholder: 'https://your-proxy.example.com',
validate: (value) => {
if (!value) return 'Endpoint URL is required';
try {
new URL(value);
} catch {
return 'Must be a valid URL';
}
return undefined;
},
});
if (p.isCancel(baseUrl)) return cancelAndExit();
const authToken = await promptSecret('Enter the auth token for the custom endpoint');
const config: ShannonConfig = {
custom_base_url: { base_url: baseUrl, auth_token: authToken },
};
const customizeModels = await p.confirm({
message:
'Do you want to change the default models?\n' +
' Small - claude-haiku-4-5-20251001\n' +
' Medium - claude-sonnet-4-6\n' +
' Large - claude-opus-4-8',
initialValue: false,
});
if (p.isCancel(customizeModels)) return cancelAndExit();
if (customizeModels) {
const small = await p.text({
message: 'Small model ID',
initialValue: 'claude-haiku-4-5-20251001',
validate: required('Small model ID is required'),
});
if (p.isCancel(small)) return cancelAndExit();
const medium = await p.text({
message: 'Medium model ID',
initialValue: 'claude-sonnet-4-6',
validate: required('Medium model ID is required'),
});
if (p.isCancel(medium)) return cancelAndExit();
const large = await p.text({
message: 'Large model ID',
initialValue: 'claude-opus-4-8',
validate: required('Large model ID is required'),
});
if (p.isCancel(large)) return cancelAndExit();
config.models = { small, medium, large };
}
return config;
const apiKey = await promptSecret('Enter your Anthropic API key');
return { anthropic: { api_key: apiKey } };
}
async function setupBedrock(): Promise<ShannonConfig> {
@@ -184,49 +139,121 @@ async function setupBedrock(): Promise<ShannonConfig> {
const token = await promptSecret('Enter your AWS Bearer Token');
const small = await p.text({
message: 'Small model ID',
placeholder: 'us.anthropic.claude-haiku-4-5-20251001-v1:0',
validate: required('Small model ID is required'),
});
if (p.isCancel(small)) return cancelAndExit();
return { bedrock: { region, token } };
}
const medium = await p.text({
message: 'Medium model ID',
placeholder: 'us.anthropic.claude-sonnet-4-6',
validate: required('Medium model ID is required'),
});
if (p.isCancel(medium)) return cancelAndExit();
interface GatewaySetup {
provider: ProviderId;
config: ShannonConfig;
baseUrl: string;
format?: OpenAiFormat;
}
const large = await p.text({
message: 'Large model ID',
placeholder: 'us.anthropic.claude-opus-4-8',
validate: required('Large model ID is required'),
/**
* Gateway route: the endpoint decides where requests go, but the format still
* picks a real provider, because that is what supplies the credential and the
* wire protocol.
*/
async function setupGateway(): Promise<GatewaySetup> {
const choice = await p.select({
message: 'API format',
options: GATEWAY_DIALECTS.map(({ value, label }) => ({ value, label })),
});
if (p.isCancel(large)) return cancelAndExit();
if (p.isCancel(choice)) return cancelAndExit();
return {
bedrock: { use: true, region, token },
models: { small, medium, large },
};
const dialect = GATEWAY_DIALECTS.find((entry) => entry.value === choice);
if (!dialect) return cancelAndExit();
const provider = dialect.provider;
const baseUrl = await p.text({
message: 'Endpoint URL',
placeholder: 'https://llm-gateway.example.com',
validate: (value) => {
if (!value) return 'Endpoint URL is required';
try {
new URL(value);
} catch {
return 'Must be a valid URL';
}
return undefined;
},
});
if (p.isCancel(baseUrl)) return cancelAndExit();
const authToken = await promptSecret('Enter the auth token for the endpoint');
const config: ShannonConfig =
provider === 'anthropic'
? { anthropic: { api_key: authToken } }
: { openai: { api_key: authToken, ...(dialect.format && { format: dialect.format }) } };
return { provider, config, baseUrl, ...(dialect.format && { format: dialect.format }) };
}
// === Model Selection ===
/**
* Ask for the one model that runs every phase. Providers with suggestions offer a
* pick list with a free-text escape hatch; the rest go straight to free text.
*/
async function promptModel(provider: ProviderId): Promise<string> {
const suggestions = MODEL_SUGGESTIONS[provider];
if (suggestions.length === 0) {
return promptModelId(provider, MODEL_ID_PLACEHOLDER[provider]);
}
const choice = await p.select({
message: 'Model',
options: [
...suggestions.map((model) => ({ value: model, label: model })),
{ value: CUSTOM_MODEL, label: 'Enter a model ID…' },
],
});
if (p.isCancel(choice)) return cancelAndExit();
if (choice === CUSTOM_MODEL) {
return promptModelId(provider, MODEL_ID_PLACEHOLDER[provider]);
}
return choice as string;
}
/**
* A leading `<provider>:` naming a supported provider other than the selected
* one. Bedrock model IDs carry their own colons (`…-v1:0`), so only a genuine
* provider id counts as a prefix.
*/
function conflictingProviderPrefix(provider: ProviderId, value: string): string | undefined {
const separator = value.indexOf(':');
if (separator === -1) return undefined;
const head = value.slice(0, separator);
if (head === provider) return undefined;
return (SUPPORTED_PROVIDERS as readonly string[]).includes(head) ? head : undefined;
}
/**
* Ask for a model ID. The provider is already chosen, so this takes the bare ID
* and the caller pairs it with the provider — pasting a full `<provider>:<model>`
* spec just has its redundant prefix dropped.
*/
async function promptModelId(provider: ProviderId, placeholder: string): Promise<string> {
const modelId = await p.text({
message: 'Model ID',
placeholder,
validate: (value) => {
if (!value) return 'Model ID is required';
const conflicting = conflictingProviderPrefix(provider, value);
if (conflicting) return `That model ID is for ${conflicting}, but you selected ${provider}.`;
return undefined;
},
});
if (p.isCancel(modelId)) return cancelAndExit();
return modelId.startsWith(`${provider}:`) ? modelId.slice(provider.length + 1) : modelId;
}
// === Helpers ===
async function maybePromptAdaptiveThinking(config: ShannonConfig): Promise<void> {
const m = config.models;
const hasAdaptiveModel = !m || [m.small, m.medium, m.large].some((v) => v && /opus-4-[678]/.test(v));
if (!hasAdaptiveModel) return;
const enable = await p.confirm({
message: 'Enable adaptive thinking on Opus 4.6/4.7/4.8? Claude decides when and how deeply to reason.',
initialValue: true,
});
if (p.isCancel(enable)) return cancelAndExit();
config.core = { ...config.core, adaptive_thinking: enable };
}
async function promptSecret(message: string): Promise<string> {
const value = await p.password({
message,
apps/cli/src/commands/start.ts
+4 -13
View File
@@ -12,6 +12,7 @@ import { ensureImage, ensureInfra, randomSuffix, spawnWorker } from '../docker.j
import { buildEnvFlags, loadEnv, validateCredentials } from '../env.js';
import { getWorkspacesDir, initHome } from '../home.js';
import { isLocal } from '../mode.js';
import { resolveModelSpec } from '../model-spec.js';
import { FINAL_REPORT_FILENAME, INTERNAL_DIR, resolveConfig, resolveRepo, resolveRunFile } from '../paths.js';
import { displaySplash } from '../splash.js';
import { stdoutIsTerminal } from '../tty.js';
@@ -261,19 +262,9 @@ function printInfo(
console.log(' Mode: Pipeline Testing');
}
// Surface Fable usage: its safety classifiers route cybersecurity tasks to
// Opus 4.8, so those phases run on Opus 4.8 regardless of the tier setting.
const fableTiers = (
[
['small', process.env.ANTHROPIC_SMALL_MODEL],
['medium', process.env.ANTHROPIC_MEDIUM_MODEL],
['large', process.env.ANTHROPIC_LARGE_MODEL],
] as const
).filter(([, model]) => model && /fable/i.test(model));
if (fableTiers.length > 0) {
const tierList = fableTiers.map(([tier, model]) => `${tier} (${model})`).join(', ');
console.log(` Note: ${tierList} set to a Fable model. Fable's safety classifiers`);
console.log(' route cybersecurity tasks to Opus 4.8, so those phases run on Opus 4.8.');
const spec = resolveModelSpec();
if (typeof spec !== 'string') {
console.log(` Model: ${spec.providerId}:${spec.modelId}`);
}
console.log('');
apps/cli/src/config/resolver.ts
+52 -68
View File
@@ -9,6 +9,7 @@ import fs from 'node:fs';
import { parse as parseTOML } from 'smol-toml';
import { getConfigFile } from '../home.js';
import { getMode } from '../mode.js';
import { DEFAULT_MODEL_SPEC, type ProviderId, parseModelSpec } from '../model-spec.js';
// === TOML ↔ Env Mapping ===
@@ -23,28 +24,34 @@ interface ConfigMapping {
/** Maps every supported env var to its TOML path (section.key) and expected type. */
const CONFIG_MAP: readonly ConfigMapping[] = [
// Core
{ env: 'CLAUDE_ADAPTIVE_THINKING', toml: 'core.adaptive_thinking', type: 'boolean', boolFormat: 'literal' },
// Core — base_url points any provider at a proxy or gateway
{ env: 'SHANNON_AI_MODEL', toml: 'core.model', type: 'string' },
{ env: 'SHANNON_AI_BASE_URL', toml: 'core.base_url', type: 'string' },
// Anthropic
{ env: 'ANTHROPIC_API_KEY', toml: 'anthropic.api_key', type: 'string' },
{ env: 'CLAUDE_CODE_OAUTH_TOKEN', toml: 'anthropic.oauth_token', type: 'string' },
// OpenAI — format picks the wire API a gateway serves
{ env: 'OPENAI_API_KEY', toml: 'openai.api_key', type: 'string' },
{ env: 'SHANNON_AI_OPENAI_FORMAT', toml: 'openai.format', type: 'string' },
// xAI
{ env: 'XAI_API_KEY', toml: 'xai.api_key', type: 'string' },
// Bedrock
{ env: 'CLAUDE_CODE_USE_BEDROCK', toml: 'bedrock.use', type: 'boolean' },
{ env: 'AWS_REGION', toml: 'bedrock.region', type: 'string' },
{ env: 'AWS_BEARER_TOKEN_BEDROCK', toml: 'bedrock.token', type: 'string' },
// Custom Base URL
{ env: 'ANTHROPIC_BASE_URL', toml: 'custom_base_url.base_url', type: 'string' },
{ env: 'ANTHROPIC_AUTH_TOKEN', toml: 'custom_base_url.auth_token', type: 'string' },
// Model tiers
{ env: 'ANTHROPIC_SMALL_MODEL', toml: 'models.small', type: 'string' },
{ env: 'ANTHROPIC_MEDIUM_MODEL', toml: 'models.medium', type: 'string' },
{ env: 'ANTHROPIC_LARGE_MODEL', toml: 'models.large', type: 'string' },
] as const;
/** TOML section holding each provider's credentials, keyed by provider id. */
const PROVIDER_SECTIONS: Readonly<Record<ProviderId, string>> = {
anthropic: 'anthropic',
openai: 'openai',
xai: 'xai',
'amazon-bedrock': 'bedrock',
};
// === TOML Parsing ===
type TOMLValue = string | number | boolean;
@@ -118,52 +125,33 @@ function buildSchema(): Map<string, Map<string, TOMLType>> {
return schema;
}
/** Check that a provider section has all required fields and dependencies. */
function validateProviderFields(config: TOMLConfig, provider: string, errors: string[]): void {
const section = config[provider] as Record<string, unknown> | undefined;
if (!section) return;
const keys = Object.keys(section);
/**
* Check that the section backing the selected provider carries a usable
* credential. `core.model` names the provider, so only that section is required;
* other providers' sections are ignored and never forwarded.
*/
function validateProviderFields(config: TOMLConfig, providerId: ProviderId, errors: string[]): void {
const sectionName = PROVIDER_SECTIONS[providerId];
const section = config[sectionName] as Record<string, unknown> | undefined;
const keys = section ? Object.keys(section) : [];
switch (provider) {
case 'anthropic':
if (!keys.includes('api_key') && !keys.includes('oauth_token')) {
errors.push('[anthropic] requires either api_key or oauth_token');
}
break;
case 'custom_base_url': {
const required = ['base_url', 'auth_token'];
const missing = required.filter((k) => !keys.includes(k));
if (missing.length > 0) {
errors.push(`[custom_base_url] missing required keys: ${missing.join(', ')}`);
}
break;
if (providerId === 'amazon-bedrock') {
const missing = ['region', 'token'].filter((k) => !keys.includes(k));
if (missing.length > 0) {
errors.push(`[bedrock] missing required keys: ${missing.join(', ')}`);
}
case 'bedrock': {
const required = ['use', 'region', 'token'];
const missing = required.filter((k) => !keys.includes(k));
if (missing.length > 0) {
errors.push(`[bedrock] missing required keys: ${missing.join(', ')}`);
}
validateModelTiers(config, 'bedrock', errors);
break;
}
}
}
/** Bedrock requires a [models] section with all three tiers. */
function validateModelTiers(config: TOMLConfig, provider: string, errors: string[]): void {
const models = config.models as Record<string, unknown> | undefined;
if (!models || typeof models !== 'object') {
errors.push(`[${provider}] requires a [models] section with small, medium, and large`);
return;
}
const required = ['small', 'medium', 'large'];
const missing = required.filter((k) => !Object.keys(models).includes(k));
if (missing.length > 0) {
errors.push(`[models] missing required keys for ${provider}: ${missing.join(', ')}`);
if (providerId === 'anthropic') {
if (!keys.includes('api_key') && !keys.includes('oauth_token')) {
errors.push('[anthropic] requires either api_key or oauth_token');
}
return;
}
if (!keys.includes('api_key')) {
errors.push(`[${sectionName}] requires api_key`);
}
}
@@ -211,23 +199,19 @@ function validateConfig(config: TOMLConfig): string[] {
}
}
// 4. Only one provider section allowed (ignore empty sections)
const PROVIDER_SECTIONS = ['anthropic', 'custom_base_url', 'bedrock'] as const;
const present = PROVIDER_SECTIONS.filter((s) => {
const section = config[s];
return section && typeof section === 'object' && Object.keys(section).length > 0;
});
if (present.length > 1) {
errors.push(
`Multiple providers configured: [${present.join('], [')}]. Only one provider section is allowed at a time`,
);
// 4. core.model must parse and name a supported provider
const modelValue = config.core?.model;
if (modelValue !== undefined && typeof modelValue !== 'string') {
return errors;
}
const spec = parseModelSpec(modelValue || DEFAULT_MODEL_SPEC);
if (typeof spec === 'string') {
errors.push(`[core].model — ${spec}`);
return errors;
}
// 5. Required fields per provider
const singleProvider = present.length === 1 ? present[0] : undefined;
if (singleProvider) {
validateProviderFields(config, singleProvider, errors);
}
// 5. The selected provider's section must carry a credential
validateProviderFields(config, spec.providerId, errors);
return errors;
}
apps/cli/src/config/writer.ts
+4 -4
View File
@@ -8,11 +8,11 @@ import { getConfigFile } from '../home.js';
// === Types ===
export interface ShannonConfig {
core?: { adaptive_thinking?: boolean };
core?: { model?: string; base_url?: string };
anthropic?: { api_key?: string; oauth_token?: string };
custom_base_url?: { base_url?: string; auth_token?: string };
bedrock?: { use?: boolean; region?: string; token?: string };
models?: { small?: string; medium?: string; large?: string };
openai?: { api_key?: string; format?: string };
xai?: { api_key?: string };
bedrock?: { region?: string; token?: string };
}
// === File Operations ===
apps/cli/src/docker.ts
+22 -9
View File
@@ -12,7 +12,7 @@ import os from 'node:os';
import path from 'node:path';
import { setTimeout as sleep } from 'node:timers/promises';
import { fileURLToPath } from 'node:url';
import { getMode } from './mode.js';
import { getMode, isDevMode } from './mode.js';
import { INTERNAL_DIR } from './paths.js';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
@@ -24,6 +24,17 @@ export function getWorkerImage(version: string): string {
return getMode() === 'local' ? DEV_IMAGE : `${NPX_IMAGE_REPO}:${version}`;
}
/** True when the working directory supplies a Dockerfile and build context. */
export function canBuildImage(): boolean {
if (getMode() === 'local') return true;
if (!isDevMode()) return false;
const hasDockerfile = fs.existsSync(path.resolve('Dockerfile'));
const hasCompose = fs.existsSync(path.resolve('docker-compose.yml'));
return hasDockerfile && hasCompose;
}
function getComposeFile(): string {
return getMode() === 'local'
? path.resolve('docker-compose.yml')
@@ -96,29 +107,31 @@ export async function ensureInfra(): Promise<void> {
}
/**
* Build the worker image locally (local mode only).
* Build the worker image from the repository, tagged with the name this mode
* resolves at run time.
*/
export function buildImage(noCache: boolean): void {
console.log(`Building ${DEV_IMAGE}...`);
export function buildImage(noCache: boolean, version: string): void {
const image = getWorkerImage(version);
console.log(`Building ${image}...`);
const args = ['build'];
if (noCache) args.push('--no-cache');
args.push('-t', DEV_IMAGE, '.');
args.push('-t', image, '.');
execFileSync('docker', args, { stdio: 'inherit' });
console.log(`Build complete: ${DEV_IMAGE}`);
console.log(`Build complete: ${image}`);
}
/**
* Ensure the worker image is available.
* Local mode: auto-builds if missing. NPX mode: pulls from Docker Hub.
* Buildable checkout: auto-builds if missing. Otherwise: pulls from Docker Hub.
*/
export function ensureImage(version: string): void {
const image = getWorkerImage(version);
const exists = runQuiet('docker', ['image', 'inspect', image]);
if (exists) return;
if (getMode() === 'local') {
if (canBuildImage()) {
console.log('Shannon image not found, building...');
buildImage(false);
buildImage(false, version);
} else {
console.log(`Pulling ${image}...`);
try {
apps/cli/src/env.ts
+66 -71
View File
@@ -8,21 +8,28 @@
import dotenv from 'dotenv';
import { resolveConfig } from './config/resolver.js';
import { getMode } from './mode.js';
import {
PROVIDER_API_KEY_ENV,
PROVIDER_CREDENTIAL_HINT,
PROVIDER_EXTRA_ENV,
type ProviderId,
resolveModelSpec,
SUPPORTED_PROVIDERS,
} from './model-spec.js';
/** Environment variables forwarded to worker containers. */
const FORWARD_VARS = [
'ANTHROPIC_API_KEY',
'ANTHROPIC_BASE_URL',
'ANTHROPIC_AUTH_TOKEN',
'CLAUDE_CODE_OAUTH_TOKEN',
'CLAUDE_CODE_USE_BEDROCK',
'AWS_REGION',
'AWS_BEARER_TOKEN_BEDROCK',
'ANTHROPIC_SMALL_MODEL',
'ANTHROPIC_MEDIUM_MODEL',
'ANTHROPIC_LARGE_MODEL',
'CLAUDE_ADAPTIVE_THINKING',
] as const;
/**
* Variables forwarded to every worker container regardless of provider. Each is
* forwarded only when set, so an unused one never appears in the container.
*/
const COMMON_FORWARD_VARS = ['SHANNON_AI_MODEL', 'SHANNON_AI_BASE_URL', 'SHANNON_AI_OPENAI_FORMAT'] as const;
/**
* Credential variables for one provider. Only the selected provider's entries are
* forwarded, so a key for an unused provider never enters the scan container.
*/
function providerForwardVars(providerId: ProviderId): readonly string[] {
return [...PROVIDER_API_KEY_ENV[providerId], ...PROVIDER_EXTRA_ENV[providerId]];
}
/**
* Load credentials into process.env.
@@ -39,12 +46,16 @@ export function loadEnv(): void {
}
/**
* Build `-e KEY=VALUE` flags for docker run, only for set variables.
* Build `-e KEY=VALUE` flags for docker run. Forwards the common vars plus only
* the selected provider's credentials.
*/
export function buildEnvFlags(): string[] {
const flags: string[] = ['-e', 'TEMPORAL_ADDRESS=shannon-temporal:7233'];
for (const key of FORWARD_VARS) {
const spec = resolveModelSpec();
const providerVars = typeof spec === 'string' ? [] : providerForwardVars(spec.providerId);
for (const key of [...COMMON_FORWARD_VARS, ...providerVars]) {
const value = process.env[key];
if (value) {
flags.push('-e', `${key}=${value}`);
@@ -57,71 +68,55 @@ export function buildEnvFlags(): string[] {
interface CredentialValidation {
valid: boolean;
error?: string;
mode: 'api-key' | 'oauth' | 'custom-base-url' | 'bedrock';
}
/** Check if a custom Anthropic-compatible base URL is configured. */
function isCustomBaseUrlConfigured(): boolean {
return !!(process.env.ANTHROPIC_BASE_URL && process.env.ANTHROPIC_AUTH_TOKEN);
}
/** Detect which providers are configured via environment variables. */
function detectProviders(): string[] {
const providers: string[] = [];
if (process.env.ANTHROPIC_API_KEY) providers.push('Anthropic API key');
if (process.env.CLAUDE_CODE_OAUTH_TOKEN) providers.push('Anthropic OAuth');
if (isCustomBaseUrlConfigured()) providers.push('Custom Base URL');
if (process.env.CLAUDE_CODE_USE_BEDROCK === '1') providers.push('AWS Bedrock');
return providers;
}
/**
* Validate that exactly one authentication method is configured.
* Whether the selected provider has a usable credential in the environment. Any
* one API key satisfies a key-based provider; Bedrock instead needs every one of
* its AWS_ vars.
*/
function hasCredential(providerId: ProviderId): boolean {
const apiKeys = PROVIDER_API_KEY_ENV[providerId];
if (apiKeys.length > 0 && !apiKeys.some((name) => Boolean(process.env[name]))) {
return false;
}
return PROVIDER_EXTRA_ENV[providerId].every((name) => Boolean(process.env[name]));
}
/** Every provider that currently has a complete credential in the environment. */
function configuredProviders(): ProviderId[] {
return SUPPORTED_PROVIDERS.filter((providerId) => hasCredential(providerId));
}
/**
* Validate that the model selection parses and its provider has a credential.
* Runs before any Docker work so mistakes fail immediately.
*/
export function validateCredentials(): CredentialValidation {
// Reject multiple providers
const providers = detectProviders();
if (providers.length > 1) {
// 1. Model selection must parse and name a supported provider
const spec = resolveModelSpec();
if (typeof spec === 'string') {
return { valid: false, error: spec };
}
// 2. The selected provider must have a credential
if (!hasCredential(spec.providerId)) {
const hint =
getMode() === 'local'
? `Set ${PROVIDER_CREDENTIAL_HINT[spec.providerId]} in .env or export it.`
: `Export the variables or run 'npx @keygraph/shannon setup'.`;
return {
valid: false,
mode: 'api-key',
error: `Multiple providers detected: ${providers.join(', ')}. Only one provider can be active at a time.`,
error: `No credentials found for provider "${spec.providerId}". ${hint}`,
};
}
if (process.env.ANTHROPIC_API_KEY) {
return { valid: true, mode: 'api-key' };
}
if (process.env.CLAUDE_CODE_OAUTH_TOKEN) {
return { valid: true, mode: 'oauth' };
}
if (isCustomBaseUrlConfigured()) {
return { valid: true, mode: 'custom-base-url' };
}
if (process.env.CLAUDE_CODE_USE_BEDROCK === '1') {
const missing: string[] = [];
if (!process.env.AWS_REGION) missing.push('AWS_REGION');
if (!process.env.AWS_BEARER_TOKEN_BEDROCK) missing.push('AWS_BEARER_TOKEN_BEDROCK');
if (!process.env.ANTHROPIC_SMALL_MODEL) missing.push('ANTHROPIC_SMALL_MODEL');
if (!process.env.ANTHROPIC_MEDIUM_MODEL) missing.push('ANTHROPIC_MEDIUM_MODEL');
if (!process.env.ANTHROPIC_LARGE_MODEL) missing.push('ANTHROPIC_LARGE_MODEL');
if (missing.length > 0) {
return {
valid: false,
mode: 'bedrock',
error: `Bedrock mode requires: ${missing.join(', ')}`,
};
}
return { valid: true, mode: 'bedrock' };
// 3. Exactly one provider may be configured. Several complete credentials make
// the scan's provider depend on SHANNON_AI_MODEL alone, which is too easy to
// misread as "both are in play" and too easy to redirect by editing one line.
if (configuredProviders().length > 1) {
return { valid: false, error: 'Credentials for more than one provider are set.' };
}
const hint =
getMode() === 'local'
? `No credentials found. Set ANTHROPIC_API_KEY in .env or export it.`
: `Authentication not configured. Export variables or run 'npx @keygraph/shannon setup'.`;
return {
valid: false,
mode: 'api-key',
error: hint,
};
return { valid: true };
}
apps/cli/src/index.ts
+1 -1
View File
@@ -220,7 +220,7 @@ switch (command) {
setup();
break;
case 'build':
build(args.includes('--no-cache'));
build(args.includes('--no-cache'), getVersion());
break;
case 'uninstall':
if (getMode() === 'local') {
apps/cli/src/mode.ts
+4
View File
@@ -23,3 +23,7 @@ export function setMode(mode: Mode): void {
export function isLocal(): boolean {
return getMode() === 'local';
}
export function isDevMode(): boolean {
return process.env.SHANNON_DEV === '1';
}
apps/cli/src/model-spec.ts
+86
View File
@@ -0,0 +1,86 @@
/**
* Parsing for the single model setting, `SHANNON_AI_MODEL=<provider>:<model-id>`.
*
* Mirrors apps/worker/src/ai/models.ts. The CLI cannot import from the worker
* package (it ships as a standalone bundle), so the provider list and the parse
* rule are duplicated here deliberately and must stay in sync.
*/
/** Providers Shannon can currently reach. Each is a pi-ai provider id. */
export const SUPPORTED_PROVIDERS = ['anthropic', 'openai', 'xai', 'amazon-bedrock'] as const;
export type ProviderId = (typeof SUPPORTED_PROVIDERS)[number];
/**
* Env vars carrying each provider's API key, in precedence order. Any one of them
* satisfies the provider. Mirrors PROVIDER_API_KEY_ENV in apps/worker/src/ai/models.ts.
*/
export const PROVIDER_API_KEY_ENV: Readonly<Record<ProviderId, readonly string[]>> = {
anthropic: ['ANTHROPIC_API_KEY', 'CLAUDE_CODE_OAUTH_TOKEN'],
openai: ['OPENAI_API_KEY'],
xai: ['XAI_API_KEY'],
'amazon-bedrock': ['AWS_BEARER_TOKEN_BEDROCK'],
};
/** Additional env vars a provider requires beyond its API key. All must be set. */
export const PROVIDER_EXTRA_ENV: Readonly<Record<ProviderId, readonly string[]>> = {
anthropic: [],
openai: [],
xai: [],
'amazon-bedrock': ['AWS_REGION'],
};
/** Human-readable credential requirement, used in "nothing configured" errors. */
export const PROVIDER_CREDENTIAL_HINT: Readonly<Record<ProviderId, string>> = {
anthropic: 'ANTHROPIC_API_KEY (or CLAUDE_CODE_OAUTH_TOKEN)',
openai: 'OPENAI_API_KEY',
xai: 'XAI_API_KEY',
'amazon-bedrock': 'AWS_REGION and AWS_BEARER_TOKEN_BEDROCK',
};
/** Model used when SHANNON_AI_MODEL is unset. */
export const DEFAULT_MODEL_SPEC = 'anthropic:claude-sonnet-4-6';
/**
* Values SHANNON_AI_OPENAI_FORMAT accepts, selecting the wire format an
* OpenAI-compatible gateway serves. Mirrors OPENAI_FORMATS in
* apps/worker/src/ai/models.ts; the worker validates and applies it.
*/
export const OPENAI_FORMATS = ['chat-completions', 'responses'] as const;
export type OpenAiFormat = (typeof OPENAI_FORMATS)[number];
export interface ModelSpec {
providerId: ProviderId;
modelId: string;
}
function isSupportedProvider(value: string): value is ProviderId {
return (SUPPORTED_PROVIDERS as readonly string[]).includes(value);
}
/**
* Parse a `<provider>:<model-id>` spec. Splits on the first colon only, so colons
* inside a model ID survive (`amazon-bedrock:us.anthropic.claude-opus-4-5-20251101-v1:0`).
* Returns an error string rather than throwing, for the CLI's validation flow.
*/
export function parseModelSpec(spec: string): ModelSpec | string {
const trimmed = spec.trim();
const separator = trimmed.indexOf(':');
const malformed = `SHANNON_AI_MODEL must be "<provider>:<model-id>", got "${trimmed}". Example: ${DEFAULT_MODEL_SPEC}`;
if (separator === -1) return malformed;
const providerId = trimmed.slice(0, separator).trim();
const modelId = trimmed.slice(separator + 1).trim();
if (!providerId || !modelId) return malformed;
if (!isSupportedProvider(providerId)) {
return `Unsupported provider "${providerId}" in SHANNON_AI_MODEL. Supported providers: ${SUPPORTED_PROVIDERS.join(', ')}`;
}
return { providerId, modelId };
}
/** Resolve the run's model spec from the environment, or an error string. */
export function resolveModelSpec(): ModelSpec | string {
return parseModelSpec(process.env.SHANNON_AI_MODEL || DEFAULT_MODEL_SPEC);
}
apps/worker/configs/config-schema.json
+5 -17
View File
@@ -102,23 +102,6 @@
"required": ["login_type", "login_url", "credentials", "success_condition"],
"additionalProperties": false
},
"pipeline": {
"type": "object",
"description": "Pipeline execution settings for retry behavior and concurrency",
"properties": {
"retry_preset": {
"type": "string",
"enum": ["default", "subscription"],
"description": "Retry preset. 'subscription' extends timeouts for Anthropic subscription rate limit windows (5h+)."
},
"max_concurrent_pipelines": {
"type": "string",
"pattern": "^[1-5]$",
"description": "Max concurrent vulnerability pipelines (1-5, default: 5)"
}
},
"additionalProperties": false
},
"rules": {
"type": "object",
"description": "Testing rules that define what to focus on or avoid during penetration testing",
@@ -177,6 +160,11 @@
"minLength": 1,
"maxLength": 500,
"description": "Free-text guidance to the report agent (e.g., 'Drop findings about missing security headers')."
},
"sarif": {
"type": "string",
"enum": ["true", "false"],
"description": "Emit a SARIF 2.1.0 log (report.sarif) beside the report. Requires exploit=true; ignored otherwise."
}
},
"additionalProperties": false
apps/worker/configs/example-config.yaml
+2 -5
View File
@@ -96,13 +96,10 @@ rules:
# Report filters applied by the report agent when assembling the final report (optional).
# Example below is illustrative; edit, remove, or add sections as needed.
# report:
# # Emit a SARIF 2.1.0 log (report.sarif) beside the report. Requires exploit: "true".
# sarif: "true"
# min_severity: low
# min_confidence: low
# guidance: |
# Drop findings about missing security headers and rate-limit gaps.
# ...
# Pipeline execution settings (optional)
# pipeline:
# retry_preset: subscription # 'default' or 'subscription' (6h max retry for rate limit recovery)
# max_concurrent_pipelines: 2 # 1-5, default: 5 (reduce to lower API usage spikes)
apps/worker/package.json
+3 -3
View File
@@ -19,9 +19,9 @@
"clean": "rm -rf dist"
},
"dependencies": {
"@earendil-works/pi-agent-core": "^0.79.1",
"@earendil-works/pi-ai": "^0.79.1",
"@earendil-works/pi-coding-agent": "^0.79.1",
"@earendil-works/pi-agent-core": "^0.82.1",
"@earendil-works/pi-ai": "^0.82.1",
"@earendil-works/pi-coding-agent": "^0.82.1",
"@gotgenes/pi-permission-system": "^10.9.0",
"@temporalio/activity": "^1.11.0",
"@temporalio/client": "^1.11.0",
apps/worker/prompts/report-executive.txt
+177 -94
View File
@@ -1,112 +1,195 @@
<role>
You are an Executive Summary Writer and Report Cleaner for security assessments. Your job is to:
1. MODIFY the existing concatenated report by adding an executive summary at the top
2. CLEAN UP hallucinated or extraneous sections throughout the report
<exploit_mode_role>
You are the Security Report Writer for a multi-agent security assessment pipeline. Upstream agents have already explored the target application, generated security hypotheses, and verified them by exploitation. Your job is to synthesize the verified findings into structured data that downstream renderers will use to produce reports and persist to the database.
</exploit_mode_role>
<analysis_mode_role>
You are the Security Report Writer for a multi-agent security assessment pipeline. Upstream agents have explored the target application, generated security hypotheses, and assessed them against the source code. Your job is to synthesize those findings into structured data that downstream renderers will use to produce reports and persist to the database.
</analysis_mode_role>
</role>
<audience>
Technical leadership (CTOs, CISOs, Engineering VPs) who need both technical accuracy and executive brevity.
</audience>
<task>
Record all findings as structured data using the `add_finding` tool. You do NOT write a markdown report — a downstream renderer produces the report from your structured output.
<objective>
The orchestrator has already concatenated all per-class deliverables into `comprehensive_security_assessment_report.md`. Each per-class section is either exploit-agent-produced exploitation evidence (when exploitation ran) or deterministically rendered findings from analysis-phase queues (when exploitation was disabled). The cleanup rules below apply uniformly to either source.
Your task is to:
1. Read this existing concatenated report
2. Add an Executive Summary (vulnerability overview) at the top
3. Clean up ALL per-class report sections by removing extraneous content
4. Save the modified version back to the same file
1. **Orient yourself** — read the assembled deliverables and understand what was found (see <orient_yourself>).
2. **Filter and clean** — identify real findings, remove noise, rewrite weak titles (see <filter_and_clean>).
3. **Record report metadata** — run `set-report-meta` once (see <record_report_meta>).
4. **Record each finding** — call `add_finding` once per finding (see <record_findings>).
</task>
IMPORTANT: You are MODIFYING an existing file, not creating a new one.
</objective>
<tools_reference>
You have two tools for recording findings:
<target>
URL: {{WEB_URL}}
- **set-report-meta** (CLI via `bash`) — Write top-level report metadata. Call once before recording findings.
`set-report-meta --target "https://..." --assessment-date "YYYY-MM-DD" --scope "..." --executive-summary "..."`
Returns: `{"status":"success"}`
Shell quoting: wrap flag values in double quotes. Escape any literal double quotes as \", dollar signs as \$, and backticks as \`.
Filesystem:
- {{REPO_PATH}}/ (read only)
- {{REPO_PATH}}/.shannon/deliverables/ (read-write)
- {{REPO_PATH}}/.shannon/scratchpad/ (read-write) - screenshots, scripts, scratch work, etc.
</target>
- **add_finding** (tool) — Record a single finding as structured data. Call once per finding. Rejects duplicate finding_ids. The tool schema describes all required and optional fields — fill them in directly.
</tools_reference>
<context>
Authentication Context:
{{AUTH_CONTEXT}}
<orient_yourself>
Before recording anything, read and understand your inputs.
Vulnerability classes tested in this run: {{VULN_CLASSES_TESTED}}
### Your goal
<exploit_mode_orient>
You are the final agent in the pipeline. Upstream agents have already performed reconnaissance, analyzed vulnerabilities, and exploited them. Their evidence has been assembled into a concatenated report. Your job is to read that report, identify the real findings, and emit each one as structured data via the `add_finding` tool.
</exploit_mode_orient>
<analysis_mode_orient>
You are the final agent in the pipeline. Upstream agents have performed reconnaissance and analyzed vulnerabilities in the source code. **No exploitation phase ran** — nothing was executed against the target and no vulnerability was confirmed by attack. Their analysis has been assembled into a concatenated report. Your job is to read that report, identify the real findings, and emit each one as structured data via the `add_finding` tool.
</analysis_mode_orient>
### Your inputs
Read these files:
- `.shannon/deliverables/comprehensive_security_assessment_report.md` — The concatenated per-class deliverables. This is your primary input. Each per-class section contains vulnerability entries with IDs.
- `.shannon/deliverables/pre_recon_deliverable.md` — Initial reconnaissance and technology stack (for executive summary context).
- `.shannon/deliverables/recon_deliverable.md` — Attack surface mapping and endpoint discovery (for executive summary context).
### Vulnerability ID patterns
Findings have IDs matching `[TYPE]-VULN-[NUMBER]` (e.g., INJ-VULN-01, AUTH-VULN-03).
### Context
Target URL: {{WEB_URL}}
Vulnerability classes tested: {{VULN_CLASSES_TESTED}}
Exploitation: {{EXPLOITATION}}
</context>
{{AUTH_CONTEXT}}
</orient_yourself>
{{NOT_ASSESSED_CLASSES}}
{{REPORT_FILTERS_BLOCK}}
<input_files>
You will analyze the following deliverable files:
- `.shannon/deliverables/pre_recon_deliverable.md` - Initial reconnaissance and technology stack
- `.shannon/deliverables/recon_deliverable.md` - Attack surface mapping and endpoint discovery
- `.shannon/deliverables/comprehensive_security_assessment_report.md` - The already-concatenated report that you will modify
</input_files>
<filter_and_clean>
Read through the concatenated report and identify which vulnerability entries to record. Apply these rules:
<deliverable_instructions>
MODIFY the existing file `.shannon/deliverables/comprehensive_security_assessment_report.md` by:
1. ADDING these sections at the top:
# Security Assessment Report
## Executive Summary
- Target: {{WEB_URL}}
- Assessment Date: {current date}
- Scope: {{VULN_CLASSES_TESTED}}
- Exploitation: {{EXPLOITATION}}
## Summary by Vulnerability Type
{For each vulnerability type below, examine the comprehensive_security_assessment_report.md sections and provide a summary. If no section exists for that type or no vulnerabilities are listed, explicitly state that none were found.}
{{VULN_SUMMARY_SUBSECTIONS}}
2. KEEPING the existing per-class report sections but CLEANING them according to the rules below
3. The final structure should be:
- Executive Summary (new)
- All existing per-class report sections (cleaned)
IMPORTANT: Do NOT reorder the existing per-class report sections. Maintain the exact order they appear in the concatenated report. Only remove sections that do not match the defined criteria above.
</deliverable_instructions>
<instructions>
1. Read the pre_recon and recon deliverable files to gather security-relevant information:
- Skip basic information such as technology stack information (the team knows their own stack)
- Use technical leadership tone - precise but concise
- Use the current date for the assessment date
2. Create the Executive Summary content:
- Executive Summary: Technical overview with actionable findings for engineering leaders
3. Clean the per-class report sections in `.shannon/deliverables/comprehensive_security_assessment_report.md` by applying these rules:
- KEEP these specific section headings:
NOTE: these sections will contain vulnerability lists with IDs matching pattern `### [TYPE]-VULN-[NUMBER]`
* `# [Type] {{REPORT_VULN_HEADING}}`
* `## {{REPORT_VULN_SUBHEADING}}`
### KEEP — these are real findings to record via `add_finding`
- Vulnerability entries under `## {{REPORT_VULN_SUBHEADING}}` sections with IDs matching `### [TYPE]-VULN-[NUMBER]`
{{REPORT_FILTER_RULES}}
- REMOVE ANY OTHER SECTIONS (even if they contain vulnerability IDs), such as:
* `## Potential Vulnerabilities (Validation Blocked)` (All agents)
* Standalone "Recommendations" sections
* "Conclusion" sections
* "Summary" sections
* "Next Steps" sections
* "Additional Analysis" sections
* Any other meta-commentary sections without vulnerability IDs
* False positives sections
* any intros in the sections
* any counts in the sections
- Preserve exact vulnerability IDs (`### [TYPE]-VULN-NN:`); if the title after the colon is only a short category label rather than a descriptive phrase, rewrite it to a concise human-readable descriptor derived from the finding's Vulnerable location and Overview.
4. Combine the content:
- Place the Executive Summary and Network Reconnaissance sections at the top
- Follow with the cleaned per-class report sections
- Save as the modified `.shannon/deliverables/comprehensive_security_assessment_report.md`
### SKIP — do not record these
<exploit_mode_skip>
- `## Potential Vulnerabilities (Validation Blocked)` entries
</exploit_mode_skip>
- Standalone "Recommendations", "Conclusion", "Summary", "Next Steps", "Additional Analysis" sections
- False positives sections
- Introductory text, vulnerability counts, or meta-commentary without vulnerability IDs
- Any section that does not contain a finding with a valid vulnerability ID
CRITICAL: You are modifying the existing concatenated report at `.shannon/deliverables/comprehensive_security_assessment_report.md` IN-PLACE, not creating a separate file.
</instructions>
### Title cleanup
If a finding's title (the text after the colon in `### TYPE-VULN-NN: Title`) is only a short category label rather than a descriptive phrase, rewrite it to a concise descriptor derived from the finding's "Vulnerable location" and "Overview" fields. Use the improved title when calling `add_finding`.
</filter_and_clean>
<record_report_meta>
Run `set-report-meta` once before recording any individual findings (see <tools_reference> for usage).
Fields:
- `target`: `{{WEB_URL}}`
- `assessment_date`: Use the current date in ISO format (YYYY-MM-DD)
- `scope`: `{{VULN_CLASSES_TESTED}}`
<exploit_mode_summary>
- `executive_summary`: 2-3 sentences summarizing the security posture for technical leadership (CTOs, CISOs, Engineering VPs). Must include the target URL and assessment date. Provide a high-level characterization based on the findings — severity distribution, most critical issues, and overall risk demonstrated by exploitation. If no vulnerabilities were confirmed in the assessed classes, state that scope clearly. A clean report is valid only when no <not_assessed_classes> block is present. If that block is present, explicitly say the listed classes were not assessed and do not assert they are free of vulnerabilities.
</exploit_mode_summary>
<analysis_mode_summary>
- `executive_summary`: 2-3 sentences summarizing the security posture for technical leadership (CTOs, CISOs, Engineering VPs). Must include the target URL and assessment date. Provide a high-level characterization based on the findings — confidence distribution, the most serious weaknesses identified, and overall risk. State plainly that this was an analysis-only assessment and that no finding was confirmed by exploitation; do not describe risk as demonstrated or proven. Findings carry no severity rating in this mode, so do not assert one. If no vulnerabilities were identified in the assessed classes, state that scope clearly. A clean report is valid only when no <not_assessed_classes> block is present. If that block is present, explicitly say the listed classes were not assessed and do not assert they are free of vulnerabilities.
</analysis_mode_summary>
</record_report_meta>
<record_findings>
For each finding identified in <filter_and_clean>, call `add_finding` once.
Record findings in the order they appear in the concatenated report (which groups by vulnerability class: injection, xss, auth, ssrf, authz).
Each `finding_id` may only be recorded once — duplicate calls are rejected.
### How to fill in each field
Map the finding's content from the per-class deliverable sections to `add_finding` fields:
- `finding_id`: The vulnerability ID exactly as it appears (e.g., `"INJ-VULN-01"`, `"AUTH-VULN-07"`)
- `title`: The cleaned-up title (see title cleanup rules in <filter_and_clean>)
- `category`: Derived from the finding type prefix — `INJ` → `"Injection"`, `XSS` → `"XSS"`, `AUTH` → `"Authentication"`, `AUTHZ` → `"Authorization"`, `SSRF` → `"SSRF"`
<exploit_mode_fields>
- `severity`: From the finding's "Severity" field. Use as-is; do not reassess.
</exploit_mode_fields>
<analysis_mode_fields>
- `confidence`: From the finding's "Confidence" field. Use as-is; do not reassess.
</analysis_mode_fields>
- `owasp_category`: Map to the appropriate OWASP Top 10 (2025) category:
- `"A01:2025 — Broken Access Control"`
- `"A02:2025 — Security Misconfiguration"`
- `"A03:2025 — Software Supply Chain Failures"`
- `"A04:2025 — Cryptographic Failures"`
- `"A05:2025 — Injection"`
- `"A06:2025 — Insecure Design"`
- `"A07:2025 — Authentication Failures"`
- `"A08:2025 — Software or Data Integrity Failures"`
- `"A09:2025 — Security Logging and Alerting Failures"`
- `"A10:2025 — Mishandling of Exceptional Conditions"`
- `vulnerable_location`: From the finding's "Vulnerable location" field
- `http_location`: The HTTP request the finding is reached through, when the deliverable names one (e.g. `"GET /api/products?id="` gives `method: "GET"`, `url: "{{WEB_URL}}/api/products"`, `parameter: "id"`). Omit for findings with no network entry point.
- `overview`: Synthesize from the finding's "Overview" field into professional prose. Do not paste verbatim.
- `remediation`: Specific, actionable fix guidance from the finding. Code-level or configuration-level. Avoid generic advice.
<exploit_mode_fields>
- `impact`: From the finding's "Impact" field if present, otherwise derive from the overview and proof of impact
- `auth_state`: From the finding's authentication context or prerequisites
- `prerequisites`: From the finding's "Prerequisites" field, or `"None"` if not specified
- `exploitation_steps`: From the finding's exploitation steps or proof-of-concept. Each step gets a title and ordered prose/code items. Use `"bash"` for shell commands, `"http"` for raw HTTP, `"json"` for response bodies.
- `proof_of_impact`: From the finding's "Proof of Impact" or evidence section. What the exploit demonstrably achieved.
- `status`: Optional. Use `"exploited"` for confirmed exploits.
</exploit_mode_fields>
<analysis_mode_fields>
- `impact`: What an attacker could achieve if this vulnerability were exploited. Derive it from the finding's "Impact" and "Overview" fields. Write it as assessed, never as achieved.
This run had no exploitation phase. Nothing was executed against the target, nothing was demonstrated, and no exploit evidence exists. Accordingly `severity`, `auth_state`, `prerequisites`, `exploitation_steps`, `proof_of_impact` and `status` are **not** part of your tool schema — the deliverables contain no source for any of them. `confidence` is the only rating this run produces; take it straight from the deliverable. Do not compensate for the missing fields by describing attack execution in `overview`, `impact` or `notes`. Report the weakness and how to fix it; that is the whole deliverable for this run.
</analysis_mode_fields>
**Optional fields:**
- `notes`: From the finding's "Notes" section if present
- `additional_sections`: Any extra subsections on the finding that don't fit the fields above
### Zero findings
If no valid findings exist after filtering, do not call `add_finding` at all. The `set-report-meta` executive summary should state that no vulnerabilities were identified in the assessed classes. If a <not_assessed_classes> block is present, it must also state that those listed classes were not assessed.
</record_findings>
<constraints>
<exploit_mode_constraints>
- **No Fabrications:** Do not invent exploitation steps, evidence, or impact. Every piece of data must come from the deliverable files. If a finding has incomplete data, include it but note the gap in `overview`.
- **No Severity Changes:** Use the severity from the deliverable as-is. Do not inflate or deflate.
</exploit_mode_constraints>
<analysis_mode_constraints>
- **No Fabrications:** Every piece of data must come from the deliverable files. If a finding has incomplete data, include it but note the gap in `overview`.
- **Nothing Was Demonstrated:** No exploit ran. Do not write that a vulnerability was confirmed, proven, exploited, or verified against the running target, and do not describe payloads, requests, or responses as having been sent.
- **No Confidence Changes:** Use the confidence from the deliverable as-is. Do not raise or lower it.
</analysis_mode_constraints>
- **No Speculation:** Only record findings that appear in the deliverables with valid vulnerability IDs. Do not add your own assessments.
- **OWASP 2025:** Map all findings to OWASP Top 10 (2025) categories.
- **Remediation Quality:** Provide specific, actionable remediation — code-level or configuration-level fixes. Avoid generic advice like "validate input" or "follow best practices".
</constraints>
<self_check>
Before finalizing, verify:
- [ ] Did I run `set-report-meta` exactly once with target, assessment_date, scope, and executive_summary?
- [ ] For each valid finding in the deliverables, did I call `add_finding` exactly once with the correct `finding_id`?
<exploit_mode_skipcheck>
- [ ] Did I skip all entries from "Potential Vulnerabilities (Validation Blocked)", false positives, and meta-commentary sections?
</exploit_mode_skipcheck>
<analysis_mode_skipcheck>
- [ ] Did I skip false positives and meta-commentary sections?
</analysis_mode_skipcheck>
- [ ] Does every finding have a valid `owasp_category` using the 2025 categorization?
- [ ] Does every finding have `overview`, `impact`, and `remediation`?
<exploit_mode_checks>
- [ ] Does every finding have `auth_state` and `prerequisites`?
- [ ] Does every finding have `exploitation_steps` with prose/code items?
- [ ] Does every finding have `proof_of_impact`?
- [ ] Are severity ratings unchanged from the source deliverables?
</exploit_mode_checks>
<analysis_mode_checks>
- [ ] Does every finding have `confidence` carried over unchanged from the deliverable?
- [ ] Is every `impact` phrased as assessed rather than demonstrated, with no claim that anything was executed?
</analysis_mode_checks>
- [ ] Are remediation recommendations specific and actionable (not generic)?
If any answer is NO, fix it before finalizing.
</self_check>
apps/worker/prompts/vuln-auth.txt
+5 -1
View File
@@ -100,7 +100,11 @@ An **exploitable vulnerability** is a logical flaw in the code that represents a
"missing_defense": "Concise description of the core problem (e.g., 'No rate limit on POST /login', 'Session ID not rotated after login').",
"exploitation_hypothesis": "A clear statement describing the expected outcome of a successful exploit. This should describe an active attack outcome, not just vulnerability confirmation (e.g., 'An attacker can successfully log into an account by guessing a simple password,' 'An attacker can hijack a user's session by replaying a stolen cookie').",
"suggested_exploit_technique": "The specific attack pattern to attempt, derived from the methodology. The exploitation agent should actively execute this attack, not just confirm it's possible (e.g., 'brute_force_login', 'credential_stuffing', 'session_hijacking', 'session_fixation').",
"confidence": "High | Medium | Low",
"confidence": "high | medium | low",
"code_locations": [
{ "file": "lib/insecurity.ts", "start_line": 21, "role": "sink", "symbol": "verify" },
{ "file": "routes/login.ts", "start_line": 34, "role": "guard" }
],
"notes": "Relevant details about required session state, applicable roles, observed headers, or links to related findings."
}
</exploitation_queue_format>
apps/worker/prompts/vuln-authz.txt
+6 -2
View File
@@ -107,7 +107,11 @@ An **exploitable vulnerability** is a logical flaw in the code that represents a
"side_effect": "specific unauthorized action possible (e.g., 'read other users profiles', 'delete any file', 'skip payment')",
"reason": "1-2 lines explaining why this is vulnerable",
"minimal_witness": "brief description of how to trigger (e.g., 'change user_id param to another user', 'call admin endpoint as regular user')",
"confidence": "high | med | low",
"confidence": "high | medium | low",
"code_locations": [
{ "file": "server.ts", "start_line": 365, "role": "sink", "symbol": "getUserById" },
{ "file": "lib/insecurity.ts", "start_line": 92, "role": "guard" }
],
"notes": "any assumptions, related findings, or special conditions"
}
</exploitation_queue_format>
@@ -220,7 +224,7 @@ An **exploitable vulnerability** is a logical flaw in the code that represents a
- `guard_evidence` (missing/misplaced),
- `side_effect` observed,
- `reason` (1–2 lines: e.g., "ownership check absent"),
- `confidence` (high/med/low),
- `confidence` (high/medium/low),
- `minimal_witness` (sketch for exploit agent).
---
apps/worker/prompts/vuln-injection.txt
+6 -2
View File
@@ -111,7 +111,11 @@ An **exploitable vulnerability** is a confirmed source-to-sink path where the en
"verdict": "safe | vulnerable.",
"mismatch_reason": "if vulnerable, 1–2 lines in plain language.",
"witness_payload": "minimal input you'd use later to show structure influence (e.g., ' for SQLi, ; ls -la for Command Injection, ../../../../etc/passwd for LFI, {{7*7}} for SSTI).",
"confidence": "high | med | low.",
"confidence": "high | medium | low",
"code_locations": [
{ "file": "routes/search.ts", "start_line": 42, "role": "sink", "symbol": "searchProducts" },
{ "file": "routes/search.ts", "start_line": 38, "role": "source" }
],
"notes": "assumptions, untraversed branches, anything unusual."
}
</exploitation_queue_format>
@@ -164,7 +168,7 @@ An **exploitable vulnerability** is a confirmed source-to-sink path where the en
- `verdict` (`safe` / `vulnerable`)
- `mismatch_reason` (plain-language, 1–2 lines)
- `witness_payload` (minimal input to demonstrate structure influence — **for later exploit phase**)
- `confidence` (`high` / `med` / `low`)
- `confidence` (`high` / `medium` / `low`)
- `notes` (assumptions, untraversed branches, unusual conditions)
- **7) Score confidence**
- **High:** binds on value/like/numeric; strict casts; whitelists for all syntax slots; **no** post-sanitization concat.
apps/worker/prompts/vuln-ssrf.txt
+5 -1
View File
@@ -102,7 +102,11 @@ An **exploitable vulnerability** is a data flow where user-controlled input infl
"missing_defense": "Concise description of the core problem (e.g., 'No URL allowlist validation', 'Missing protocol restriction').",
"exploitation_hypothesis": "A clear statement describing the expected outcome of a successful exploit (e.g., 'An attacker can access internal services by providing internal URLs', 'An attacker can retrieve cloud metadata through URL manipulation').",
"suggested_exploit_technique": "The specific attack pattern to attempt, derived from the methodology (e.g., 'internal_service_access', 'cloud_metadata_retrieval', 'port_scanning', 'webhook_abuse').",
"confidence": "High | Medium | Low",
"confidence": "high | medium | low",
"code_locations": [
{ "file": "routes/profileImageUrlUpload.ts", "start_line": 24, "role": "sink", "symbol": "profileImageUrlUpload" },
{ "file": "routes/profileImageUrlUpload.ts", "start_line": 18, "role": "source" }
],
"notes": "Relevant details about request format, authentication requirements, observed timeouts, or links to related findings."
}
</exploitation_queue_format>
apps/worker/prompts/vuln-xss.txt
+5 -1
View File
@@ -108,7 +108,11 @@ Structure: The vulnerability JSON object MUST follow this exact format:
"verdict": "vulnerable | safe.",
"mismatch_reason": "If vulnerable, explain why the observed encoding is wrong for the render context (e.g., 'URL encoding used in an HTML attribute context, allowing event handler injection.').",
"witness_payload": "A minimal, non-malicious payload that proves context control (e.g., '><script>alert(1)</script>', '" onmouseover=alert(1) ').",
"confidence": "high | med | low.",
"confidence": "high | medium | low",
"code_locations": [
{ "file": "frontend/src/app/search-result/search-result.component.ts", "start_line": 121, "role": "sink", "symbol": "filterTable" },
{ "file": "frontend/src/app/search-result/search-result.component.ts", "start_line": 115, "role": "source" }
],
"notes": "Relevant CSP, HttpOnly flags, WAF behavior, or other environmental factors."
}
</exploitation_queue_format>
apps/worker/src/ai/extensions/bash-timeout/index.ts
+1 -1
View File
@@ -23,7 +23,7 @@ function evaluateBashTimeout(timeout: number | undefined): ToolCallEventResult |
if (!hasValidTimeout) {
return {
block: true,
reason: `Set bash 'timeout' (seconds). Default ${DEFAULT_TIMEOUT_SECONDS}s, max ${MAX_TIMEOUT_SECONDS}s.`,
reason: `A timeout in seconds is required for the bash tool. The bash tool was not executed. Use the default of ${DEFAULT_TIMEOUT_SECONDS} seconds, or up to a maximum of ${MAX_TIMEOUT_SECONDS} seconds.`,
};
}
apps/worker/src/ai/models.ts
+267 -120
View File
@@ -5,157 +5,304 @@
// as published by the Free Software Foundation.
/**
* Model tier definitions and resolution for the pi harness.
* Model selection and resolution for the pi harness.
*
* Three tiers mapped to capability levels:
* - "small" (Haiku — summarization, structured extraction)
* - "medium" (Sonnet — tool use, general analysis)
* - "large" (Opus — deep reasoning, complex analysis)
* One model runs the entire workflow. Users name it with a single setting:
*
* Users override per tier via ANTHROPIC_SMALL_MODEL / ANTHROPIC_MEDIUM_MODEL /
* ANTHROPIC_LARGE_MODEL, which works across all providers (Anthropic, Bedrock,
* custom base URL).
* SHANNON_AI_MODEL=<provider>:<model-id>
*
* The active provider is chosen from the env-var contract the CLI forwards
* (`CLAUDE_CODE_USE_BEDROCK`, `ANTHROPIC_BASE_URL`+`ANTHROPIC_AUTH_TOKEN`, else
* direct Anthropic). Resolution returns a pi `Model` via `ModelRegistry.find`, the
* `thinkingLevel`, and an `AuthStorage` primed with the right credential. Bedrock
* authenticates from the AWS_ env vars via pi-ai.
* The provider half decides the endpoint, the credential, and the API dialect;
* the model half is passed to pi's registry as-is. The separator is a colon
* because model IDs routinely contain slashes, and it is the *first* colon that
* splits, because Bedrock model IDs contain colons of their own
* (`amazon-bedrock:us.anthropic.claude-opus-4-5-20251101-v1:0`).
*
* Resolution returns a pi `Model` plus the `ModelRuntime` that owns its auth,
* built over an in-memory credential store primed from the environment.
*/
import type { ThinkingLevel } from '@earendil-works/pi-agent-core';
import type { Api, Model } from '@earendil-works/pi-ai';
import { AuthStorage, type ModelRegistry } from '@earendil-works/pi-coding-agent';
import type { Api, Credential, CredentialInfo, CredentialStore, Model } from '@earendil-works/pi-ai';
import { ModelRuntime } from '@earendil-works/pi-coding-agent';
export type ModelTier = 'small' | 'medium' | 'large';
/** Providers Shannon can currently reach. Each is a pi-ai provider id. */
export const SUPPORTED_PROVIDERS = ['anthropic', 'openai', 'xai', 'amazon-bedrock'] as const;
const DEFAULT_MODELS: Readonly<Record<ModelTier, string>> = {
small: 'claude-haiku-4-5-20251001',
medium: 'claude-sonnet-4-6',
large: 'claude-opus-4-8',
export type ProviderId = (typeof SUPPORTED_PROVIDERS)[number];
/**
* Env vars carrying each provider's API key, in precedence order. Shannon does not
* invent credential names — these are the variables each provider's own tooling
* uses. Bedrock pairs its bearer token with AWS_REGION, which is provider config
* rather than a credential.
*/
export const PROVIDER_API_KEY_ENV: Readonly<Record<ProviderId, readonly string[]>> = {
anthropic: ['ANTHROPIC_API_KEY', 'CLAUDE_CODE_OAUTH_TOKEN'],
openai: ['OPENAI_API_KEY'],
xai: ['XAI_API_KEY'],
'amazon-bedrock': ['AWS_BEARER_TOKEN_BEDROCK'],
};
export interface EffectiveProvider {
/** pi-ai provider id: 'anthropic' or 'amazon-bedrock'. */
providerId: string;
/** Custom-base-URL override applied to the resolved anthropic model. */
/** Model used when SHANNON_AI_MODEL is unset. */
export const DEFAULT_MODEL_SPEC = 'anthropic:claude-sonnet-4-6';
/**
* Wire formats an OpenAI-compatible gateway may serve, named by
* SHANNON_AI_OPENAI_FORMAT. Only `openai` offers a choice: every other supported
* provider has exactly one API in pi's registry.
*/
export const OPENAI_FORMATS = {
'chat-completions': 'openai-completions',
responses: 'openai-responses',
} as const;
export type OpenAiFormat = keyof typeof OPENAI_FORMATS;
/** Format assumed when a gateway is configured but no format is named. */
export const DEFAULT_OPENAI_FORMAT: OpenAiFormat = 'chat-completions';
function isOpenAiFormat(value: string): value is OpenAiFormat {
return value in OPENAI_FORMATS;
}
/**
* Read SHANNON_AI_OPENAI_FORMAT. Unset returns undefined, which lets the caller
* distinguish "not configured" from an explicit choice and reject the variable
* where it has no effect.
*/
export function resolveOpenAiFormat(): OpenAiFormat | undefined {
const raw = process.env.SHANNON_AI_OPENAI_FORMAT?.trim();
if (!raw) return undefined;
if (!isOpenAiFormat(raw)) {
throw new Error(
`SHANNON_AI_OPENAI_FORMAT must be one of: ${Object.keys(OPENAI_FORMATS).join(', ')}. Got "${raw}".`,
);
}
return raw;
}
export interface ModelSpec {
providerId: ProviderId;
modelId: string;
}
function isSupportedProvider(value: string): value is ProviderId {
return (SUPPORTED_PROVIDERS as readonly string[]).includes(value);
}
/**
* Parse a `<provider>:<model-id>` spec. Splits on the first colon only, so
* colons inside a model ID survive. Throws with the supported provider list on
* a malformed or unknown provider.
*/
export function parseModelSpec(spec: string): ModelSpec {
const trimmed = spec.trim();
const separator = trimmed.indexOf(':');
if (separator === -1) {
throw new Error(
`SHANNON_AI_MODEL must be "<provider>:<model-id>", got "${trimmed}". Example: ${DEFAULT_MODEL_SPEC}`,
);
}
const providerId = trimmed.slice(0, separator).trim();
const modelId = trimmed.slice(separator + 1).trim();
if (!providerId || !modelId) {
throw new Error(
`SHANNON_AI_MODEL must be "<provider>:<model-id>", got "${trimmed}". Example: ${DEFAULT_MODEL_SPEC}`,
);
}
if (!isSupportedProvider(providerId)) {
throw new Error(
`Unsupported provider "${providerId}" in SHANNON_AI_MODEL. Supported providers: ${SUPPORTED_PROVIDERS.join(', ')}`,
);
}
return { providerId, modelId };
}
/** Resolve the run's model from SHANNON_AI_MODEL, falling back to the default. */
export function resolveModelSpec(): ModelSpec {
return parseModelSpec(process.env.SHANNON_AI_MODEL || DEFAULT_MODEL_SPEC);
}
export interface ProviderCredentials {
/** Endpoint override, applied whatever the provider (proxies, gateways). */
baseUrl?: string;
/** Runtime credential to prime on AuthStorage for the 'anthropic' provider. */
anthropicToken?: string;
/** Runtime API key primed into the ModelRuntime's credential store. */
apiKey?: string;
}
/** Collect the API key and optional endpoint override for a provider. */
export function resolveProviderCredentials(providerId: ProviderId): ProviderCredentials {
const credentials: ProviderCredentials = {};
for (const name of PROVIDER_API_KEY_ENV[providerId]) {
const value = process.env[name];
if (value) {
credentials.apiKey = value;
break;
}
}
if (process.env.SHANNON_AI_BASE_URL) credentials.baseUrl = process.env.SHANNON_AI_BASE_URL;
return credentials;
}
/**
* Determine the active provider + auth from the env-var contract the CLI forwards:
* `CLAUDE_CODE_USE_BEDROCK` → Bedrock; `ANTHROPIC_BASE_URL`+`ANTHROPIC_AUTH_TOKEN`
* → custom base URL; else direct Anthropic (`ANTHROPIC_API_KEY`, or
* `CLAUDE_CODE_OAUTH_TOKEN`). Bedrock authenticates from the AWS_ env vars via
* pi-ai, so it needs no anthropic token.
*/
export function resolveEffectiveProvider(): EffectiveProvider {
// Bedrock — env flag.
if (process.env.CLAUDE_CODE_USE_BEDROCK === '1') {
return { providerId: 'amazon-bedrock' };
}
// Custom base URL — env contract.
if (process.env.ANTHROPIC_BASE_URL && process.env.ANTHROPIC_AUTH_TOKEN) {
return {
providerId: 'anthropic',
baseUrl: process.env.ANTHROPIC_BASE_URL,
anthropicToken: process.env.ANTHROPIC_AUTH_TOKEN,
};
}
// Direct Anthropic (API key, or OAuth token).
const eff: EffectiveProvider = { providerId: 'anthropic' };
const token = process.env.ANTHROPIC_API_KEY ?? process.env.CLAUDE_CODE_OAUTH_TOKEN;
if (token) eff.anthropicToken = token;
return eff;
}
/** Resolve a model tier to a concrete model ID (env override → default). */
export function resolveModelId(tier: ModelTier = 'medium'): string {
switch (tier) {
case 'small':
return process.env.ANTHROPIC_SMALL_MODEL || DEFAULT_MODELS.small;
case 'large':
return process.env.ANTHROPIC_LARGE_MODEL || DEFAULT_MODELS.large;
default:
return process.env.ANTHROPIC_MEDIUM_MODEL || DEFAULT_MODELS.medium;
}
}
/** Whether a model supports adaptive thinking. Opus 4.6, 4.7, and 4.8 only. */
export function supportsAdaptiveThinking(model: string): boolean {
return /opus-4-[678]/.test(model);
}
/**
* Resolve the thinking level for a run.
* In-memory credential store holding the selected provider's API key.
*
* Adaptive thinking is enabled only on capable models (Opus 4.6/4.7/4.8), mapped to
* pi's 'medium' level; every other model runs with thinking 'off'. The
* CLAUDE_ADAPTIVE_THINKING=false kill switch forces 'off' regardless of model.
* pi ships the `CredentialStore` interface but no in-memory implementation — its
* own store reads `auth.json` from disk. Shannon's credentials arrive as env vars
* in an ephemeral container, so nothing may be read from or written to disk.
*/
export function resolveThinkingLevel(modelId: string): ThinkingLevel {
if (process.env.CLAUDE_ADAPTIVE_THINKING === 'false') return 'off';
return supportsAdaptiveThinking(modelId) ? 'medium' : 'off';
class RuntimeCredentialStore implements CredentialStore {
private readonly credentials = new Map<string, Credential>();
constructor(providerId: string, apiKey: string | undefined) {
if (apiKey) {
this.credentials.set(providerId, { type: 'api_key', key: apiKey });
}
}
async read(providerId: string): Promise<Credential | undefined> {
return this.credentials.get(providerId);
}
async list(): Promise<readonly CredentialInfo[]> {
return [...this.credentials].map(([providerId, credential]) => ({ providerId, type: credential.type }));
}
/** Serialized read-modify-write. `fn` returning undefined leaves the entry alone. */
async modify(
providerId: string,
fn: (current: Credential | undefined) => Promise<Credential | undefined>,
): Promise<Credential | undefined> {
const next = await fn(this.credentials.get(providerId));
if (next !== undefined) {
this.credentials.set(providerId, next);
}
return this.credentials.get(providerId);
}
async delete(providerId: string): Promise<void> {
this.credentials.delete(providerId);
}
}
/**
* Build a ModelRuntime whose only credential is the one supplied. Model catalogs
* stay offline (`allowModelNetwork` defaults to false) so a scan never blocks on
* a catalog refresh.
*/
export async function createModelRuntime(providerId: string, apiKey: string | undefined): Promise<ModelRuntime> {
return ModelRuntime.create({ credentials: new RuntimeCredentialStore(providerId, apiKey) });
}
export interface ModelSelection {
model: Model<Api>;
thinkingLevel: ThinkingLevel;
authStorage: AuthStorage;
modelRuntime: ModelRuntime;
modelId: string;
providerId: string;
providerId: ProviderId;
}
/**
* Resolve the active provider (see resolveEffectiveProvider), prime an AuthStorage
* with its credential, and resolve the tier's model from a fresh ModelRegistry.
* Anthropic / custom-base-URL use a runtime anthropic key; Bedrock authenticates
* from the AWS_ env vars (bearer token primed explicitly as a belt-and-suspenders).
* Point a model descriptor at a gateway.
*
* An OpenAI gateway may serve either wire format, named by
* SHANNON_AI_OPENAI_FORMAT and defaulting to chat completions, which is what
* most gateway software exposes. Switching to completions also drops the stored
* `compat` block: the catalogue's block describes Responses, and an explicit
* entry outranks pi's `detectCompat`, so leaving it would apply Responses
* settings to a completions request. Staying on Responses keeps it, since it
* then describes the format in use. Every other provider has one API and only
* changes address.
*/
export function resolveModelSelection(
registryFactory: (authStorage: AuthStorage) => ModelRegistry,
modelTier: ModelTier,
): ModelSelection {
const eff = resolveEffectiveProvider();
const modelId = resolveModelId(modelTier);
function pointAtGateway(model: Model<Api>, providerId: ProviderId, baseUrl: string, format: OpenAiFormat): Model<Api> {
if (providerId !== 'openai') return { ...model, baseUrl };
if (format === 'responses') return { ...model, baseUrl, api: OPENAI_FORMATS.responses };
const authStorage = AuthStorage.inMemory();
if (eff.providerId === 'anthropic' && eff.anthropicToken) {
authStorage.setRuntimeApiKey('anthropic', eff.anthropicToken);
}
// Bedrock auth flows from the AWS_ env vars; prime the bearer token explicitly so
// it resolves via AuthStorage in addition to pi-ai's own env fallback.
if (eff.providerId === 'amazon-bedrock' && process.env.AWS_BEARER_TOKEN_BEDROCK) {
authStorage.setRuntimeApiKey('amazon-bedrock', process.env.AWS_BEARER_TOKEN_BEDROCK);
}
const { compat: _responsesCompat, ...withoutCompat } = model;
return { ...withoutCompat, baseUrl, api: OPENAI_FORMATS['chat-completions'] };
}
const registry = registryFactory(authStorage);
const found = registry.find(eff.providerId, modelId);
if (!found) {
throw new Error(`Model not found in pi registry: provider="${eff.providerId}" model="${modelId}"`);
/**
* Resolve a model against a runtime.
*
* Direct to a provider, the model must exist in the catalogue. Behind a custom
* endpoint it need not: a gateway may serve models under its own names, so an
* unknown id is passed through on a descriptor borrowed from the provider's
* catalogue for its API dialect. Cost and context window on such a descriptor
* are the reference model's, so spend figures are approximate there.
*
* Returns undefined when the id is unresolvable — unknown with no endpoint
* override, or a provider carrying no models at all.
*/
export function resolveModel(
modelRuntime: ModelRuntime,
providerId: ProviderId,
modelId: string,
baseUrl: string | undefined,
format: OpenAiFormat = DEFAULT_OPENAI_FORMAT,
): Model<Api> | undefined {
const found = modelRuntime.getModel(providerId, modelId);
if (found) {
return baseUrl ? pointAtGateway(found, providerId, baseUrl, format) : found;
}
if (!baseUrl) return undefined;
// Custom base URL: override the resolved model's endpoint.
const model: Model<Api> = eff.baseUrl ? { ...found, baseUrl: eff.baseUrl } : found;
const reference = modelRuntime.getModels(providerId)[0];
if (!reference) return undefined;
return pointAtGateway({ ...reference, id: modelId, name: modelId }, providerId, baseUrl, format);
}
/**
* Validate SHANNON_AI_OPENAI_FORMAT against the rest of the configuration and
* return the format a gateway run should use.
*
* The variable only reaches a request when both an OpenAI model and a gateway
* are configured, so it is rejected outside that combination rather than
* silently ignored.
*/
export function resolveGatewayFormat(providerId: ProviderId, baseUrl: string | undefined): OpenAiFormat {
const configured = resolveOpenAiFormat();
if (!configured) return DEFAULT_OPENAI_FORMAT;
if (providerId !== 'openai') {
throw new Error(
`SHANNON_AI_OPENAI_FORMAT applies to openai models only, but SHANNON_AI_MODEL selects "${providerId}". ` +
`${providerId} serves a single API, so there is no format to choose.`,
);
}
if (!baseUrl) {
throw new Error(
'SHANNON_AI_OPENAI_FORMAT applies to gateway runs only. Set SHANNON_AI_BASE_URL, or unset the format to call OpenAI directly.',
);
}
return configured;
}
/**
* Resolve SHANNON_AI_MODEL, build a ModelRuntime primed with the provider's
* credential, and look the model up in it.
*/
export async function resolveModelSelection(): Promise<ModelSelection> {
const { providerId, modelId } = resolveModelSpec();
const credentials = resolveProviderCredentials(providerId);
const format = resolveGatewayFormat(providerId, credentials.baseUrl);
const modelRuntime = await createModelRuntime(providerId, credentials.apiKey);
const model = resolveModel(modelRuntime, providerId, modelId, credentials.baseUrl, format);
if (!model) {
throw new Error(`Model not found in pi registry: provider="${providerId}" model="${modelId}"`);
}
return {
model,
thinkingLevel: resolveThinkingLevel(modelId),
authStorage,
modelRuntime,
modelId,
providerId: eff.providerId,
providerId,
};
}
/**
* Whether a model is in the Fable family. Fable's safety classifiers flag
* cybersecurity tasks and route them to Opus 4.8, so a security scan on Fable
* largely runs on Opus 4.8 anyway.
*/
export function isFableModel(model: string): boolean {
return /fable/i.test(model);
}
apps/worker/src/ai/pi/pi-executor.ts
+69 -69
View File
@@ -9,11 +9,11 @@
import os from 'node:os';
import type { AgentMessage } from '@earendil-works/pi-agent-core';
import {
type AgentSession,
type AgentSessionEvent,
createAgentSession,
DefaultResourceLoader,
getAgentDir,
ModelRegistry,
type ResourceLoader,
SessionManager,
SettingsManager,
@@ -23,16 +23,14 @@ import {
import { fs, path } from 'zx';
import type { AuditSession } from '../../audit/index.js';
import { BASH_TIMEOUT_EXTENSION_DIR, deliverablesDir } from '../../paths.js';
import { isRetryableError, PentestError } from '../../services/error-handling.js';
import { isRetryableFailure, PentestError } from '../../services/error-handling.js';
import { AGENT_VALIDATORS } from '../../session-manager.js';
import type { ActivityLogger } from '../../types/activity-logger.js';
import { ErrorCode } from '../../types/errors.js';
import { isSpendingCapBehavior, matchesBillingTextPattern } from '../../utils/billing-detection.js';
import { isBrowserAgent } from '../../utils/browser-agents.js';
import { formatTimestamp } from '../../utils/formatting.js';
import { Timer } from '../../utils/metrics.js';
import { createAuditLogger } from '../audit-logger.js';
import { type ModelTier, resolveModelSelection } from '../models.js';
import { resolveModelSelection } from '../models.js';
import {
detectExecutionContext,
formatAssistantOutput,
@@ -43,8 +41,10 @@ import {
import { createProgressManager } from '../progress-manager.js';
import type { CapturedSubmitTool } from '../submit-tool.js';
import { permissionSystemConfigExists, permissionSystemPackageDir } from './permission-system.js';
import { PI_RETRY_SETTINGS } from './retry-settings.js';
import { createGlobTool, createTodoWriteTool } from './session-tools.js';
import { createTaskTool } from './task-tool.js';
import { providerTurnError } from './turn-error.js';
declare global {
var SHANNON_DISABLE_LOADER: boolean | undefined;
@@ -105,15 +105,41 @@ async function buildResourceLoader(
return loader;
}
interface ChildUsage {
cost: number;
inputTokens: number;
outputTokens: number;
cacheReadTokens: number;
cacheWriteTokens: number;
}
/**
* Usage for one agent: the parent session plus every `task` sub-session it
* spawned. Sub-sessions keep their own stats, so their spend is accumulated
* separately and added here.
*/
function totalUsage(session: AgentSession | undefined, childUsage: ChildUsage) {
const stats = session?.getSessionStats();
return {
cost: (stats?.cost ?? 0) + childUsage.cost,
inputTokens: (stats?.tokens.input ?? 0) + childUsage.inputTokens,
outputTokens: (stats?.tokens.output ?? 0) + childUsage.outputTokens,
cacheReadTokens: (stats?.tokens.cacheRead ?? 0) + childUsage.cacheReadTokens,
cacheWriteTokens: (stats?.tokens.cacheWrite ?? 0) + childUsage.cacheWriteTokens,
};
}
export interface PiPromptResult {
result?: string | null | undefined;
success: boolean;
duration: number;
turns?: number | undefined;
cost: number;
inputTokens?: number | undefined;
outputTokens?: number | undefined;
cacheReadTokens?: number | undefined;
cacheWriteTokens?: number | undefined;
model?: string | undefined;
partialCost?: number | undefined;
apiErrorDetected?: boolean | undefined;
error?: string | undefined;
errorType?: string | undefined;
prompt?: string | undefined;
@@ -138,7 +164,7 @@ async function writeErrorLog(
timestamp: formatTimestamp(),
agent: 'pi-executor',
error: { name: err.constructor.name, message: err.message, code: err.code, status: err.status, stack: err.stack },
context: { sourceDir, prompt: `${fullPrompt.slice(0, 200)}...`, retryable: isRetryableError(err) },
context: { sourceDir, prompt: `${fullPrompt.slice(0, 200)}...`, retryable: isRetryableFailure(err) },
duration,
};
const logPath = path.join(deliverablesDir(sourceDir), 'error.log');
@@ -190,28 +216,6 @@ function extractAssistantText(message: AgentMessage): string {
.join('\n');
}
/**
* Classify error-bearing text into a PentestError, mirroring the prior provider error
* handling. Spending-cap / billing text is retryable (Temporal backs off and
* recovers when the cap resets); session limit is permanent.
*/
function classifyErrorText(content: string): PentestError | null {
if (!content) return null;
if (matchesBillingTextPattern(content)) {
return new PentestError(
`Billing limit reached: ${content.slice(0, 100)}`,
'billing',
true,
{},
ErrorCode.SPENDING_CAP_REACHED,
);
}
if (content.toLowerCase().includes('session limit reached')) {
return new PentestError('Session limit reached', 'billing', false);
}
return null;
}
// Low-level pi execution. Drives one agent session to completion with progress and
// audit logging. Exported for Temporal activities to call single-attempt execution.
export async function runPiPrompt(
@@ -222,7 +226,6 @@ export async function runPiPrompt(
agentName: string | null = null,
auditSession: AuditSession | null = null,
logger: ActivityLogger,
modelTier: ModelTier = 'medium',
callerTools?: ToolDefinition[],
deliverablesSubdir?: string,
cancellationSignal?: AbortSignal,
@@ -254,21 +257,22 @@ export async function runPiPrompt(
// 4. Resolve model + auth, then assemble the tool set (universal task/todo tools
// plus any caller-supplied collector/submit tools).
const selection = resolveModelSelection((auth) => ModelRegistry.create(auth), modelTier);
const selection = await resolveModelSelection();
const resourceLoader = await buildResourceLoader(sourceDir, logger, agentName);
// Accumulates usage from in-process `task` child sessions so the parent's reported
// cost includes sub-agent spend (their getSessionStats is separate from ours).
const childUsage = { cost: 0, inputTokens: 0, outputTokens: 0 };
const childUsage: ChildUsage = { cost: 0, inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 };
const customTools: ToolDefinition[] = [
createTaskTool({
model: selection.model,
thinkingLevel: selection.thinkingLevel,
authStorage: selection.authStorage,
modelRuntime: selection.modelRuntime,
cwd: sourceDir,
onUsage: (usage) => {
childUsage.cost += usage.cost;
childUsage.inputTokens += usage.inputTokens;
childUsage.outputTokens += usage.outputTokens;
childUsage.cacheReadTokens += usage.cacheReadTokens;
childUsage.cacheWriteTokens += usage.cacheWriteTokens;
},
resourceLoader,
...(cancellationSignal && { cancellationSignal }),
@@ -283,24 +287,25 @@ export async function runPiPrompt(
let turnCount = 0;
let pendingError: PentestError | null = null;
let apiErrorDetected = false;
// Declared out here so the catch can bill spend accrued before a failure.
let session: AgentSession | undefined;
progress.start();
try {
const { session } = await createAgentSession({
({ session } = await createAgentSession({
cwd: sourceDir,
model: selection.model,
thinkingLevel: selection.thinkingLevel,
tools,
customTools,
authStorage: selection.authStorage,
modelRuntime: selection.modelRuntime,
sessionManager: SessionManager.inMemory(),
// Temporal owns retry; pi compaction stays on (no analog previously, guards
// against context overflow on long agent runs).
settingsManager: SettingsManager.inMemory({ retry: { enabled: false }, compaction: { enabled: true } }),
// Temporal owns agent restarts, pi absorbs transport faults (see
// PI_RETRY_SETTINGS); compaction stays on to guard against context overflow
// on long agent runs.
settingsManager: SettingsManager.inMemory({ retry: PI_RETRY_SETTINGS, compaction: { enabled: true } }),
resourceLoader,
});
}));
// 5. Map pi events to audit logging + progress + error capture.
session.subscribe((event: AgentSessionEvent) => {
@@ -314,15 +319,9 @@ export async function runPiPrompt(
progress.stop();
outputLines(formatAssistantOutput(text, execContext, turnCount, description));
progress.start();
const billing = classifyErrorText(text);
if (billing) pendingError = billing;
}
if (msg.role === 'assistant' && msg.stopReason === 'error') {
apiErrorDetected = true;
pendingError =
pendingError ??
classifyErrorText(msg.errorMessage ?? '') ??
new PentestError(`Agent error: ${(msg.errorMessage ?? 'unknown').slice(0, 200)}`, 'unknown', true);
pendingError = pendingError ?? providerTurnError(msg, 'Agent error', selection.model.contextWindow);
}
break;
}
@@ -348,7 +347,6 @@ export async function runPiPrompt(
if (!event.aborted && !event.willRetry && event.errorMessage) {
pendingError =
pendingError ??
classifyErrorText(event.errorMessage) ??
new PentestError(`Context compaction failed: ${event.errorMessage.slice(0, 200)}`, 'unknown', true);
}
break;
@@ -365,19 +363,9 @@ export async function runPiPrompt(
if (pendingError) throw pendingError;
// 8. Read usage/cost and final text.
const stats = session.getSessionStats();
const totalCost = stats.cost + childUsage.cost;
const usage = totalUsage(session, childUsage);
const result = session.getLastAssistantText() ?? null;
// 9. Defense-in-depth: detect a spending cap that produced an empty/cheap run.
if (isSpendingCapBehavior(turnCount, totalCost, result || '')) {
throw new PentestError(
`Spending cap likely reached (turns=${turnCount}, cost=$0): ${result?.slice(0, 100)}`,
'billing',
true,
);
}
const duration = timer.stop();
progress.finish(formatCompletionMessage(execContext, description, turnCount, duration));
@@ -390,10 +378,12 @@ export async function runPiPrompt(
success: true,
duration,
turns: turnCount,
cost: totalCost,
cost: usage.cost,
inputTokens: usage.inputTokens,
outputTokens: usage.outputTokens,
cacheReadTokens: usage.cacheReadTokens,
cacheWriteTokens: usage.cacheWriteTokens,
model: selection.model.id,
partialCost: totalCost,
apiErrorDetected,
...(structuredOutput !== undefined && { structuredOutput }),
};
} catch (error) {
@@ -402,17 +392,27 @@ export async function runPiPrompt(
const err = error as Error & { code?: string; status?: number };
await auditLogger.logError(err, duration, turnCount);
progress.stop();
outputLines(formatErrorOutput(err, execContext, description, duration, sourceDir, isRetryableError(err)));
outputLines(formatErrorOutput(err, execContext, description, duration, sourceDir, isRetryableFailure(err)));
await writeErrorLog(err, sourceDir, fullPrompt, duration);
// A failed agent still spent money — on its own turns and, since Shannon's
// prompts delegate the heavy work, mostly on `task` sub-agents. Both count
// toward the run's usage.
const usage = totalUsage(session, childUsage);
return {
error: err.message,
errorType: err.constructor.name,
errorType: err instanceof PentestError && err.code ? err.code : err.constructor.name,
prompt: `${fullPrompt.slice(0, 100)}...`,
success: false,
duration,
cost: 0,
retryable: isRetryableError(err),
turns: turnCount,
cost: usage.cost,
inputTokens: usage.inputTokens,
outputTokens: usage.outputTokens,
cacheReadTokens: usage.cacheReadTokens,
cacheWriteTokens: usage.cacheWriteTokens,
retryable: isRetryableFailure(err),
};
}
}
apps/worker/src/ai/pi/retry-settings.ts
+28
View File
@@ -0,0 +1,28 @@
// Copyright (C) 2025 Keygraph, Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License version 3
// as published by the Free Software Foundation.
/**
* Retry split between the two layers that can restart work.
*
* `enabled: false` turns off pi's own agent-level retry loop — Temporal owns
* agent restarts, and both retrying the same turn would compound. `provider`
* settings are read independently of that flag, so transport faults
* (408/409/429/5xx) are still absorbed inside the session, which is far cheaper
* than a Temporal retry that re-runs the agent and respends its tokens.
*
* `maxRetries` is handed to the selected vendor's SDK, which owns the backoff, so
* the schedule varies by provider rather than following one formula.
*
* NOTE: pi recommends keeping this at 0, since SDK-level retries consume
* out-of-usage-limit responses before pi's classifier can mark them terminal.
* Shannon accepts that trade for the transport-fault coverage. `maxRetryDelayMs`
* is left at pi's 60s default so a server asking for a longer wait fails fast
* instead of parking the activity.
*/
export const PI_RETRY_SETTINGS = {
enabled: false,
provider: { maxRetries: 8 },
} as const;
apps/worker/src/ai/pi/task-tool.ts
+20 -17
View File
@@ -16,28 +16,25 @@
* resource loader, and a fixed child tool surface.
*/
import type { ThinkingLevel } from '@earendil-works/pi-agent-core';
import { type AssistantMessage, type Model, Type } from '@earendil-works/pi-ai';
import {
type AuthStorage,
createAgentSession,
defineTool,
getAgentDir,
type ModelRegistry,
type ModelRuntime,
type ResourceLoader,
SessionManager,
SettingsManager,
type ToolDefinition,
} from '@earendil-works/pi-coding-agent';
import { PI_RETRY_SETTINGS } from './retry-settings.js';
export interface TaskToolContext {
cwd: string;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
model: Model<any>;
thinkingLevel?: ThinkingLevel;
authStorage: AuthStorage;
/** Explicit model registry for sub-session resolution. Omit to inherit the parent's default. */
modelRegistry?: ModelRegistry;
/** Parent's model/auth runtime, reused so sub-agents share its resolved credential. */
modelRuntime: ModelRuntime;
resourceLoader: ResourceLoader;
cancellationSignal?: AbortSignal | undefined;
/**
@@ -46,7 +43,13 @@ export interface TaskToolContext {
* so without this their spend (the bulk of a whitebox run, since Shannon
* prompts delegate the heavy work) is invisible to billing.
*/
onUsage?: (usage: { cost: number; inputTokens: number; outputTokens: number }) => void;
onUsage?: (usage: {
cost: number;
inputTokens: number;
outputTokens: number;
cacheReadTokens: number;
cacheWriteTokens: number;
}) => void;
}
const CHILD_TOOLS = ['read', 'grep', 'find', 'ls', 'write', 'bash'];
@@ -83,13 +86,11 @@ export function createTaskTool(config: TaskToolContext): ToolDefinition {
agentDir,
resourceLoader: config.resourceLoader,
model: config.model,
...(config.thinkingLevel && { thinkingLevel: config.thinkingLevel }),
tools: CHILD_TOOLS,
authStorage: config.authStorage,
...(config.modelRegistry && { modelRegistry: config.modelRegistry }),
modelRuntime: config.modelRuntime,
sessionManager: SessionManager.inMemory(config.cwd),
settingsManager: SettingsManager.inMemory({
retry: { enabled: false },
retry: PI_RETRY_SETTINGS,
compaction: { enabled: true },
}),
});
@@ -109,8 +110,6 @@ export function createTaskTool(config: TaskToolContext): ToolDefinition {
let resultText = '';
let subCost = 0;
let subInputTokens = 0;
let subOutputTokens = 0;
subSession.subscribe((event) => {
if (event.type === 'turn_end') {
const msg = event.message as AssistantMessage | undefined;
@@ -120,8 +119,6 @@ export function createTaskTool(config: TaskToolContext): ToolDefinition {
}
}
if (msg?.usage?.cost?.total != null) subCost += msg.usage.cost.total;
subInputTokens += msg?.usage?.input ?? 0;
subOutputTokens += msg?.usage?.output ?? 0;
}
});
@@ -138,7 +135,13 @@ export function createTaskTool(config: TaskToolContext): ToolDefinition {
// Read stats before dispose; reconcile cost the same way the parent does.
const subStats = subSession.getSessionStats();
if (subStats.cost > subCost) subCost = subStats.cost;
config.onUsage?.({ cost: subCost, inputTokens: subInputTokens, outputTokens: subOutputTokens });
config.onUsage?.({
cost: subCost,
inputTokens: subStats.tokens.input,
outputTokens: subStats.tokens.output,
cacheReadTokens: subStats.tokens.cacheRead,
cacheWriteTokens: subStats.tokens.cacheWrite,
});
} finally {
config.cancellationSignal?.removeEventListener('abort', onCancellation);
subSession.dispose();
apps/worker/src/ai/pi/turn-error.ts
+44
View File
@@ -0,0 +1,44 @@
// Copyright (C) 2025 Keygraph, Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License version 3
// as published by the Free Software Foundation.
import { type AssistantMessage, isContextOverflow, isRetryableAssistantError } from '@earendil-works/pi-ai';
import { PentestError } from '../../services/error-handling.js';
import { ErrorCode } from '../../types/errors.js';
/**
* Wrap a failed assistant turn, taking the verdict from pi.
*
* Overflow is separated first, as pi's retry contract requires: it means the
* request was too large, not that the provider faltered, so an identical retry
* would overflow again. Everything else goes to pi's classifier, which treats
* quota, billing, and auth exhaustion as terminal and load, throttling, and
* transport faults as transient — those were already retried in-session, so
* reaching here means the attempts were exhausted.
*
* `contextWindow` is omitted where overflow cannot apply, such as a one-word
* credential probe.
*/
export function providerTurnError(message: AssistantMessage, label: string, contextWindow?: number): PentestError {
const detail = (message.errorMessage ?? 'unknown provider error').slice(0, 300);
if (contextWindow !== undefined && isContextOverflow(message, contextWindow)) {
return new PentestError(
`${label}: context window exceeded after compaction: ${detail}`,
'unknown',
false,
{ contextWindow },
ErrorCode.AGENT_EXECUTION_FAILED,
);
}
return new PentestError(
`${label}: ${detail}`,
'unknown',
isRetryableAssistantError(message),
{},
ErrorCode.AGENT_EXECUTION_FAILED,
);
}
apps/worker/src/ai/queue-schemas.ts
+30 -2
View File
@@ -14,6 +14,7 @@
import { defineTool } from '@earendil-works/pi-coding-agent';
import { type Static, type TObject, Type } from 'typebox';
import { stringEnum } from '../collectors/schema.js';
import type { AgentName } from '../types/agents.js';
import type { CapturedSubmitTool } from './submit-tool.js';
@@ -23,13 +24,38 @@ function optStr(description?: string) {
return Type.Optional(Type.String(description === undefined ? {} : { description }));
}
/** Base fields shared by every queue entry. `notes` gains guidance in analysis mode. */
/**
* Base fields shared by every queue entry. `notes` gains guidance in analysis mode.
*
* `confidence` is enumerated so it reaches the report agent in the same casing the report
* schema accepts — an analysis-only run carries it through verbatim as its only rating.
*/
function baseFields(exploit: boolean) {
return {
ID: Type.String(),
vulnerability_type: Type.String(),
externally_exploitable: Type.Boolean(),
confidence: Type.String(),
confidence: stringEnum(['high', 'medium', 'low'], {
description: 'Confidence that this is a real, reachable vulnerability.',
}),
code_locations: Type.Optional(
Type.Array(
Type.Object({
file: Type.String({ description: 'Repository-relative path, no leading slash.' }),
start_line: Type.Optional(Type.Integer({ minimum: 1 })),
end_line: Type.Optional(Type.Integer({ minimum: 1, description: 'Set when the flaw spans a range.' })),
role: stringEnum(['sink', 'source', 'guard'], {
description:
'sink where the flaw manifests, source where untrusted input enters, guard for a check ' +
'that is missing or misplaced.',
}),
symbol: Type.Optional(
Type.String({ description: 'Enclosing function or method, named as written in the code.' }),
),
}),
{ description: 'Every code site this finding touches, sink first.' },
),
),
notes: exploit ? optStr() : optStr(ANALYSIS_NOTES_DESCRIPTION),
};
}
@@ -94,6 +120,8 @@ const authEntry = () => Type.Object({ ...baseFields(true), ...authFields });
const ssrfEntry = () => Type.Object({ ...baseFields(true), ...ssrfFields });
const authzEntry = () => Type.Object({ ...baseFields(true), ...authzFields });
export type QueueCodeLocation = NonNullable<Static<ReturnType<typeof injectionEntry>>['code_locations']>[number];
export type InjectionFinding = Static<ReturnType<typeof injectionEntry>>;
export type XssFinding = Static<ReturnType<typeof xssEntry>>;
export type AuthFinding = Static<ReturnType<typeof authEntry>>;
apps/worker/src/audit/metrics-tracker.ts
+23 -1
View File
@@ -23,6 +23,11 @@ interface AttemptData {
attempt_number: number;
duration_ms: number;
cost_usd: number;
input_tokens?: number | undefined;
output_tokens?: number | undefined;
cache_read_tokens?: number | undefined;
cache_write_tokens?: number | undefined;
turns?: number | undefined;
success: boolean;
timestamp: string;
model?: string | undefined;
@@ -34,6 +39,10 @@ interface AgentAuditMetrics {
attempts: AttemptData[];
final_duration_ms: number;
total_cost_usd: number;
total_input_tokens: number;
total_output_tokens: number;
total_cache_read_tokens: number;
total_cache_write_tokens: number;
model?: string | undefined;
checkpoint?: string | undefined;
}
@@ -174,6 +183,10 @@ export class MetricsTracker {
attempts: [],
final_duration_ms: 0,
total_cost_usd: 0,
total_input_tokens: 0,
total_output_tokens: 0,
total_cache_read_tokens: 0,
total_cache_write_tokens: 0,
};
this.data.metrics.agents[agentName] = agent;
@@ -184,6 +197,11 @@ export class MetricsTracker {
cost_usd: result.cost_usd,
success: result.success,
timestamp: formatTimestamp(),
...(result.input_tokens !== undefined && { input_tokens: result.input_tokens }),
...(result.output_tokens !== undefined && { output_tokens: result.output_tokens }),
...(result.cache_read_tokens !== undefined && { cache_read_tokens: result.cache_read_tokens }),
...(result.cache_write_tokens !== undefined && { cache_write_tokens: result.cache_write_tokens }),
...(result.turns !== undefined && { turns: result.turns }),
};
if (result.model) {
@@ -197,8 +215,12 @@ export class MetricsTracker {
// 3. Append attempt to history
agent.attempts.push(attempt);
// 4. Recalculate total cost across all attempts (includes failures)
// 4. Recalculate totals across all attempts (includes failures)
agent.total_cost_usd = agent.attempts.reduce((sum, a) => sum + a.cost_usd, 0);
agent.total_input_tokens = agent.attempts.reduce((sum, a) => sum + (a.input_tokens ?? 0), 0);
agent.total_output_tokens = agent.attempts.reduce((sum, a) => sum + (a.output_tokens ?? 0), 0);
agent.total_cache_read_tokens = agent.attempts.reduce((sum, a) => sum + (a.cache_read_tokens ?? 0), 0);
agent.total_cache_write_tokens = agent.attempts.reduce((sum, a) => sum + (a.cache_write_tokens ?? 0), 0);
// 5. Update agent status based on outcome
if (result.success) {
apps/worker/src/audit/workflow-logger.ts
-14
View File
@@ -12,7 +12,6 @@
*/
import fs from 'node:fs/promises';
import { isFableModel, resolveModelId } from '../ai/models.js';
import { formatDuration, formatTimestamp } from '../utils/formatting.js';
import { LogStream } from './log-stream.js';
import { generateWorkflowLogPath, type SessionMetadata } from './utils.js';
@@ -87,19 +86,6 @@ export class WorkflowLogger {
`Started: ${formatTimestamp()}`,
];
// Surface Fable usage: its safety classifiers route cybersecurity tasks to
// Opus 4.8, so those phases run on Opus 4.8 regardless of the tier setting.
const fableTiers = (['small', 'medium', 'large'] as const)
.map((tier) => ({ tier, model: resolveModelId(tier) }))
.filter(({ model }) => isFableModel(model));
if (fableTiers.length > 0) {
const tierList = fableTiers.map(({ tier, model }) => `${tier} (${model})`).join(', ');
lines.push(
`Note: ${tierList} set to a Fable model. Fable's safety classifiers`,
` route cybersecurity tasks to Opus 4.8, so those phases run on Opus 4.8.`,
);
}
lines.push(`================================================================================`, ``);
return this.logStream.write(lines.join('\n'));
apps/worker/src/collectors/exploit-collector.ts
+1 -2
View File
@@ -122,8 +122,7 @@ export function buildSchemas(validIds: ReadonlySet<string>) {
const vulnerableLocationField = Type.String({
minLength: 1,
description:
'Endpoint or mechanism where the vulnerability exists (e.g. "GET /api/products?id=", ' +
'"POST /login", or a code location like "controllers/userController.js:42").',
'Endpoint or mechanism where the vulnerability exists (e.g. "GET /api/products?id=", ' + '"POST /login").',
});
const overviewField = Type.String({
apps/worker/src/collectors/finding-collector.ts
+329
View File
@@ -0,0 +1,329 @@
// Copyright (C) 2025 Keygraph, Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License version 3
// as published by the Free Software Foundation.
/**
* Finding Collector tools
*
* Collects structured findings from the report agent via a pi tool. The agent
* calls `add_finding` once per finding with TypeBox-validated parameters. After
* the agent finishes, the caller retrieves collected findings via `getAll()`
* for downstream rendering (markdown, PDF, DB).
*
* The tool schema is mode-dependent: fields describing a demonstrated attack have no source in
* an analysis-only run, and offering them would only make the agent invent them.
*/
import { defineTool, type ToolDefinition } from '@earendil-works/pi-coding-agent';
import { type Static, Type } from 'typebox';
import { cleanInput, stringEnum } from './schema.js';
// ============================================================================
// SCHEMA
// ============================================================================
const OWASP_CATEGORY_VALUES = [
'A01:2025 — Broken Access Control',
'A02:2025 — Security Misconfiguration',
'A03:2025 — Software Supply Chain Failures',
'A04:2025 — Cryptographic Failures',
'A05:2025 — Injection',
'A06:2025 — Insecure Design',
'A07:2025 — Authentication Failures',
'A08:2025 — Software or Data Integrity Failures',
'A09:2025 — Security Logging and Alerting Failures',
'A10:2025 — Mishandling of Exceptional Conditions',
] as const;
const SEVERITY_VALUES = ['critical', 'high', 'medium', 'low', 'informational'] as const;
const STATUS_VALUES = ['exploited', 'out_of_scope', 'blocked_by_constraints', 'false_positive'] as const;
const CONFIDENCE_VALUES = ['high', 'medium', 'low'] as const;
const StepItemSchema = Type.Union([
Type.Object({
kind: Type.Literal('prose'),
text: Type.String({ minLength: 1, description: 'Narrative prose for this item.' }),
}),
Type.Object({
kind: Type.Literal('code'),
block: Type.Object({
language: Type.String({
description: 'Language identifier for syntax highlighting (e.g., "bash", "http", "json").',
}),
content: Type.String({ minLength: 1, description: 'The code content.' }),
}),
}),
]);
const StructuredStepSchema = Type.Object({
title: Type.Optional(
Type.Union([Type.String(), Type.Null()], {
description: 'Optional title for this step (e.g., "Send malicious payload").',
}),
),
items: Type.Array(StepItemSchema, {
minItems: 1,
description: 'Ordered list of prose and code items that make up this step.',
}),
});
const CodeLocationSchema = Type.Object({
file: Type.String({
minLength: 1,
description: 'Repository-relative path, no leading slash (e.g., "routes/search.ts").',
}),
start_line: Type.Optional(
Type.Union([Type.Integer({ minimum: 1 }), Type.Null()], {
description: '1-indexed line number. Omit when the deliverable gives only a file.',
}),
),
end_line: Type.Optional(
Type.Union([Type.Integer({ minimum: 1 }), Type.Null()], {
description: 'End of the range, when the finding spans multiple lines.',
}),
),
role: stringEnum(['sink', 'source', 'guard'], {
description:
'What this location is in the data flow. `sink` is where the vulnerability manifests, `source` ' +
'where untrusted input enters, `guard` a check that is missing or misplaced.',
}),
symbol: Type.Optional(
Type.Union([Type.String(), Type.Null()], {
description: 'Enclosing function or method name, when known.',
}),
),
});
const HttpLocationSchema = Type.Object({
method: Type.String({ minLength: 1, description: 'HTTP method (e.g., "GET", "POST").' }),
url: Type.String({ minLength: 1, description: 'Full URL of the affected endpoint.' }),
parameter: Type.Optional(
Type.Union([Type.String(), Type.Null()], {
description: 'The specific parameter carrying the payload, when the finding names one.',
}),
),
});
const AdditionalSectionSchema = Type.Object({
heading: Type.String({
minLength: 1,
description: 'Section heading (e.g., "Real-World Attack Scenario").',
}),
items: Type.Array(StepItemSchema, {
minItems: 1,
description: 'Ordered prose and code items for this section.',
}),
});
function identityFields() {
return {
finding_id: Type.String({
minLength: 1,
description: 'Finding identifier (e.g., "AUTH-VULN-07", "INJ-VULN-03"). Must be unique per report.',
}),
title: Type.String({
minLength: 1,
description:
'Descriptive name (e.g., "SQL Injection — User Search", "IDOR — Unauthorized Access to User Orders").',
}),
category: stringEnum(['Injection', 'XSS', 'Authentication', 'Authorization', 'SSRF'], {
description:
'From the finding_id prefix: INJ-VULN-xxx Injection, ' +
'XSS-VULN-xxx XSS, AUTH-VULN-xxx Authentication, AUTHZ-VULN-xxx Authorization, ' +
'SSRF-VULN-xxx SSRF.',
}),
owasp_category: stringEnum(OWASP_CATEGORY_VALUES, {
description: 'OWASP Top Ten 2025 category.',
}),
};
}
function locationFields() {
return {
vulnerable_location: Type.String({
minLength: 1,
description: 'Endpoint or code location where the vulnerability exists.',
}),
http_location: Type.Optional(
Type.Union([HttpLocationSchema, Type.Null()], {
description:
'The HTTP request the finding is reached through, when the deliverable names one. Omit for ' +
'findings with no network entry point.',
}),
),
};
}
/** `impact` is described per mode: an analysis run demonstrated nothing, and implying otherwise invites fabrication. */
function narrativeFields(exploit: boolean) {
const impactDescription = exploit
? 'What the exploit demonstrably achieved.'
: 'What an attacker could achieve if this were exploited. State it as assessed, not demonstrated.';
return {
overview: Type.String({
minLength: 1,
description: 'What the vulnerability is and why it matters. 2-3 sentences of professional prose.',
}),
impact: Type.String({ minLength: 1, description: impactDescription }),
remediation: Type.String({
minLength: 1,
description: 'Specific, actionable fix guidance. Code-level or configuration-level.',
}),
};
}
/** Fields that only mean something once an exploit has run. Absent from the analysis schema. */
function exploitOnlyFields() {
return {
severity: stringEnum(SEVERITY_VALUES, {
description: 'Severity of the finding, based on the impact the exploit demonstrated.',
}),
auth_state: Type.String({
minLength: 1,
description: 'Authentication state during testing (e.g., "Unauthenticated", "Any authenticated user").',
}),
prerequisites: Type.String({
minLength: 1,
description: 'What is needed to exploit the vulnerability (or "None").',
}),
exploitation_steps: Type.Array(StructuredStepSchema, {
minItems: 1,
description: 'Ordered exploitation steps. Each step has an optional title and prose/code items.',
}),
proof_of_impact: Type.Array(StepItemSchema, {
minItems: 1,
description: 'Evidence of what the exploit achieved — prose and code items.',
}),
status: Type.Optional(
Type.Union([stringEnum(STATUS_VALUES), Type.Null()], {
description: 'Finding status. Use "exploited" for confirmed exploits.',
}),
),
};
}
/** Replaces `severity` when nothing was exploited. */
function analysisOnlyFields() {
return {
confidence: stringEnum(CONFIDENCE_VALUES, {
description:
'Confidence that this is a real, reachable vulnerability. Carry it over from the analysis ' +
'deliverable rather than reassessing.',
}),
};
}
function sharedOptionalFields() {
return {
notes: Type.Optional(
Type.Union([Type.Array(StepItemSchema), Type.Null()], {
description: 'Additional context as prose/code items.',
}),
),
additional_sections: Type.Optional(
Type.Union([Type.Array(AdditionalSectionSchema), Type.Null()], {
description: 'Extra report sections that do not fit into other fields (e.g., "Real-World Attack Scenario").',
}),
),
};
}
export function buildAddFindingSchema(exploit: boolean) {
return Type.Object({
...identityFields(),
...(exploit ? exploitOnlyFields() : analysisOnlyFields()),
...locationFields(),
...narrativeFields(exploit),
...sharedOptionalFields(),
});
}
/**
* Superset of both modes, for typing only. Consumers must check presence rather than assume:
* `report.json` from an analysis run has no `severity` or `exploitation_steps` key at all.
*/
const AddFindingSupersetSchema = Type.Object({
...identityFields(),
code_locations: Type.Optional(Type.Array(CodeLocationSchema)),
severity: Type.Optional(stringEnum(SEVERITY_VALUES)),
auth_state: Type.Optional(Type.String()),
prerequisites: Type.Optional(Type.String()),
exploitation_steps: Type.Optional(Type.Array(StructuredStepSchema)),
proof_of_impact: Type.Optional(Type.Array(StepItemSchema)),
status: Type.Optional(Type.Union([stringEnum(STATUS_VALUES), Type.Null()])),
confidence: Type.Optional(Type.Union([stringEnum(CONFIDENCE_VALUES), Type.Null()])),
...locationFields(),
...narrativeFields(true),
...sharedOptionalFields(),
});
export type AddFindingInput = Static<typeof AddFindingSupersetSchema>;
// Re-export schema types for downstream consumers
export type CodeLocation = Static<typeof CodeLocationSchema>;
export type HttpLocation = Static<typeof HttpLocationSchema>;
export type StepItem = Static<typeof StepItemSchema>;
export type StructuredStep = Static<typeof StructuredStepSchema>;
export type AdditionalSection = Static<typeof AdditionalSectionSchema>;
// ============================================================================
// RESPONSE HELPERS
// ============================================================================
function toolResult(payload: Record<string, unknown>) {
return {
content: [{ type: 'text' as const, text: JSON.stringify(payload, null, 2) }],
details: undefined,
};
}
function successResult(data: Record<string, unknown>) {
return toolResult({ status: 'success', ...data });
}
function errorResult(message: string, errorType = 'ValidationError', retryable = true) {
return toolResult({ status: 'error', message, errorType, retryable });
}
// ============================================================================
// COLLECTOR FACTORY
// ============================================================================
export interface FindingCollector {
tools: ToolDefinition[];
getAll(): AddFindingInput[];
}
export function createFindingCollector(exploit: boolean): FindingCollector {
const findings: AddFindingInput[] = [];
const schema = buildAddFindingSchema(exploit);
const addFindingTool = defineTool({
name: 'add_finding',
label: 'Add Finding',
description:
'Record a single finding as structured data for report rendering and DB persistence. Call once per finding after grouping/dedup. Duplicate finding_ids are rejected.',
parameters: schema,
async execute(_toolCallId, input) {
const existing = findings.find((f) => f.finding_id === input.finding_id);
if (existing) {
return errorResult(
`Finding ${input.finding_id} has already been recorded. Each finding may only be added once.`,
'DuplicateError',
false,
);
}
const typed = cleanInput(schema, input) as AddFindingInput;
findings.push(typed);
return successResult({ added: [typed.finding_id] });
},
});
return {
tools: [addFindingTool],
getAll: (): AddFindingInput[] => [...findings],
};
}
apps/worker/src/config-parser.ts
+1
View File
@@ -675,6 +675,7 @@ export const distributeConfig = (config: Config | null): DistributedConfig => {
const exploit = config?.exploit !== undefined ? config.exploit === 'true' : true;
const report = {
sarif: config?.report?.sarif === 'true',
...(config?.report?.min_severity && { min_severity: config.report.min_severity }),
...(config?.report?.min_confidence && { min_confidence: config.report.min_confidence }),
...(config?.report?.guidance && { guidance: config.report.guidance.trim() }),
apps/worker/src/paths.ts
+6
View File
@@ -31,6 +31,12 @@ export const ASSEMBLED_REPORT_FILENAME = 'comprehensive_security_assessment_repo
/** Filename of the human-facing final report surfaced at the run directory root */
export const FINAL_REPORT_FILENAME = 'Security-Assessment-Report.md';
/** Structured findings the report agent emits; the markdown report is rendered from it. */
export const REPORT_JSON_FILENAME = 'report.json';
/** SARIF 2.1.0 log, written only for exploit=true runs when report.sarif is enabled. */
export const SARIF_FILENAME = 'report.sarif';
/**
* Resolve the session.json path for a run directory, preferring the current
* `.shannon/` location and falling back to the legacy run-root location so
apps/worker/src/scripts/set-report-meta.ts
+139
View File
@@ -0,0 +1,139 @@
#!/usr/bin/env node
// Copyright (C) 2025 Keygraph, Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License version 3
// as published by the Free Software Foundation.
/**
* set-report-meta CLI
*
* Writes top-level report metadata to report.json.
* Called once by the report agent before recording individual findings.
* Overwrites any existing report_meta — idempotent.
*
* Usage:
* set-report-meta --target "https://example.com" --assessment-date "2026-05-07" \
* --scope "injection, xss, auth, authz, ssrf" --executive-summary "..."
*
* Output (JSON to stdout):
* { "status": "success" }
* { "status": "error", "message": "...", "retryable": true }
*/
import { existsSync, mkdirSync, readFileSync, renameSync, unlinkSync, writeFileSync } from 'node:fs';
import { resolve } from 'node:path';
const REPORT_FILENAME = 'report.json';
interface ReportMeta {
target: string;
assessment_date: string;
scope: string;
executive_summary: string;
}
interface ReportFile {
report_meta?: ReportMeta;
findings: Array<Record<string, unknown>>;
}
const HELP = `set-report-meta — write top-level report metadata to report.json
Usage:
set-report-meta --target "https://example.com" --assessment-date "2026-05-07" \\
--scope "injection, xss, auth" --executive-summary "..."
Required flags: --target, --assessment-date, --scope, --executive-summary
Output: JSON to stdout with status "success" or "error".`;
function getFlag(argv: string[], flag: string): string | undefined {
for (let i = 2; i < argv.length; i++) {
if (argv[i] === flag && argv[i + 1] && !argv[i + 1]!.startsWith('--')) {
return argv[i + 1]!;
}
}
return undefined;
}
function readReportFile(filePath: string): ReportFile {
if (!existsSync(filePath)) {
return { findings: [] };
}
const raw = readFileSync(filePath, 'utf-8');
return JSON.parse(raw) as ReportFile;
}
function writeReportFile(filePath: string, data: ReportFile): void {
const tmpPath = `${filePath}.tmp`;
const payload = JSON.stringify(data, null, 2);
try {
writeFileSync(tmpPath, payload, 'utf-8');
renameSync(tmpPath, filePath);
} catch (err) {
try {
unlinkSync(tmpPath);
} catch {
/* best-effort */
}
throw err;
}
}
function main(): void {
if (process.argv[2] === '--help' || process.argv[2] === '-h') {
console.log(HELP);
return;
}
const target = getFlag(process.argv, '--target');
const assessmentDate = getFlag(process.argv, '--assessment-date');
const scope = getFlag(process.argv, '--scope');
const executiveSummary = getFlag(process.argv, '--executive-summary');
if (!target) {
console.log(JSON.stringify({ status: 'error', message: 'Missing required --target flag', retryable: true }));
process.exit(1);
}
if (!assessmentDate) {
console.log(
JSON.stringify({ status: 'error', message: 'Missing required --assessment-date flag', retryable: true }),
);
process.exit(1);
}
if (!scope) {
console.log(JSON.stringify({ status: 'error', message: 'Missing required --scope flag', retryable: true }));
process.exit(1);
}
if (!executiveSummary) {
console.log(
JSON.stringify({ status: 'error', message: 'Missing required --executive-summary flag', retryable: true }),
);
process.exit(1);
}
const subdir = process.env.SHANNON_DELIVERABLES_SUBDIR || '.shannon/deliverables';
const deliverablesDir = resolve(process.cwd(), ...subdir.split('/'));
mkdirSync(deliverablesDir, { recursive: true });
const filePath = resolve(deliverablesDir, REPORT_FILENAME);
const data = readReportFile(filePath);
data.report_meta = {
target,
assessment_date: assessmentDate,
scope,
executive_summary: executiveSummary,
};
writeReportFile(filePath, data);
console.log(JSON.stringify({ status: 'success' }));
}
try {
main();
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
console.log(JSON.stringify({ status: 'error', message, retryable: true }));
process.exit(1);
}
apps/worker/src/services/agent-execution.ts
+63 -55
View File
@@ -13,7 +13,6 @@
* - Create git checkpoint
* - Start audit logging
* - Invoke the pi agent via runPiPrompt
* - Spending cap check using isSpendingCapBehavior
* - Handle failure (rollback, audit)
* - Validate output using AGENTS[agentName].deliverableFilename
* - Render the deliverable to disk via the writeDeliverable hook (if provided)
@@ -34,7 +33,6 @@ import type { AgentEndResult } from '../types/audit.js';
import { ErrorCode, type PentestErrorType } from '../types/errors.js';
import type { AgentMetrics } from '../types/metrics.js';
import { err, isErr, ok, type Result } from '../types/result.js';
import { isSpendingCapBehavior } from '../utils/billing-detection.js';
import { getAgentGitPaths } from './agent-git-paths.js';
import type { ConfigLoaderService } from './config-loader.js';
import { PentestError } from './error-handling.js';
@@ -55,6 +53,7 @@ export interface AgentExecutionInput {
attemptNumber: number;
promptDir?: string | undefined;
customTools?: import('@earendil-works/pi-coding-agent').ToolDefinition[];
failedClasses?: readonly import('../types/config.js').VulnClass[] | undefined;
// Renders the deliverable to disk; invoked after validation, before the success commit.
writeDeliverable?: (deliverablesPath: string) => Promise<void>;
cancellationSignal?: AbortSignal | undefined;
@@ -80,11 +79,6 @@ function errorCodeFromResult(result: PiPromptResult): ErrorCode {
function categoryForErrorCode(code: ErrorCode): PentestErrorType {
switch (code) {
case ErrorCode.SPENDING_CAP_REACHED:
case ErrorCode.INSUFFICIENT_CREDITS:
case ErrorCode.BILLING_ERROR:
case ErrorCode.API_RATE_LIMITED:
return 'billing';
case ErrorCode.GIT_CHECKPOINT_FAILED:
case ErrorCode.GIT_ROLLBACK_FAILED:
return 'filesystem';
@@ -153,6 +147,7 @@ export class AgentExecutionService {
attemptNumber,
promptDir,
customTools,
failedClasses,
writeDeliverable,
cancellationSignal,
} = input;
@@ -171,7 +166,12 @@ export class AgentExecutionService {
try {
prompt = await loadPrompt(
promptTemplate,
{ webUrl, repoPath, AUTH_STATE_FILE: authStateFile(auditSession.sessionMetadata) },
{
webUrl,
repoPath,
AUTH_STATE_FILE: authStateFile(auditSession.sessionMetadata),
...(failedClasses !== undefined && { failedClasses }),
},
distributedConfig,
pipelineTestingMode,
logger,
@@ -227,31 +227,13 @@ export class AgentExecutionService {
agentName,
auditSession,
logger,
AGENTS[agentName].modelTier,
customTools,
path.relative(repoPath, deliverablesPath),
cancellationSignal,
submitTool,
);
// 6. Spending cap check - defense-in-depth
if (result.success && (result.turns ?? 0) <= 2 && (result.cost || 0) === 0) {
const resultText = result.result || '';
if (isSpendingCapBehavior(result.turns ?? 0, result.cost || 0, resultText)) {
return this.failAgent(agentName, deliverablesPath, auditSession, logger, {
attemptNumber,
result,
rollbackReason: 'spending cap detected',
errorMessage: `Spending cap likely reached: ${resultText.slice(0, 100)}`,
errorCode: ErrorCode.SPENDING_CAP_REACHED,
category: 'billing',
retryable: true,
context: { agentName, turns: result.turns, cost: result.cost },
});
}
}
// 7. Handle execution failure
// 6. Handle execution failure
if (!result.success) {
const errorCode = errorCodeFromResult(result);
return this.failAgent(agentName, deliverablesPath, auditSession, logger, {
@@ -270,39 +252,53 @@ export class AgentExecutionService {
// the write→validate→commit sequence is atomic against concurrent sibling agents.
let commitHash: string | undefined;
const finalizationError = await withGitRepoLock(async (): Promise<PentestError | null> => {
// 8. Write structured output to disk (vuln agents only) from the executor's capture
const queueFilename = getQueueFilename(agentName);
if (submitTool && queueFilename && result.structuredOutput !== undefined) {
await fs.ensureDir(deliverablesPath);
const queuePath = path.join(deliverablesPath, queueFilename);
await fs.writeFile(queuePath, JSON.stringify(result.structuredOutput, null, 2), 'utf8');
logger.info(`Wrote structured output queue to ${queueFilename}`);
}
// Every step below must surface as a returned error rather than a throw: only the
// returned path rolls the workspace back and records the failed attempt.
try {
// 8. Write structured output to disk (vuln agents only) from the executor's capture
const queueFilename = getQueueFilename(agentName);
if (submitTool && queueFilename && result.structuredOutput !== undefined) {
await fs.ensureDir(deliverablesPath);
const queuePath = path.join(deliverablesPath, queueFilename);
await fs.writeFile(queuePath, JSON.stringify(result.structuredOutput, null, 2), 'utf8');
logger.info(`Wrote structured output queue to ${queueFilename}`);
}
// 9. Validate output
const validationPassed = await validateAgentOutput(result, agentName, deliverablesPath, logger);
if (!validationPassed) {
// 9. Validate output
const validationPassed = await validateAgentOutput(result, agentName, deliverablesPath, logger);
if (!validationPassed) {
return new PentestError(
`Agent ${agentName} failed output validation`,
'validation',
true,
{ agentName, deliverableFilename: AGENTS[agentName].deliverableFilename },
ErrorCode.OUTPUT_VALIDATION_FAILED,
);
}
// 10. Render the deliverable to disk so the success commit below stages it
if (writeDeliverable) {
await writeDeliverable(deliverablesPath);
}
// 11. Success - commit deliverables (scoped) and capture the checkpoint hash
const commitResult = await commitGitSuccess(deliverablesPath, agentName, logger, gitPaths);
if (!commitResult.success) {
return gitFailureForAgent(agentName, 'commit successful results', commitResult.error);
}
commitHash = commitResult.commitHash;
return null;
} catch (error) {
if (error instanceof PentestError) return error;
const errorMessage = error instanceof Error ? error.message : String(error);
return new PentestError(
`Agent ${agentName} failed output validation`,
`Agent ${agentName} post-processing failed: ${errorMessage}`,
'validation',
true,
{ agentName, deliverableFilename: AGENTS[agentName].deliverableFilename },
{ agentName, originalError: errorMessage },
ErrorCode.OUTPUT_VALIDATION_FAILED,
);
}
// 10. Render the deliverable to disk so the success commit below stages it
if (writeDeliverable) {
await writeDeliverable(deliverablesPath);
}
// 11. Success - commit deliverables (scoped) and capture the checkpoint hash
const commitResult = await commitGitSuccess(deliverablesPath, agentName, logger, gitPaths);
if (!commitResult.success) {
return gitFailureForAgent(agentName, 'commit successful results', commitResult.error);
}
commitHash = commitResult.commitHash;
return null;
});
if (finalizationError) {
@@ -326,6 +322,11 @@ export class AgentExecutionService {
attemptNumber,
duration_ms: result.duration,
cost_usd: result.cost || 0,
input_tokens: result.inputTokens,
output_tokens: result.outputTokens,
cache_read_tokens: result.cacheReadTokens,
cache_write_tokens: result.cacheWriteTokens,
turns: result.turns,
success: true,
model: result.model,
...(commitHash && { checkpoint: commitHash }),
@@ -353,6 +354,11 @@ export class AgentExecutionService {
attemptNumber: opts.attemptNumber,
duration_ms: opts.result.duration,
cost_usd: opts.result.cost || 0,
input_tokens: opts.result.inputTokens,
output_tokens: opts.result.outputTokens,
cache_read_tokens: opts.result.cacheReadTokens,
cache_write_tokens: opts.result.cacheWriteTokens,
turns: opts.result.turns,
success: false,
model: opts.result.model,
error: opts.errorMessage,
@@ -406,8 +412,10 @@ export class AgentExecutionService {
static toMetrics(endResult: AgentEndResult, result: PiPromptResult): AgentMetrics {
return {
durationMs: endResult.duration_ms,
inputTokens: null, // Not currently exposed by the pi executor
outputTokens: null,
inputTokens: result.inputTokens ?? null,
outputTokens: result.outputTokens ?? null,
cacheReadTokens: result.cacheReadTokens ?? null,
cacheWriteTokens: result.cacheWriteTokens ?? null,
costUsd: endResult.cost_usd,
numTurns: result.turns ?? null,
model: result.model,
apps/worker/src/services/agent-git-paths.ts
+8
View File
@@ -14,6 +14,7 @@
*/
import { getQueueFilename } from '../ai/queue-schemas.js';
import { REPORT_JSON_FILENAME, SARIF_FILENAME } from '../paths.js';
import { AGENTS } from '../session-manager.js';
import type { AgentName } from '../types/agents.js';
@@ -27,5 +28,12 @@ export function getAgentGitPaths(agentName: AgentName): string[] {
if (queueFilename) {
paths.push(queueFilename);
}
// The report agent also emits the structured findings the markdown is rendered from, and the
// SARIF log when enabled. Listing the log unconditionally is harmless when it was not written,
// and keeps a stale one from surviving the rollback of a failed attempt.
if (agentName === 'report') {
paths.push(REPORT_JSON_FILENAME);
paths.push(SARIF_FILENAME);
}
return [...new Set(paths)];
}
apps/worker/src/services/code-location-join.ts
+78
View File
@@ -0,0 +1,78 @@
// Copyright (C) 2025 Keygraph, Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License version 3
// as published by the Free Software Foundation.
/**
* Attach vuln-queue code locations to collected findings.
*
* The vuln agent authors `code_locations` once, into its queue. Every stage after that used to
* re-transcribe them — the exploit agent into its evidence, the report agent into `add_finding` —
* and each hop lost some: 100% in the queue, 98% in the evidence, 42-63% by the report. Nothing
* about the copy is a judgement call, and `finding_id` matches the queue `ID` exactly, so the
* locations are joined here instead of being asked for again.
*/
import { fs, path } from 'zx';
import type { QueueCodeLocation } from '../ai/queue-schemas.js';
import type { AddFindingInput } from '../collectors/finding-collector.js';
import type { ActivityLogger } from '../types/activity-logger.js';
import { ALL_VULN_CLASSES } from '../types/config.js';
interface QueueEntry {
ID?: string;
code_locations?: QueueCodeLocation[];
}
/** Read every per-class queue in the deliverables dir into an ID-to-locations map. */
async function loadQueueLocations(
deliverablesPath: string,
logger: ActivityLogger,
): Promise<Map<string, QueueCodeLocation[]>> {
const locations = new Map<string, QueueCodeLocation[]>();
for (const vulnClass of ALL_VULN_CLASSES) {
const queuePath = path.join(deliverablesPath, `${vulnClass}_exploitation_queue.json`);
if (!(await fs.pathExists(queuePath))) continue;
try {
const doc = (await fs.readJson(queuePath)) as { vulnerabilities?: QueueEntry[] };
for (const entry of doc.vulnerabilities ?? []) {
if (entry.ID && entry.code_locations && entry.code_locations.length > 0) {
locations.set(entry.ID, entry.code_locations);
}
}
} catch (error) {
logger.warn(`Could not read ${vulnClass} queue for code locations: ${(error as Error).message}`);
}
}
return locations;
}
/**
* Return the findings with `code_locations` filled in from the queue.
*
* A finding with no matching queue entry keeps none — the join never invents one. Findings are
* copied rather than mutated so the collector's own state stays untouched.
*/
export async function attachQueueCodeLocations(
findings: readonly AddFindingInput[],
deliverablesPath: string,
logger: ActivityLogger,
): Promise<AddFindingInput[]> {
const byId = await loadQueueLocations(deliverablesPath, logger);
if (byId.size === 0) return [...findings];
let matched = 0;
const joined = findings.map((finding) => {
const locations = byId.get(finding.finding_id);
if (!locations) return finding;
matched += 1;
return { ...finding, code_locations: locations };
});
logger.info(`Attached code locations to ${matched}/${findings.length} finding(s) from the vuln queues`);
return joined;
}
apps/worker/src/services/config-loader.ts
+3 -7
View File
@@ -38,13 +38,9 @@ export class ConfigLoaderService {
} catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
// Determine appropriate error code based on error message
let errorCode = ErrorCode.CONFIG_PARSE_ERROR;
if (errorMessage.includes('not found') || errorMessage.includes('ENOENT')) {
errorCode = ErrorCode.CONFIG_NOT_FOUND;
} else if (errorMessage.includes('validation failed')) {
errorCode = ErrorCode.CONFIG_VALIDATION_FAILED;
}
// parseConfig throws PentestErrors that already name the failure; anything
// else reaching here is a parse-time fault.
const errorCode = error instanceof PentestError && error.code ? error.code : ErrorCode.CONFIG_PARSE_ERROR;
return err(
new PentestError(
apps/worker/src/services/error-handling.ts
+28 -157
View File
@@ -4,8 +4,8 @@
// it under the terms of the GNU Affero General Public License version 3
// as published by the Free Software Foundation.
import { type AssistantMessage, isRetryableAssistantError } from '@earendil-works/pi-ai';
import { ErrorCode, type PentestErrorContext, type PentestErrorType, type PromptErrorResult } from '../types/errors.js';
import { matchesBillingApiPattern, matchesBillingTextPattern } from '../utils/billing-detection.js';
export class PentestError extends Error {
override name = 'PentestError' as const;
@@ -44,53 +44,23 @@ export function handlePromptError(promptName: string, error: Error): PromptError
};
}
const RETRYABLE_PATTERNS = [
// Network and connection errors
'network',
'connection',
'timeout',
'econnreset',
'enotfound',
'econnrefused',
// Rate limiting
'rate limit',
'429',
'too many requests',
// Server errors
'server error',
'5xx',
'internal server error',
'service unavailable',
'bad gateway',
// Provider API errors
'model unavailable',
'service temporarily unavailable',
'api error',
'terminated',
// Max turns
'max turns',
'maximum turns',
];
/**
* Whether a failed agent attempt is worth retrying.
*
* A PentestError already carries a verdict — for provider turns that verdict
* comes from pi — so it is taken as given. Anything else is raw text, judged by
* pi's classifier: transient for load, throttling, and transport failures,
* terminal for quota, billing, and auth. Unrecognised errors are not retried, so
* a permanent fault fails fast.
*/
export function isRetryableFailure(error: Error): boolean {
if (error instanceof PentestError) return error.retryable;
// Patterns that indicate non-retryable errors (checked before default)
const NON_RETRYABLE_PATTERNS = [
'authentication',
'invalid prompt',
'out of memory',
'permission denied',
'session limit reached',
'invalid api key',
];
// Conservative retry classification - unknown errors don't retry (fail-safe default)
export function isRetryableError(error: Error): boolean {
const message = error.message.toLowerCase();
if (NON_RETRYABLE_PATTERNS.some((pattern) => message.includes(pattern))) {
return false;
}
return RETRYABLE_PATTERNS.some((pattern) => message.includes(pattern));
return isRetryableAssistantError({
role: 'assistant',
stopReason: 'error',
errorMessage: error.message,
} as AssistantMessage);
}
/**
@@ -99,14 +69,6 @@ export function isRetryableError(error: Error): boolean {
*/
function classifyByErrorCode(code: ErrorCode, retryableFromError: boolean): { type: string; retryable: boolean } {
switch (code) {
// Billing errors - retryable (wait for cap reset or credits added)
case ErrorCode.SPENDING_CAP_REACHED:
case ErrorCode.INSUFFICIENT_CREDITS:
return { type: 'BillingError', retryable: true };
case ErrorCode.API_RATE_LIMITED:
return { type: 'RateLimitError', retryable: true };
// Config errors - non-retryable (need manual fix)
case ErrorCode.CONFIG_NOT_FOUND:
case ErrorCode.CONFIG_VALIDATION_FAILED:
@@ -143,11 +105,10 @@ function classifyByErrorCode(code: ErrorCode, retryableFromError: boolean): { ty
case ErrorCode.AUTH_LOGIN_FAILED:
return { type: 'AuthLoginFailedError', retryable: false };
case ErrorCode.BILLING_ERROR:
return { type: 'BillingError', retryable: true };
case ErrorCode.TARGET_UNREACHABLE:
return { type: 'InvalidTargetError', retryable: false };
default:
// Unknown code - fall through to string matching
return { type: 'UnknownError', retryable: retryableFromError };
}
}
@@ -161,8 +122,8 @@ function classifyByErrorCode(code: ErrorCode, retryableFromError: boolean): { ty
* - Non-retryable errors: Temporal fails immediately
*
* Classification priority:
* 1. If error is PentestError with ErrorCode, classify by code (reliable)
* 2. Fall through to string matching for external errors (provider, network, etc.)
* 1. A PentestError carrying an ErrorCode is classified by that code.
* 2. Anything else falls through to isRetryableFailure.
*/
export function classifyErrorForTemporal(error: unknown): { type: string; retryable: boolean } {
// === CODE-BASED CLASSIFICATION (Preferred for internal errors) ===
@@ -170,101 +131,11 @@ export function classifyErrorForTemporal(error: unknown): { type: string; retrya
return classifyByErrorCode(error.code, error.retryable);
}
// === STRING-BASED CLASSIFICATION (Fallback for external errors) ===
const message = (error instanceof Error ? error.message : String(error)).toLowerCase();
// === BILLING ERRORS (Retryable with long backoff) ===
// Anthropic returns billing as 400 invalid_request_error
// Human can add credits OR wait for spending cap to reset (5-30 min backoff)
// Check both API patterns and text patterns for comprehensive detection
if (matchesBillingApiPattern(message) || matchesBillingTextPattern(message)) {
return { type: 'BillingError', retryable: true };
}
// === PERMANENT ERRORS (Non-retryable) ===
// Authentication (401) - bad API key won't fix itself
if (
message.includes('authentication') ||
message.includes('api key') ||
message.includes('401') ||
message.includes('authentication_error')
) {
return { type: 'AuthenticationError', retryable: false };
}
// Permission (403) - access won't be granted
if (message.includes('permission') || message.includes('forbidden') || message.includes('403')) {
return { type: 'PermissionError', retryable: false };
}
// Out of memory - deterministic resource exhaustion, retrying won't help
if (message.includes('out of memory')) {
return { type: 'OutOfMemoryError', retryable: false };
}
// Invalid prompt - malformed/rejected prompt content won't fix itself on retry
if (message.includes('invalid prompt')) {
return { type: 'InvalidPromptError', retryable: false };
}
// Session limit reached - distinct from billing/rate-limit; needs manual intervention
if (message.includes('session limit reached')) {
return { type: 'SessionLimitError', retryable: false };
}
// Overloaded - provider's own error-type token is authoritative regardless of the
// HTTP status it arrives under (seen in production under 400, not just 529)
if (message.includes('overloaded_error') || message.includes('overloaded')) {
return { type: 'OverloadedError', retryable: true };
}
// === OUTPUT VALIDATION ERRORS (Retryable) ===
// Agent didn't produce expected deliverables - retry may succeed
// IMPORTANT: Must come BEFORE generic 'validation' check below
if (message.includes('failed output validation') || message.includes('output validation failed')) {
return { type: 'OutputValidationError', retryable: true };
}
// Invalid Request (400) - malformed request is permanent
// Note: Checked AFTER billing and AFTER output validation
if (message.includes('invalid_request_error') || message.includes('malformed') || message.includes('validation')) {
return { type: 'InvalidRequestError', retryable: false };
}
// Request Too Large (413) - won't fit no matter how many retries
if (message.includes('request_too_large') || message.includes('too large') || message.includes('413')) {
return { type: 'RequestTooLargeError', retryable: false };
}
// Configuration errors - missing files need manual fix
if (message.includes('enoent') || message.includes('no such file') || message.includes('cli not installed')) {
return { type: 'ConfigurationError', retryable: false };
}
// Execution limits - max turns/budget reached
if (
message.includes('max turns') ||
message.includes('budget') ||
message.includes('execution limit') ||
message.includes('error_max_turns') ||
message.includes('error_max_budget')
) {
return { type: 'ExecutionLimitError', retryable: false };
}
// Invalid target URL - bad URL format won't fix itself
if (
message.includes('invalid url') ||
message.includes('invalid target') ||
message.includes('malformed url') ||
message.includes('invalid uri')
) {
return { type: 'InvalidTargetError', retryable: false };
}
// === TRANSIENT ERRORS (Retryable) ===
// Rate limits (429), server errors (5xx), network issues
// Let Temporal retry with configured backoff
return { type: 'TransientError', retryable: true };
// === FALLBACK ===
// Everything else is a raw throw: a library error, or a PentestError carrying no
// code. isRetryableFailure decides — pi's classifier for provider text, the
// error's own verdict when it has one, and no retry for anything unrecognised.
const err = error instanceof Error ? error : new Error(String(error));
const retryable = isRetryableFailure(err);
return { type: retryable ? 'TransientError' : 'PermanentError', retryable };
}
apps/worker/src/services/findings-renderer.ts
+12
View File
@@ -53,9 +53,15 @@ function formatLocation(endpoint: string | undefined, codeLocation: string | und
return endpoint ?? codeLocation ?? '';
}
/** The analysis queue carries no severity, so confidence is the only rating. */
interface CommonEntryFields {
readonly confidence: string;
}
function buildEntry(
id: string,
title: string,
common: CommonEntryFields,
summaryRows: ReadonlyArray<string | null>,
notes: string | undefined,
): string {
@@ -63,6 +69,7 @@ function buildEntry(
lines.push(`### ${id}: ${title}`);
lines.push('');
lines.push('**Summary:**');
lines.push(`- **Confidence:** ${common.confidence}`);
for (const row of summaryRows) {
if (row !== null) lines.push(row);
}
@@ -79,6 +86,7 @@ function renderAuthEntry(e: AuthFinding): string {
return buildEntry(
e.ID,
e.vulnerability_type,
{ confidence: e.confidence },
[
summaryRow('Vulnerable location', formatLocation(e.source_endpoint, e.vulnerable_code_location)),
summaryRow('Overview', e.missing_defense),
@@ -92,6 +100,7 @@ function renderSsrfEntry(e: SsrfFinding): string {
return buildEntry(
e.ID,
e.vulnerability_type,
{ confidence: e.confidence },
[
summaryRow('Vulnerable location', formatLocation(e.source_endpoint, e.vulnerable_code_location)),
summaryRow('Overview', e.missing_defense),
@@ -105,6 +114,7 @@ function renderAuthzEntry(e: AuthzFinding): string {
return buildEntry(
e.ID,
e.vulnerability_type,
{ confidence: e.confidence },
[
summaryRow('Vulnerable location', formatLocation(e.endpoint, e.vulnerable_code_location)),
summaryRow('Overview', e.guard_evidence),
@@ -119,6 +129,7 @@ function renderInjectionEntry(e: InjectionFinding): string {
return buildEntry(
e.ID,
e.vulnerability_type,
{ confidence: e.confidence },
[summaryRow('Vulnerable location', location), summaryRow('Overview', e.mismatch_reason)],
e.notes,
);
@@ -129,6 +140,7 @@ function renderXssEntry(e: XssFinding): string {
return buildEntry(
e.ID,
e.vulnerability_type,
{ confidence: e.confidence },
[summaryRow('Vulnerable location', location), summaryRow('Overview', e.mismatch_reason)],
e.notes,
);
apps/worker/src/services/index.ts
+2
View File
@@ -20,4 +20,6 @@ export type { ContainerDependencies } from './container.js';
export { Container, getContainer, getOrCreateContainer, removeContainer, setContainerFactory } from './container.js';
export { ExploitationCheckerService } from './exploitation-checker.js';
export { loadPrompt } from './prompt-manager.js';
export type { ReportData, ReportMeta } from './report-renderer.js';
export { renderReport } from './report-renderer.js';
export { assembleFinalReport, copyReportToRunRoot, injectModelIntoReport } from './reporting.js';
apps/worker/src/services/preflight.ts
+136 -138
View File
@@ -15,7 +15,7 @@
* 1. Repository path exists and is a directory
* 2. Config file parses and validates (if provided)
* 3. code_path rules match real entries in the repo (filesystem only)
* 4. Credentials validate via a minimal pi session (API key, OAuth, or Bedrock)
* 4. Credentials validate via a minimal pi session against the run's own model
* 5. Target URL resolves, is not link-local (cloud metadata), and is reachable (DNS + HTTP)
*/
@@ -26,22 +26,33 @@ import http from 'node:http';
import https from 'node:https';
import net, { type LookupFunction } from 'node:net';
import os from 'node:os';
import type { Api, AssistantMessage, Model } from '@earendil-works/pi-ai';
import {
AuthStorage,
type AgentSession,
createAgentSession,
ModelRegistry,
type ModelRuntime,
SessionManager,
SettingsManager,
} from '@earendil-works/pi-coding-agent';
import { glob } from 'zx';
import { resolveEffectiveProvider, resolveModelId } from '../ai/models.js';
import {
createModelRuntime,
type ModelSpec,
type OpenAiFormat,
type ProviderId,
resolveGatewayFormat,
resolveModel,
resolveModelSpec,
resolveProviderCredentials,
} from '../ai/models.js';
import { PI_RETRY_SETTINGS } from '../ai/pi/retry-settings.js';
import { providerTurnError } from '../ai/pi/turn-error.js';
import { parseConfig } from '../config-parser.js';
import type { ActivityLogger } from '../types/activity-logger.js';
import type { Config, Rule } from '../types/config.js';
import { ErrorCode } from '../types/errors.js';
import { err, isErr, ok, type Result } from '../types/result.js';
import { matchesBillingTextPattern } from '../utils/billing-detection.js';
import { PentestError } from './error-handling.js';
import { isRetryableFailure, PentestError } from './error-handling.js';
const TARGET_URL_TIMEOUT_MS = 10_000;
@@ -215,157 +226,79 @@ async function validateCodePathsExist(
// === Credential Validation ===
/** Map provider error text to a human-readable preflight PentestError. */
/** Classify a provider error message (thrown or from a failed turn) into a PentestError. */
function classifyCredentialError(text: string, authType: string): Result<void, PentestError> {
const lower = text.toLowerCase();
if (matchesBillingTextPattern(text)) {
return err(
new PentestError(
`Anthropic account has a billing or rate-limit issue during ${authType} validation. Add credits or wait and retry.`,
'billing',
true,
{ authType },
ErrorCode.BILLING_ERROR,
),
);
}
if (/401|403|invalid[ _-]?api[ _-]?key|unauthorized|authentication|forbidden|not allowed|x-api-key/.test(lower)) {
return err(
new PentestError(
`Invalid ${authType}. Check your credentials in .env and try again.`,
'config',
false,
{ authType },
ErrorCode.AUTH_FAILED,
),
);
}
if (/model/.test(lower) && /not found|not available|unknown/.test(lower)) {
return err(
new PentestError(
`Configured model is not available for this account. Check ANTHROPIC_*_MODEL in .env.`,
'config',
false,
{ authType },
),
);
}
if (
/network|timeout|enotfound|econnrefused|fetch failed|getaddrinfo|socket|overloaded|unavailable|50\d/.test(lower)
) {
return err(
new PentestError(`Anthropic API unreachable or temporarily unavailable. Try again shortly.`, 'network', true, {
authType,
}),
);
}
return err(
new PentestError(
`${authType} validation failed: ${text.slice(0, 150)}`,
'config',
false,
{ authType },
ErrorCode.AUTH_FAILED,
),
);
}
/** Minimal pi session probe to validate credentials. An optional baseUrl overrides the endpoint. */
/**
* Minimal pi session probe against the model the scan will use, so credentials the
* account cannot use fail here rather than partway through the run. The descriptor
* already carries the run's endpoint and wire format, so the probe exercises the
* same path the scan will.
*/
async function probeCredentialsWithPi(
model: Model<Api>,
modelRuntime: ModelRuntime,
authType: string,
token?: string,
baseUrl?: string,
): Promise<Result<void, PentestError>> {
const authStorage = AuthStorage.inMemory();
if (token) authStorage.setRuntimeApiKey('anthropic', token);
const baseModel = ModelRegistry.create(authStorage).find('anthropic', resolveModelId('small'));
if (!baseModel) {
return err(
new PentestError(
`Model not found in pi registry: ${resolveModelId('small')}`,
'config',
false,
{},
ErrorCode.AUTH_FAILED,
),
);
}
const model = baseUrl ? { ...baseModel, baseUrl } : baseModel;
let errText: string | undefined;
let failedTurn: AssistantMessage | undefined;
let session: AgentSession | undefined;
try {
const { session } = await createAgentSession({
({ session } = await createAgentSession({
cwd: os.tmpdir(),
model,
thinkingLevel: 'off',
noTools: 'all',
authStorage,
modelRuntime,
sessionManager: SessionManager.inMemory(),
settingsManager: SettingsManager.inMemory({ retry: { enabled: false }, compaction: { enabled: false } }),
});
settingsManager: SettingsManager.inMemory({ retry: PI_RETRY_SETTINGS, compaction: { enabled: false } }),
}));
session.subscribe((e) => {
if (e.type === 'turn_end' && e.message.role === 'assistant' && e.message.stopReason === 'error') {
errText = e.message.errorMessage ?? 'unknown provider error';
failedTurn = e.message;
}
});
await session.prompt('hi');
session.dispose();
} catch (error) {
errText = error instanceof Error ? error.message : String(error);
const thrown = error instanceof Error ? error : new Error(String(error));
return err(
new PentestError(
`${authType} validation failed: ${thrown.message.slice(0, 300)}`,
'unknown',
isRetryableFailure(thrown),
{ authType },
ErrorCode.AGENT_EXECUTION_FAILED,
),
);
} finally {
session?.dispose();
}
if (errText) return classifyCredentialError(errText, authType);
if (failedTurn) return err(providerTurnError(failedTurn, `${authType} validation failed`));
return ok(undefined);
}
/** Validate credentials via a minimal pi session. */
/** Credential env var a provider reads, for "credential missing" messages. */
const PROVIDER_CREDENTIAL_HINT: Readonly<Record<ProviderId, string>> = {
anthropic: 'ANTHROPIC_API_KEY (or CLAUDE_CODE_OAUTH_TOKEN)',
openai: 'OPENAI_API_KEY',
xai: 'XAI_API_KEY',
'amazon-bedrock': 'AWS_BEARER_TOKEN_BEDROCK and AWS_REGION',
};
/** Human-readable label for which credential path a run is using. */
function describeAuth(providerId: ProviderId, baseUrl: string | undefined): string {
if (baseUrl) return `custom endpoint (${baseUrl})`;
if (providerId === 'amazon-bedrock') return 'Bedrock bearer token';
return `${providerId} API key`;
}
/** Validate the model selection and its credentials via a minimal pi session. */
async function validateCredentials(logger: ActivityLogger): Promise<Result<void, PentestError>> {
// Resolve the active provider through the same precedence the executor uses, so
// preflight validates exactly the credentials the run will use (no drift).
const eff = resolveEffectiveProvider();
// 1. Bedrock mode — validate required AWS credentials are present (pi-ai owns the
// live AWS auth, so there is no cheap session probe here)
if (eff.providerId === 'amazon-bedrock') {
const required = [
'AWS_REGION',
'AWS_BEARER_TOKEN_BEDROCK',
'ANTHROPIC_SMALL_MODEL',
'ANTHROPIC_MEDIUM_MODEL',
'ANTHROPIC_LARGE_MODEL',
];
const missing = required.filter((v) => !process.env[v]);
if (missing.length > 0) {
return err(
new PentestError(
`Bedrock mode requires the following env vars in .env: ${missing.join(', ')}`,
'config',
false,
{ missing },
ErrorCode.AUTH_FAILED,
),
);
}
logger.info('Bedrock credentials OK');
return ok(undefined);
}
// 2. Custom base URL — validate the endpoint via a minimal pi session
if (eff.baseUrl) {
logger.info('Validating custom base URL');
const probe = await probeCredentialsWithPi(`custom endpoint (${eff.baseUrl})`, eff.anthropicToken, eff.baseUrl);
if (isErr(probe)) return probe;
logger.info('Custom base URL OK');
return ok(undefined);
}
// 3. Direct Anthropic — require a credential, then validate via a minimal pi session
if (!eff.anthropicToken) {
// 1. Resolve the run's model. A malformed spec or unknown provider fails here,
// before any scan work begins.
let spec: ModelSpec;
try {
spec = resolveModelSpec();
} catch (error) {
return err(
new PentestError(
'No API credentials found. Set ANTHROPIC_API_KEY or CLAUDE_CODE_OAUTH_TOKEN in .env (or use CLAUDE_CODE_USE_BEDROCK=1 for AWS Bedrock)',
error instanceof Error ? error.message : String(error),
'config',
false,
{},
@@ -373,11 +306,76 @@ async function validateCredentials(logger: ActivityLogger): Promise<Result<void,
),
);
}
logger.info(`Model: ${spec.providerId}:${spec.modelId}`);
const usingApiKey = Boolean(process.env.ANTHROPIC_API_KEY);
const authType = usingApiKey ? 'API key' : 'OAuth token';
// 2. Credential presence. Bedrock needs both AWS_ vars; every other provider
// needs one API key.
const credentials = resolveProviderCredentials(spec.providerId);
// 3. Wire format for an OpenAI gateway. Rejects a format named where it cannot
// take effect, rather than letting the run proceed on the wrong API.
let format: OpenAiFormat;
try {
format = resolveGatewayFormat(spec.providerId, credentials.baseUrl);
} catch (error) {
return err(
new PentestError(
error instanceof Error ? error.message : String(error),
'config',
false,
{ providerId: spec.providerId },
ErrorCode.AUTH_FAILED,
),
);
}
const isBedrock = spec.providerId === 'amazon-bedrock';
const missing = isBedrock ? ['AWS_REGION', 'AWS_BEARER_TOKEN_BEDROCK'].filter((n) => !process.env[n]) : [];
if (missing.length > 0 || (!isBedrock && !credentials.apiKey)) {
return err(
new PentestError(
`No credentials found for provider "${spec.providerId}". Set ${PROVIDER_CREDENTIAL_HINT[spec.providerId]} in .env.`,
'config',
false,
{ providerId: spec.providerId, ...(missing.length > 0 && { missing }) },
ErrorCode.AUTH_FAILED,
),
);
}
// 4. Model must exist in the registry, for every provider — Bedrock IDs are the
// easiest to get wrong, since region prefixes and version suffixes differ per
// model (`us.anthropic.claude-opus-5` exists, bare `anthropic.` does not).
// A custom endpoint is exempt: it may serve models under its own names.
const modelRuntime = await createModelRuntime(spec.providerId, credentials.apiKey);
const baseModel = resolveModel(modelRuntime, spec.providerId, spec.modelId, credentials.baseUrl, format);
if (!baseModel) {
return err(
new PentestError(
`Model not found in pi registry: provider="${spec.providerId}" model="${spec.modelId}". Check SHANNON_AI_MODEL.`,
'config',
false,
{ providerId: spec.providerId, modelId: spec.modelId },
ErrorCode.AUTH_FAILED,
),
);
}
if (!modelRuntime.getModel(spec.providerId, spec.modelId)) {
logger.warn(
`Model "${spec.modelId}" is not in the ${spec.providerId} catalogue; passing it to the custom endpoint as given. Cost figures will be approximate.`,
);
}
if (credentials.baseUrl && spec.providerId === 'openai') {
logger.info(`Gateway API: ${format} (${baseModel.api})`);
}
// 5. One real request, so a credential the account cannot use fails here
// rather than partway through the run. Bedrock included: pi resolves the
// bearer token from the primed credential and the region from AWS_REGION,
// so the probe exercises the same auth path the scan will.
const authType = describeAuth(spec.providerId, credentials.baseUrl);
logger.info(`Validating ${authType} via pi...`);
const probe = await probeCredentialsWithPi(authType, eff.anthropicToken);
const probe = await probeCredentialsWithPi(baseModel, modelRuntime, authType);
if (isErr(probe)) return probe;
logger.info(`${authType} OK`);
return ok(undefined);
apps/worker/src/services/prompt-manager.ts
+94 -10
View File
@@ -8,7 +8,7 @@ import { fs, path } from 'zx';
import { PROMPTS_DIR } from '../paths.js';
import { PLAYWRIGHT_SESSION_MAPPING } from '../session-manager.js';
import type { ActivityLogger } from '../types/activity-logger.js';
import type { Authentication, DistributedConfig, ReportConfig, Rule, VulnClass } from '../types/config.js';
import type { Authentication, DistributedConfig, DistributedReportConfig, Rule, VulnClass } from '../types/config.js';
import { isGlobPattern } from '../utils/glob.js';
import { handlePromptError, PentestError } from './error-handling.js';
@@ -67,27 +67,76 @@ function renderVulnSummarySubsections(selected: readonly VulnClass[]): string {
.join('\n\n');
}
/**
* Renders the <not_assessed_classes> block. Empty when every class completed.
*
* A class whose analysis failed was never assessed, so the report must not present its
* absence of findings as a clean result. The block is authoritative for that caveat.
*/
function renderNotAssessedClassesBlock(failed: readonly VulnClass[] = []): string {
if (failed.length === 0) {
return '';
}
const classes = [...new Set(failed)];
const lines: string[] = [
'<not_assessed_classes>',
'The following vulnerability classes did not complete and were NOT assessed in this run. Treat this list as authoritative for completeness caveats.',
'',
];
for (const cls of classes) {
const spec = VULN_SUMMARY_SPECS[cls];
lines.push(
`- ${spec.heading}: analysis did not complete; this class was NOT assessed. Absence of findings here does not indicate the class is clean.`,
);
}
lines.push(
'',
'When writing report_meta.executive_summary, scope any no-findings statement to the classes that were assessed and mention these not-assessed classes. Do not state or imply that the target is clean for these classes.',
'</not_assessed_classes>',
);
return lines.join('\n');
}
/**
* Which configured filters this run can actually enforce.
*
* The two ratings are mode-exclusive (see ../collectors/finding-collector.ts): an exploited
* finding carries `severity`, an analysed one carries `confidence`. Handing the agent a
* threshold for the rating its findings do not have is a directive it cannot honor.
*/
function applicableFilters(report: DistributedReportConfig | undefined, exploitEnabled: boolean) {
return {
severity: Boolean(report?.min_severity) && exploitEnabled,
confidence: Boolean(report?.min_confidence) && !exploitEnabled,
guidance: Boolean(report?.guidance?.trim()),
};
}
/**
* Renders the top-level <report_filters> block. Empty when no filters are set —
* each filter is included only when the operator configured it, so the agent
* never sees `none` placeholders or instructions for filters that don't apply.
*/
function renderReportFiltersBlock(report: ReportConfig | undefined): string {
function renderReportFiltersBlock(report: DistributedReportConfig | undefined, exploitEnabled: boolean): string {
if (!report) return '';
const guidance = report.guidance?.trim();
if (!report.min_severity && !report.min_confidence && !guidance) return '';
const applies = applicableFilters(report, exploitEnabled);
if (!applies.severity && !applies.confidence && !applies.guidance) return '';
const lines: string[] = [
'<report_filters>',
'The filters below are user-supplied and binding for this assessment. Honor each strictly when assembling the final report.',
'',
];
if (report.min_severity) {
if (applies.severity) {
lines.push(
`- Minimum severity: ${report.min_severity} — keep only findings rated this severity or higher (scale: low < medium < high < critical).`,
);
}
if (report.min_confidence) {
if (applies.confidence) {
lines.push(
`- Minimum confidence: ${report.min_confidence} — keep only findings rated this confidence or higher (scale: low < medium < high).`,
);
@@ -106,10 +155,11 @@ function renderReportFiltersBlock(report: ReportConfig | undefined): string {
* confidence inline as concrete thresholds; guidance is referenced by pointer
* so the actual text only lives in <report_filters>, avoiding double-statement.
*/
function renderReportFilterRules(report: ReportConfig | undefined): string {
function renderReportFilterRules(report: DistributedReportConfig | undefined, exploitEnabled: boolean): string {
const applies = applicableFilters(report, exploitEnabled);
const drops: string[] = [];
if (report?.min_severity) drops.push(`* severity is below ${report.min_severity}`);
if (report?.min_confidence) drops.push(`* confidence is below ${report.min_confidence}`);
if (applies.severity) drops.push(`* severity is below ${report?.min_severity}`);
if (applies.confidence) drops.push(`* confidence is below ${report?.min_confidence}`);
if (report?.guidance?.trim()) drops.push('* topic matches an exclusion in the user guidance');
if (drops.length === 0) return '';
return [' - DROP any `### [TYPE]-VULN-[NUMBER]` finding whose:', ...drops.map((d) => ` ${d}`)].join('\n');
@@ -118,6 +168,8 @@ function renderReportFilterRules(report: ReportConfig | undefined): string {
interface PromptVariables {
webUrl: string;
repoPath: string;
/** Classes whose analysis did not complete, so the report can mark them not assessed. */
failedClasses?: readonly VulnClass[];
AUTH_STATE_FILE: string;
PLAYWRIGHT_SESSION?: string;
}
@@ -365,8 +417,20 @@ async function interpolateVariables(
vulnClasses.length > 0 ? vulnClasses.join(', ') : 'injection, xss, auth, authz, ssrf',
);
result = replaceLiteral(result, /{{VULN_SUMMARY_SUBSECTIONS}}/g, renderVulnSummarySubsections(vulnClasses));
result = replaceLiteral(
result,
/{{NOT_ASSESSED_CLASSES}}/g,
renderNotAssessedClassesBlock(variables.failedClasses ?? []),
);
const exploitEnabled = config?.exploit ?? true;
// Drop every block belonging to the mode this run is not in, so the prompt never documents
// a field the tool would reject. The backreference pins each match to a closed pair.
const droppedMode = exploitEnabled ? 'analysis' : 'exploit';
result = result.replace(new RegExp(`<(${droppedMode}_mode_[a-z_]+)>[\\s\\S]*?</\\1>\\n?`, 'g'), '');
result = result.replace(/<\/?(?:exploit|analysis)_mode_[a-z_]+>\n?/g, '');
result = replaceLiteral(result, /{{EXPLOITATION}}/g, exploitEnabled ? 'enabled' : 'disabled');
result = replaceLiteral(result, /{{REPORT_VULN_HEADING}}/g, exploitEnabled ? 'Exploitation Evidence' : 'Findings');
result = replaceLiteral(
@@ -375,8 +439,28 @@ async function interpolateVariables(
exploitEnabled ? 'Successfully Exploited Vulnerabilities' : 'Identified Vulnerabilities',
);
result = replaceLiteral(result, /{{REPORT_FILTERS_BLOCK}}/g, renderReportFiltersBlock(config?.report));
result = replaceLiteral(result, /{{REPORT_FILTER_RULES}}/g, renderReportFilterRules(config?.report));
if (config?.report?.min_severity && !exploitEnabled) {
logger.warn(
`report.min_severity="${config.report.min_severity}" is ignored when exploit=false: an ` +
'analysis-only run rates findings by confidence, not severity. Use report.min_confidence.',
);
}
if (config?.report?.min_confidence && exploitEnabled) {
logger.warn(
`report.min_confidence="${config.report.min_confidence}" is ignored when exploit=true: an ` +
'exploited finding is rated by severity, not confidence. Use report.min_severity.',
);
}
result = replaceLiteral(
result,
/{{REPORT_FILTERS_BLOCK}}/g,
renderReportFiltersBlock(config?.report, exploitEnabled),
);
result = replaceLiteral(
result,
/{{REPORT_FILTER_RULES}}/g,
renderReportFilterRules(config?.report, exploitEnabled),
);
// Collapse runs of 3+ newlines (left behind by tag-strip and empty-fragment substitutions).
result = result.replace(/\n{3,}/g, '\n\n');
apps/worker/src/services/report-renderer.ts
+289
View File
@@ -0,0 +1,289 @@
// Copyright (C) 2025 Keygraph, Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License version 3
// as published by the Free Software Foundation.
/**
* Deterministic report.json → markdown renderer.
*
* Converts the structured report output (produced by the finding-collector
* tool + set-report-meta CLI) into the same markdown format that the
* report agent previously wrote by hand. No LLM in the loop.
*/
import type { AddFindingInput, AdditionalSection, StepItem, StructuredStep } from '../collectors/finding-collector.js';
import type { VulnClass } from '../types/config.js';
// ============================================================================
// TYPES
// ============================================================================
export interface ReportMeta {
readonly target: string;
readonly assessment_date: string;
readonly scope: string;
readonly executive_summary: string;
readonly exploit?: boolean;
readonly model?: string;
}
export interface ReportData {
readonly report_meta: ReportMeta;
readonly findings: readonly AddFindingInput[];
// Vuln classes whose pipeline failed and were not assessed this run. Rendered as an explicit
// caveat so an un-assessed class is never presented as a clean result.
readonly not_assessed?: readonly VulnClass[];
}
// Without this, an analysis-only report reads as though the impact was demonstrated.
const ANALYSIS_ONLY_DISCLAIMER = [
'> Exploitation was not run for this assessment. Each finding documents a vulnerability',
'> identified through analysis; impact is assessed rather than demonstrated, and no live',
'> exploitation steps or proof of impact are included.',
].join('\n');
const NOT_ASSESSED_LABELS: Record<VulnClass, string> = {
auth: 'Authentication',
authz: 'Authorization',
xss: 'Cross-Site Scripting (XSS)',
injection: 'SQL/Command Injection',
ssrf: 'Server-Side Request Forgery (SSRF)',
};
function renderNotAssessedSection(notAssessed: readonly VulnClass[]): string {
const lines: string[] = ['## Not Assessed', ''];
lines.push(
'The following vulnerability classes were NOT assessed in this run because their analysis did ' +
'not complete. Absence of findings for these classes does not indicate they are clean — re-run ' +
'to assess them:',
);
lines.push('');
for (const cls of notAssessed) {
lines.push(`- ${NOT_ASSESSED_LABELS[cls]} — analysis did not complete; not assessed.`);
}
return lines.join('\n');
}
// ============================================================================
// STEP ITEM RENDERING
// ============================================================================
function renderStepItem(item: StepItem): string {
if (item.kind === 'prose') {
return item.text;
}
const lang = item.block.language || '';
return `\`\`\`${lang}\n${item.block.content}\n\`\`\``;
}
function renderStepItems(items: readonly StepItem[]): string {
return items.map(renderStepItem).join('\n\n');
}
function renderStructuredStep(step: StructuredStep, index: number): string {
const lines: string[] = [];
const title = step.title ? `**Step ${index + 1}: ${step.title}**` : `**Step ${index + 1}**`;
lines.push(title);
lines.push('');
lines.push(renderStepItems(step.items));
return lines.join('\n');
}
function renderAdditionalSection(section: AdditionalSection): string {
const lines: string[] = [];
lines.push(`#### ${section.heading}`);
lines.push('');
lines.push(renderStepItems(section.items));
return lines.join('\n');
}
// ============================================================================
// FINDING RENDERING
// ============================================================================
function titleCase(s: string): string {
return s.charAt(0).toUpperCase() + s.slice(1);
}
function renderFinding(finding: AddFindingInput, exploitEnabled: boolean): string {
const lines: string[] = [];
// Heading
lines.push(`### ${finding.finding_id}: ${finding.title}`);
lines.push('');
// Each row is emitted only when the mode that produced the finding supplied its field.
lines.push('**Summary:**');
if (finding.severity) {
lines.push(`- **Severity:** ${titleCase(finding.severity)}`);
}
if (finding.confidence) {
lines.push(`- **Confidence:** ${titleCase(finding.confidence)}`);
}
lines.push(`- **OWASP:** ${finding.owasp_category}`);
lines.push(`- **Vulnerable location:** ${finding.vulnerable_location}`);
if (finding.auth_state) {
lines.push(`- **Auth state:** ${finding.auth_state}`);
}
if (exploitEnabled && finding.status) {
lines.push(`- **Status:** ${titleCase(finding.status)}`);
}
if (finding.prerequisites) {
lines.push(`- **Prerequisites:** ${finding.prerequisites}`);
}
lines.push('');
// Overview
lines.push('**Overview:**');
lines.push(finding.overview);
lines.push('');
// Impact
lines.push('**Impact:**');
lines.push(finding.impact);
lines.push('');
if (finding.exploitation_steps && finding.exploitation_steps.length > 0) {
lines.push('**Exploitation Steps:**');
lines.push('');
for (let i = 0; i < finding.exploitation_steps.length; i++) {
lines.push(renderStructuredStep(finding.exploitation_steps[i]!, i));
lines.push('');
}
}
if (finding.proof_of_impact && finding.proof_of_impact.length > 0) {
lines.push('**Proof of Impact:**');
lines.push('');
lines.push(renderStepItems(finding.proof_of_impact));
lines.push('');
}
// Remediation
lines.push('**Remediation:**');
lines.push(finding.remediation);
lines.push('');
// Notes
if (finding.notes && finding.notes.length > 0) {
lines.push('**Notes:**');
lines.push('');
lines.push(renderStepItems(finding.notes));
lines.push('');
}
// Additional sections
if (finding.additional_sections && finding.additional_sections.length > 0) {
for (const section of finding.additional_sections) {
lines.push(renderAdditionalSection(section));
lines.push('');
}
}
return lines.join('\n').trimEnd();
}
// ============================================================================
// CATEGORY GROUPING
// ============================================================================
const CATEGORY_ORDER: readonly string[] = ['Injection', 'XSS', 'Authentication', 'SSRF', 'Authorization'];
function categorySort(a: string, b: string): number {
const ai = CATEGORY_ORDER.indexOf(a);
const bi = CATEGORY_ORDER.indexOf(b);
if (ai !== -1 && bi !== -1) return ai - bi;
if (ai !== -1) return -1;
if (bi !== -1) return 1;
return a.localeCompare(b);
}
// ============================================================================
// REPORT RENDERING
// ============================================================================
export function renderReport(data: ReportData): string {
const { report_meta, findings, not_assessed = [] } = data;
const notAssessedClasses = [...new Set(not_assessed)];
const exploitEnabled = report_meta.exploit ?? true;
const sections: string[] = [];
// 1. Executive Summary
sections.push('# Security Assessment Report');
sections.push('');
sections.push('## Executive Summary');
sections.push(`- Target: ${report_meta.target}`);
sections.push(`- Assessment Date: ${report_meta.assessment_date}`);
sections.push(`- Scope: ${report_meta.scope}`);
sections.push(`- Exploitation: ${exploitEnabled ? 'enabled' : 'disabled'}`);
if (report_meta.model) {
sections.push(`- Model: ${report_meta.model}`);
}
sections.push('');
sections.push(report_meta.executive_summary);
sections.push('');
if (!exploitEnabled) {
sections.push(ANALYSIS_ONLY_DISCLAIMER);
sections.push('');
}
if (findings.length === 0) {
if (notAssessedClasses.length > 0) {
// Some classes were not assessed — a blanket "no vulnerabilities" statement would be a false
// clean bill of health. Scope the clean statement to assessed classes and list the gaps.
sections.push('No vulnerabilities were identified in the classes that were assessed.');
sections.push('');
sections.push(renderNotAssessedSection(notAssessedClasses));
} else {
sections.push('No vulnerabilities were identified during this assessment.');
}
return sections.join('\n').trimEnd() + '\n';
}
if (notAssessedClasses.length > 0) {
sections.push(renderNotAssessedSection(notAssessedClasses));
sections.push('');
}
// 2. Summary by Vulnerability Type
const byCategory = new Map<string, AddFindingInput[]>();
for (const f of findings) {
const list = byCategory.get(f.category) ?? [];
list.push(f);
byCategory.set(f.category, list);
}
const sortedCategories = [...byCategory.keys()].sort(categorySort);
sections.push('## Summary by Vulnerability Type');
sections.push('');
for (const cat of sortedCategories) {
const catFindings = byCategory.get(cat)!;
sections.push(`### ${cat}`);
sections.push('');
for (const f of catFindings) {
const suffix = f.severity ? ` (${titleCase(f.severity)})` : '';
sections.push(`- **${f.finding_id}:** ${f.title}${suffix}`);
}
sections.push('');
}
// 3. Per-category finding sections
const subheading = exploitEnabled ? 'Successfully Exploited Vulnerabilities' : 'Identified Vulnerabilities';
const heading = exploitEnabled ? 'Exploitation Evidence' : 'Findings';
for (const cat of sortedCategories) {
const catFindings = byCategory.get(cat)!;
sections.push(`# ${cat} ${heading}`);
sections.push('');
sections.push(`## ${subheading}`);
sections.push('');
for (const f of catFindings) {
sections.push(renderFinding(f, exploitEnabled));
sections.push('');
}
}
return sections.join('\n').trimEnd() + '\n';
}
apps/worker/src/services/reporting.ts
+27 -10
View File
@@ -5,7 +5,13 @@
// as published by the Free Software Foundation.
import { fs, path } from 'zx';
import { ASSEMBLED_REPORT_FILENAME, deliverablesDir, FINAL_REPORT_FILENAME, resolveSessionJsonPath } from '../paths.js';
import {
ASSEMBLED_REPORT_FILENAME,
deliverablesDir,
FINAL_REPORT_FILENAME,
resolveSessionJsonPath,
SARIF_FILENAME,
} from '../paths.js';
import type { ActivityLogger } from '../types/activity-logger.js';
import { ErrorCode } from '../types/errors.js';
import { PentestError } from './error-handling.js';
@@ -166,9 +172,13 @@ export async function injectModelIntoReport(
}
/**
* Surface the assembled report at the run directory's top level as the single
* human-facing deliverable, so a customer opening the run folder sees only the
* report. The source stays in the deliverables dir (git-checkpointed, used by resume).
* Surface the run's deliverables at the run directory's top level, so a customer opening the run
* folder sees the report without digging through internals. Sources stay in the deliverables dir
* (git-checkpointed, used by resume).
*
* The SARIF log is surfaced beside it when present, since a CI step consuming it needs a stable
* path and cannot be expected to reach into the internals directory. It is absent whenever the
* run was analysis-only or `report.sarif` was not enabled.
*/
export async function copyReportToRunRoot(
repoPath: string,
@@ -176,14 +186,21 @@ export async function copyReportToRunRoot(
runDir: string,
logger: ActivityLogger,
): Promise<void> {
const source = path.join(deliverablesDir(repoPath, deliverablesSubdir), ASSEMBLED_REPORT_FILENAME);
const dir = deliverablesDir(repoPath, deliverablesSubdir);
if (!(await fs.pathExists(source))) {
const source = path.join(dir, ASSEMBLED_REPORT_FILENAME);
if (await fs.pathExists(source)) {
const destination = path.join(runDir, FINAL_REPORT_FILENAME);
await fs.copy(source, destination, { overwrite: true });
logger.info(`Surfaced report at ${destination}`);
} else {
logger.warn(`Final report not found, skipping ${FINAL_REPORT_FILENAME}`);
return;
}
const destination = path.join(runDir, FINAL_REPORT_FILENAME);
await fs.copy(source, destination, { overwrite: true });
logger.info(`Surfaced report at ${destination}`);
const sarifSource = path.join(dir, SARIF_FILENAME);
if (await fs.pathExists(sarifSource)) {
const sarifDestination = path.join(runDir, SARIF_FILENAME);
await fs.copy(sarifSource, sarifDestination, { overwrite: true });
logger.info(`Surfaced SARIF log at ${sarifDestination}`);
}
}
apps/worker/src/services/sarif-renderer.ts
+293
View File
@@ -0,0 +1,293 @@
// Copyright (C) 2025 Keygraph, Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License version 3
// as published by the Free Software Foundation.
/** Deterministic report.json to SARIF 2.1.0 renderer, for `exploit=true` runs only. */
import type { AddFindingInput, CodeLocation } from '../collectors/finding-collector.js';
import type { ReportData } from './report-renderer.js';
export interface SarifOptions {
readonly workspaceName: string;
}
interface SarifRule {
readonly id: string;
readonly name: string;
readonly shortDescription: { text: string };
readonly fullDescription: { text: string };
readonly help: { text: string };
readonly properties: { tags: string[] };
}
const TOOL_NAME = 'Shannon';
const TOOL_URI = 'https://github.com/KeygraphHQ/shannon';
/** Taxonomy identity. A reference resolves the component by name, so this must not be reworded. */
const OWASP_TAXONOMY_NAME = 'OWASP Top Ten 2025';
/**
* One rule per vulnerability class, keyed by `finding.category`.
*
* Rule IDs are the unit of alert grouping: renaming one detaches every alert filed under it.
* `fullDescription` and `help` describe the class, never the instance, and GitHub requires the
* `text` of both.
*/
const RULES: Record<string, SarifRule> = {
Injection: {
id: 'shannon/injection',
name: 'Injection',
shortDescription: { text: 'Injection' },
fullDescription: {
text: 'Untrusted input reaches an interpreter sink (SQL, OS command, template, file path or deserializer) at a position where it can alter the structure of the statement rather than only supply data.',
},
help: {
text: 'Separate code from data at the sink: bind SQL parameters, pass command arguments as an array, and allowlist file paths. Escaping is a weaker control than parameterisation and breaks whenever the sink context changes.',
},
properties: { tags: ['security', 'shannon'] },
},
XSS: {
id: 'shannon/xss',
name: 'Cross-Site Scripting',
shortDescription: { text: 'Cross-Site Scripting' },
fullDescription: {
text: 'Untrusted input reaches a browser rendering context without the encoding that context requires.',
},
help: {
text: 'Encode at the point of output for the specific context (HTML body, attribute, URL, script or style); no single encoder is correct for all of them. Prefer APIs that treat input as text, such as textContent over innerHTML.',
},
properties: { tags: ['security', 'shannon'] },
},
Authentication: {
id: 'shannon/auth',
name: 'Authentication',
shortDescription: { text: 'Authentication' },
fullDescription: {
text: 'A weakness in credential verification or session lifecycle that lets an attacker assume another identity or retain access they should have lost.',
},
help: {
text: 'Issue a fresh session identifier on every privilege change, set HttpOnly, Secure and SameSite on session cookies, rate-limit credential endpoints, and verify the signature and algorithm of externally issued tokens.',
},
properties: { tags: ['security', 'shannon'] },
},
Authorization: {
id: 'shannon/authz',
name: 'Authorization',
shortDescription: { text: 'Authorization' },
fullDescription: {
text: 'An access control decision is missing, evaluated in the client, or applied at the wrong layer, letting a caller act on resources they do not own.',
},
help: {
text: 'Check ownership and role on the server for every object reference, and enforce it in the data-access layer rather than per route, denying by default. An unguessable identifier is not an access control.',
},
properties: { tags: ['security', 'shannon'] },
},
SSRF: {
id: 'shannon/ssrf',
name: 'Server-Side Request Forgery',
shortDescription: { text: 'Server-Side Request Forgery' },
fullDescription: {
text: 'A server-side request takes its destination from untrusted input, letting an attacker reach hosts the server can see but they cannot.',
},
help: {
text: 'Allowlist destination hosts and schemes, resolve DNS before validating the address so rebinding cannot slip through, and block loopback, private and link-local ranges including cloud metadata. Do not follow redirects.',
},
properties: { tags: ['security', 'shannon'] },
},
};
const CATEGORY_ORDER: readonly string[] = ['Injection', 'XSS', 'Authentication', 'SSRF', 'Authorization'];
/**
* Five severities collapse into SARIF's three usable levels, so `critical` and `high` are
* indistinguishable. `security-severity` would separate them but lives on the rule, which would
* flatten every finding of a class to one score instead.
*/
function severityToLevel(severity: string | undefined): string {
switch (severity) {
case 'critical':
case 'high':
return 'error';
case 'medium':
return 'warning';
default:
return 'note';
}
}
function toPhysicalLocation(location: CodeLocation) {
const region: Record<string, number> = {};
if (location.start_line) region.startLine = location.start_line;
if (location.end_line) region.endLine = location.end_line;
return {
physicalLocation: {
artifactLocation: { uri: location.file },
...(Object.keys(region).length > 0 && { region }),
},
...(location.symbol && { logicalLocations: [{ name: location.symbol, kind: 'function' }] }),
message: { text: location.role },
};
}
/**
* Fall back to the HTTP entry point when a finding names no file: a result with no location is
* silently discarded downstream. No `uriBaseId`, since the path does not resolve in the repo.
*/
function syntheticLocationFromHttp(finding: AddFindingInput) {
if (!finding.http_location) return undefined;
let uri = finding.http_location.url;
try {
const parsed = new URL(finding.http_location.url);
uri = `${parsed.pathname}${parsed.hash}`;
} catch {}
return {
physicalLocation: { artifactLocation: { uri } },
message: { text: `${finding.http_location.method} ${finding.http_location.url}` },
};
}
function buildMessageMarkdown(finding: AddFindingInput): string {
const parts = [`**${finding.title}**`, '', finding.overview, '', '**Impact**', '', finding.impact];
parts.push('', '**Remediation**', '', finding.remediation);
// Exploitation steps and proof of impact are deliberately absent: SARIF has no structural home
// for them, and flattening them into prose would imply this file carries the evidence.
parts.push('', 'Full exploitation evidence: `Security-Assessment-Report.md`');
return parts.join('\n');
}
/**
* `owasp_category` is one label, `A05:2025 <separator> Injection`; SARIF wants the id and the name
* as separate fields. The enum in ../collectors/finding-collector.ts fixes the shape, so the
* separator is dropped by position rather than matched.
*/
function splitOwaspCategory(label: string): { id: string; name: string } {
const [id, , ...nameParts] = label.split(' ');
return { id: id ?? label, name: nameParts.join(' ') };
}
interface RenderedResult {
readonly result: Record<string, unknown>;
readonly category: string;
readonly owaspId: string;
}
function renderResult(finding: AddFindingInput, ruleId: string): RenderedResult | null {
const codeLocations = finding.code_locations ?? [];
const sinks = codeLocations.filter((l) => l.role === 'sink');
const related = codeLocations.filter((l) => l.role !== 'sink');
const primary = sinks[0] ?? codeLocations[0];
const locations = primary ? [toPhysicalLocation(primary)] : [syntheticLocationFromHttp(finding)].filter(Boolean);
if (locations.length === 0) return null;
const properties: Record<string, unknown> = { findingId: finding.finding_id };
if (finding.http_location?.parameter) properties.parameter = finding.http_location.parameter;
if (finding.status) properties.status = finding.status;
if (finding.auth_state) properties.authState = finding.auth_state;
if (finding.prerequisites) properties.prerequisites = finding.prerequisites;
const owaspId = splitOwaspCategory(finding.owasp_category).id;
return {
category: finding.category,
owaspId,
result: {
ruleId,
level: severityToLevel(finding.severity),
message: {
text: `${finding.title}. ${finding.overview}`,
markdown: buildMessageMarkdown(finding),
},
locations,
...(related.length > 0 && {
relatedLocations: related.map((l, i) => ({ id: i + 1, ...toPhysicalLocation(l) })),
}),
...(finding.http_location && {
// No `parameters`: SARIF wants a name-to-value map and the deliverable names only the
// parameter, so any value here would be invented. It travels in `properties` instead.
webRequest: { method: finding.http_location.method, target: finding.http_location.url },
}),
taxa: [
{
id: owaspId,
toolComponent: { name: OWASP_TAXONOMY_NAME },
},
],
properties,
},
};
}
/** Render a SARIF 2.1.0 log from the structured report. Findings with no location are omitted. */
export function renderSarif(data: ReportData, options: SarifOptions): string {
const { report_meta, findings, not_assessed = [] } = data;
const rendered: RenderedResult[] = [];
for (const finding of findings) {
const rule = RULES[finding.category];
if (!rule) continue;
const result = renderResult(finding, rule.id);
if (result !== null) rendered.push(result);
}
// Only classes that produced a result are declared, and `ruleIndex` is the position in this list.
const usedRules = CATEGORY_ORDER.flatMap((category) => {
const rule = RULES[category];
if (!rule || !rendered.some((r) => r.category === category)) return [];
return [{ category, rule }];
});
const rules = usedRules.map((u) => u.rule);
const results: Record<string, unknown>[] = usedRules.flatMap(({ category }, ruleIndex) =>
rendered.filter((r) => r.category === category).map((r) => ({ ...r.result, ruleIndex })),
);
const owaspCategories = [...new Set(findings.map((f) => f.owasp_category))]
.map(splitOwaspCategory)
.filter((c) => rendered.some((r) => r.owaspId === c.id))
.sort((a, b) => a.id.localeCompare(b.id));
const log = {
$schema: 'https://json.schemastore.org/sarif-2.1.0.json',
version: '2.1.0',
runs: [
{
tool: {
driver: {
name: TOOL_NAME,
informationUri: TOOL_URI,
rules,
},
},
// Scoped to the exploit pipeline: an analysis run of the same target has a different
// finding population, which would read as alerts resolved.
automationDetails: { id: `shannon/exploit/${options.workspaceName}` },
invocations: [
{
// A failed class produced no results; reporting success would read as resolved alerts.
executionSuccessful: not_assessed.length === 0,
},
],
...(owaspCategories.length > 0 && {
taxonomies: [
{
name: OWASP_TAXONOMY_NAME,
organization: 'OWASP',
informationUri: 'https://owasp.org/Top10/',
shortDescription: { text: 'OWASP Top Ten 2025 categories.' },
taxa: owaspCategories.map((c) => ({ id: c.id, name: c.name })),
},
],
}),
results,
properties: { target: report_meta.target, assessmentDate: report_meta.assessment_date },
},
],
};
return `${JSON.stringify(log, null, 2)}\n`;
}
apps/worker/src/services/validate-authentication.ts
-1
View File
@@ -145,7 +145,6 @@ export async function validateAuthentication(input: ValidateAuthInput): Promise<
AGENT_NAME,
auditSession,
logger,
'medium',
undefined, // callerTools
deliverablesSubdir,
cancellationSignal,
apps/worker/src/session-manager.ts
-1
View File
@@ -17,7 +17,6 @@ export const AGENTS: Readonly<Record<AgentName, AgentDefinition>> = Object.freez
prerequisites: [],
promptTemplate: 'pre-recon-code',
deliverableFilename: 'pre_recon_deliverable.md',
modelTier: 'large',
},
recon: {
name: 'recon',
apps/worker/src/temporal/activities.ts
+108 -6
View File
@@ -25,7 +25,14 @@ import type { ResumeAttempt } from '../audit/metrics-tracker.js';
import { authStateFile, generateAuditPath, generateSessionJsonPath, type SessionMetadata } from '../audit/utils.js';
import type { WorkflowSummary } from '../audit/workflow-logger.js';
import type { CheckpointContext } from '../interfaces/checkpoint-provider.js';
import { DEFAULT_DELIVERABLES_SUBDIR, deliverablesDir, resolveSessionJsonPath } from '../paths.js';
import {
ASSEMBLED_REPORT_FILENAME,
DEFAULT_DELIVERABLES_SUBDIR,
deliverablesDir,
REPORT_JSON_FILENAME,
resolveSessionJsonPath,
SARIF_FILENAME,
} from '../paths.js';
import { getAgentGitPaths } from '../services/agent-git-paths.js';
import { getContainer, getOrCreateContainer, removeContainer } from '../services/container.js';
import { classifyErrorForTemporal, PentestError } from '../services/error-handling.js';
@@ -34,6 +41,7 @@ import { renderFindingsFromQueues } from '../services/findings-renderer.js';
import { executeGitCommandWithRetry } from '../services/git-manager.js';
import { runPreflightChecks } from '../services/preflight.js';
import type { ExploitationDecision, VulnType } from '../services/queue-validation.js';
import type { ReportData, ReportMeta } from '../services/report-renderer.js';
import { assembleFinalReport, copyReportToRunRoot, injectModelIntoReport } from '../services/reporting.js';
import { validateAuthentication } from '../services/validate-authentication.js';
import { AGENTS } from '../session-manager.js';
@@ -76,6 +84,10 @@ export interface ActivityInput {
auditDir?: string;
promptDir?: string;
sastSarifPath?: string;
// Vuln classes whose pipeline failed. Set before the report stage on a partial run so the
// report marks them "not assessed" instead of asserting no findings were present.
failedClasses?: VulnClass[];
}
/**
@@ -187,6 +199,7 @@ async function runAgentActivity(
attemptNumber,
...(input.promptDir !== undefined && { promptDir: input.promptDir }),
...(input.configYAML !== undefined && { configYAML: input.configYAML }),
...(input.failedClasses !== undefined && { failedClasses: input.failedClasses }),
...(customTools && { customTools }),
...(writeDeliverable && { writeDeliverable }),
cancellationSignal: Context.current().cancellationSignal,
@@ -198,10 +211,12 @@ async function runAgentActivity(
// 4. Return metrics
return {
durationMs: Date.now() - startTime,
inputTokens: null,
outputTokens: null,
inputTokens: endResult.input_tokens ?? null,
outputTokens: endResult.output_tokens ?? null,
cacheReadTokens: endResult.cache_read_tokens ?? null,
cacheWriteTokens: endResult.cache_write_tokens ?? null,
costUsd: endResult.cost_usd,
numTurns: null,
numTurns: endResult.turns ?? null,
model: endResult.model,
};
} catch (error) {
@@ -432,8 +447,95 @@ export async function runAuthzExploitAgent(input: ActivityInput): Promise<AgentM
return runExploitAgentWithCollector('authz-exploit', 'authz', input);
}
export async function runReportAgent(input: ActivityInput): Promise<AgentMetrics> {
return runAgentActivity('report', input);
/**
* Write report.sarif when the run is exploitative and the operator asked for it.
*
* Skipped entirely for analysis-only runs: those findings carry no severity, so every
* `result.level` would be invented. Failures are logged and swallowed — the SARIF log is a
* secondary artifact and must not fail a run whose report is already written.
*/
async function writeSarifIfEnabled(
input: ActivityInput,
exploit: boolean,
reportData: ReportData,
deliverablesPath: string,
logger: ReturnType<typeof createActivityLogger>,
): Promise<void> {
if (!exploit) return;
const container = getOrCreateContainer(input.workflowId, buildSessionMetadata(input), buildContainerConfig(input));
const configResult = await container.configLoader.loadOptional(input.configPath, undefined, input.configYAML);
if (isErr(configResult) || configResult.value?.report?.sarif !== true) return;
try {
const { renderSarif } = await import('../services/sarif-renderer.js');
const sarif = renderSarif(reportData, { workspaceName: input.sessionId });
await atomicWrite(path.join(deliverablesPath, SARIF_FILENAME), sarif);
logger.info(`Wrote ${SARIF_FILENAME}`);
} catch (error) {
logger.warn(`Failed to write ${SARIF_FILENAME}: ${(error as Error).message}`);
}
}
export async function runReportAgent(input: ActivityInput, exploit: boolean): Promise<AgentMetrics> {
const { createFindingCollector } = await import('../collectors/finding-collector.js');
const { renderReport } = await import('../services/report-renderer.js');
const collector = createFindingCollector(exploit);
const writeDeliverable = async (deliverablesPath: string): Promise<void> => {
const logger = createActivityLogger();
const { attachQueueCodeLocations } = await import('../services/code-location-join.js');
const collected = collector.getAll();
logger.info(`Collected ${collected.length} finding(s) from report agent`);
const findings = await attachQueueCodeLocations(collected, deliverablesPath, logger);
// report_meta is written by the set-report-meta CLI while the agent runs; read it back so
// the two halves of report.json end up in one document.
const reportJsonPath = path.join(deliverablesPath, REPORT_JSON_FILENAME);
let reportMeta: ReportMeta = {
target: input.webUrl,
assessment_date: new Date().toISOString().split('T')[0]!,
scope: '',
executive_summary: '',
exploit,
};
if (await fileExists(reportJsonPath)) {
try {
const existing = await readJson<{ report_meta?: Record<string, unknown> }>(reportJsonPath);
if (existing.report_meta) {
reportMeta = {
target: String(existing.report_meta.target ?? input.webUrl),
assessment_date: String(existing.report_meta.assessment_date ?? reportMeta.assessment_date),
scope: String(existing.report_meta.scope ?? ''),
executive_summary: String(existing.report_meta.executive_summary ?? ''),
// Run scope, not agent output — keeps the rendered report and the schema the agent
// was given in agreement.
exploit,
...(existing.report_meta.model !== undefined && { model: String(existing.report_meta.model) }),
};
}
} catch {
logger.warn('Failed to read report_meta from report.json, using defaults');
}
}
const reportData: ReportData = {
report_meta: reportMeta,
findings,
...(input.failedClasses && input.failedClasses.length > 0 && { not_assessed: input.failedClasses }),
};
await atomicWrite(reportJsonPath, JSON.stringify(reportData, null, 2));
logger.info(`Wrote ${REPORT_JSON_FILENAME} with ${findings.length} finding(s)`);
await atomicWrite(path.join(deliverablesPath, ASSEMBLED_REPORT_FILENAME), renderReport(reportData));
logger.info(`Wrote ${ASSEMBLED_REPORT_FILENAME} from structured data`);
await writeSarifIfEnabled(input, exploit, reportData, deliverablesPath, logger);
};
return runAgentActivity('report', input, collector.tools, writeDeliverable);
}
/**
apps/worker/src/temporal/shared.ts
+1 -2
View File
@@ -2,7 +2,7 @@ import { defineQuery } from '@temporalio/workflow';
export type { AgentMetrics } from '../types/metrics.js';
import type { DistributedConfig, PipelineConfig, VulnClass } from '../types/config.js';
import type { DistributedConfig, VulnClass } from '../types/config.js';
import type { ErrorCode } from '../types/errors.js';
import type { AgentMetrics } from '../types/metrics.js';
@@ -12,7 +12,6 @@ export interface PipelineInput {
configPath?: string;
outputPath?: string;
pipelineTestingMode?: boolean;
pipelineConfig?: PipelineConfig;
workflowId?: string; // Used for audit correlation
sessionId?: string; // Workspace directory name (distinct from workflowId for named workspaces)
resumeFromWorkspace?: string; // Workspace name to resume from
apps/worker/src/temporal/worker.ts
+2 -13
View File
@@ -36,7 +36,7 @@ import dotenv from 'dotenv';
import { sanitizeHostname } from '../audit/utils.js';
import { parseConfig } from '../config-parser.js';
import { ASSEMBLED_REPORT_FILENAME, deliverablesDir, FINAL_REPORT_FILENAME, resolveSessionJsonPath } from '../paths.js';
import type { PipelineConfig, VulnClass } from '../types/config.js';
import type { VulnClass } from '../types/config.js';
import { fileExists, readJson } from '../utils/file-io.js';
import * as activities from './activities.js';
import type { PipelineInput, PipelineProgress, PipelineState } from './shared.js';
@@ -276,26 +276,16 @@ async function resolveWorkspace(client: Client, args: CliArgs): Promise<Workspac
// === Pipeline Input Construction ===
interface OrchestrationConfig {
pipelineConfig: PipelineConfig;
vulnClasses?: VulnClass[];
exploit?: boolean;
}
async function loadOrchestrationConfig(configPath: string | undefined): Promise<OrchestrationConfig> {
if (!configPath) return { pipelineConfig: {} };
if (!configPath) return {};
try {
const config = await parseConfig(configPath);
const pipelineConfig: PipelineConfig = {};
if (config.pipeline?.retry_preset !== undefined) {
pipelineConfig.retry_preset = config.pipeline.retry_preset;
}
if (config.pipeline?.max_concurrent_pipelines !== undefined) {
pipelineConfig.max_concurrent_pipelines = Number(config.pipeline.max_concurrent_pipelines);
}
return {
pipelineConfig,
...(config.vuln_classes && config.vuln_classes.length > 0 && { vulnClasses: [...config.vuln_classes] }),
...(config.exploit !== undefined && { exploit: config.exploit === 'true' }),
};
@@ -322,7 +312,6 @@ function buildPipelineInput(
...(args.pipelineTestingMode && { pipelineTestingMode: args.pipelineTestingMode }),
...(workspace.isResume && args.resumeFromWorkspace && { resumeFromWorkspace: args.resumeFromWorkspace }),
...(workspace.terminatedWorkflows.length > 0 && { terminatedWorkflows: workspace.terminatedWorkflows }),
...(Object.keys(orchestration.pipelineConfig).length > 0 && { pipelineConfig: orchestration.pipelineConfig }),
...(orchestration.vulnClasses && { vulnClasses: orchestration.vulnClasses }),
...(orchestration.exploit !== undefined && { exploit: orchestration.exploit }),
};
apps/worker/src/temporal/workflow-errors.ts
+1 -6
View File
@@ -21,8 +21,6 @@ import { ErrorCode } from '../types/errors.js';
*/
const ERROR_TYPE_TO_CODE: Record<string, ErrorCode> = {
AuthenticationError: ErrorCode.AUTH_FAILED,
BillingError: ErrorCode.BILLING_ERROR,
RateLimitError: ErrorCode.API_RATE_LIMITED,
ConfigurationError: ErrorCode.CONFIG_VALIDATION_FAILED,
OutputValidationError: ErrorCode.OUTPUT_VALIDATION_FAILED,
AgentExecutionError: ErrorCode.AGENT_EXECUTION_FAILED,
@@ -44,13 +42,10 @@ export function classifyErrorCode(error: unknown): ErrorCode | undefined {
/** Maps Temporal error type strings to actionable remediation hints. */
const REMEDIATION_HINTS: Record<string, string> = {
AuthenticationError: 'Verify ANTHROPIC_API_KEY or CLAUDE_CODE_OAUTH_TOKEN in .env is valid and not expired.',
AuthenticationError: "Verify the selected provider's API key is valid and not expired.",
ConfigurationError: 'Check your CONFIG file path and contents.',
BillingError: 'Check your Anthropic billing dashboard. Add credits or wait for spending cap reset.',
GitError: 'Check repository path and git state.',
InvalidTargetError: 'Verify the target URL is correct and accessible.',
PermissionError: 'Check file and network permissions.',
ExecutionLimitError: 'Agent exceeded maximum turns or budget. Review prompt complexity.',
};
/**
apps/worker/src/temporal/workflows.ts
+18 -34
View File
@@ -17,7 +17,7 @@
*
* Features:
* - Queryable state via getProgress
* - Automatic retry with backoff for transient/billing errors
* - Automatic retry with backoff for transient errors
* - Non-retryable classification for permanent errors
* - Audit correlation via workflowId
* - Graceful failure handling: pipelines continue if one fails
@@ -64,21 +64,21 @@ function computeExpectedAgents(vulnClasses: readonly VulnClass[], exploit: boole
return expected;
}
// Retry configuration for production (long intervals for billing recovery)
// Retry configuration for production (long intervals so a rate-limit window can clear)
const PRODUCTION_RETRY = {
initialInterval: '5 minutes',
maximumInterval: '30 minutes',
backoffCoefficient: 2,
maximumAttempts: 50,
// Belt-and-braces: activities already throw non-retryable ApplicationFailures for
// these. Only types that are always permanent belong here — GitError and
// AgentExecutionError carry a per-error verdict and must not be listed.
nonRetryableErrorTypes: [
'AuthenticationError',
'PermissionError',
'InvalidRequestError',
'RequestTooLargeError',
'ConfigurationError',
'InvalidTargetError',
'ExecutionLimitError',
'AuthLoginFailedError',
'PermanentError',
],
};
@@ -105,22 +105,6 @@ const testActs = proxyActivities<typeof activities>({
retry: TESTING_RETRY,
});
// Retry configuration for subscription plans (5h+ rolling rate limit windows)
const SUBSCRIPTION_RETRY = {
initialInterval: '5 minutes',
maximumInterval: '6 hours',
backoffCoefficient: 2,
maximumAttempts: 100,
nonRetryableErrorTypes: PRODUCTION_RETRY.nonRetryableErrorTypes,
};
// Activity proxy for subscription plan recovery (extended timeouts)
const subscriptionActs = proxyActivities<typeof activities>({
startToCloseTimeout: '8 hours',
heartbeatTimeout: '2 hours',
retry: SUBSCRIPTION_RETRY,
});
// Retry configuration for preflight validation (short timeout, few retries)
const PREFLIGHT_RETRY = {
initialInterval: '10 seconds',
@@ -167,6 +151,9 @@ function computeSummary(state: PipelineState): PipelineSummary {
};
}
/** One pipeline per vulnerability class, all five in flight together. */
const MAX_CONCURRENT_PIPELINES = 5;
const MAX_PIPELINE_ERROR_MESSAGE_LENGTH = 2000;
function truncatePipelineErrorMessage(message: string): string {
@@ -200,14 +187,7 @@ export async function pentestPipeline(input: PipelineInput): Promise<PipelineSta
const { workflowId } = workflowInfo();
// Select activity proxy based on mode: testing (fast), subscription (extended), or default
function selectActivityProxy(pipelineInput: PipelineInput) {
if (pipelineInput.pipelineTestingMode) return testActs;
if (pipelineInput.pipelineConfig?.retry_preset === 'subscription') return subscriptionActs;
return acts;
}
const a = selectActivityProxy(input);
const a = input.pipelineTestingMode ? testActs : acts;
const state: PipelineState = {
status: 'running',
@@ -611,8 +591,6 @@ export async function pentestPipeline(input: PipelineInput): Promise<PipelineSta
}
}
const maxConcurrent = input.pipelineConfig?.max_concurrent_pipelines ?? 5;
const pipelineConfigs = buildPipelineConfigs();
const pipelineThunks: Array<() => Promise<VulnExploitPipelineResult>> = [];
let alreadyCompletedPipelineCount = 0;
@@ -632,9 +610,15 @@ export async function pentestPipeline(input: PipelineInput): Promise<PipelineSta
}
}
const pipelineResults = await runWithConcurrencyLimit(pipelineThunks, maxConcurrent);
const pipelineResults = await runWithConcurrencyLimit(pipelineThunks, MAX_CONCURRENT_PIPELINES);
aggregatePipelineResults(pipelineResults, alreadyCompletedPipelineCount);
// Surface the not-assessed classes to the report stage so a failed class renders as
// "analysis did not complete" rather than the absence assertion "no findings".
if (state.failedPipelines.length > 0) {
activityInput.failedClasses = state.failedPipelines.map((f) => f.vulnType);
}
state.currentPhase = 'exploitation';
state.currentAgent = null;
await a.logPhaseTransition(activityInput, 'vulnerability-exploitation', 'complete');
@@ -649,7 +633,7 @@ export async function pentestPipeline(input: PipelineInput): Promise<PipelineSta
await a.assembleReportActivity(activityInput, exploit);
// Then run the report agent to add executive summary and clean up
state.agentMetrics.report = await a.runReportAgent(activityInput);
state.agentMetrics.report = await a.runReportAgent(activityInput, exploit);
state.completedAgents.push('report');
if (input.checkpointsEnabled) {
await a.saveCheckpoint(activityInput, 'report', 'reporting', state);
apps/worker/src/types/agents.ts
-1
View File
@@ -48,7 +48,6 @@ export interface AgentDefinition {
prerequisites: AgentName[];
promptTemplate: string;
deliverableFilename: string;
modelTier?: 'small' | 'medium' | 'large';
}
/**
apps/worker/src/types/audit.ts
+5
View File
@@ -27,6 +27,11 @@ export interface AgentEndResult {
attemptNumber: number;
duration_ms: number;
cost_usd: number;
input_tokens?: number | undefined;
output_tokens?: number | undefined;
cache_read_tokens?: number | undefined;
cache_write_tokens?: number | undefined;
turns?: number | undefined;
success: boolean;
model?: string | undefined;
error?: string | undefined;
apps/worker/src/types/config.ts
+5 -8
View File
@@ -32,6 +32,8 @@ export interface ReportConfig {
min_severity?: Severity;
min_confidence?: Confidence;
guidance?: string;
/** Emit report.sarif alongside the markdown report. Ignored when exploit is false. */
sarif?: 'true' | 'false';
}
export type LoginType = 'form' | 'sso' | 'api' | 'basic';
@@ -65,7 +67,6 @@ export interface Authentication {
export interface Config {
rules?: Rules;
authentication?: Authentication;
pipeline?: PipelineConfig;
description?: string;
vuln_classes?: VulnClass[];
exploit?: 'true' | 'false';
@@ -73,12 +74,8 @@ export interface Config {
rules_of_engagement?: string;
}
export type RetryPreset = 'default' | 'subscription';
export interface PipelineConfig {
retry_preset?: RetryPreset;
max_concurrent_pipelines?: number;
}
/** Report config after coercion. The YAML form of `sarif` is a string (see ReportConfig). */
export type DistributedReportConfig = Omit<ReportConfig, 'sarif'> & { sarif: boolean };
export interface DistributedConfig {
avoid: Rule[];
@@ -87,7 +84,7 @@ export interface DistributedConfig {
description: string;
vuln_classes: VulnClass[];
exploit: boolean;
report: ReportConfig;
report: DistributedReportConfig;
rules_of_engagement: string;
}
apps/worker/src/types/errors.ts
+1 -7
View File
@@ -25,11 +25,6 @@ export enum ErrorCode {
AGENT_EXECUTION_FAILED = 'AGENT_EXECUTION_FAILED',
OUTPUT_VALIDATION_FAILED = 'OUTPUT_VALIDATION_FAILED',
// Billing errors (PentestErrorType: 'billing')
API_RATE_LIMITED = 'API_RATE_LIMITED',
SPENDING_CAP_REACHED = 'SPENDING_CAP_REACHED',
INSUFFICIENT_CREDITS = 'INSUFFICIENT_CREDITS',
// Git errors (PentestErrorType: 'filesystem')
GIT_CHECKPOINT_FAILED = 'GIT_CHECKPOINT_FAILED',
GIT_ROLLBACK_FAILED = 'GIT_ROLLBACK_FAILED',
@@ -45,10 +40,9 @@ export enum ErrorCode {
TARGET_UNREACHABLE = 'TARGET_UNREACHABLE',
AUTH_FAILED = 'AUTH_FAILED',
AUTH_LOGIN_FAILED = 'AUTH_LOGIN_FAILED',
BILLING_ERROR = 'BILLING_ERROR',
}
export type PentestErrorType = 'config' | 'network' | 'prompt' | 'filesystem' | 'validation' | 'billing' | 'unknown';
export type PentestErrorType = 'config' | 'network' | 'prompt' | 'filesystem' | 'validation' | 'unknown';
export interface PentestErrorContext {
[key: string]: unknown;
apps/worker/src/types/metrics.ts
+2
View File
@@ -13,6 +13,8 @@ export interface AgentMetrics {
durationMs: number;
inputTokens: number | null;
outputTokens: number | null;
cacheReadTokens: number | null;
cacheWriteTokens: number | null;
costUsd: number | null;
numTurns: number | null;
model?: string | undefined;
apps/worker/src/utils/billing-detection.ts
-90
View File
@@ -1,90 +0,0 @@
// Copyright (C) 2025 Keygraph, Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License version 3
// as published by the Free Software Foundation.
/**
* Consolidated billing/spending cap detection utilities.
*
* Anthropic's spending cap behavior is inconsistent:
* - Sometimes a proper provider error (billing_error)
* - Sometimes the model responds with text about the cap
* - Sometimes partial billing before cutoff
*
* This module provides defense-in-depth detection with shared pattern lists
* to prevent drift between detection points.
*/
/**
* Text patterns for model-output sniffing (what the model says).
* Used by the pi executor and the behavioral heuristic.
*/
export const BILLING_TEXT_PATTERNS = [
'spending cap',
'spending limit',
'cap reached',
'budget exceeded',
'usage limit',
] as const;
/**
* API patterns for error message classification (what the API returns).
* Used by classifyErrorForTemporal in error-handling.ts.
*/
export const BILLING_API_PATTERNS = [
'billing_error',
'credit balance is too low',
'insufficient credits',
'usage is blocked due to insufficient credits',
'please visit plans & billing',
'please visit plans and billing',
'usage limit reached',
'quota exceeded',
'daily rate limit',
'limit will reset',
'billing limit reached',
] as const;
/**
* Checks if text matches any billing text pattern.
* Used for sniffing model output content for spending cap messages.
*/
export function matchesBillingTextPattern(text: string): boolean {
const lowerText = text.toLowerCase();
return BILLING_TEXT_PATTERNS.some((pattern) => lowerText.includes(pattern));
}
/**
* Checks if an error message matches any billing API pattern.
* Used for classifying API error messages.
*/
export function matchesBillingApiPattern(message: string): boolean {
const lowerMessage = message.toLowerCase();
return BILLING_API_PATTERNS.some((pattern) => lowerMessage.includes(pattern));
}
/**
* Behavioral heuristic for detecting spending cap.
*
* When the model hits a spending cap, it often returns a short message
* with $0 cost. Legitimate agent work NEVER costs $0 with only 1-2 turns.
*
* This combines three signals:
* 1. Very low turn count (<=2)
* 2. Zero cost ($0)
* 3. Text matches billing patterns
*
* @param turns - Number of turns the agent took
* @param cost - Total cost in USD
* @param resultText - The result text from the agent
* @returns true if this looks like a spending cap hit
*/
export function isSpendingCapBehavior(turns: number, cost: number, resultText: string): boolean {
// Only check if turns <= 2 AND cost is exactly 0
if (turns > 2 || cost !== 0) {
return false;
}
return matchesBillingTextPattern(resultText);
}