mirror of
https://github.com/KeygraphHQ/shannon.git
synced 2026-09-22 17:50:50 +02:00
feat: multi-provider model support, SARIF output, and exploit-mode fixes (#402)
* feat(worker): record token, cache, and turn usage per agent * feat: replace model tiers with a single SHANNON_AI_MODEL across five providers * feat(cli): rebuild the setup wizard for provider and model selection * docs: document single-model selection and supported providers * feat(worker): use chat completions for OpenAI behind a custom base URL * feat: add SHANNON_AI_OPENAI_FORMAT to pick the wire API for OpenAI gateways * refactor(cli): drop endpoint path hints from the gateway format picker * feat(worker): enable pi in-session provider retry with retry-after backoff * refactor(worker): hand provider error classification to pi and drop the Anthropic ladders * refactor: remove the subscription retry preset and pipeline config section * fix(worker): validate Bedrock credentials with the same live probe as other providers * feat(worker): render the report from structured findings instead of agent-written markdown * fix(worker): dispose the credential probe session on every path * fix(worker): refuse to replace the assembled report with an empty one * refactor(worker): catch post-processing throws across the whole finalization block * revert(worker): drop the report zero-findings guard * docs(worker): correct the retry split and Bedrock credential claims * docs: regenerate llms-full.txt from current sources * feat(cli): build and run the npx flow from a clone * refactor(cli): flatten the setup summary output * feat(cli): reject runs with more than one provider configured * fix(worker): say a rejected bash call never ran * chore(cli): drop grok-4.3 and gpt-5.6-luna from the setup suggestions * feat(worker): capture structured finding locations for SARIF output * fix(worker): enumerate queue confidence so the report inherits it verbatim * feat(worker): give the reporting phase a mode-specific output schema * feat(worker): emit a SARIF 2.1.0 log for exploitative runs * fix(worker): correct SARIF locations and defer fingerprinting to the upload action * fix(worker): drop the confidence suffix from the analysis-mode summary list * feat(worker): give exploit findings a dedicated code location field * feat(worker): carry structured code locations from the vuln queue to the report * fix(worker): join code locations from the vuln queue instead of re-asking agents * fix(worker): spell out the finding_id to category mapping in the tool schema * feat: drop Google/Gemini as a supported AI provider * fix(worker): stop asking the report agent for code locations * docs: correct the provider list and drop the removed rate-limit settings * docs: add provider cyber safeguards and suggested models per provider * docs: document the SARIF output and the report rating thresholds
This commit is contained in:
ezl-keygraph
@@ -13,7 +13,6 @@
|
||||
* - Create git checkpoint
|
||||
* - Start audit logging
|
||||
* - Invoke the pi agent via runPiPrompt
|
||||
* - Spending cap check using isSpendingCapBehavior
|
||||
* - Handle failure (rollback, audit)
|
||||
* - Validate output using AGENTS[agentName].deliverableFilename
|
||||
* - Render the deliverable to disk via the writeDeliverable hook (if provided)
|
||||
@@ -34,7 +33,6 @@ import type { AgentEndResult } from '../types/audit.js';
|
||||
import { ErrorCode, type PentestErrorType } from '../types/errors.js';
|
||||
import type { AgentMetrics } from '../types/metrics.js';
|
||||
import { err, isErr, ok, type Result } from '../types/result.js';
|
||||
import { isSpendingCapBehavior } from '../utils/billing-detection.js';
|
||||
import { getAgentGitPaths } from './agent-git-paths.js';
|
||||
import type { ConfigLoaderService } from './config-loader.js';
|
||||
import { PentestError } from './error-handling.js';
|
||||
@@ -55,6 +53,7 @@ export interface AgentExecutionInput {
|
||||
attemptNumber: number;
|
||||
promptDir?: string | undefined;
|
||||
customTools?: import('@earendil-works/pi-coding-agent').ToolDefinition[];
|
||||
failedClasses?: readonly import('../types/config.js').VulnClass[] | undefined;
|
||||
// Renders the deliverable to disk; invoked after validation, before the success commit.
|
||||
writeDeliverable?: (deliverablesPath: string) => Promise<void>;
|
||||
cancellationSignal?: AbortSignal | undefined;
|
||||
@@ -80,11 +79,6 @@ function errorCodeFromResult(result: PiPromptResult): ErrorCode {
|
||||
|
||||
function categoryForErrorCode(code: ErrorCode): PentestErrorType {
|
||||
switch (code) {
|
||||
case ErrorCode.SPENDING_CAP_REACHED:
|
||||
case ErrorCode.INSUFFICIENT_CREDITS:
|
||||
case ErrorCode.BILLING_ERROR:
|
||||
case ErrorCode.API_RATE_LIMITED:
|
||||
return 'billing';
|
||||
case ErrorCode.GIT_CHECKPOINT_FAILED:
|
||||
case ErrorCode.GIT_ROLLBACK_FAILED:
|
||||
return 'filesystem';
|
||||
@@ -153,6 +147,7 @@ export class AgentExecutionService {
|
||||
attemptNumber,
|
||||
promptDir,
|
||||
customTools,
|
||||
failedClasses,
|
||||
writeDeliverable,
|
||||
cancellationSignal,
|
||||
} = input;
|
||||
@@ -171,7 +166,12 @@ export class AgentExecutionService {
|
||||
try {
|
||||
prompt = await loadPrompt(
|
||||
promptTemplate,
|
||||
{ webUrl, repoPath, AUTH_STATE_FILE: authStateFile(auditSession.sessionMetadata) },
|
||||
{
|
||||
webUrl,
|
||||
repoPath,
|
||||
AUTH_STATE_FILE: authStateFile(auditSession.sessionMetadata),
|
||||
...(failedClasses !== undefined && { failedClasses }),
|
||||
},
|
||||
distributedConfig,
|
||||
pipelineTestingMode,
|
||||
logger,
|
||||
@@ -227,31 +227,13 @@ export class AgentExecutionService {
|
||||
agentName,
|
||||
auditSession,
|
||||
logger,
|
||||
AGENTS[agentName].modelTier,
|
||||
customTools,
|
||||
path.relative(repoPath, deliverablesPath),
|
||||
cancellationSignal,
|
||||
submitTool,
|
||||
);
|
||||
|
||||
// 6. Spending cap check - defense-in-depth
|
||||
if (result.success && (result.turns ?? 0) <= 2 && (result.cost || 0) === 0) {
|
||||
const resultText = result.result || '';
|
||||
if (isSpendingCapBehavior(result.turns ?? 0, result.cost || 0, resultText)) {
|
||||
return this.failAgent(agentName, deliverablesPath, auditSession, logger, {
|
||||
attemptNumber,
|
||||
result,
|
||||
rollbackReason: 'spending cap detected',
|
||||
errorMessage: `Spending cap likely reached: ${resultText.slice(0, 100)}`,
|
||||
errorCode: ErrorCode.SPENDING_CAP_REACHED,
|
||||
category: 'billing',
|
||||
retryable: true,
|
||||
context: { agentName, turns: result.turns, cost: result.cost },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// 7. Handle execution failure
|
||||
// 6. Handle execution failure
|
||||
if (!result.success) {
|
||||
const errorCode = errorCodeFromResult(result);
|
||||
return this.failAgent(agentName, deliverablesPath, auditSession, logger, {
|
||||
@@ -270,39 +252,53 @@ export class AgentExecutionService {
|
||||
// the write→validate→commit sequence is atomic against concurrent sibling agents.
|
||||
let commitHash: string | undefined;
|
||||
const finalizationError = await withGitRepoLock(async (): Promise<PentestError | null> => {
|
||||
// 8. Write structured output to disk (vuln agents only) from the executor's capture
|
||||
const queueFilename = getQueueFilename(agentName);
|
||||
if (submitTool && queueFilename && result.structuredOutput !== undefined) {
|
||||
await fs.ensureDir(deliverablesPath);
|
||||
const queuePath = path.join(deliverablesPath, queueFilename);
|
||||
await fs.writeFile(queuePath, JSON.stringify(result.structuredOutput, null, 2), 'utf8');
|
||||
logger.info(`Wrote structured output queue to ${queueFilename}`);
|
||||
}
|
||||
// Every step below must surface as a returned error rather than a throw: only the
|
||||
// returned path rolls the workspace back and records the failed attempt.
|
||||
try {
|
||||
// 8. Write structured output to disk (vuln agents only) from the executor's capture
|
||||
const queueFilename = getQueueFilename(agentName);
|
||||
if (submitTool && queueFilename && result.structuredOutput !== undefined) {
|
||||
await fs.ensureDir(deliverablesPath);
|
||||
const queuePath = path.join(deliverablesPath, queueFilename);
|
||||
await fs.writeFile(queuePath, JSON.stringify(result.structuredOutput, null, 2), 'utf8');
|
||||
logger.info(`Wrote structured output queue to ${queueFilename}`);
|
||||
}
|
||||
|
||||
// 9. Validate output
|
||||
const validationPassed = await validateAgentOutput(result, agentName, deliverablesPath, logger);
|
||||
if (!validationPassed) {
|
||||
// 9. Validate output
|
||||
const validationPassed = await validateAgentOutput(result, agentName, deliverablesPath, logger);
|
||||
if (!validationPassed) {
|
||||
return new PentestError(
|
||||
`Agent ${agentName} failed output validation`,
|
||||
'validation',
|
||||
true,
|
||||
{ agentName, deliverableFilename: AGENTS[agentName].deliverableFilename },
|
||||
ErrorCode.OUTPUT_VALIDATION_FAILED,
|
||||
);
|
||||
}
|
||||
|
||||
// 10. Render the deliverable to disk so the success commit below stages it
|
||||
if (writeDeliverable) {
|
||||
await writeDeliverable(deliverablesPath);
|
||||
}
|
||||
|
||||
// 11. Success - commit deliverables (scoped) and capture the checkpoint hash
|
||||
const commitResult = await commitGitSuccess(deliverablesPath, agentName, logger, gitPaths);
|
||||
if (!commitResult.success) {
|
||||
return gitFailureForAgent(agentName, 'commit successful results', commitResult.error);
|
||||
}
|
||||
commitHash = commitResult.commitHash;
|
||||
return null;
|
||||
} catch (error) {
|
||||
if (error instanceof PentestError) return error;
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
return new PentestError(
|
||||
`Agent ${agentName} failed output validation`,
|
||||
`Agent ${agentName} post-processing failed: ${errorMessage}`,
|
||||
'validation',
|
||||
true,
|
||||
{ agentName, deliverableFilename: AGENTS[agentName].deliverableFilename },
|
||||
{ agentName, originalError: errorMessage },
|
||||
ErrorCode.OUTPUT_VALIDATION_FAILED,
|
||||
);
|
||||
}
|
||||
|
||||
// 10. Render the deliverable to disk so the success commit below stages it
|
||||
if (writeDeliverable) {
|
||||
await writeDeliverable(deliverablesPath);
|
||||
}
|
||||
|
||||
// 11. Success - commit deliverables (scoped) and capture the checkpoint hash
|
||||
const commitResult = await commitGitSuccess(deliverablesPath, agentName, logger, gitPaths);
|
||||
if (!commitResult.success) {
|
||||
return gitFailureForAgent(agentName, 'commit successful results', commitResult.error);
|
||||
}
|
||||
commitHash = commitResult.commitHash;
|
||||
return null;
|
||||
});
|
||||
|
||||
if (finalizationError) {
|
||||
@@ -326,6 +322,11 @@ export class AgentExecutionService {
|
||||
attemptNumber,
|
||||
duration_ms: result.duration,
|
||||
cost_usd: result.cost || 0,
|
||||
input_tokens: result.inputTokens,
|
||||
output_tokens: result.outputTokens,
|
||||
cache_read_tokens: result.cacheReadTokens,
|
||||
cache_write_tokens: result.cacheWriteTokens,
|
||||
turns: result.turns,
|
||||
success: true,
|
||||
model: result.model,
|
||||
...(commitHash && { checkpoint: commitHash }),
|
||||
@@ -353,6 +354,11 @@ export class AgentExecutionService {
|
||||
attemptNumber: opts.attemptNumber,
|
||||
duration_ms: opts.result.duration,
|
||||
cost_usd: opts.result.cost || 0,
|
||||
input_tokens: opts.result.inputTokens,
|
||||
output_tokens: opts.result.outputTokens,
|
||||
cache_read_tokens: opts.result.cacheReadTokens,
|
||||
cache_write_tokens: opts.result.cacheWriteTokens,
|
||||
turns: opts.result.turns,
|
||||
success: false,
|
||||
model: opts.result.model,
|
||||
error: opts.errorMessage,
|
||||
@@ -406,8 +412,10 @@ export class AgentExecutionService {
|
||||
static toMetrics(endResult: AgentEndResult, result: PiPromptResult): AgentMetrics {
|
||||
return {
|
||||
durationMs: endResult.duration_ms,
|
||||
inputTokens: null, // Not currently exposed by the pi executor
|
||||
outputTokens: null,
|
||||
inputTokens: result.inputTokens ?? null,
|
||||
outputTokens: result.outputTokens ?? null,
|
||||
cacheReadTokens: result.cacheReadTokens ?? null,
|
||||
cacheWriteTokens: result.cacheWriteTokens ?? null,
|
||||
costUsd: endResult.cost_usd,
|
||||
numTurns: result.turns ?? null,
|
||||
model: result.model,
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
*/
|
||||
|
||||
import { getQueueFilename } from '../ai/queue-schemas.js';
|
||||
import { REPORT_JSON_FILENAME, SARIF_FILENAME } from '../paths.js';
|
||||
import { AGENTS } from '../session-manager.js';
|
||||
import type { AgentName } from '../types/agents.js';
|
||||
|
||||
@@ -27,5 +28,12 @@ export function getAgentGitPaths(agentName: AgentName): string[] {
|
||||
if (queueFilename) {
|
||||
paths.push(queueFilename);
|
||||
}
|
||||
// The report agent also emits the structured findings the markdown is rendered from, and the
|
||||
// SARIF log when enabled. Listing the log unconditionally is harmless when it was not written,
|
||||
// and keeps a stale one from surviving the rollback of a failed attempt.
|
||||
if (agentName === 'report') {
|
||||
paths.push(REPORT_JSON_FILENAME);
|
||||
paths.push(SARIF_FILENAME);
|
||||
}
|
||||
return [...new Set(paths)];
|
||||
}
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
// Copyright (C) 2025 Keygraph, Inc.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License version 3
|
||||
// as published by the Free Software Foundation.
|
||||
|
||||
/**
|
||||
* Attach vuln-queue code locations to collected findings.
|
||||
*
|
||||
* The vuln agent authors `code_locations` once, into its queue. Every stage after that used to
|
||||
* re-transcribe them — the exploit agent into its evidence, the report agent into `add_finding` —
|
||||
* and each hop lost some: 100% in the queue, 98% in the evidence, 42-63% by the report. Nothing
|
||||
* about the copy is a judgement call, and `finding_id` matches the queue `ID` exactly, so the
|
||||
* locations are joined here instead of being asked for again.
|
||||
*/
|
||||
|
||||
import { fs, path } from 'zx';
|
||||
import type { QueueCodeLocation } from '../ai/queue-schemas.js';
|
||||
import type { AddFindingInput } from '../collectors/finding-collector.js';
|
||||
import type { ActivityLogger } from '../types/activity-logger.js';
|
||||
import { ALL_VULN_CLASSES } from '../types/config.js';
|
||||
|
||||
interface QueueEntry {
|
||||
ID?: string;
|
||||
code_locations?: QueueCodeLocation[];
|
||||
}
|
||||
|
||||
/** Read every per-class queue in the deliverables dir into an ID-to-locations map. */
|
||||
async function loadQueueLocations(
|
||||
deliverablesPath: string,
|
||||
logger: ActivityLogger,
|
||||
): Promise<Map<string, QueueCodeLocation[]>> {
|
||||
const locations = new Map<string, QueueCodeLocation[]>();
|
||||
|
||||
for (const vulnClass of ALL_VULN_CLASSES) {
|
||||
const queuePath = path.join(deliverablesPath, `${vulnClass}_exploitation_queue.json`);
|
||||
if (!(await fs.pathExists(queuePath))) continue;
|
||||
|
||||
try {
|
||||
const doc = (await fs.readJson(queuePath)) as { vulnerabilities?: QueueEntry[] };
|
||||
for (const entry of doc.vulnerabilities ?? []) {
|
||||
if (entry.ID && entry.code_locations && entry.code_locations.length > 0) {
|
||||
locations.set(entry.ID, entry.code_locations);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warn(`Could not read ${vulnClass} queue for code locations: ${(error as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
return locations;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the findings with `code_locations` filled in from the queue.
|
||||
*
|
||||
* A finding with no matching queue entry keeps none — the join never invents one. Findings are
|
||||
* copied rather than mutated so the collector's own state stays untouched.
|
||||
*/
|
||||
export async function attachQueueCodeLocations(
|
||||
findings: readonly AddFindingInput[],
|
||||
deliverablesPath: string,
|
||||
logger: ActivityLogger,
|
||||
): Promise<AddFindingInput[]> {
|
||||
const byId = await loadQueueLocations(deliverablesPath, logger);
|
||||
if (byId.size === 0) return [...findings];
|
||||
|
||||
let matched = 0;
|
||||
const joined = findings.map((finding) => {
|
||||
const locations = byId.get(finding.finding_id);
|
||||
if (!locations) return finding;
|
||||
matched += 1;
|
||||
return { ...finding, code_locations: locations };
|
||||
});
|
||||
|
||||
logger.info(`Attached code locations to ${matched}/${findings.length} finding(s) from the vuln queues`);
|
||||
return joined;
|
||||
}
|
||||
@@ -38,13 +38,9 @@ export class ConfigLoaderService {
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
|
||||
// Determine appropriate error code based on error message
|
||||
let errorCode = ErrorCode.CONFIG_PARSE_ERROR;
|
||||
if (errorMessage.includes('not found') || errorMessage.includes('ENOENT')) {
|
||||
errorCode = ErrorCode.CONFIG_NOT_FOUND;
|
||||
} else if (errorMessage.includes('validation failed')) {
|
||||
errorCode = ErrorCode.CONFIG_VALIDATION_FAILED;
|
||||
}
|
||||
// parseConfig throws PentestErrors that already name the failure; anything
|
||||
// else reaching here is a parse-time fault.
|
||||
const errorCode = error instanceof PentestError && error.code ? error.code : ErrorCode.CONFIG_PARSE_ERROR;
|
||||
|
||||
return err(
|
||||
new PentestError(
|
||||
|
||||
@@ -4,8 +4,8 @@
|
||||
// it under the terms of the GNU Affero General Public License version 3
|
||||
// as published by the Free Software Foundation.
|
||||
|
||||
import { type AssistantMessage, isRetryableAssistantError } from '@earendil-works/pi-ai';
|
||||
import { ErrorCode, type PentestErrorContext, type PentestErrorType, type PromptErrorResult } from '../types/errors.js';
|
||||
import { matchesBillingApiPattern, matchesBillingTextPattern } from '../utils/billing-detection.js';
|
||||
|
||||
export class PentestError extends Error {
|
||||
override name = 'PentestError' as const;
|
||||
@@ -44,53 +44,23 @@ export function handlePromptError(promptName: string, error: Error): PromptError
|
||||
};
|
||||
}
|
||||
|
||||
const RETRYABLE_PATTERNS = [
|
||||
// Network and connection errors
|
||||
'network',
|
||||
'connection',
|
||||
'timeout',
|
||||
'econnreset',
|
||||
'enotfound',
|
||||
'econnrefused',
|
||||
// Rate limiting
|
||||
'rate limit',
|
||||
'429',
|
||||
'too many requests',
|
||||
// Server errors
|
||||
'server error',
|
||||
'5xx',
|
||||
'internal server error',
|
||||
'service unavailable',
|
||||
'bad gateway',
|
||||
// Provider API errors
|
||||
'model unavailable',
|
||||
'service temporarily unavailable',
|
||||
'api error',
|
||||
'terminated',
|
||||
// Max turns
|
||||
'max turns',
|
||||
'maximum turns',
|
||||
];
|
||||
/**
|
||||
* Whether a failed agent attempt is worth retrying.
|
||||
*
|
||||
* A PentestError already carries a verdict — for provider turns that verdict
|
||||
* comes from pi — so it is taken as given. Anything else is raw text, judged by
|
||||
* pi's classifier: transient for load, throttling, and transport failures,
|
||||
* terminal for quota, billing, and auth. Unrecognised errors are not retried, so
|
||||
* a permanent fault fails fast.
|
||||
*/
|
||||
export function isRetryableFailure(error: Error): boolean {
|
||||
if (error instanceof PentestError) return error.retryable;
|
||||
|
||||
// Patterns that indicate non-retryable errors (checked before default)
|
||||
const NON_RETRYABLE_PATTERNS = [
|
||||
'authentication',
|
||||
'invalid prompt',
|
||||
'out of memory',
|
||||
'permission denied',
|
||||
'session limit reached',
|
||||
'invalid api key',
|
||||
];
|
||||
|
||||
// Conservative retry classification - unknown errors don't retry (fail-safe default)
|
||||
export function isRetryableError(error: Error): boolean {
|
||||
const message = error.message.toLowerCase();
|
||||
|
||||
if (NON_RETRYABLE_PATTERNS.some((pattern) => message.includes(pattern))) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return RETRYABLE_PATTERNS.some((pattern) => message.includes(pattern));
|
||||
return isRetryableAssistantError({
|
||||
role: 'assistant',
|
||||
stopReason: 'error',
|
||||
errorMessage: error.message,
|
||||
} as AssistantMessage);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -99,14 +69,6 @@ export function isRetryableError(error: Error): boolean {
|
||||
*/
|
||||
function classifyByErrorCode(code: ErrorCode, retryableFromError: boolean): { type: string; retryable: boolean } {
|
||||
switch (code) {
|
||||
// Billing errors - retryable (wait for cap reset or credits added)
|
||||
case ErrorCode.SPENDING_CAP_REACHED:
|
||||
case ErrorCode.INSUFFICIENT_CREDITS:
|
||||
return { type: 'BillingError', retryable: true };
|
||||
|
||||
case ErrorCode.API_RATE_LIMITED:
|
||||
return { type: 'RateLimitError', retryable: true };
|
||||
|
||||
// Config errors - non-retryable (need manual fix)
|
||||
case ErrorCode.CONFIG_NOT_FOUND:
|
||||
case ErrorCode.CONFIG_VALIDATION_FAILED:
|
||||
@@ -143,11 +105,10 @@ function classifyByErrorCode(code: ErrorCode, retryableFromError: boolean): { ty
|
||||
case ErrorCode.AUTH_LOGIN_FAILED:
|
||||
return { type: 'AuthLoginFailedError', retryable: false };
|
||||
|
||||
case ErrorCode.BILLING_ERROR:
|
||||
return { type: 'BillingError', retryable: true };
|
||||
case ErrorCode.TARGET_UNREACHABLE:
|
||||
return { type: 'InvalidTargetError', retryable: false };
|
||||
|
||||
default:
|
||||
// Unknown code - fall through to string matching
|
||||
return { type: 'UnknownError', retryable: retryableFromError };
|
||||
}
|
||||
}
|
||||
@@ -161,8 +122,8 @@ function classifyByErrorCode(code: ErrorCode, retryableFromError: boolean): { ty
|
||||
* - Non-retryable errors: Temporal fails immediately
|
||||
*
|
||||
* Classification priority:
|
||||
* 1. If error is PentestError with ErrorCode, classify by code (reliable)
|
||||
* 2. Fall through to string matching for external errors (provider, network, etc.)
|
||||
* 1. A PentestError carrying an ErrorCode is classified by that code.
|
||||
* 2. Anything else falls through to isRetryableFailure.
|
||||
*/
|
||||
export function classifyErrorForTemporal(error: unknown): { type: string; retryable: boolean } {
|
||||
// === CODE-BASED CLASSIFICATION (Preferred for internal errors) ===
|
||||
@@ -170,101 +131,11 @@ export function classifyErrorForTemporal(error: unknown): { type: string; retrya
|
||||
return classifyByErrorCode(error.code, error.retryable);
|
||||
}
|
||||
|
||||
// === STRING-BASED CLASSIFICATION (Fallback for external errors) ===
|
||||
const message = (error instanceof Error ? error.message : String(error)).toLowerCase();
|
||||
|
||||
// === BILLING ERRORS (Retryable with long backoff) ===
|
||||
// Anthropic returns billing as 400 invalid_request_error
|
||||
// Human can add credits OR wait for spending cap to reset (5-30 min backoff)
|
||||
// Check both API patterns and text patterns for comprehensive detection
|
||||
if (matchesBillingApiPattern(message) || matchesBillingTextPattern(message)) {
|
||||
return { type: 'BillingError', retryable: true };
|
||||
}
|
||||
|
||||
// === PERMANENT ERRORS (Non-retryable) ===
|
||||
|
||||
// Authentication (401) - bad API key won't fix itself
|
||||
if (
|
||||
message.includes('authentication') ||
|
||||
message.includes('api key') ||
|
||||
message.includes('401') ||
|
||||
message.includes('authentication_error')
|
||||
) {
|
||||
return { type: 'AuthenticationError', retryable: false };
|
||||
}
|
||||
|
||||
// Permission (403) - access won't be granted
|
||||
if (message.includes('permission') || message.includes('forbidden') || message.includes('403')) {
|
||||
return { type: 'PermissionError', retryable: false };
|
||||
}
|
||||
|
||||
// Out of memory - deterministic resource exhaustion, retrying won't help
|
||||
if (message.includes('out of memory')) {
|
||||
return { type: 'OutOfMemoryError', retryable: false };
|
||||
}
|
||||
|
||||
// Invalid prompt - malformed/rejected prompt content won't fix itself on retry
|
||||
if (message.includes('invalid prompt')) {
|
||||
return { type: 'InvalidPromptError', retryable: false };
|
||||
}
|
||||
|
||||
// Session limit reached - distinct from billing/rate-limit; needs manual intervention
|
||||
if (message.includes('session limit reached')) {
|
||||
return { type: 'SessionLimitError', retryable: false };
|
||||
}
|
||||
|
||||
// Overloaded - provider's own error-type token is authoritative regardless of the
|
||||
// HTTP status it arrives under (seen in production under 400, not just 529)
|
||||
if (message.includes('overloaded_error') || message.includes('overloaded')) {
|
||||
return { type: 'OverloadedError', retryable: true };
|
||||
}
|
||||
|
||||
// === OUTPUT VALIDATION ERRORS (Retryable) ===
|
||||
// Agent didn't produce expected deliverables - retry may succeed
|
||||
// IMPORTANT: Must come BEFORE generic 'validation' check below
|
||||
if (message.includes('failed output validation') || message.includes('output validation failed')) {
|
||||
return { type: 'OutputValidationError', retryable: true };
|
||||
}
|
||||
|
||||
// Invalid Request (400) - malformed request is permanent
|
||||
// Note: Checked AFTER billing and AFTER output validation
|
||||
if (message.includes('invalid_request_error') || message.includes('malformed') || message.includes('validation')) {
|
||||
return { type: 'InvalidRequestError', retryable: false };
|
||||
}
|
||||
|
||||
// Request Too Large (413) - won't fit no matter how many retries
|
||||
if (message.includes('request_too_large') || message.includes('too large') || message.includes('413')) {
|
||||
return { type: 'RequestTooLargeError', retryable: false };
|
||||
}
|
||||
|
||||
// Configuration errors - missing files need manual fix
|
||||
if (message.includes('enoent') || message.includes('no such file') || message.includes('cli not installed')) {
|
||||
return { type: 'ConfigurationError', retryable: false };
|
||||
}
|
||||
|
||||
// Execution limits - max turns/budget reached
|
||||
if (
|
||||
message.includes('max turns') ||
|
||||
message.includes('budget') ||
|
||||
message.includes('execution limit') ||
|
||||
message.includes('error_max_turns') ||
|
||||
message.includes('error_max_budget')
|
||||
) {
|
||||
return { type: 'ExecutionLimitError', retryable: false };
|
||||
}
|
||||
|
||||
// Invalid target URL - bad URL format won't fix itself
|
||||
if (
|
||||
message.includes('invalid url') ||
|
||||
message.includes('invalid target') ||
|
||||
message.includes('malformed url') ||
|
||||
message.includes('invalid uri')
|
||||
) {
|
||||
return { type: 'InvalidTargetError', retryable: false };
|
||||
}
|
||||
|
||||
// === TRANSIENT ERRORS (Retryable) ===
|
||||
// Rate limits (429), server errors (5xx), network issues
|
||||
// Let Temporal retry with configured backoff
|
||||
return { type: 'TransientError', retryable: true };
|
||||
// === FALLBACK ===
|
||||
// Everything else is a raw throw: a library error, or a PentestError carrying no
|
||||
// code. isRetryableFailure decides — pi's classifier for provider text, the
|
||||
// error's own verdict when it has one, and no retry for anything unrecognised.
|
||||
const err = error instanceof Error ? error : new Error(String(error));
|
||||
const retryable = isRetryableFailure(err);
|
||||
return { type: retryable ? 'TransientError' : 'PermanentError', retryable };
|
||||
}
|
||||
|
||||
@@ -53,9 +53,15 @@ function formatLocation(endpoint: string | undefined, codeLocation: string | und
|
||||
return endpoint ?? codeLocation ?? '';
|
||||
}
|
||||
|
||||
/** The analysis queue carries no severity, so confidence is the only rating. */
|
||||
interface CommonEntryFields {
|
||||
readonly confidence: string;
|
||||
}
|
||||
|
||||
function buildEntry(
|
||||
id: string,
|
||||
title: string,
|
||||
common: CommonEntryFields,
|
||||
summaryRows: ReadonlyArray<string | null>,
|
||||
notes: string | undefined,
|
||||
): string {
|
||||
@@ -63,6 +69,7 @@ function buildEntry(
|
||||
lines.push(`### ${id}: ${title}`);
|
||||
lines.push('');
|
||||
lines.push('**Summary:**');
|
||||
lines.push(`- **Confidence:** ${common.confidence}`);
|
||||
for (const row of summaryRows) {
|
||||
if (row !== null) lines.push(row);
|
||||
}
|
||||
@@ -79,6 +86,7 @@ function renderAuthEntry(e: AuthFinding): string {
|
||||
return buildEntry(
|
||||
e.ID,
|
||||
e.vulnerability_type,
|
||||
{ confidence: e.confidence },
|
||||
[
|
||||
summaryRow('Vulnerable location', formatLocation(e.source_endpoint, e.vulnerable_code_location)),
|
||||
summaryRow('Overview', e.missing_defense),
|
||||
@@ -92,6 +100,7 @@ function renderSsrfEntry(e: SsrfFinding): string {
|
||||
return buildEntry(
|
||||
e.ID,
|
||||
e.vulnerability_type,
|
||||
{ confidence: e.confidence },
|
||||
[
|
||||
summaryRow('Vulnerable location', formatLocation(e.source_endpoint, e.vulnerable_code_location)),
|
||||
summaryRow('Overview', e.missing_defense),
|
||||
@@ -105,6 +114,7 @@ function renderAuthzEntry(e: AuthzFinding): string {
|
||||
return buildEntry(
|
||||
e.ID,
|
||||
e.vulnerability_type,
|
||||
{ confidence: e.confidence },
|
||||
[
|
||||
summaryRow('Vulnerable location', formatLocation(e.endpoint, e.vulnerable_code_location)),
|
||||
summaryRow('Overview', e.guard_evidence),
|
||||
@@ -119,6 +129,7 @@ function renderInjectionEntry(e: InjectionFinding): string {
|
||||
return buildEntry(
|
||||
e.ID,
|
||||
e.vulnerability_type,
|
||||
{ confidence: e.confidence },
|
||||
[summaryRow('Vulnerable location', location), summaryRow('Overview', e.mismatch_reason)],
|
||||
e.notes,
|
||||
);
|
||||
@@ -129,6 +140,7 @@ function renderXssEntry(e: XssFinding): string {
|
||||
return buildEntry(
|
||||
e.ID,
|
||||
e.vulnerability_type,
|
||||
{ confidence: e.confidence },
|
||||
[summaryRow('Vulnerable location', location), summaryRow('Overview', e.mismatch_reason)],
|
||||
e.notes,
|
||||
);
|
||||
|
||||
@@ -20,4 +20,6 @@ export type { ContainerDependencies } from './container.js';
|
||||
export { Container, getContainer, getOrCreateContainer, removeContainer, setContainerFactory } from './container.js';
|
||||
export { ExploitationCheckerService } from './exploitation-checker.js';
|
||||
export { loadPrompt } from './prompt-manager.js';
|
||||
export type { ReportData, ReportMeta } from './report-renderer.js';
|
||||
export { renderReport } from './report-renderer.js';
|
||||
export { assembleFinalReport, copyReportToRunRoot, injectModelIntoReport } from './reporting.js';
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
* 1. Repository path exists and is a directory
|
||||
* 2. Config file parses and validates (if provided)
|
||||
* 3. code_path rules match real entries in the repo (filesystem only)
|
||||
* 4. Credentials validate via a minimal pi session (API key, OAuth, or Bedrock)
|
||||
* 4. Credentials validate via a minimal pi session against the run's own model
|
||||
* 5. Target URL resolves, is not link-local (cloud metadata), and is reachable (DNS + HTTP)
|
||||
*/
|
||||
|
||||
@@ -26,22 +26,33 @@ import http from 'node:http';
|
||||
import https from 'node:https';
|
||||
import net, { type LookupFunction } from 'node:net';
|
||||
import os from 'node:os';
|
||||
import type { Api, AssistantMessage, Model } from '@earendil-works/pi-ai';
|
||||
import {
|
||||
AuthStorage,
|
||||
type AgentSession,
|
||||
createAgentSession,
|
||||
ModelRegistry,
|
||||
type ModelRuntime,
|
||||
SessionManager,
|
||||
SettingsManager,
|
||||
} from '@earendil-works/pi-coding-agent';
|
||||
import { glob } from 'zx';
|
||||
import { resolveEffectiveProvider, resolveModelId } from '../ai/models.js';
|
||||
import {
|
||||
createModelRuntime,
|
||||
type ModelSpec,
|
||||
type OpenAiFormat,
|
||||
type ProviderId,
|
||||
resolveGatewayFormat,
|
||||
resolveModel,
|
||||
resolveModelSpec,
|
||||
resolveProviderCredentials,
|
||||
} from '../ai/models.js';
|
||||
import { PI_RETRY_SETTINGS } from '../ai/pi/retry-settings.js';
|
||||
import { providerTurnError } from '../ai/pi/turn-error.js';
|
||||
import { parseConfig } from '../config-parser.js';
|
||||
import type { ActivityLogger } from '../types/activity-logger.js';
|
||||
import type { Config, Rule } from '../types/config.js';
|
||||
import { ErrorCode } from '../types/errors.js';
|
||||
import { err, isErr, ok, type Result } from '../types/result.js';
|
||||
import { matchesBillingTextPattern } from '../utils/billing-detection.js';
|
||||
import { PentestError } from './error-handling.js';
|
||||
import { isRetryableFailure, PentestError } from './error-handling.js';
|
||||
|
||||
const TARGET_URL_TIMEOUT_MS = 10_000;
|
||||
|
||||
@@ -215,157 +226,79 @@ async function validateCodePathsExist(
|
||||
|
||||
// === Credential Validation ===
|
||||
|
||||
/** Map provider error text to a human-readable preflight PentestError. */
|
||||
/** Classify a provider error message (thrown or from a failed turn) into a PentestError. */
|
||||
function classifyCredentialError(text: string, authType: string): Result<void, PentestError> {
|
||||
const lower = text.toLowerCase();
|
||||
if (matchesBillingTextPattern(text)) {
|
||||
return err(
|
||||
new PentestError(
|
||||
`Anthropic account has a billing or rate-limit issue during ${authType} validation. Add credits or wait and retry.`,
|
||||
'billing',
|
||||
true,
|
||||
{ authType },
|
||||
ErrorCode.BILLING_ERROR,
|
||||
),
|
||||
);
|
||||
}
|
||||
if (/401|403|invalid[ _-]?api[ _-]?key|unauthorized|authentication|forbidden|not allowed|x-api-key/.test(lower)) {
|
||||
return err(
|
||||
new PentestError(
|
||||
`Invalid ${authType}. Check your credentials in .env and try again.`,
|
||||
'config',
|
||||
false,
|
||||
{ authType },
|
||||
ErrorCode.AUTH_FAILED,
|
||||
),
|
||||
);
|
||||
}
|
||||
if (/model/.test(lower) && /not found|not available|unknown/.test(lower)) {
|
||||
return err(
|
||||
new PentestError(
|
||||
`Configured model is not available for this account. Check ANTHROPIC_*_MODEL in .env.`,
|
||||
'config',
|
||||
false,
|
||||
{ authType },
|
||||
),
|
||||
);
|
||||
}
|
||||
if (
|
||||
/network|timeout|enotfound|econnrefused|fetch failed|getaddrinfo|socket|overloaded|unavailable|50\d/.test(lower)
|
||||
) {
|
||||
return err(
|
||||
new PentestError(`Anthropic API unreachable or temporarily unavailable. Try again shortly.`, 'network', true, {
|
||||
authType,
|
||||
}),
|
||||
);
|
||||
}
|
||||
return err(
|
||||
new PentestError(
|
||||
`${authType} validation failed: ${text.slice(0, 150)}`,
|
||||
'config',
|
||||
false,
|
||||
{ authType },
|
||||
ErrorCode.AUTH_FAILED,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
/** Minimal pi session probe to validate credentials. An optional baseUrl overrides the endpoint. */
|
||||
/**
|
||||
* Minimal pi session probe against the model the scan will use, so credentials the
|
||||
* account cannot use fail here rather than partway through the run. The descriptor
|
||||
* already carries the run's endpoint and wire format, so the probe exercises the
|
||||
* same path the scan will.
|
||||
*/
|
||||
async function probeCredentialsWithPi(
|
||||
model: Model<Api>,
|
||||
modelRuntime: ModelRuntime,
|
||||
authType: string,
|
||||
token?: string,
|
||||
baseUrl?: string,
|
||||
): Promise<Result<void, PentestError>> {
|
||||
const authStorage = AuthStorage.inMemory();
|
||||
if (token) authStorage.setRuntimeApiKey('anthropic', token);
|
||||
|
||||
const baseModel = ModelRegistry.create(authStorage).find('anthropic', resolveModelId('small'));
|
||||
if (!baseModel) {
|
||||
return err(
|
||||
new PentestError(
|
||||
`Model not found in pi registry: ${resolveModelId('small')}`,
|
||||
'config',
|
||||
false,
|
||||
{},
|
||||
ErrorCode.AUTH_FAILED,
|
||||
),
|
||||
);
|
||||
}
|
||||
const model = baseUrl ? { ...baseModel, baseUrl } : baseModel;
|
||||
|
||||
let errText: string | undefined;
|
||||
let failedTurn: AssistantMessage | undefined;
|
||||
let session: AgentSession | undefined;
|
||||
try {
|
||||
const { session } = await createAgentSession({
|
||||
({ session } = await createAgentSession({
|
||||
cwd: os.tmpdir(),
|
||||
model,
|
||||
thinkingLevel: 'off',
|
||||
noTools: 'all',
|
||||
authStorage,
|
||||
modelRuntime,
|
||||
sessionManager: SessionManager.inMemory(),
|
||||
settingsManager: SettingsManager.inMemory({ retry: { enabled: false }, compaction: { enabled: false } }),
|
||||
});
|
||||
settingsManager: SettingsManager.inMemory({ retry: PI_RETRY_SETTINGS, compaction: { enabled: false } }),
|
||||
}));
|
||||
session.subscribe((e) => {
|
||||
if (e.type === 'turn_end' && e.message.role === 'assistant' && e.message.stopReason === 'error') {
|
||||
errText = e.message.errorMessage ?? 'unknown provider error';
|
||||
failedTurn = e.message;
|
||||
}
|
||||
});
|
||||
await session.prompt('hi');
|
||||
session.dispose();
|
||||
} catch (error) {
|
||||
errText = error instanceof Error ? error.message : String(error);
|
||||
const thrown = error instanceof Error ? error : new Error(String(error));
|
||||
return err(
|
||||
new PentestError(
|
||||
`${authType} validation failed: ${thrown.message.slice(0, 300)}`,
|
||||
'unknown',
|
||||
isRetryableFailure(thrown),
|
||||
{ authType },
|
||||
ErrorCode.AGENT_EXECUTION_FAILED,
|
||||
),
|
||||
);
|
||||
} finally {
|
||||
session?.dispose();
|
||||
}
|
||||
|
||||
if (errText) return classifyCredentialError(errText, authType);
|
||||
if (failedTurn) return err(providerTurnError(failedTurn, `${authType} validation failed`));
|
||||
return ok(undefined);
|
||||
}
|
||||
|
||||
/** Validate credentials via a minimal pi session. */
|
||||
/** Credential env var a provider reads, for "credential missing" messages. */
|
||||
const PROVIDER_CREDENTIAL_HINT: Readonly<Record<ProviderId, string>> = {
|
||||
anthropic: 'ANTHROPIC_API_KEY (or CLAUDE_CODE_OAUTH_TOKEN)',
|
||||
openai: 'OPENAI_API_KEY',
|
||||
xai: 'XAI_API_KEY',
|
||||
'amazon-bedrock': 'AWS_BEARER_TOKEN_BEDROCK and AWS_REGION',
|
||||
};
|
||||
|
||||
/** Human-readable label for which credential path a run is using. */
|
||||
function describeAuth(providerId: ProviderId, baseUrl: string | undefined): string {
|
||||
if (baseUrl) return `custom endpoint (${baseUrl})`;
|
||||
if (providerId === 'amazon-bedrock') return 'Bedrock bearer token';
|
||||
return `${providerId} API key`;
|
||||
}
|
||||
|
||||
/** Validate the model selection and its credentials via a minimal pi session. */
|
||||
async function validateCredentials(logger: ActivityLogger): Promise<Result<void, PentestError>> {
|
||||
// Resolve the active provider through the same precedence the executor uses, so
|
||||
// preflight validates exactly the credentials the run will use (no drift).
|
||||
const eff = resolveEffectiveProvider();
|
||||
|
||||
// 1. Bedrock mode — validate required AWS credentials are present (pi-ai owns the
|
||||
// live AWS auth, so there is no cheap session probe here)
|
||||
if (eff.providerId === 'amazon-bedrock') {
|
||||
const required = [
|
||||
'AWS_REGION',
|
||||
'AWS_BEARER_TOKEN_BEDROCK',
|
||||
'ANTHROPIC_SMALL_MODEL',
|
||||
'ANTHROPIC_MEDIUM_MODEL',
|
||||
'ANTHROPIC_LARGE_MODEL',
|
||||
];
|
||||
const missing = required.filter((v) => !process.env[v]);
|
||||
if (missing.length > 0) {
|
||||
return err(
|
||||
new PentestError(
|
||||
`Bedrock mode requires the following env vars in .env: ${missing.join(', ')}`,
|
||||
'config',
|
||||
false,
|
||||
{ missing },
|
||||
ErrorCode.AUTH_FAILED,
|
||||
),
|
||||
);
|
||||
}
|
||||
logger.info('Bedrock credentials OK');
|
||||
return ok(undefined);
|
||||
}
|
||||
|
||||
// 2. Custom base URL — validate the endpoint via a minimal pi session
|
||||
if (eff.baseUrl) {
|
||||
logger.info('Validating custom base URL');
|
||||
const probe = await probeCredentialsWithPi(`custom endpoint (${eff.baseUrl})`, eff.anthropicToken, eff.baseUrl);
|
||||
if (isErr(probe)) return probe;
|
||||
logger.info('Custom base URL OK');
|
||||
return ok(undefined);
|
||||
}
|
||||
|
||||
// 3. Direct Anthropic — require a credential, then validate via a minimal pi session
|
||||
if (!eff.anthropicToken) {
|
||||
// 1. Resolve the run's model. A malformed spec or unknown provider fails here,
|
||||
// before any scan work begins.
|
||||
let spec: ModelSpec;
|
||||
try {
|
||||
spec = resolveModelSpec();
|
||||
} catch (error) {
|
||||
return err(
|
||||
new PentestError(
|
||||
'No API credentials found. Set ANTHROPIC_API_KEY or CLAUDE_CODE_OAUTH_TOKEN in .env (or use CLAUDE_CODE_USE_BEDROCK=1 for AWS Bedrock)',
|
||||
error instanceof Error ? error.message : String(error),
|
||||
'config',
|
||||
false,
|
||||
{},
|
||||
@@ -373,11 +306,76 @@ async function validateCredentials(logger: ActivityLogger): Promise<Result<void,
|
||||
),
|
||||
);
|
||||
}
|
||||
logger.info(`Model: ${spec.providerId}:${spec.modelId}`);
|
||||
|
||||
const usingApiKey = Boolean(process.env.ANTHROPIC_API_KEY);
|
||||
const authType = usingApiKey ? 'API key' : 'OAuth token';
|
||||
// 2. Credential presence. Bedrock needs both AWS_ vars; every other provider
|
||||
// needs one API key.
|
||||
const credentials = resolveProviderCredentials(spec.providerId);
|
||||
|
||||
// 3. Wire format for an OpenAI gateway. Rejects a format named where it cannot
|
||||
// take effect, rather than letting the run proceed on the wrong API.
|
||||
let format: OpenAiFormat;
|
||||
try {
|
||||
format = resolveGatewayFormat(spec.providerId, credentials.baseUrl);
|
||||
} catch (error) {
|
||||
return err(
|
||||
new PentestError(
|
||||
error instanceof Error ? error.message : String(error),
|
||||
'config',
|
||||
false,
|
||||
{ providerId: spec.providerId },
|
||||
ErrorCode.AUTH_FAILED,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
const isBedrock = spec.providerId === 'amazon-bedrock';
|
||||
const missing = isBedrock ? ['AWS_REGION', 'AWS_BEARER_TOKEN_BEDROCK'].filter((n) => !process.env[n]) : [];
|
||||
if (missing.length > 0 || (!isBedrock && !credentials.apiKey)) {
|
||||
return err(
|
||||
new PentestError(
|
||||
`No credentials found for provider "${spec.providerId}". Set ${PROVIDER_CREDENTIAL_HINT[spec.providerId]} in .env.`,
|
||||
'config',
|
||||
false,
|
||||
{ providerId: spec.providerId, ...(missing.length > 0 && { missing }) },
|
||||
ErrorCode.AUTH_FAILED,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
// 4. Model must exist in the registry, for every provider — Bedrock IDs are the
|
||||
// easiest to get wrong, since region prefixes and version suffixes differ per
|
||||
// model (`us.anthropic.claude-opus-5` exists, bare `anthropic.` does not).
|
||||
// A custom endpoint is exempt: it may serve models under its own names.
|
||||
const modelRuntime = await createModelRuntime(spec.providerId, credentials.apiKey);
|
||||
const baseModel = resolveModel(modelRuntime, spec.providerId, spec.modelId, credentials.baseUrl, format);
|
||||
if (!baseModel) {
|
||||
return err(
|
||||
new PentestError(
|
||||
`Model not found in pi registry: provider="${spec.providerId}" model="${spec.modelId}". Check SHANNON_AI_MODEL.`,
|
||||
'config',
|
||||
false,
|
||||
{ providerId: spec.providerId, modelId: spec.modelId },
|
||||
ErrorCode.AUTH_FAILED,
|
||||
),
|
||||
);
|
||||
}
|
||||
if (!modelRuntime.getModel(spec.providerId, spec.modelId)) {
|
||||
logger.warn(
|
||||
`Model "${spec.modelId}" is not in the ${spec.providerId} catalogue; passing it to the custom endpoint as given. Cost figures will be approximate.`,
|
||||
);
|
||||
}
|
||||
if (credentials.baseUrl && spec.providerId === 'openai') {
|
||||
logger.info(`Gateway API: ${format} (${baseModel.api})`);
|
||||
}
|
||||
|
||||
// 5. One real request, so a credential the account cannot use fails here
|
||||
// rather than partway through the run. Bedrock included: pi resolves the
|
||||
// bearer token from the primed credential and the region from AWS_REGION,
|
||||
// so the probe exercises the same auth path the scan will.
|
||||
const authType = describeAuth(spec.providerId, credentials.baseUrl);
|
||||
logger.info(`Validating ${authType} via pi...`);
|
||||
const probe = await probeCredentialsWithPi(authType, eff.anthropicToken);
|
||||
const probe = await probeCredentialsWithPi(baseModel, modelRuntime, authType);
|
||||
if (isErr(probe)) return probe;
|
||||
logger.info(`${authType} OK`);
|
||||
return ok(undefined);
|
||||
|
||||
@@ -8,7 +8,7 @@ import { fs, path } from 'zx';
|
||||
import { PROMPTS_DIR } from '../paths.js';
|
||||
import { PLAYWRIGHT_SESSION_MAPPING } from '../session-manager.js';
|
||||
import type { ActivityLogger } from '../types/activity-logger.js';
|
||||
import type { Authentication, DistributedConfig, ReportConfig, Rule, VulnClass } from '../types/config.js';
|
||||
import type { Authentication, DistributedConfig, DistributedReportConfig, Rule, VulnClass } from '../types/config.js';
|
||||
import { isGlobPattern } from '../utils/glob.js';
|
||||
import { handlePromptError, PentestError } from './error-handling.js';
|
||||
|
||||
@@ -67,27 +67,76 @@ function renderVulnSummarySubsections(selected: readonly VulnClass[]): string {
|
||||
.join('\n\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Renders the <not_assessed_classes> block. Empty when every class completed.
|
||||
*
|
||||
* A class whose analysis failed was never assessed, so the report must not present its
|
||||
* absence of findings as a clean result. The block is authoritative for that caveat.
|
||||
*/
|
||||
function renderNotAssessedClassesBlock(failed: readonly VulnClass[] = []): string {
|
||||
if (failed.length === 0) {
|
||||
return '';
|
||||
}
|
||||
|
||||
const classes = [...new Set(failed)];
|
||||
const lines: string[] = [
|
||||
'<not_assessed_classes>',
|
||||
'The following vulnerability classes did not complete and were NOT assessed in this run. Treat this list as authoritative for completeness caveats.',
|
||||
'',
|
||||
];
|
||||
|
||||
for (const cls of classes) {
|
||||
const spec = VULN_SUMMARY_SPECS[cls];
|
||||
lines.push(
|
||||
`- ${spec.heading}: analysis did not complete; this class was NOT assessed. Absence of findings here does not indicate the class is clean.`,
|
||||
);
|
||||
}
|
||||
|
||||
lines.push(
|
||||
'',
|
||||
'When writing report_meta.executive_summary, scope any no-findings statement to the classes that were assessed and mention these not-assessed classes. Do not state or imply that the target is clean for these classes.',
|
||||
'</not_assessed_classes>',
|
||||
);
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Which configured filters this run can actually enforce.
|
||||
*
|
||||
* The two ratings are mode-exclusive (see ../collectors/finding-collector.ts): an exploited
|
||||
* finding carries `severity`, an analysed one carries `confidence`. Handing the agent a
|
||||
* threshold for the rating its findings do not have is a directive it cannot honor.
|
||||
*/
|
||||
function applicableFilters(report: DistributedReportConfig | undefined, exploitEnabled: boolean) {
|
||||
return {
|
||||
severity: Boolean(report?.min_severity) && exploitEnabled,
|
||||
confidence: Boolean(report?.min_confidence) && !exploitEnabled,
|
||||
guidance: Boolean(report?.guidance?.trim()),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Renders the top-level <report_filters> block. Empty when no filters are set —
|
||||
* each filter is included only when the operator configured it, so the agent
|
||||
* never sees `none` placeholders or instructions for filters that don't apply.
|
||||
*/
|
||||
function renderReportFiltersBlock(report: ReportConfig | undefined): string {
|
||||
function renderReportFiltersBlock(report: DistributedReportConfig | undefined, exploitEnabled: boolean): string {
|
||||
if (!report) return '';
|
||||
const guidance = report.guidance?.trim();
|
||||
if (!report.min_severity && !report.min_confidence && !guidance) return '';
|
||||
const applies = applicableFilters(report, exploitEnabled);
|
||||
if (!applies.severity && !applies.confidence && !applies.guidance) return '';
|
||||
|
||||
const lines: string[] = [
|
||||
'<report_filters>',
|
||||
'The filters below are user-supplied and binding for this assessment. Honor each strictly when assembling the final report.',
|
||||
'',
|
||||
];
|
||||
if (report.min_severity) {
|
||||
if (applies.severity) {
|
||||
lines.push(
|
||||
`- Minimum severity: ${report.min_severity} — keep only findings rated this severity or higher (scale: low < medium < high < critical).`,
|
||||
);
|
||||
}
|
||||
if (report.min_confidence) {
|
||||
if (applies.confidence) {
|
||||
lines.push(
|
||||
`- Minimum confidence: ${report.min_confidence} — keep only findings rated this confidence or higher (scale: low < medium < high).`,
|
||||
);
|
||||
@@ -106,10 +155,11 @@ function renderReportFiltersBlock(report: ReportConfig | undefined): string {
|
||||
* confidence inline as concrete thresholds; guidance is referenced by pointer
|
||||
* so the actual text only lives in <report_filters>, avoiding double-statement.
|
||||
*/
|
||||
function renderReportFilterRules(report: ReportConfig | undefined): string {
|
||||
function renderReportFilterRules(report: DistributedReportConfig | undefined, exploitEnabled: boolean): string {
|
||||
const applies = applicableFilters(report, exploitEnabled);
|
||||
const drops: string[] = [];
|
||||
if (report?.min_severity) drops.push(`* severity is below ${report.min_severity}`);
|
||||
if (report?.min_confidence) drops.push(`* confidence is below ${report.min_confidence}`);
|
||||
if (applies.severity) drops.push(`* severity is below ${report?.min_severity}`);
|
||||
if (applies.confidence) drops.push(`* confidence is below ${report?.min_confidence}`);
|
||||
if (report?.guidance?.trim()) drops.push('* topic matches an exclusion in the user guidance');
|
||||
if (drops.length === 0) return '';
|
||||
return [' - DROP any `### [TYPE]-VULN-[NUMBER]` finding whose:', ...drops.map((d) => ` ${d}`)].join('\n');
|
||||
@@ -118,6 +168,8 @@ function renderReportFilterRules(report: ReportConfig | undefined): string {
|
||||
interface PromptVariables {
|
||||
webUrl: string;
|
||||
repoPath: string;
|
||||
/** Classes whose analysis did not complete, so the report can mark them not assessed. */
|
||||
failedClasses?: readonly VulnClass[];
|
||||
AUTH_STATE_FILE: string;
|
||||
PLAYWRIGHT_SESSION?: string;
|
||||
}
|
||||
@@ -365,8 +417,20 @@ async function interpolateVariables(
|
||||
vulnClasses.length > 0 ? vulnClasses.join(', ') : 'injection, xss, auth, authz, ssrf',
|
||||
);
|
||||
result = replaceLiteral(result, /{{VULN_SUMMARY_SUBSECTIONS}}/g, renderVulnSummarySubsections(vulnClasses));
|
||||
result = replaceLiteral(
|
||||
result,
|
||||
/{{NOT_ASSESSED_CLASSES}}/g,
|
||||
renderNotAssessedClassesBlock(variables.failedClasses ?? []),
|
||||
);
|
||||
|
||||
const exploitEnabled = config?.exploit ?? true;
|
||||
|
||||
// Drop every block belonging to the mode this run is not in, so the prompt never documents
|
||||
// a field the tool would reject. The backreference pins each match to a closed pair.
|
||||
const droppedMode = exploitEnabled ? 'analysis' : 'exploit';
|
||||
result = result.replace(new RegExp(`<(${droppedMode}_mode_[a-z_]+)>[\\s\\S]*?</\\1>\\n?`, 'g'), '');
|
||||
result = result.replace(/<\/?(?:exploit|analysis)_mode_[a-z_]+>\n?/g, '');
|
||||
|
||||
result = replaceLiteral(result, /{{EXPLOITATION}}/g, exploitEnabled ? 'enabled' : 'disabled');
|
||||
result = replaceLiteral(result, /{{REPORT_VULN_HEADING}}/g, exploitEnabled ? 'Exploitation Evidence' : 'Findings');
|
||||
result = replaceLiteral(
|
||||
@@ -375,8 +439,28 @@ async function interpolateVariables(
|
||||
exploitEnabled ? 'Successfully Exploited Vulnerabilities' : 'Identified Vulnerabilities',
|
||||
);
|
||||
|
||||
result = replaceLiteral(result, /{{REPORT_FILTERS_BLOCK}}/g, renderReportFiltersBlock(config?.report));
|
||||
result = replaceLiteral(result, /{{REPORT_FILTER_RULES}}/g, renderReportFilterRules(config?.report));
|
||||
if (config?.report?.min_severity && !exploitEnabled) {
|
||||
logger.warn(
|
||||
`report.min_severity="${config.report.min_severity}" is ignored when exploit=false: an ` +
|
||||
'analysis-only run rates findings by confidence, not severity. Use report.min_confidence.',
|
||||
);
|
||||
}
|
||||
if (config?.report?.min_confidence && exploitEnabled) {
|
||||
logger.warn(
|
||||
`report.min_confidence="${config.report.min_confidence}" is ignored when exploit=true: an ` +
|
||||
'exploited finding is rated by severity, not confidence. Use report.min_severity.',
|
||||
);
|
||||
}
|
||||
result = replaceLiteral(
|
||||
result,
|
||||
/{{REPORT_FILTERS_BLOCK}}/g,
|
||||
renderReportFiltersBlock(config?.report, exploitEnabled),
|
||||
);
|
||||
result = replaceLiteral(
|
||||
result,
|
||||
/{{REPORT_FILTER_RULES}}/g,
|
||||
renderReportFilterRules(config?.report, exploitEnabled),
|
||||
);
|
||||
|
||||
// Collapse runs of 3+ newlines (left behind by tag-strip and empty-fragment substitutions).
|
||||
result = result.replace(/\n{3,}/g, '\n\n');
|
||||
|
||||
@@ -0,0 +1,289 @@
|
||||
// Copyright (C) 2025 Keygraph, Inc.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License version 3
|
||||
// as published by the Free Software Foundation.
|
||||
|
||||
/**
|
||||
* Deterministic report.json → markdown renderer.
|
||||
*
|
||||
* Converts the structured report output (produced by the finding-collector
|
||||
* tool + set-report-meta CLI) into the same markdown format that the
|
||||
* report agent previously wrote by hand. No LLM in the loop.
|
||||
*/
|
||||
|
||||
import type { AddFindingInput, AdditionalSection, StepItem, StructuredStep } from '../collectors/finding-collector.js';
|
||||
import type { VulnClass } from '../types/config.js';
|
||||
|
||||
// ============================================================================
|
||||
// TYPES
|
||||
// ============================================================================
|
||||
|
||||
export interface ReportMeta {
|
||||
readonly target: string;
|
||||
readonly assessment_date: string;
|
||||
readonly scope: string;
|
||||
readonly executive_summary: string;
|
||||
readonly exploit?: boolean;
|
||||
readonly model?: string;
|
||||
}
|
||||
|
||||
export interface ReportData {
|
||||
readonly report_meta: ReportMeta;
|
||||
readonly findings: readonly AddFindingInput[];
|
||||
// Vuln classes whose pipeline failed and were not assessed this run. Rendered as an explicit
|
||||
// caveat so an un-assessed class is never presented as a clean result.
|
||||
readonly not_assessed?: readonly VulnClass[];
|
||||
}
|
||||
|
||||
// Without this, an analysis-only report reads as though the impact was demonstrated.
|
||||
const ANALYSIS_ONLY_DISCLAIMER = [
|
||||
'> Exploitation was not run for this assessment. Each finding documents a vulnerability',
|
||||
'> identified through analysis; impact is assessed rather than demonstrated, and no live',
|
||||
'> exploitation steps or proof of impact are included.',
|
||||
].join('\n');
|
||||
|
||||
const NOT_ASSESSED_LABELS: Record<VulnClass, string> = {
|
||||
auth: 'Authentication',
|
||||
authz: 'Authorization',
|
||||
xss: 'Cross-Site Scripting (XSS)',
|
||||
injection: 'SQL/Command Injection',
|
||||
ssrf: 'Server-Side Request Forgery (SSRF)',
|
||||
};
|
||||
|
||||
function renderNotAssessedSection(notAssessed: readonly VulnClass[]): string {
|
||||
const lines: string[] = ['## Not Assessed', ''];
|
||||
lines.push(
|
||||
'The following vulnerability classes were NOT assessed in this run because their analysis did ' +
|
||||
'not complete. Absence of findings for these classes does not indicate they are clean — re-run ' +
|
||||
'to assess them:',
|
||||
);
|
||||
lines.push('');
|
||||
for (const cls of notAssessed) {
|
||||
lines.push(`- ${NOT_ASSESSED_LABELS[cls]} — analysis did not complete; not assessed.`);
|
||||
}
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// STEP ITEM RENDERING
|
||||
// ============================================================================
|
||||
|
||||
function renderStepItem(item: StepItem): string {
|
||||
if (item.kind === 'prose') {
|
||||
return item.text;
|
||||
}
|
||||
const lang = item.block.language || '';
|
||||
return `\`\`\`${lang}\n${item.block.content}\n\`\`\``;
|
||||
}
|
||||
|
||||
function renderStepItems(items: readonly StepItem[]): string {
|
||||
return items.map(renderStepItem).join('\n\n');
|
||||
}
|
||||
|
||||
function renderStructuredStep(step: StructuredStep, index: number): string {
|
||||
const lines: string[] = [];
|
||||
const title = step.title ? `**Step ${index + 1}: ${step.title}**` : `**Step ${index + 1}**`;
|
||||
lines.push(title);
|
||||
lines.push('');
|
||||
lines.push(renderStepItems(step.items));
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
function renderAdditionalSection(section: AdditionalSection): string {
|
||||
const lines: string[] = [];
|
||||
lines.push(`#### ${section.heading}`);
|
||||
lines.push('');
|
||||
lines.push(renderStepItems(section.items));
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// FINDING RENDERING
|
||||
// ============================================================================
|
||||
|
||||
function titleCase(s: string): string {
|
||||
return s.charAt(0).toUpperCase() + s.slice(1);
|
||||
}
|
||||
|
||||
function renderFinding(finding: AddFindingInput, exploitEnabled: boolean): string {
|
||||
const lines: string[] = [];
|
||||
|
||||
// Heading
|
||||
lines.push(`### ${finding.finding_id}: ${finding.title}`);
|
||||
lines.push('');
|
||||
|
||||
// Each row is emitted only when the mode that produced the finding supplied its field.
|
||||
lines.push('**Summary:**');
|
||||
if (finding.severity) {
|
||||
lines.push(`- **Severity:** ${titleCase(finding.severity)}`);
|
||||
}
|
||||
if (finding.confidence) {
|
||||
lines.push(`- **Confidence:** ${titleCase(finding.confidence)}`);
|
||||
}
|
||||
lines.push(`- **OWASP:** ${finding.owasp_category}`);
|
||||
lines.push(`- **Vulnerable location:** ${finding.vulnerable_location}`);
|
||||
if (finding.auth_state) {
|
||||
lines.push(`- **Auth state:** ${finding.auth_state}`);
|
||||
}
|
||||
if (exploitEnabled && finding.status) {
|
||||
lines.push(`- **Status:** ${titleCase(finding.status)}`);
|
||||
}
|
||||
if (finding.prerequisites) {
|
||||
lines.push(`- **Prerequisites:** ${finding.prerequisites}`);
|
||||
}
|
||||
lines.push('');
|
||||
|
||||
// Overview
|
||||
lines.push('**Overview:**');
|
||||
lines.push(finding.overview);
|
||||
lines.push('');
|
||||
|
||||
// Impact
|
||||
lines.push('**Impact:**');
|
||||
lines.push(finding.impact);
|
||||
lines.push('');
|
||||
|
||||
if (finding.exploitation_steps && finding.exploitation_steps.length > 0) {
|
||||
lines.push('**Exploitation Steps:**');
|
||||
lines.push('');
|
||||
for (let i = 0; i < finding.exploitation_steps.length; i++) {
|
||||
lines.push(renderStructuredStep(finding.exploitation_steps[i]!, i));
|
||||
lines.push('');
|
||||
}
|
||||
}
|
||||
|
||||
if (finding.proof_of_impact && finding.proof_of_impact.length > 0) {
|
||||
lines.push('**Proof of Impact:**');
|
||||
lines.push('');
|
||||
lines.push(renderStepItems(finding.proof_of_impact));
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
// Remediation
|
||||
lines.push('**Remediation:**');
|
||||
lines.push(finding.remediation);
|
||||
lines.push('');
|
||||
|
||||
// Notes
|
||||
if (finding.notes && finding.notes.length > 0) {
|
||||
lines.push('**Notes:**');
|
||||
lines.push('');
|
||||
lines.push(renderStepItems(finding.notes));
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
// Additional sections
|
||||
if (finding.additional_sections && finding.additional_sections.length > 0) {
|
||||
for (const section of finding.additional_sections) {
|
||||
lines.push(renderAdditionalSection(section));
|
||||
lines.push('');
|
||||
}
|
||||
}
|
||||
|
||||
return lines.join('\n').trimEnd();
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// CATEGORY GROUPING
|
||||
// ============================================================================
|
||||
|
||||
const CATEGORY_ORDER: readonly string[] = ['Injection', 'XSS', 'Authentication', 'SSRF', 'Authorization'];
|
||||
|
||||
function categorySort(a: string, b: string): number {
|
||||
const ai = CATEGORY_ORDER.indexOf(a);
|
||||
const bi = CATEGORY_ORDER.indexOf(b);
|
||||
if (ai !== -1 && bi !== -1) return ai - bi;
|
||||
if (ai !== -1) return -1;
|
||||
if (bi !== -1) return 1;
|
||||
return a.localeCompare(b);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// REPORT RENDERING
|
||||
// ============================================================================
|
||||
|
||||
export function renderReport(data: ReportData): string {
|
||||
const { report_meta, findings, not_assessed = [] } = data;
|
||||
const notAssessedClasses = [...new Set(not_assessed)];
|
||||
const exploitEnabled = report_meta.exploit ?? true;
|
||||
const sections: string[] = [];
|
||||
|
||||
// 1. Executive Summary
|
||||
sections.push('# Security Assessment Report');
|
||||
sections.push('');
|
||||
sections.push('## Executive Summary');
|
||||
sections.push(`- Target: ${report_meta.target}`);
|
||||
sections.push(`- Assessment Date: ${report_meta.assessment_date}`);
|
||||
sections.push(`- Scope: ${report_meta.scope}`);
|
||||
sections.push(`- Exploitation: ${exploitEnabled ? 'enabled' : 'disabled'}`);
|
||||
if (report_meta.model) {
|
||||
sections.push(`- Model: ${report_meta.model}`);
|
||||
}
|
||||
sections.push('');
|
||||
sections.push(report_meta.executive_summary);
|
||||
sections.push('');
|
||||
if (!exploitEnabled) {
|
||||
sections.push(ANALYSIS_ONLY_DISCLAIMER);
|
||||
sections.push('');
|
||||
}
|
||||
|
||||
if (findings.length === 0) {
|
||||
if (notAssessedClasses.length > 0) {
|
||||
// Some classes were not assessed — a blanket "no vulnerabilities" statement would be a false
|
||||
// clean bill of health. Scope the clean statement to assessed classes and list the gaps.
|
||||
sections.push('No vulnerabilities were identified in the classes that were assessed.');
|
||||
sections.push('');
|
||||
sections.push(renderNotAssessedSection(notAssessedClasses));
|
||||
} else {
|
||||
sections.push('No vulnerabilities were identified during this assessment.');
|
||||
}
|
||||
return sections.join('\n').trimEnd() + '\n';
|
||||
}
|
||||
|
||||
if (notAssessedClasses.length > 0) {
|
||||
sections.push(renderNotAssessedSection(notAssessedClasses));
|
||||
sections.push('');
|
||||
}
|
||||
|
||||
// 2. Summary by Vulnerability Type
|
||||
const byCategory = new Map<string, AddFindingInput[]>();
|
||||
for (const f of findings) {
|
||||
const list = byCategory.get(f.category) ?? [];
|
||||
list.push(f);
|
||||
byCategory.set(f.category, list);
|
||||
}
|
||||
|
||||
const sortedCategories = [...byCategory.keys()].sort(categorySort);
|
||||
|
||||
sections.push('## Summary by Vulnerability Type');
|
||||
sections.push('');
|
||||
for (const cat of sortedCategories) {
|
||||
const catFindings = byCategory.get(cat)!;
|
||||
sections.push(`### ${cat}`);
|
||||
sections.push('');
|
||||
for (const f of catFindings) {
|
||||
const suffix = f.severity ? ` (${titleCase(f.severity)})` : '';
|
||||
sections.push(`- **${f.finding_id}:** ${f.title}${suffix}`);
|
||||
}
|
||||
sections.push('');
|
||||
}
|
||||
|
||||
// 3. Per-category finding sections
|
||||
const subheading = exploitEnabled ? 'Successfully Exploited Vulnerabilities' : 'Identified Vulnerabilities';
|
||||
const heading = exploitEnabled ? 'Exploitation Evidence' : 'Findings';
|
||||
|
||||
for (const cat of sortedCategories) {
|
||||
const catFindings = byCategory.get(cat)!;
|
||||
sections.push(`# ${cat} ${heading}`);
|
||||
sections.push('');
|
||||
sections.push(`## ${subheading}`);
|
||||
sections.push('');
|
||||
for (const f of catFindings) {
|
||||
sections.push(renderFinding(f, exploitEnabled));
|
||||
sections.push('');
|
||||
}
|
||||
}
|
||||
|
||||
return sections.join('\n').trimEnd() + '\n';
|
||||
}
|
||||
@@ -5,7 +5,13 @@
|
||||
// as published by the Free Software Foundation.
|
||||
|
||||
import { fs, path } from 'zx';
|
||||
import { ASSEMBLED_REPORT_FILENAME, deliverablesDir, FINAL_REPORT_FILENAME, resolveSessionJsonPath } from '../paths.js';
|
||||
import {
|
||||
ASSEMBLED_REPORT_FILENAME,
|
||||
deliverablesDir,
|
||||
FINAL_REPORT_FILENAME,
|
||||
resolveSessionJsonPath,
|
||||
SARIF_FILENAME,
|
||||
} from '../paths.js';
|
||||
import type { ActivityLogger } from '../types/activity-logger.js';
|
||||
import { ErrorCode } from '../types/errors.js';
|
||||
import { PentestError } from './error-handling.js';
|
||||
@@ -166,9 +172,13 @@ export async function injectModelIntoReport(
|
||||
}
|
||||
|
||||
/**
|
||||
* Surface the assembled report at the run directory's top level as the single
|
||||
* human-facing deliverable, so a customer opening the run folder sees only the
|
||||
* report. The source stays in the deliverables dir (git-checkpointed, used by resume).
|
||||
* Surface the run's deliverables at the run directory's top level, so a customer opening the run
|
||||
* folder sees the report without digging through internals. Sources stay in the deliverables dir
|
||||
* (git-checkpointed, used by resume).
|
||||
*
|
||||
* The SARIF log is surfaced beside it when present, since a CI step consuming it needs a stable
|
||||
* path and cannot be expected to reach into the internals directory. It is absent whenever the
|
||||
* run was analysis-only or `report.sarif` was not enabled.
|
||||
*/
|
||||
export async function copyReportToRunRoot(
|
||||
repoPath: string,
|
||||
@@ -176,14 +186,21 @@ export async function copyReportToRunRoot(
|
||||
runDir: string,
|
||||
logger: ActivityLogger,
|
||||
): Promise<void> {
|
||||
const source = path.join(deliverablesDir(repoPath, deliverablesSubdir), ASSEMBLED_REPORT_FILENAME);
|
||||
const dir = deliverablesDir(repoPath, deliverablesSubdir);
|
||||
|
||||
if (!(await fs.pathExists(source))) {
|
||||
const source = path.join(dir, ASSEMBLED_REPORT_FILENAME);
|
||||
if (await fs.pathExists(source)) {
|
||||
const destination = path.join(runDir, FINAL_REPORT_FILENAME);
|
||||
await fs.copy(source, destination, { overwrite: true });
|
||||
logger.info(`Surfaced report at ${destination}`);
|
||||
} else {
|
||||
logger.warn(`Final report not found, skipping ${FINAL_REPORT_FILENAME}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const destination = path.join(runDir, FINAL_REPORT_FILENAME);
|
||||
await fs.copy(source, destination, { overwrite: true });
|
||||
logger.info(`Surfaced report at ${destination}`);
|
||||
const sarifSource = path.join(dir, SARIF_FILENAME);
|
||||
if (await fs.pathExists(sarifSource)) {
|
||||
const sarifDestination = path.join(runDir, SARIF_FILENAME);
|
||||
await fs.copy(sarifSource, sarifDestination, { overwrite: true });
|
||||
logger.info(`Surfaced SARIF log at ${sarifDestination}`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,293 @@
|
||||
// Copyright (C) 2025 Keygraph, Inc.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License version 3
|
||||
// as published by the Free Software Foundation.
|
||||
|
||||
/** Deterministic report.json to SARIF 2.1.0 renderer, for `exploit=true` runs only. */
|
||||
|
||||
import type { AddFindingInput, CodeLocation } from '../collectors/finding-collector.js';
|
||||
import type { ReportData } from './report-renderer.js';
|
||||
|
||||
export interface SarifOptions {
|
||||
readonly workspaceName: string;
|
||||
}
|
||||
|
||||
interface SarifRule {
|
||||
readonly id: string;
|
||||
readonly name: string;
|
||||
readonly shortDescription: { text: string };
|
||||
readonly fullDescription: { text: string };
|
||||
readonly help: { text: string };
|
||||
readonly properties: { tags: string[] };
|
||||
}
|
||||
|
||||
const TOOL_NAME = 'Shannon';
|
||||
const TOOL_URI = 'https://github.com/KeygraphHQ/shannon';
|
||||
|
||||
/** Taxonomy identity. A reference resolves the component by name, so this must not be reworded. */
|
||||
const OWASP_TAXONOMY_NAME = 'OWASP Top Ten 2025';
|
||||
|
||||
/**
|
||||
* One rule per vulnerability class, keyed by `finding.category`.
|
||||
*
|
||||
* Rule IDs are the unit of alert grouping: renaming one detaches every alert filed under it.
|
||||
* `fullDescription` and `help` describe the class, never the instance, and GitHub requires the
|
||||
* `text` of both.
|
||||
*/
|
||||
const RULES: Record<string, SarifRule> = {
|
||||
Injection: {
|
||||
id: 'shannon/injection',
|
||||
name: 'Injection',
|
||||
shortDescription: { text: 'Injection' },
|
||||
fullDescription: {
|
||||
text: 'Untrusted input reaches an interpreter sink (SQL, OS command, template, file path or deserializer) at a position where it can alter the structure of the statement rather than only supply data.',
|
||||
},
|
||||
help: {
|
||||
text: 'Separate code from data at the sink: bind SQL parameters, pass command arguments as an array, and allowlist file paths. Escaping is a weaker control than parameterisation and breaks whenever the sink context changes.',
|
||||
},
|
||||
properties: { tags: ['security', 'shannon'] },
|
||||
},
|
||||
XSS: {
|
||||
id: 'shannon/xss',
|
||||
name: 'Cross-Site Scripting',
|
||||
shortDescription: { text: 'Cross-Site Scripting' },
|
||||
fullDescription: {
|
||||
text: 'Untrusted input reaches a browser rendering context without the encoding that context requires.',
|
||||
},
|
||||
help: {
|
||||
text: 'Encode at the point of output for the specific context (HTML body, attribute, URL, script or style); no single encoder is correct for all of them. Prefer APIs that treat input as text, such as textContent over innerHTML.',
|
||||
},
|
||||
properties: { tags: ['security', 'shannon'] },
|
||||
},
|
||||
Authentication: {
|
||||
id: 'shannon/auth',
|
||||
name: 'Authentication',
|
||||
shortDescription: { text: 'Authentication' },
|
||||
fullDescription: {
|
||||
text: 'A weakness in credential verification or session lifecycle that lets an attacker assume another identity or retain access they should have lost.',
|
||||
},
|
||||
help: {
|
||||
text: 'Issue a fresh session identifier on every privilege change, set HttpOnly, Secure and SameSite on session cookies, rate-limit credential endpoints, and verify the signature and algorithm of externally issued tokens.',
|
||||
},
|
||||
properties: { tags: ['security', 'shannon'] },
|
||||
},
|
||||
Authorization: {
|
||||
id: 'shannon/authz',
|
||||
name: 'Authorization',
|
||||
shortDescription: { text: 'Authorization' },
|
||||
fullDescription: {
|
||||
text: 'An access control decision is missing, evaluated in the client, or applied at the wrong layer, letting a caller act on resources they do not own.',
|
||||
},
|
||||
help: {
|
||||
text: 'Check ownership and role on the server for every object reference, and enforce it in the data-access layer rather than per route, denying by default. An unguessable identifier is not an access control.',
|
||||
},
|
||||
properties: { tags: ['security', 'shannon'] },
|
||||
},
|
||||
SSRF: {
|
||||
id: 'shannon/ssrf',
|
||||
name: 'Server-Side Request Forgery',
|
||||
shortDescription: { text: 'Server-Side Request Forgery' },
|
||||
fullDescription: {
|
||||
text: 'A server-side request takes its destination from untrusted input, letting an attacker reach hosts the server can see but they cannot.',
|
||||
},
|
||||
help: {
|
||||
text: 'Allowlist destination hosts and schemes, resolve DNS before validating the address so rebinding cannot slip through, and block loopback, private and link-local ranges including cloud metadata. Do not follow redirects.',
|
||||
},
|
||||
properties: { tags: ['security', 'shannon'] },
|
||||
},
|
||||
};
|
||||
|
||||
const CATEGORY_ORDER: readonly string[] = ['Injection', 'XSS', 'Authentication', 'SSRF', 'Authorization'];
|
||||
|
||||
/**
|
||||
* Five severities collapse into SARIF's three usable levels, so `critical` and `high` are
|
||||
* indistinguishable. `security-severity` would separate them but lives on the rule, which would
|
||||
* flatten every finding of a class to one score instead.
|
||||
*/
|
||||
function severityToLevel(severity: string | undefined): string {
|
||||
switch (severity) {
|
||||
case 'critical':
|
||||
case 'high':
|
||||
return 'error';
|
||||
case 'medium':
|
||||
return 'warning';
|
||||
default:
|
||||
return 'note';
|
||||
}
|
||||
}
|
||||
|
||||
function toPhysicalLocation(location: CodeLocation) {
|
||||
const region: Record<string, number> = {};
|
||||
if (location.start_line) region.startLine = location.start_line;
|
||||
if (location.end_line) region.endLine = location.end_line;
|
||||
|
||||
return {
|
||||
physicalLocation: {
|
||||
artifactLocation: { uri: location.file },
|
||||
...(Object.keys(region).length > 0 && { region }),
|
||||
},
|
||||
...(location.symbol && { logicalLocations: [{ name: location.symbol, kind: 'function' }] }),
|
||||
message: { text: location.role },
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fall back to the HTTP entry point when a finding names no file: a result with no location is
|
||||
* silently discarded downstream. No `uriBaseId`, since the path does not resolve in the repo.
|
||||
*/
|
||||
function syntheticLocationFromHttp(finding: AddFindingInput) {
|
||||
if (!finding.http_location) return undefined;
|
||||
let uri = finding.http_location.url;
|
||||
try {
|
||||
const parsed = new URL(finding.http_location.url);
|
||||
uri = `${parsed.pathname}${parsed.hash}`;
|
||||
} catch {}
|
||||
return {
|
||||
physicalLocation: { artifactLocation: { uri } },
|
||||
message: { text: `${finding.http_location.method} ${finding.http_location.url}` },
|
||||
};
|
||||
}
|
||||
|
||||
function buildMessageMarkdown(finding: AddFindingInput): string {
|
||||
const parts = [`**${finding.title}**`, '', finding.overview, '', '**Impact**', '', finding.impact];
|
||||
parts.push('', '**Remediation**', '', finding.remediation);
|
||||
// Exploitation steps and proof of impact are deliberately absent: SARIF has no structural home
|
||||
// for them, and flattening them into prose would imply this file carries the evidence.
|
||||
parts.push('', 'Full exploitation evidence: `Security-Assessment-Report.md`');
|
||||
return parts.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* `owasp_category` is one label, `A05:2025 <separator> Injection`; SARIF wants the id and the name
|
||||
* as separate fields. The enum in ../collectors/finding-collector.ts fixes the shape, so the
|
||||
* separator is dropped by position rather than matched.
|
||||
*/
|
||||
function splitOwaspCategory(label: string): { id: string; name: string } {
|
||||
const [id, , ...nameParts] = label.split(' ');
|
||||
return { id: id ?? label, name: nameParts.join(' ') };
|
||||
}
|
||||
|
||||
interface RenderedResult {
|
||||
readonly result: Record<string, unknown>;
|
||||
readonly category: string;
|
||||
readonly owaspId: string;
|
||||
}
|
||||
|
||||
function renderResult(finding: AddFindingInput, ruleId: string): RenderedResult | null {
|
||||
const codeLocations = finding.code_locations ?? [];
|
||||
const sinks = codeLocations.filter((l) => l.role === 'sink');
|
||||
const related = codeLocations.filter((l) => l.role !== 'sink');
|
||||
const primary = sinks[0] ?? codeLocations[0];
|
||||
|
||||
const locations = primary ? [toPhysicalLocation(primary)] : [syntheticLocationFromHttp(finding)].filter(Boolean);
|
||||
if (locations.length === 0) return null;
|
||||
|
||||
const properties: Record<string, unknown> = { findingId: finding.finding_id };
|
||||
if (finding.http_location?.parameter) properties.parameter = finding.http_location.parameter;
|
||||
if (finding.status) properties.status = finding.status;
|
||||
if (finding.auth_state) properties.authState = finding.auth_state;
|
||||
if (finding.prerequisites) properties.prerequisites = finding.prerequisites;
|
||||
|
||||
const owaspId = splitOwaspCategory(finding.owasp_category).id;
|
||||
|
||||
return {
|
||||
category: finding.category,
|
||||
owaspId,
|
||||
result: {
|
||||
ruleId,
|
||||
level: severityToLevel(finding.severity),
|
||||
message: {
|
||||
text: `${finding.title}. ${finding.overview}`,
|
||||
markdown: buildMessageMarkdown(finding),
|
||||
},
|
||||
locations,
|
||||
...(related.length > 0 && {
|
||||
relatedLocations: related.map((l, i) => ({ id: i + 1, ...toPhysicalLocation(l) })),
|
||||
}),
|
||||
...(finding.http_location && {
|
||||
// No `parameters`: SARIF wants a name-to-value map and the deliverable names only the
|
||||
// parameter, so any value here would be invented. It travels in `properties` instead.
|
||||
webRequest: { method: finding.http_location.method, target: finding.http_location.url },
|
||||
}),
|
||||
taxa: [
|
||||
{
|
||||
id: owaspId,
|
||||
toolComponent: { name: OWASP_TAXONOMY_NAME },
|
||||
},
|
||||
],
|
||||
properties,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Render a SARIF 2.1.0 log from the structured report. Findings with no location are omitted. */
|
||||
export function renderSarif(data: ReportData, options: SarifOptions): string {
|
||||
const { report_meta, findings, not_assessed = [] } = data;
|
||||
|
||||
const rendered: RenderedResult[] = [];
|
||||
|
||||
for (const finding of findings) {
|
||||
const rule = RULES[finding.category];
|
||||
if (!rule) continue;
|
||||
const result = renderResult(finding, rule.id);
|
||||
if (result !== null) rendered.push(result);
|
||||
}
|
||||
|
||||
// Only classes that produced a result are declared, and `ruleIndex` is the position in this list.
|
||||
const usedRules = CATEGORY_ORDER.flatMap((category) => {
|
||||
const rule = RULES[category];
|
||||
if (!rule || !rendered.some((r) => r.category === category)) return [];
|
||||
return [{ category, rule }];
|
||||
});
|
||||
const rules = usedRules.map((u) => u.rule);
|
||||
|
||||
const results: Record<string, unknown>[] = usedRules.flatMap(({ category }, ruleIndex) =>
|
||||
rendered.filter((r) => r.category === category).map((r) => ({ ...r.result, ruleIndex })),
|
||||
);
|
||||
|
||||
const owaspCategories = [...new Set(findings.map((f) => f.owasp_category))]
|
||||
.map(splitOwaspCategory)
|
||||
.filter((c) => rendered.some((r) => r.owaspId === c.id))
|
||||
.sort((a, b) => a.id.localeCompare(b.id));
|
||||
|
||||
const log = {
|
||||
$schema: 'https://json.schemastore.org/sarif-2.1.0.json',
|
||||
version: '2.1.0',
|
||||
runs: [
|
||||
{
|
||||
tool: {
|
||||
driver: {
|
||||
name: TOOL_NAME,
|
||||
informationUri: TOOL_URI,
|
||||
rules,
|
||||
},
|
||||
},
|
||||
// Scoped to the exploit pipeline: an analysis run of the same target has a different
|
||||
// finding population, which would read as alerts resolved.
|
||||
automationDetails: { id: `shannon/exploit/${options.workspaceName}` },
|
||||
invocations: [
|
||||
{
|
||||
// A failed class produced no results; reporting success would read as resolved alerts.
|
||||
executionSuccessful: not_assessed.length === 0,
|
||||
},
|
||||
],
|
||||
...(owaspCategories.length > 0 && {
|
||||
taxonomies: [
|
||||
{
|
||||
name: OWASP_TAXONOMY_NAME,
|
||||
organization: 'OWASP',
|
||||
informationUri: 'https://owasp.org/Top10/',
|
||||
shortDescription: { text: 'OWASP Top Ten 2025 categories.' },
|
||||
taxa: owaspCategories.map((c) => ({ id: c.id, name: c.name })),
|
||||
},
|
||||
],
|
||||
}),
|
||||
results,
|
||||
properties: { target: report_meta.target, assessmentDate: report_meta.assessment_date },
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
return `${JSON.stringify(log, null, 2)}\n`;
|
||||
}
|
||||
@@ -145,7 +145,6 @@ export async function validateAuthentication(input: ValidateAuthInput): Promise<
|
||||
AGENT_NAME,
|
||||
auditSession,
|
||||
logger,
|
||||
'medium',
|
||||
undefined, // callerTools
|
||||
deliverablesSubdir,
|
||||
cancellationSignal,
|
||||
|
||||
Reference in New Issue
Block a user