feat: add preflight validation phase with structured error reporting

- Add preflight activity that validates repo path, config, and credentials before agent execution
- Add formatWorkflowError() with pipe-delimited segments for multi-line log rendering
- Add remediation hints for common failures (auth, billing, config errors)
- Add REPO_NOT_FOUND, AUTH_FAILED, BILLING_ERROR codes with error classification
- Add formatErrorBlock() in WorkflowLogger for indented error display
This commit is contained in:
ajmallesh
2026-02-19 19:09:02 -08:00
parent afa0e9b701
commit c0d46cb6b9
6 changed files with 590 additions and 2 deletions
+19 -1
View File
@@ -311,6 +311,24 @@ export class WorkflowLogger {
await this.logStream.write(line);
}
/**
* Format a pipe-delimited error string into indented multi-line display.
*
* Input: "phase context|ErrorType|message|Hint: ..."
* Output: "Error: phase context\n ErrorType\n ..."
*/
private formatErrorBlock(errorString: string): string {
const segments = errorString.split('|');
const label = 'Error: ';
const indent = ' '.repeat(label.length);
const lines = segments.map((segment, i) =>
i === 0 ? `${label}${segment.trim()}` : `${indent}${segment.trim()}`
);
return lines.join('\n') + '\n';
}
/**
* Log workflow completion with full summary
*/
@@ -330,7 +348,7 @@ export class WorkflowLogger {
await this.logStream.write(`Agents: ${summary.completedAgents.length} completed\n`);
if (summary.error) {
await this.logStream.write(`Error: ${summary.error}\n`);
await this.logStream.write(this.formatErrorBlock(summary.error));
}
await this.logStream.write(`\n`);
+10
View File
@@ -148,6 +148,16 @@ function classifyByErrorCode(
case ErrorCode.AGENT_EXECUTION_FAILED:
return { type: 'AgentExecutionError', retryable: retryableFromError };
// Preflight validation errors
case ErrorCode.REPO_NOT_FOUND:
return { type: 'ConfigurationError', retryable: false };
case ErrorCode.AUTH_FAILED:
return { type: 'AuthenticationError', retryable: false };
case ErrorCode.BILLING_ERROR:
return { type: 'BillingError', retryable: true };
default:
// Unknown code - fall through to string matching
return { type: 'UnknownError', retryable: retryableFromError };
+379
View File
@@ -0,0 +1,379 @@
// Copyright (C) 2025 Keygraph, Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License version 3
// as published by the Free Software Foundation.
/**
* Preflight Validation Service
*
* Runs cheap, fast checks before any agent execution begins.
* Catches configuration and credential problems early, saving
* time and API costs compared to failing mid-pipeline.
*
* Checks run sequentially, cheapest first:
* 1. Repository path exists and contains .git
* 2. Config file parses and validates (if provided)
* 3. Credentials validate (API key, OAuth token, or router mode)
*/
import fs from 'fs/promises';
import { PentestError } from './error-handling.js';
import { ErrorCode } from '../types/errors.js';
import { type Result, ok, err } from '../types/result.js';
import { parseConfig } from '../config-parser.js';
import type { ActivityLogger } from '../types/activity-logger.js';
const VALIDATION_MODEL = 'claude-haiku-3-5-20241022';
const ANTHROPIC_MESSAGES_URL = 'https://api.anthropic.com/v1/messages';
const ANTHROPIC_OAUTH_USAGE_URL = 'https://api.anthropic.com/api/oauth/usage';
const FETCH_TIMEOUT_MS = 30_000;
// === Repository Validation ===
async function validateRepo(
repoPath: string,
logger: ActivityLogger
): Promise<Result<void, PentestError>> {
logger.info('Checking repository path...', { repoPath });
// 1. Check repo directory exists
try {
const stats = await fs.stat(repoPath);
if (!stats.isDirectory()) {
return err(
new PentestError(
`Repository path is not a directory: ${repoPath}`,
'config',
false,
{ repoPath },
ErrorCode.REPO_NOT_FOUND
)
);
}
} catch {
return err(
new PentestError(
`Repository path does not exist: ${repoPath}`,
'config',
false,
{ repoPath },
ErrorCode.REPO_NOT_FOUND
)
);
}
// 2. Check .git directory exists
try {
const gitStats = await fs.stat(`${repoPath}/.git`);
if (!gitStats.isDirectory()) {
return err(
new PentestError(
`Not a git repository (no .git directory): ${repoPath}`,
'config',
false,
{ repoPath },
ErrorCode.REPO_NOT_FOUND
)
);
}
} catch {
return err(
new PentestError(
`Not a git repository (no .git directory): ${repoPath}`,
'config',
false,
{ repoPath },
ErrorCode.REPO_NOT_FOUND
)
);
}
logger.info('Repository path OK');
return ok(undefined);
}
// === Config Validation ===
async function validateConfig(
configPath: string,
logger: ActivityLogger
): Promise<Result<void, PentestError>> {
logger.info('Validating configuration file...', { configPath });
try {
await parseConfig(configPath);
logger.info('Configuration file OK');
return ok(undefined);
} catch (error) {
if (error instanceof PentestError) {
return err(error);
}
const message = error instanceof Error ? error.message : String(error);
return err(
new PentestError(
`Configuration validation failed: ${message}`,
'config',
false,
{ configPath },
ErrorCode.CONFIG_VALIDATION_FAILED
)
);
}
}
// === Credential Validation ===
/**
* Validate a direct Anthropic API key via minimal Messages API call.
* Costs ~$0.000025 (1 input token + 1 output token on Haiku).
*/
async function validateApiKey(
apiKey: string,
logger: ActivityLogger
): Promise<Result<void, PentestError>> {
logger.info('Validating Anthropic API key...');
try {
const response = await fetch(ANTHROPIC_MESSAGES_URL, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'x-api-key': apiKey,
'anthropic-version': '2023-06-01',
},
body: JSON.stringify({
model: VALIDATION_MODEL,
max_tokens: 1,
messages: [{ role: 'user', content: 'hi' }],
}),
signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
});
if (response.ok) {
logger.info('API key OK');
return ok(undefined);
}
let errorBody: string;
try {
errorBody = await response.text();
} catch {
errorBody = '';
}
if (response.status === 401) {
return err(
new PentestError(
`API authentication failed: invalid x-api-key`,
'config',
false,
{ status: response.status },
ErrorCode.AUTH_FAILED
)
);
}
if (response.status === 402 || response.status === 403) {
return err(
new PentestError(
`Anthropic billing error (HTTP ${response.status}): ${errorBody.slice(0, 200)}`,
'billing',
true,
{ status: response.status },
ErrorCode.BILLING_ERROR
)
);
}
if (response.status === 429) {
return err(
new PentestError(
`Spending cap or rate limit reached (HTTP 429)`,
'billing',
true,
{ status: response.status },
ErrorCode.BILLING_ERROR
)
);
}
// Other status codes (5xx, etc) - transient
return err(
new PentestError(
`Anthropic API error (HTTP ${response.status}): ${errorBody.slice(0, 200)}`,
'network',
true,
{ status: response.status }
)
);
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
return err(
new PentestError(
`Failed to reach Anthropic API: ${message}`,
'network',
true,
{ originalError: message }
)
);
}
}
/**
* Validate an OAuth token via the Anthropic usage endpoint.
* Confirms the token is valid and checks quota availability.
*/
async function validateOAuthToken(
token: string,
logger: ActivityLogger
): Promise<Result<void, PentestError>> {
logger.info('Validating OAuth token...');
try {
const response = await fetch(ANTHROPIC_OAUTH_USAGE_URL, {
method: 'GET',
headers: {
Authorization: `Bearer ${token}`,
},
signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
});
if (response.ok) {
logger.info('OAuth token OK');
return ok(undefined);
}
let errorBody: string;
try {
errorBody = await response.text();
} catch {
errorBody = '';
}
if (response.status === 401) {
return err(
new PentestError(
`OAuth token is invalid or expired`,
'config',
false,
{ status: response.status },
ErrorCode.AUTH_FAILED
)
);
}
if (response.status === 403 || response.status === 429) {
return err(
new PentestError(
`OAuth billing/quota error (HTTP ${response.status}): ${errorBody.slice(0, 200)}`,
'billing',
true,
{ status: response.status },
ErrorCode.BILLING_ERROR
)
);
}
return err(
new PentestError(
`OAuth validation error (HTTP ${response.status}): ${errorBody.slice(0, 200)}`,
'network',
true,
{ status: response.status }
)
);
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
return err(
new PentestError(
`Failed to reach Anthropic OAuth endpoint: ${message}`,
'network',
true,
{ originalError: message }
)
);
}
}
/**
* Validate credentials based on detected auth mode.
*
* Auth modes (mutually exclusive):
* - Router mode (ANTHROPIC_BASE_URL set): skip validation, log warning
* - OAuth (CLAUDE_CODE_OAUTH_TOKEN set): validate via /api/oauth/usage
* - API key (ANTHROPIC_API_KEY set): validate via Messages API
* - None: error
*/
async function validateCredentials(
logger: ActivityLogger
): Promise<Result<void, PentestError>> {
// 1. Router mode — can't validate provider keys, just warn
if (process.env.ANTHROPIC_BASE_URL) {
logger.warn('Router mode detected — skipping API credential validation');
return ok(undefined);
}
// 2. OAuth token
const oauthToken = process.env.CLAUDE_CODE_OAUTH_TOKEN;
if (oauthToken) {
return validateOAuthToken(oauthToken, logger);
}
// 3. Direct API key
const apiKey = process.env.ANTHROPIC_API_KEY;
if (apiKey) {
return validateApiKey(apiKey, logger);
}
// 4. No credentials
return err(
new PentestError(
'No API credentials found. Set ANTHROPIC_API_KEY or CLAUDE_CODE_OAUTH_TOKEN in .env',
'config',
false,
{},
ErrorCode.AUTH_FAILED
)
);
}
// === Preflight Orchestrator ===
/**
* Run all preflight checks sequentially (cheapest first).
*
* 1. Repository path exists and contains .git
* 2. Config file parses and validates (if configPath provided)
* 3. Credentials validate (API key, OAuth, or router mode)
*
* Returns on first failure.
*/
export async function runPreflightChecks(
repoPath: string,
configPath: string | undefined,
logger: ActivityLogger
): Promise<Result<void, PentestError>> {
// 1. Repository check (free — filesystem only)
const repoResult = await validateRepo(repoPath, logger);
if (!repoResult.ok) {
return repoResult;
}
// 2. Config check (free — filesystem + CPU)
if (configPath) {
const configResult = await validateConfig(configPath, logger);
if (!configResult.ok) {
return configResult;
}
}
// 3. Credential check (cheap — 1 token or single GET)
const credResult = await validateCredentials(logger);
if (!credResult.ok) {
return credResult;
}
logger.info('All preflight checks passed');
return ok(undefined);
}
+68
View File
@@ -36,6 +36,8 @@ import { AGENTS } from '../session-manager.js';
import { executeGitCommandWithRetry } from '../services/git-manager.js';
import type { ResumeAttempt } from '../audit/metrics-tracker.js';
import { createActivityLogger } from './activity-logger.js';
import { runPreflightChecks } from '../services/preflight.js';
import { isErr } from '../types/result.js';
// Max lengths to prevent Temporal protobuf buffer overflow
const MAX_ERROR_MESSAGE_LENGTH = 2000;
@@ -246,6 +248,72 @@ export async function runReportAgent(input: ActivityInput): Promise<AgentMetrics
return runAgentActivity('report', input);
}
/**
* Preflight validation activity.
*
* Runs cheap checks before any agent execution:
* 1. Repository path exists with .git
* 2. Config file validates (if provided)
* 3. Credential validation (API key, OAuth, or router mode)
*
* NOT using runAgentActivity — preflight doesn't run an agent via the SDK.
*/
export async function runPreflightValidation(input: ActivityInput): Promise<void> {
const startTime = Date.now();
const attemptNumber = Context.current().info.attempt;
const heartbeatInterval = setInterval(() => {
const elapsed = Math.floor((Date.now() - startTime) / 1000);
heartbeat({ phase: 'preflight', elapsedSeconds: elapsed, attempt: attemptNumber });
}, HEARTBEAT_INTERVAL_MS);
try {
const logger = createActivityLogger();
logger.info('Running preflight validation...', { attempt: attemptNumber });
const result = await runPreflightChecks(input.repoPath, input.configPath, logger);
if (isErr(result)) {
const classified = classifyErrorForTemporal(result.error);
const message = truncateErrorMessage(result.error.message);
if (classified.retryable) {
const failure = ApplicationFailure.create({
message,
type: classified.type,
details: [{ phase: 'preflight', attemptNumber, elapsed: Date.now() - startTime }],
});
truncateStackTrace(failure);
throw failure;
} else {
const failure = ApplicationFailure.nonRetryable(message, classified.type, [
{ phase: 'preflight', attemptNumber, elapsed: Date.now() - startTime },
]);
truncateStackTrace(failure);
throw failure;
}
}
logger.info('Preflight validation passed');
} catch (error) {
if (error instanceof ApplicationFailure) {
throw error;
}
const classified = classifyErrorForTemporal(error);
const rawMessage = error instanceof Error ? error.message : String(error);
const message = truncateErrorMessage(rawMessage);
const failure = ApplicationFailure.nonRetryable(message, classified.type, [
{ phase: 'preflight', attemptNumber, elapsed: Date.now() - startTime },
]);
truncateStackTrace(failure);
throw failure;
} finally {
clearInterval(heartbeatInterval);
}
}
/**
* Assemble the final report by concatenating exploitation evidence files.
*/
+109 -1
View File
@@ -86,6 +86,106 @@ const testActs = proxyActivities<typeof activities>({
retry: TESTING_RETRY,
});
// Retry configuration for preflight validation (short timeout, few retries)
const PREFLIGHT_RETRY = {
initialInterval: '10 seconds',
maximumInterval: '1 minute',
backoffCoefficient: 2,
maximumAttempts: 3,
nonRetryableErrorTypes: PRODUCTION_RETRY.nonRetryableErrorTypes,
};
// Activity proxy for preflight validation (short timeout)
const preflightActs = proxyActivities<typeof activities>({
startToCloseTimeout: '2 minutes',
heartbeatTimeout: '2 minutes',
retry: PREFLIGHT_RETRY,
});
/** Maps Temporal error type strings to actionable remediation hints. */
const REMEDIATION_HINTS: Record<string, string> = {
AuthenticationError:
'Verify ANTHROPIC_API_KEY or CLAUDE_CODE_OAUTH_TOKEN in .env is valid and not expired.',
ConfigurationError: 'Check your CONFIG file path and contents.',
BillingError:
'Check your Anthropic billing dashboard. Add credits or wait for spending cap reset.',
GitError: 'Check repository path and git state.',
InvalidTargetError: 'Verify the target URL is correct and accessible.',
PermissionError: 'Check file and network permissions.',
ExecutionLimitError: 'Agent exceeded maximum turns or budget. Review prompt complexity.',
};
/**
* Walk the .cause chain to find the innermost error with a .type property.
* Temporal wraps ApplicationFailure in ActivityFailure — the useful info is inside.
*
* Uses duck-typing because workflow code cannot import @temporalio/activity types.
*/
function unwrapActivityError(error: unknown): {
message: string;
type: string | null;
} {
let current: unknown = error;
let typed: { message: string; type: string } | null = null;
while (current instanceof Error) {
if ('type' in current && typeof (current as { type: unknown }).type === 'string') {
typed = {
message: current.message,
type: (current as { type: string }).type,
};
}
current = (current as { cause?: unknown }).cause;
}
if (typed) {
return typed;
}
return {
message: error instanceof Error ? error.message : String(error),
type: null,
};
}
/**
* Format a structured error string from workflow catch context.
* Segments are delimited by | for multi-line rendering by WorkflowLogger.
*/
function formatWorkflowError(
error: unknown,
currentPhase: string | null,
currentAgent: string | null
): string {
const unwrapped = unwrapActivityError(error);
// Phase context (first segment)
let phaseContext = 'Pipeline failed';
if (currentPhase && currentAgent && currentPhase !== currentAgent) {
phaseContext = `${currentPhase} failed (agent: ${currentAgent})`;
} else if (currentPhase) {
phaseContext = `${currentPhase} failed`;
}
const segments: string[] = [phaseContext];
if (unwrapped.type) {
segments.push(unwrapped.type);
}
// Sanitize pipe characters from message to preserve delimiter format
segments.push(unwrapped.message.replaceAll('|', '/'));
if (unwrapped.type) {
const hint = REMEDIATION_HINTS[unwrapped.type];
if (hint) {
segments.push(`Hint: ${hint}`);
}
}
return segments.join('|');
}
/**
* Compute aggregated metrics from the current pipeline state.
* Called on both success and failure to provide partial metrics.
@@ -298,6 +398,14 @@ export async function pentestPipelineWorkflow(
}
try {
// === Preflight Validation ===
// Quick sanity checks before committing to expensive agent runs.
// NOT using runSequentialPhase — preflight doesn't produce AgentMetrics.
state.currentPhase = 'preflight';
state.currentAgent = null;
await preflightActs.runPreflightValidation(activityInput);
log.info('Preflight validation passed');
// === Phase 1: Pre-Reconnaissance ===
await runSequentialPhase('pre-recon', 'pre-recon', a.runPreReconAgent);
@@ -409,7 +517,7 @@ export async function pentestPipelineWorkflow(
} catch (error) {
state.status = 'failed';
state.failedAgent = state.currentAgent;
state.error = error instanceof Error ? error.message : String(error);
state.error = formatWorkflowError(error, state.currentPhase, state.currentAgent);
state.summary = computeSummary(state);
// Log workflow failure summary
+5
View File
@@ -39,6 +39,11 @@ export enum ErrorCode {
// Validation errors (PentestErrorType: 'validation')
DELIVERABLE_NOT_FOUND = 'DELIVERABLE_NOT_FOUND',
// Preflight validation errors
REPO_NOT_FOUND = 'REPO_NOT_FOUND',
AUTH_FAILED = 'AUTH_FAILED',
BILLING_ERROR = 'BILLING_ERROR',
}
export type PentestErrorType =