mirror of
https://github.com/KeygraphHQ/shannon.git
synced 2026-09-28 04:22:07 +02:00
feat: add npx CLI with monorepo, CI/CD, and ephemeral worker architecture (#256)
* feat: integrate npx CLI, CI/CD, and ephemeral worker architecture
Bring in changes from shannon-npx: npx-distributable CLI package (cli/),
semantic-release CI/CD workflows, ephemeral per-scan worker containers,
TOML config support, setup wizard, and workspace management.
Preserves all shannon-only changes: security hardening (localhost-bound
ports, MCP env allowlist, path traversal guard), updated benchmarks
(XBEN 19/31/35/44), README assets, and prompt injection disclaimer.
Applies security hardening to cli/infra/compose.yml as well.
* refactor: migrate to Turborepo + pnpm + Biome monorepo
Restructure into apps/worker, apps/cli, packages/mcp-server with
Turborepo task orchestration, pnpm workspaces, Biome linting/formatting,
and tsdown CLI bundling.
Key changes:
- src/ -> apps/worker/src/, cli/ -> apps/cli/, mcp-server/ -> packages/mcp-server/
- prompts/ and configs/ moved into apps/worker/
- npm replaced with pnpm, package-lock.json replaced with pnpm-lock.yaml
- Dockerfile updated for pnpm-based builds
- CLI logs command rewritten with chokidar for cross-platform reliability
- Router health checking added for auto-detected router mode
- Centralized path resolution via apps/worker/src/paths.ts
* fix: resolve all biome warnings and formatting issues
- Remove unnecessary non-null assertions where values are guaranteed
- Replace array index access with .at() for safer element retrieval
- Use local variables to avoid repeated process.env lookups
- Replace any types with unknown in functional utilities
- Use nullish coalescing for TOTP hash byte access
- Auto-format security patches to match biome config
* fix: pin pnpm to 10.12.1 in Dockerfile for catalog support
* fix: handle Esc cancellation in Bedrock setup flow
Replace p.group() with individual prompts and per-field cancel checks,
matching the pattern used by all other provider setup flows.
* feat: add optional model customization to Anthropic setup
* fix: resolve Docker bind mount permission errors on Linux
Use entrypoint-based UID remapping instead of --user flag so the
container's pentest user matches the host UID/GID, keeping bind-mounted
volumes writable. Git config moved to --system level to survive remapping.
* fix: show resumed workflow ID in splash screen URL
When resuming a workflow, the Temporal Web UI link pointed to the old
(terminated) workflow ID. Now extracts "New Workflow ID" from the resume
header in workflow.log, falling back to the original ID for fresh scans.
* style: fix biome formatting in docker.ts
* fix: align TypeScript config types with JSON Schema
- SuccessCondition.type: use schema values (url_contains,
element_present, url_equals_exactly, text_contains) instead of
stale values (url, cookie, element, redirect)
- Authentication.login_flow: mark optional to match schema which
does not require it
* feat: mark GitHub release as latest during rollback
* fix: use native ARM64 runners for Docker multi-platform builds
Replace QEMU emulation with parallel native builds using a matrix
strategy (ubuntu-latest for amd64, ubuntu-24.04-arm for arm64).
Each platform pushes by digest, then a merge job creates the
multi-arch manifest list before signing with cosign.
* fix: resolve SessionMutex race condition with 3+ concurrent waiters
* fix: skip POSIX permission check on Windows
writeFileSync mode option is ignored on Windows, so config.toml
gets 0o666 and the guard rejects it.
* fix: resolve unsubstituted placeholders in report prompt
Remove unused {{GITHUB_URL}} placeholder and wire up {{AUTH_CONTEXT}}
with structured auth context (login type, username, URL, MFA status).
* fix: remove duplicate environment gate from merge-docker job
Move DOCKERHUB_USERNAME from vars to secrets so merge-docker can access
credentials without its own environment scope. This eliminates the
redundant double approval since build-docker already gates on
release-publish.
* fix: replace POSIX sleep binary with cross-platform async sleep
execFileSync('sleep') is unavailable on Windows. Use node:timers/promises
setTimeout instead, making ensureInfra async.
* fix: use session.json for workflow ID on resume instead of parsing workflow.log
On resume, workflow.log already exists with stale headers from the
previous run. The CLI poll found '====' immediately and extracted the
old workflow ID, producing a wrong Temporal Web UI URL.
Read the workflow ID from session.json instead — the worker writes
resume attempts there atomically. For fresh runs, poll until
originalWorkflowId appears. For resumes, poll until a new
resumeAttempts entry is appended.
* feat: add custom base URL support for Anthropic-compatible proxies
Support ANTHROPIC_BASE_URL + ANTHROPIC_AUTH_TOKEN to route SDK requests
through LiteLLM or any Anthropic-compatible proxy. Adds TUI wizard
option, TOML config mapping, credential validation, and preflight
endpoint reachability check via SDK query.
* fix: remove environment gates and add NPM_TOKEN to publish step
* feat: add beta release and rollback workflows with cosign signing
* fix: remove redundant checkout and pnpm steps from beta release workflow
* docs: normalize README commands to mode-neutral shorthand
Add a substitution note after Quick Start sections so all subsequent
examples use bare `shannon` instead of mixing `./shannon` and
`npx @keygraph/shannon`. Mode-specific commands (build, update,
uninstall) get inline annotations. Also fixes a broken command in the
Custom Base URL section.
* fix: remove redundant `update` command
Image is already auto-pulled by `ensureImage()` during `start` when the
pinned version tag is missing locally. Manual `update` was unnecessary.
* docs: add CLI package README stub
* docs: update README setup instructions for dual CLI modes
* docs: update announcement banner to npx availability
* feat: migrate from MCP tools to CLI based tools (#252)
* feat: migrate from MCP tools to CLI tools
* fix: restore browser action emoji formatters for CLI output
Adapt formatBrowserAction for playwright-cli commands, replacing the old
mcp__playwright__browser_* tool name matching removed during migration.
* fix: mount credential file to fixed container path for Vertex AI
GOOGLE_APPLICATION_CREDENTIALS was forwarded as-is to the container,
causing the relative host path to resolve against the repo mount
instead of the credentials mount. Now both local and npx modes mount
the resolved file to /app/credentials/google-sa-key.json and rewrite
the env var to match.
* feat: add git awareness and optional description field to config
* fix: drop redundant --ipc host flag from worker container
* fix: align announcement banner URL with main branch
* feat: add target URL reachability preflight check (#254)
* Moving asset benchmark graph image to this folder
* Move benchmark results to benchmark repo
Windows Defender flags exploit code in the pentest reports as false positives, forcing every Windows user to add a Defender exclusion just to clone Shannon.
* Updated README
* fix: case-insensitive grep for semantic-release version probe
* fix: harden supply chain security (#255)
* fix: patch smol-toml and tsdown vulnerabilities
Update smol-toml 1.6.0→1.6.1 (DoS via recursive comment parsing) and
tsdown 0.21.2→0.21.5 (picomatch ReDoS + method injection).
* fix: pin all unpinned dependency versions in Dockerfile
Pins subfinder v2.13.0, WhatWeb v0.6.3 (switched from git clone to
release tarball), schemathesis 4.13.0, addressable 2.8.9,
claude-code 2.1.84, and playwright-cli 0.1.1 for reproducible builds.
* fix: pin GitHub Actions to commit SHAs for supply chain security
* fix: pin GitHub Actions to commit SHAs in beta and rollback workflows
This commit is contained in:
ezl-keygraph
@@ -0,0 +1,282 @@
|
||||
// Copyright (C) 2025 Keygraph, Inc.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License version 3
|
||||
// as published by the Free Software Foundation.
|
||||
|
||||
/**
|
||||
* Audit Session - Main Facade
|
||||
*
|
||||
* Coordinates logger, metrics tracker, and concurrency control for comprehensive
|
||||
* crash-safe audit logging.
|
||||
*/
|
||||
|
||||
import { PentestError } from '../services/error-handling.js';
|
||||
import { ErrorCode } from '../types/errors.js';
|
||||
import type { AgentEndResult } from '../types/index.js';
|
||||
import { SessionMutex } from '../utils/concurrency.js';
|
||||
import { formatTimestamp } from '../utils/formatting.js';
|
||||
import { AgentLogger } from './logger.js';
|
||||
import { MetricsTracker } from './metrics-tracker.js';
|
||||
import { initializeAuditStructure, type SessionMetadata } from './utils.js';
|
||||
import { type AgentLogDetails, WorkflowLogger, type WorkflowSummary } from './workflow-logger.js';
|
||||
|
||||
// Global mutex instance
|
||||
const sessionMutex = new SessionMutex();
|
||||
|
||||
/**
|
||||
* AuditSession - Main audit system facade
|
||||
*/
|
||||
export class AuditSession {
|
||||
private sessionMetadata: SessionMetadata;
|
||||
private sessionId: string;
|
||||
private metricsTracker: MetricsTracker;
|
||||
private workflowLogger: WorkflowLogger;
|
||||
private currentLogger: AgentLogger | null = null;
|
||||
private currentAgentName: string | null = null;
|
||||
private initialized: boolean = false;
|
||||
|
||||
constructor(sessionMetadata: SessionMetadata) {
|
||||
this.sessionMetadata = sessionMetadata;
|
||||
this.sessionId = sessionMetadata.id;
|
||||
|
||||
// Validate required fields
|
||||
if (!this.sessionId) {
|
||||
throw new PentestError(
|
||||
'sessionMetadata.id is required',
|
||||
'config',
|
||||
false,
|
||||
{ field: 'sessionMetadata.id' },
|
||||
ErrorCode.CONFIG_VALIDATION_FAILED,
|
||||
);
|
||||
}
|
||||
if (!this.sessionMetadata.webUrl) {
|
||||
throw new PentestError(
|
||||
'sessionMetadata.webUrl is required',
|
||||
'config',
|
||||
false,
|
||||
{ field: 'sessionMetadata.webUrl' },
|
||||
ErrorCode.CONFIG_VALIDATION_FAILED,
|
||||
);
|
||||
}
|
||||
|
||||
// Components
|
||||
this.metricsTracker = new MetricsTracker(sessionMetadata);
|
||||
this.workflowLogger = new WorkflowLogger(sessionMetadata);
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize audit session (creates directories, session.json)
|
||||
* Idempotent and race-safe
|
||||
*
|
||||
* @param workflowId - Optional workflow ID for tracking original or resume workflows
|
||||
*/
|
||||
async initialize(workflowId?: string): Promise<void> {
|
||||
if (this.initialized) {
|
||||
return; // Already initialized
|
||||
}
|
||||
|
||||
// Create directory structure
|
||||
await initializeAuditStructure(this.sessionMetadata);
|
||||
|
||||
// Initialize metrics tracker (loads or creates session.json)
|
||||
await this.metricsTracker.initialize(workflowId);
|
||||
|
||||
// Initialize workflow logger with actual Temporal workflow ID
|
||||
await this.workflowLogger.initialize(workflowId);
|
||||
|
||||
this.initialized = true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure initialized (helper for lazy initialization)
|
||||
*/
|
||||
private async ensureInitialized(): Promise<void> {
|
||||
if (!this.initialized) {
|
||||
await this.initialize();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start agent execution
|
||||
*/
|
||||
async startAgent(agentName: string, promptContent: string, attemptNumber: number = 1): Promise<void> {
|
||||
await this.ensureInitialized();
|
||||
|
||||
// 1. Save prompt snapshot (only on first attempt)
|
||||
if (attemptNumber === 1) {
|
||||
await AgentLogger.savePrompt(this.sessionMetadata, agentName, promptContent);
|
||||
}
|
||||
|
||||
// 2. Create and initialize the per-agent logger
|
||||
this.currentAgentName = agentName;
|
||||
this.currentLogger = new AgentLogger(this.sessionMetadata, agentName, attemptNumber);
|
||||
await this.currentLogger.initialize();
|
||||
|
||||
// 3. Start metrics timer
|
||||
this.metricsTracker.startAgent(agentName, attemptNumber);
|
||||
|
||||
// 4. Log start event to both agent log and workflow log
|
||||
await this.currentLogger.logEvent('agent_start', {
|
||||
agentName,
|
||||
attemptNumber,
|
||||
timestamp: formatTimestamp(),
|
||||
});
|
||||
|
||||
await this.workflowLogger.logAgent(agentName, 'start', { attemptNumber });
|
||||
}
|
||||
|
||||
/**
|
||||
* Log event during agent execution
|
||||
*/
|
||||
async logEvent(eventType: string, eventData: unknown): Promise<void> {
|
||||
if (!this.currentLogger) {
|
||||
throw new PentestError(
|
||||
'No active logger. Call startAgent() first.',
|
||||
'validation',
|
||||
false,
|
||||
{},
|
||||
ErrorCode.AGENT_EXECUTION_FAILED,
|
||||
);
|
||||
}
|
||||
|
||||
// Log to agent-specific log file (JSON format)
|
||||
await this.currentLogger.logEvent(eventType, eventData);
|
||||
|
||||
// Also log to unified workflow log (human-readable format)
|
||||
const data = eventData as Record<string, unknown>;
|
||||
const agentName = this.currentAgentName || 'unknown';
|
||||
switch (eventType) {
|
||||
case 'tool_start':
|
||||
await this.workflowLogger.logToolStart(agentName, String(data.toolName || ''), data.parameters);
|
||||
break;
|
||||
case 'llm_response':
|
||||
await this.workflowLogger.logLlmResponse(agentName, Number(data.turn || 0), String(data.content || ''));
|
||||
break;
|
||||
// tool_end and error events are intentionally not logged to workflow log
|
||||
// to reduce noise - the agent completion message captures the outcome
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* End agent execution (mutex-protected)
|
||||
*/
|
||||
async endAgent(agentName: string, result: AgentEndResult): Promise<void> {
|
||||
// 1. Finalize agent log and close the stream
|
||||
if (this.currentLogger) {
|
||||
await this.currentLogger.logEvent('agent_end', {
|
||||
agentName,
|
||||
success: result.success,
|
||||
duration_ms: result.duration_ms,
|
||||
cost_usd: result.cost_usd,
|
||||
timestamp: formatTimestamp(),
|
||||
});
|
||||
|
||||
await this.currentLogger.close();
|
||||
this.currentLogger = null;
|
||||
}
|
||||
|
||||
// 2. Log completion to the unified workflow log
|
||||
this.currentAgentName = null;
|
||||
|
||||
const agentLogDetails: AgentLogDetails = {
|
||||
attemptNumber: result.attemptNumber,
|
||||
duration_ms: result.duration_ms,
|
||||
cost_usd: result.cost_usd,
|
||||
success: result.success,
|
||||
...(result.error !== undefined && { error: result.error }),
|
||||
};
|
||||
await this.workflowLogger.logAgent(agentName, 'end', agentLogDetails);
|
||||
|
||||
// 3. Acquire mutex before touching session.json
|
||||
const unlock = await sessionMutex.lock(this.sessionId);
|
||||
try {
|
||||
// 4. Reload-then-write inside mutex to prevent lost updates during parallel phases
|
||||
await this.metricsTracker.reload();
|
||||
await this.metricsTracker.endAgent(agentName, result);
|
||||
} finally {
|
||||
unlock();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update session status
|
||||
*/
|
||||
async updateSessionStatus(status: 'in-progress' | 'completed' | 'failed'): Promise<void> {
|
||||
await this.ensureInitialized();
|
||||
|
||||
const unlock = await sessionMutex.lock(this.sessionId);
|
||||
try {
|
||||
await this.metricsTracker.reload();
|
||||
await this.metricsTracker.updateSessionStatus(status);
|
||||
} finally {
|
||||
unlock();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current metrics (read-only)
|
||||
*/
|
||||
async getMetrics(): Promise<unknown> {
|
||||
await this.ensureInitialized();
|
||||
return this.metricsTracker.getMetrics();
|
||||
}
|
||||
|
||||
/**
|
||||
* Log phase start to unified workflow log
|
||||
*/
|
||||
async logPhaseStart(phase: string): Promise<void> {
|
||||
await this.ensureInitialized();
|
||||
await this.workflowLogger.logPhase(phase, 'start');
|
||||
}
|
||||
|
||||
/**
|
||||
* Log phase completion to unified workflow log
|
||||
*/
|
||||
async logPhaseComplete(phase: string): Promise<void> {
|
||||
await this.ensureInitialized();
|
||||
await this.workflowLogger.logPhase(phase, 'complete');
|
||||
}
|
||||
|
||||
/**
|
||||
* Log workflow completion to unified workflow log
|
||||
*/
|
||||
async logWorkflowComplete(summary: WorkflowSummary): Promise<void> {
|
||||
await this.ensureInitialized();
|
||||
await this.workflowLogger.logWorkflowComplete(summary);
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a resume attempt to the session
|
||||
* Call this when a workflow is resuming from an existing workspace
|
||||
*
|
||||
* @param workflowId - The new workflow ID for this resume attempt
|
||||
* @param terminatedWorkflows - IDs of workflows that were terminated
|
||||
* @param checkpointHash - Git checkpoint hash that was restored
|
||||
*/
|
||||
async addResumeAttempt(workflowId: string, terminatedWorkflows: string[], checkpointHash?: string): Promise<void> {
|
||||
await this.ensureInitialized();
|
||||
|
||||
const unlock = await sessionMutex.lock(this.sessionId);
|
||||
try {
|
||||
await this.metricsTracker.reload();
|
||||
await this.metricsTracker.addResumeAttempt(workflowId, terminatedWorkflows, checkpointHash);
|
||||
} finally {
|
||||
unlock();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Log resume header to workflow.log
|
||||
* Call this when a workflow is resuming to add a visual separator
|
||||
*/
|
||||
async logResumeHeader(resumeInfo: {
|
||||
previousWorkflowId: string;
|
||||
newWorkflowId: string;
|
||||
checkpointHash: string;
|
||||
completedAgents: string[];
|
||||
}): Promise<void> {
|
||||
await this.ensureInitialized();
|
||||
await this.workflowLogger.logResumeHeader(resumeInfo);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
// Copyright (C) 2025 Keygraph, Inc.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License version 3
|
||||
// as published by the Free Software Foundation.
|
||||
|
||||
/**
|
||||
* Unified Audit & Metrics System
|
||||
*
|
||||
* Public API for the audit system. Provides crash-safe, append-only logging
|
||||
* and comprehensive metrics tracking for Shannon penetration testing sessions.
|
||||
*
|
||||
* IMPORTANT: Session objects must have an 'id' field (NOT 'sessionId')
|
||||
* Example: { id: "uuid", webUrl: "...", repoPath: "..." }
|
||||
*
|
||||
* @module audit
|
||||
*/
|
||||
|
||||
export { AuditSession } from './audit-session.js';
|
||||
@@ -0,0 +1,127 @@
|
||||
// Copyright (C) 2025 Keygraph, Inc.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License version 3
|
||||
// as published by the Free Software Foundation.
|
||||
|
||||
/**
|
||||
* LogStream - Stream composition utility for append-only logging
|
||||
*
|
||||
* Encapsulates the common stream management pattern used by AgentLogger
|
||||
* and WorkflowLogger: opening streams in append mode, handling backpressure,
|
||||
* and proper cleanup.
|
||||
*/
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { ensureDirectory } from '../utils/file-io.js';
|
||||
|
||||
/**
|
||||
* LogStream - Manages a single append-only log file stream
|
||||
*/
|
||||
export class LogStream {
|
||||
private readonly filePath: string;
|
||||
private stream: fs.WriteStream | null = null;
|
||||
private _isOpen: boolean = false;
|
||||
|
||||
constructor(filePath: string) {
|
||||
this.filePath = filePath;
|
||||
}
|
||||
|
||||
/**
|
||||
* Open the stream for writing (creates parent directories, opens in append mode)
|
||||
*/
|
||||
async open(): Promise<void> {
|
||||
if (this._isOpen) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Ensure parent directory exists
|
||||
await ensureDirectory(path.dirname(this.filePath));
|
||||
|
||||
// Create write stream in append mode
|
||||
this.stream = fs.createWriteStream(this.filePath, {
|
||||
flags: 'a',
|
||||
encoding: 'utf8',
|
||||
autoClose: true,
|
||||
});
|
||||
|
||||
// Handle stream errors to prevent crashes (log and mark closed)
|
||||
this.stream.on('error', (err) => {
|
||||
console.error(`LogStream error for ${this.filePath}:`, err.message);
|
||||
this._isOpen = false;
|
||||
});
|
||||
|
||||
this._isOpen = true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Write text to the stream with backpressure handling
|
||||
*/
|
||||
async write(text: string): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
if (!this._isOpen || !this.stream) {
|
||||
reject(new Error('LogStream not open'));
|
||||
return;
|
||||
}
|
||||
|
||||
const stream = this.stream;
|
||||
let drainHandler: (() => void) | null = null;
|
||||
|
||||
const cleanup = () => {
|
||||
if (drainHandler) {
|
||||
stream.removeListener('drain', drainHandler);
|
||||
drainHandler = null;
|
||||
}
|
||||
};
|
||||
|
||||
const needsDrain = !stream.write(text, 'utf8', (error) => {
|
||||
cleanup();
|
||||
if (error) {
|
||||
reject(error);
|
||||
} else if (!needsDrain) {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
|
||||
if (needsDrain) {
|
||||
drainHandler = () => {
|
||||
cleanup();
|
||||
resolve();
|
||||
};
|
||||
stream.once('drain', drainHandler);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Close the stream (flush and close)
|
||||
*/
|
||||
async close(): Promise<void> {
|
||||
if (!this._isOpen || !this.stream) {
|
||||
return;
|
||||
}
|
||||
|
||||
return new Promise((resolve) => {
|
||||
this.stream?.end(() => {
|
||||
this._isOpen = false;
|
||||
this.stream = null;
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the stream is currently open
|
||||
*/
|
||||
get isOpen(): boolean {
|
||||
return this._isOpen;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the file path this stream writes to
|
||||
*/
|
||||
get path(): string {
|
||||
return this.filePath;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
// Copyright (C) 2025 Keygraph, Inc.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License version 3
|
||||
// as published by the Free Software Foundation.
|
||||
|
||||
/**
|
||||
* Append-Only Agent Logger
|
||||
*
|
||||
* Provides crash-safe, append-only logging for agent execution.
|
||||
* Uses LogStream for stream management with backpressure handling.
|
||||
*/
|
||||
|
||||
import { atomicWrite } from '../utils/file-io.js';
|
||||
import { formatTimestamp } from '../utils/formatting.js';
|
||||
import { LogStream } from './log-stream.js';
|
||||
import { generateLogPath, generatePromptPath, type SessionMetadata } from './utils.js';
|
||||
|
||||
interface LogEvent {
|
||||
type: string;
|
||||
timestamp: string;
|
||||
data: unknown;
|
||||
}
|
||||
|
||||
/**
|
||||
* AgentLogger - Manages append-only logging for a single agent execution
|
||||
*/
|
||||
export class AgentLogger {
|
||||
private readonly sessionMetadata: SessionMetadata;
|
||||
private readonly agentName: string;
|
||||
private readonly attemptNumber: number;
|
||||
private readonly timestamp: number;
|
||||
private readonly logStream: LogStream;
|
||||
|
||||
constructor(sessionMetadata: SessionMetadata, agentName: string, attemptNumber: number) {
|
||||
this.sessionMetadata = sessionMetadata;
|
||||
this.agentName = agentName;
|
||||
this.attemptNumber = attemptNumber;
|
||||
this.timestamp = Date.now();
|
||||
|
||||
const logPath = generateLogPath(sessionMetadata, agentName, this.timestamp, attemptNumber);
|
||||
this.logStream = new LogStream(logPath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the log stream (creates file and opens stream)
|
||||
*/
|
||||
async initialize(): Promise<void> {
|
||||
if (this.logStream.isOpen) {
|
||||
return; // Already initialized
|
||||
}
|
||||
|
||||
await this.logStream.open();
|
||||
|
||||
// Write header
|
||||
await this.writeHeader();
|
||||
}
|
||||
|
||||
/**
|
||||
* Write header to log file
|
||||
*/
|
||||
private async writeHeader(): Promise<void> {
|
||||
const header = [
|
||||
`========================================`,
|
||||
`Agent: ${this.agentName}`,
|
||||
`Attempt: ${this.attemptNumber}`,
|
||||
`Started: ${formatTimestamp(this.timestamp)}`,
|
||||
`Session: ${this.sessionMetadata.id}`,
|
||||
`Web URL: ${this.sessionMetadata.webUrl}`,
|
||||
`========================================\n`,
|
||||
].join('\n');
|
||||
|
||||
return this.logStream.write(header);
|
||||
}
|
||||
|
||||
/**
|
||||
* Log an event (tool_start, tool_end, llm_response, etc.)
|
||||
* Events are logged as JSON for parseability
|
||||
*/
|
||||
async logEvent(eventType: string, eventData: unknown): Promise<void> {
|
||||
const event: LogEvent = {
|
||||
type: eventType,
|
||||
timestamp: formatTimestamp(),
|
||||
data: eventData,
|
||||
};
|
||||
|
||||
const eventLine = `${JSON.stringify(event)}\n`;
|
||||
return this.logStream.write(eventLine);
|
||||
}
|
||||
|
||||
/**
|
||||
* Close the log stream
|
||||
*/
|
||||
async close(): Promise<void> {
|
||||
return this.logStream.close();
|
||||
}
|
||||
|
||||
/**
|
||||
* Save prompt snapshot to prompts directory
|
||||
* Static method - doesn't require logger instance
|
||||
*/
|
||||
static async savePrompt(sessionMetadata: SessionMetadata, agentName: string, promptContent: string): Promise<void> {
|
||||
const promptPath = generatePromptPath(sessionMetadata, agentName);
|
||||
|
||||
// Create header with metadata
|
||||
const header = [
|
||||
`# Prompt Snapshot: ${agentName}`,
|
||||
``,
|
||||
`**Session:** ${sessionMetadata.id}`,
|
||||
`**Web URL:** ${sessionMetadata.webUrl}`,
|
||||
`**Saved:** ${formatTimestamp()}`,
|
||||
``,
|
||||
`---`,
|
||||
``,
|
||||
].join('\n');
|
||||
|
||||
const fullContent = header + promptContent;
|
||||
|
||||
// Use atomic write for safety
|
||||
await atomicWrite(promptPath, fullContent);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,380 @@
|
||||
// Copyright (C) 2025 Keygraph, Inc.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License version 3
|
||||
// as published by the Free Software Foundation.
|
||||
|
||||
/**
|
||||
* Metrics Tracker
|
||||
*
|
||||
* Manages session.json with comprehensive timing, cost, and validation metrics.
|
||||
* Tracks attempt-level data for complete forensic trail.
|
||||
*/
|
||||
|
||||
import { PentestError } from '../services/error-handling.js';
|
||||
import { AGENT_PHASE_MAP, type PhaseName } from '../session-manager.js';
|
||||
import { ErrorCode } from '../types/errors.js';
|
||||
import type { AgentEndResult, AgentName } from '../types/index.js';
|
||||
import { atomicWrite, fileExists, readJson } from '../utils/file-io.js';
|
||||
import { calculatePercentage, formatTimestamp } from '../utils/formatting.js';
|
||||
import { generateSessionJsonPath, type SessionMetadata } from './utils.js';
|
||||
|
||||
interface AttemptData {
|
||||
attempt_number: number;
|
||||
duration_ms: number;
|
||||
cost_usd: number;
|
||||
success: boolean;
|
||||
timestamp: string;
|
||||
model?: string | undefined;
|
||||
error?: string | undefined;
|
||||
}
|
||||
|
||||
interface AgentAuditMetrics {
|
||||
status: 'in-progress' | 'success' | 'failed';
|
||||
attempts: AttemptData[];
|
||||
final_duration_ms: number;
|
||||
total_cost_usd: number;
|
||||
model?: string | undefined;
|
||||
checkpoint?: string | undefined;
|
||||
}
|
||||
|
||||
interface PhaseMetrics {
|
||||
duration_ms: number;
|
||||
duration_percentage: number;
|
||||
cost_usd: number;
|
||||
agent_count: number;
|
||||
}
|
||||
|
||||
export interface ResumeAttempt {
|
||||
workflowId: string;
|
||||
timestamp: string;
|
||||
terminatedPrevious?: string;
|
||||
resumedFromCheckpoint?: string;
|
||||
}
|
||||
|
||||
interface SessionData {
|
||||
session: {
|
||||
id: string;
|
||||
webUrl: string;
|
||||
repoPath?: string;
|
||||
status: 'in-progress' | 'completed' | 'failed';
|
||||
createdAt: string;
|
||||
completedAt?: string;
|
||||
originalWorkflowId?: string; // First workflow that created this workspace
|
||||
resumeAttempts?: ResumeAttempt[]; // Track all resume attempts
|
||||
};
|
||||
metrics: {
|
||||
total_duration_ms: number;
|
||||
total_cost_usd: number;
|
||||
phases: Record<string, PhaseMetrics>;
|
||||
agents: Record<string, AgentAuditMetrics>;
|
||||
};
|
||||
}
|
||||
|
||||
interface ActiveTimer {
|
||||
startTime: number;
|
||||
attemptNumber: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* MetricsTracker - Manages metrics for a session
|
||||
*/
|
||||
export class MetricsTracker {
|
||||
private sessionMetadata: SessionMetadata;
|
||||
private sessionJsonPath: string;
|
||||
private data: SessionData | null = null;
|
||||
private activeTimers: Map<string, ActiveTimer> = new Map();
|
||||
|
||||
constructor(sessionMetadata: SessionMetadata) {
|
||||
this.sessionMetadata = sessionMetadata;
|
||||
this.sessionJsonPath = generateSessionJsonPath(sessionMetadata);
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize session.json (idempotent)
|
||||
*
|
||||
* @param workflowId - Optional workflow ID to set as originalWorkflowId for new sessions
|
||||
*/
|
||||
async initialize(workflowId?: string): Promise<void> {
|
||||
// Check if session.json already exists
|
||||
const exists = await fileExists(this.sessionJsonPath);
|
||||
|
||||
if (exists) {
|
||||
// Load existing data
|
||||
this.data = await readJson<SessionData>(this.sessionJsonPath);
|
||||
} else {
|
||||
// Create new session.json
|
||||
this.data = this.createInitialData(workflowId);
|
||||
await this.save();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create initial session.json structure
|
||||
*
|
||||
* @param workflowId - Optional workflow ID to set as originalWorkflowId
|
||||
*/
|
||||
private createInitialData(workflowId?: string): SessionData {
|
||||
const sessionData: SessionData = {
|
||||
session: {
|
||||
id: this.sessionMetadata.id,
|
||||
webUrl: this.sessionMetadata.webUrl,
|
||||
status: 'in-progress',
|
||||
createdAt: (this.sessionMetadata as { createdAt?: string }).createdAt || formatTimestamp(),
|
||||
resumeAttempts: [],
|
||||
},
|
||||
metrics: {
|
||||
total_duration_ms: 0,
|
||||
total_cost_usd: 0,
|
||||
phases: {}, // Phase-level aggregations
|
||||
agents: {}, // Agent-level metrics
|
||||
},
|
||||
};
|
||||
|
||||
// Set originalWorkflowId if provided (for new workspaces)
|
||||
if (workflowId) {
|
||||
sessionData.session.originalWorkflowId = workflowId;
|
||||
}
|
||||
|
||||
// Only add repoPath if it exists
|
||||
if (this.sessionMetadata.repoPath) {
|
||||
sessionData.session.repoPath = this.sessionMetadata.repoPath;
|
||||
}
|
||||
return sessionData;
|
||||
}
|
||||
|
||||
/**
|
||||
* Start tracking an agent execution
|
||||
*/
|
||||
startAgent(agentName: string, attemptNumber: number): void {
|
||||
this.activeTimers.set(agentName, {
|
||||
startTime: Date.now(),
|
||||
attemptNumber,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* End agent execution and update metrics
|
||||
*/
|
||||
async endAgent(agentName: string, result: AgentEndResult): Promise<void> {
|
||||
if (!this.data) {
|
||||
throw new PentestError(
|
||||
'MetricsTracker not initialized',
|
||||
'validation',
|
||||
false,
|
||||
{},
|
||||
ErrorCode.AGENT_EXECUTION_FAILED,
|
||||
);
|
||||
}
|
||||
|
||||
// 1. Initialize agent metrics if first time seeing this agent
|
||||
const existingAgent = this.data.metrics.agents[agentName];
|
||||
const agent = existingAgent ?? {
|
||||
status: 'in-progress' as const,
|
||||
attempts: [],
|
||||
final_duration_ms: 0,
|
||||
total_cost_usd: 0,
|
||||
};
|
||||
this.data.metrics.agents[agentName] = agent;
|
||||
|
||||
// 2. Build attempt record with optional model/error fields
|
||||
const attempt: AttemptData = {
|
||||
attempt_number: result.attemptNumber,
|
||||
duration_ms: result.duration_ms,
|
||||
cost_usd: result.cost_usd,
|
||||
success: result.success,
|
||||
timestamp: formatTimestamp(),
|
||||
};
|
||||
|
||||
if (result.model) {
|
||||
attempt.model = result.model;
|
||||
}
|
||||
|
||||
if (result.error) {
|
||||
attempt.error = result.error;
|
||||
}
|
||||
|
||||
// 3. Append attempt to history
|
||||
agent.attempts.push(attempt);
|
||||
|
||||
// 4. Recalculate total cost across all attempts (includes failures)
|
||||
agent.total_cost_usd = agent.attempts.reduce((sum, a) => sum + a.cost_usd, 0);
|
||||
|
||||
// 5. Update agent status based on outcome
|
||||
if (result.success) {
|
||||
agent.status = 'success';
|
||||
agent.final_duration_ms = result.duration_ms;
|
||||
|
||||
// 6. Attach model and checkpoint metadata on success
|
||||
if (result.model) {
|
||||
agent.model = result.model;
|
||||
}
|
||||
|
||||
if (result.checkpoint) {
|
||||
agent.checkpoint = result.checkpoint;
|
||||
}
|
||||
} else {
|
||||
if (result.isFinalAttempt) {
|
||||
agent.status = 'failed';
|
||||
}
|
||||
}
|
||||
|
||||
// 7. Clear active timer
|
||||
this.activeTimers.delete(agentName);
|
||||
|
||||
// 8. Recalculate phase and session-level aggregations
|
||||
this.recalculateAggregations();
|
||||
|
||||
// 9. Persist to session.json
|
||||
await this.save();
|
||||
}
|
||||
|
||||
/**
|
||||
* Update session status
|
||||
*/
|
||||
async updateSessionStatus(status: 'in-progress' | 'completed' | 'failed'): Promise<void> {
|
||||
if (!this.data) return;
|
||||
|
||||
this.data.session.status = status;
|
||||
|
||||
if (status === 'completed' || status === 'failed') {
|
||||
this.data.session.completedAt = formatTimestamp();
|
||||
}
|
||||
|
||||
await this.save();
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a resume attempt to the session
|
||||
*
|
||||
* @param workflowId - The new workflow ID for this resume attempt
|
||||
* @param terminatedWorkflows - IDs of workflows that were terminated
|
||||
* @param checkpointHash - Git checkpoint hash that was restored
|
||||
*/
|
||||
async addResumeAttempt(workflowId: string, terminatedWorkflows: string[], checkpointHash?: string): Promise<void> {
|
||||
if (!this.data) {
|
||||
throw new PentestError(
|
||||
'MetricsTracker not initialized',
|
||||
'validation',
|
||||
false,
|
||||
{},
|
||||
ErrorCode.AGENT_EXECUTION_FAILED,
|
||||
);
|
||||
}
|
||||
|
||||
// Ensure originalWorkflowId is set (backfill if missing from old sessions)
|
||||
if (!this.data.session.originalWorkflowId) {
|
||||
this.data.session.originalWorkflowId = this.data.session.id;
|
||||
}
|
||||
|
||||
// Ensure resumeAttempts array exists
|
||||
if (!this.data.session.resumeAttempts) {
|
||||
this.data.session.resumeAttempts = [];
|
||||
}
|
||||
|
||||
// Add new resume attempt
|
||||
const resumeAttempt: ResumeAttempt = {
|
||||
workflowId,
|
||||
timestamp: formatTimestamp(),
|
||||
};
|
||||
|
||||
if (terminatedWorkflows.length > 0) {
|
||||
resumeAttempt.terminatedPrevious = terminatedWorkflows.join(',');
|
||||
}
|
||||
|
||||
if (checkpointHash) {
|
||||
resumeAttempt.resumedFromCheckpoint = checkpointHash;
|
||||
}
|
||||
|
||||
this.data.session.resumeAttempts.push(resumeAttempt);
|
||||
|
||||
await this.save();
|
||||
}
|
||||
|
||||
/**
|
||||
* Recalculate aggregations (total duration, total cost, phases)
|
||||
*/
|
||||
private recalculateAggregations(): void {
|
||||
if (!this.data) return;
|
||||
|
||||
const agents = this.data.metrics.agents;
|
||||
|
||||
// Only count successful agents
|
||||
const successfulAgents = Object.entries(agents).filter(([, data]) => data.status === 'success');
|
||||
|
||||
// Calculate total duration and cost
|
||||
const totalDuration = successfulAgents.reduce((sum, [, data]) => sum + data.final_duration_ms, 0);
|
||||
|
||||
const totalCost = successfulAgents.reduce((sum, [, data]) => sum + data.total_cost_usd, 0);
|
||||
|
||||
this.data.metrics.total_duration_ms = totalDuration;
|
||||
this.data.metrics.total_cost_usd = totalCost;
|
||||
|
||||
// Calculate phase-level metrics
|
||||
this.data.metrics.phases = this.calculatePhaseMetrics(successfulAgents);
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate phase-level metrics
|
||||
*/
|
||||
private calculatePhaseMetrics(successfulAgents: Array<[string, AgentAuditMetrics]>): Record<string, PhaseMetrics> {
|
||||
const phases: Record<PhaseName, AgentAuditMetrics[]> = {
|
||||
'pre-recon': [],
|
||||
recon: [],
|
||||
'vulnerability-analysis': [],
|
||||
exploitation: [],
|
||||
reporting: [],
|
||||
};
|
||||
|
||||
// Group agents by phase using imported AGENT_PHASE_MAP
|
||||
for (const [agentName, agentData] of successfulAgents) {
|
||||
const phase = AGENT_PHASE_MAP[agentName as AgentName];
|
||||
if (phase) {
|
||||
phases[phase].push(agentData);
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate metrics per phase
|
||||
const phaseMetrics: Record<string, PhaseMetrics> = {};
|
||||
// biome-ignore lint/style/noNonNullAssertion: called from recalculateAggregations which guards this.data
|
||||
const totalDuration = this.data!.metrics.total_duration_ms;
|
||||
|
||||
for (const [phaseName, agentList] of Object.entries(phases)) {
|
||||
if (agentList.length === 0) continue;
|
||||
|
||||
const phaseDuration = agentList.reduce((sum, agent) => sum + agent.final_duration_ms, 0);
|
||||
const phaseCost = agentList.reduce((sum, agent) => sum + agent.total_cost_usd, 0);
|
||||
|
||||
phaseMetrics[phaseName] = {
|
||||
duration_ms: phaseDuration,
|
||||
duration_percentage: calculatePercentage(phaseDuration, totalDuration),
|
||||
cost_usd: phaseCost,
|
||||
agent_count: agentList.length,
|
||||
};
|
||||
}
|
||||
|
||||
return phaseMetrics;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current metrics
|
||||
*/
|
||||
getMetrics(): SessionData {
|
||||
return JSON.parse(JSON.stringify(this.data)) as SessionData;
|
||||
}
|
||||
|
||||
/**
|
||||
* Save metrics to session.json (atomic write)
|
||||
*/
|
||||
private async save(): Promise<void> {
|
||||
if (!this.data) return;
|
||||
await atomicWrite(this.sessionJsonPath, this.data);
|
||||
}
|
||||
|
||||
/**
|
||||
* Reload metrics from disk
|
||||
*/
|
||||
async reload(): Promise<void> {
|
||||
this.data = await readJson<SessionData>(this.sessionJsonPath);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
// Copyright (C) 2025 Keygraph, Inc.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License version 3
|
||||
// as published by the Free Software Foundation.
|
||||
|
||||
/**
|
||||
* Audit System Utilities
|
||||
*
|
||||
* Core utility functions for path generation, atomic writes, and formatting.
|
||||
* All functions are pure and crash-safe.
|
||||
*/
|
||||
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { WORKSPACES_DIR } from '../paths.js';
|
||||
import { ensureDirectory } from '../utils/file-io.js';
|
||||
|
||||
export type { SessionMetadata } from '../types/audit.js';
|
||||
|
||||
import type { SessionMetadata } from '../types/audit.js';
|
||||
|
||||
/**
|
||||
* Extract and sanitize hostname from URL for use in identifiers
|
||||
*/
|
||||
export function sanitizeHostname(url: string): string {
|
||||
return new URL(url).hostname.replace(/[^a-zA-Z0-9-]/g, '-');
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate standardized session identifier from workflow ID
|
||||
* Workflow IDs already contain hostname, so we use them directly
|
||||
*/
|
||||
export function generateSessionIdentifier(sessionMetadata: SessionMetadata): string {
|
||||
return sessionMetadata.id;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate path to audit log directory for a session
|
||||
* Uses custom outputPath if provided, otherwise defaults to WORKSPACES_DIR
|
||||
*/
|
||||
export function generateAuditPath(sessionMetadata: SessionMetadata): string {
|
||||
const sessionIdentifier = generateSessionIdentifier(sessionMetadata);
|
||||
const baseDir = sessionMetadata.outputPath || WORKSPACES_DIR;
|
||||
return path.join(baseDir, sessionIdentifier);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate path to agent log file
|
||||
*/
|
||||
export function generateLogPath(
|
||||
sessionMetadata: SessionMetadata,
|
||||
agentName: string,
|
||||
timestamp: number,
|
||||
attemptNumber: number,
|
||||
): string {
|
||||
const auditPath = generateAuditPath(sessionMetadata);
|
||||
const filename = `${timestamp}_${agentName}_attempt-${attemptNumber}.log`;
|
||||
return path.join(auditPath, 'agents', filename);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate path to prompt snapshot file
|
||||
*/
|
||||
export function generatePromptPath(sessionMetadata: SessionMetadata, agentName: string): string {
|
||||
const auditPath = generateAuditPath(sessionMetadata);
|
||||
return path.join(auditPath, 'prompts', `${agentName}.md`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate path to session.json file
|
||||
*/
|
||||
export function generateSessionJsonPath(sessionMetadata: SessionMetadata): string {
|
||||
const auditPath = generateAuditPath(sessionMetadata);
|
||||
return path.join(auditPath, 'session.json');
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate path to workflow.log file
|
||||
*/
|
||||
export function generateWorkflowLogPath(sessionMetadata: SessionMetadata): string {
|
||||
const auditPath = generateAuditPath(sessionMetadata);
|
||||
return path.join(auditPath, 'workflow.log');
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize audit directory structure for a session
|
||||
* Creates: workspaces/{sessionId}/, agents/, prompts/, deliverables/
|
||||
*/
|
||||
export async function initializeAuditStructure(sessionMetadata: SessionMetadata): Promise<void> {
|
||||
const auditPath = generateAuditPath(sessionMetadata);
|
||||
const agentsPath = path.join(auditPath, 'agents');
|
||||
const promptsPath = path.join(auditPath, 'prompts');
|
||||
const deliverablesPath = path.join(auditPath, 'deliverables');
|
||||
|
||||
await ensureDirectory(auditPath);
|
||||
await ensureDirectory(agentsPath);
|
||||
await ensureDirectory(promptsPath);
|
||||
await ensureDirectory(deliverablesPath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Copy deliverable files from repo to workspaces for self-contained audit trail.
|
||||
* No-ops if source directory doesn't exist. Idempotent and parallel-safe.
|
||||
*/
|
||||
export async function copyDeliverablesToAudit(sessionMetadata: SessionMetadata, repoPath: string): Promise<void> {
|
||||
const sourceDir = path.join(repoPath, 'deliverables');
|
||||
const destDir = path.join(generateAuditPath(sessionMetadata), 'deliverables');
|
||||
|
||||
let entries: string[];
|
||||
try {
|
||||
entries = await fs.readdir(sourceDir);
|
||||
} catch {
|
||||
// Source directory doesn't exist yet — nothing to copy
|
||||
return;
|
||||
}
|
||||
|
||||
await ensureDirectory(destDir);
|
||||
|
||||
for (const entry of entries) {
|
||||
const sourcePath = path.join(sourceDir, entry);
|
||||
const destPath = path.join(destDir, entry);
|
||||
|
||||
// Only copy files, skip subdirectories
|
||||
const stat = await fs.stat(sourcePath);
|
||||
if (stat.isFile()) {
|
||||
await fs.copyFile(sourcePath, destPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,374 @@
|
||||
// Copyright (C) 2025 Keygraph, Inc.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License version 3
|
||||
// as published by the Free Software Foundation.
|
||||
|
||||
/**
|
||||
* Workflow Logger
|
||||
*
|
||||
* Provides a unified, human-readable log file per workflow.
|
||||
* Optimized for `tail -f` viewing during concurrent workflow execution.
|
||||
*/
|
||||
|
||||
import fs from 'node:fs/promises';
|
||||
import { formatDuration, formatTimestamp } from '../utils/formatting.js';
|
||||
import { LogStream } from './log-stream.js';
|
||||
import { generateWorkflowLogPath, type SessionMetadata } from './utils.js';
|
||||
|
||||
export interface AgentLogDetails {
|
||||
attemptNumber?: number;
|
||||
duration_ms?: number;
|
||||
cost_usd?: number;
|
||||
success?: boolean;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface AgentMetricsSummary {
|
||||
durationMs: number;
|
||||
costUsd: number | null;
|
||||
}
|
||||
|
||||
export interface WorkflowSummary {
|
||||
status: 'completed' | 'failed';
|
||||
totalDurationMs: number;
|
||||
totalCostUsd: number;
|
||||
completedAgents: string[];
|
||||
agentMetrics: Record<string, AgentMetricsSummary>;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* WorkflowLogger - Manages the unified workflow log file
|
||||
*/
|
||||
export class WorkflowLogger {
|
||||
private readonly sessionMetadata: SessionMetadata;
|
||||
private readonly logStream: LogStream;
|
||||
private workflowId: string | undefined;
|
||||
|
||||
constructor(sessionMetadata: SessionMetadata) {
|
||||
this.sessionMetadata = sessionMetadata;
|
||||
const logPath = generateWorkflowLogPath(sessionMetadata);
|
||||
this.logStream = new LogStream(logPath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the log stream (creates file and writes header)
|
||||
*/
|
||||
async initialize(workflowId?: string): Promise<void> {
|
||||
if (workflowId) {
|
||||
this.workflowId = workflowId;
|
||||
}
|
||||
|
||||
if (this.logStream.isOpen) {
|
||||
return;
|
||||
}
|
||||
|
||||
await this.logStream.open();
|
||||
|
||||
// Write header only if file is new (empty)
|
||||
const stats = await fs.stat(this.logStream.path).catch(() => null);
|
||||
if (!stats || stats.size === 0) {
|
||||
await this.writeHeader();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Write header to log file
|
||||
*/
|
||||
private async writeHeader(): Promise<void> {
|
||||
const header = [
|
||||
`================================================================================`,
|
||||
`Shannon Pentest - Workflow Log`,
|
||||
`================================================================================`,
|
||||
`Workflow ID: ${this.workflowId ?? this.sessionMetadata.id}`,
|
||||
`Target URL: ${this.sessionMetadata.webUrl}`,
|
||||
`Started: ${formatTimestamp()}`,
|
||||
`================================================================================`,
|
||||
``,
|
||||
].join('\n');
|
||||
|
||||
return this.logStream.write(header);
|
||||
}
|
||||
|
||||
/**
|
||||
* Write resume header to log file when workflow is resumed
|
||||
*/
|
||||
async logResumeHeader(resumeInfo: {
|
||||
previousWorkflowId: string;
|
||||
newWorkflowId: string;
|
||||
checkpointHash: string;
|
||||
completedAgents: string[];
|
||||
}): Promise<void> {
|
||||
await this.ensureInitialized();
|
||||
|
||||
const header = [
|
||||
``,
|
||||
`================================================================================`,
|
||||
`RESUMED`,
|
||||
`================================================================================`,
|
||||
`Previous Workflow ID: ${resumeInfo.previousWorkflowId}`,
|
||||
`New Workflow ID: ${resumeInfo.newWorkflowId}`,
|
||||
`Resumed At: ${formatTimestamp()}`,
|
||||
`Checkpoint: ${resumeInfo.checkpointHash}`,
|
||||
`Completed: ${resumeInfo.completedAgents.length} agents (${resumeInfo.completedAgents.join(', ')})`,
|
||||
`================================================================================`,
|
||||
``,
|
||||
].join('\n');
|
||||
|
||||
return this.logStream.write(header);
|
||||
}
|
||||
|
||||
/**
|
||||
* Format timestamp for log line (local time, human readable)
|
||||
*/
|
||||
private formatLogTime(): string {
|
||||
const now = new Date();
|
||||
return now.toISOString().replace('T', ' ').slice(0, 19);
|
||||
}
|
||||
|
||||
/**
|
||||
* Log a phase transition event
|
||||
*/
|
||||
async logPhase(phase: string, event: 'start' | 'complete'): Promise<void> {
|
||||
await this.ensureInitialized();
|
||||
|
||||
const action = event === 'start' ? 'Starting' : 'Completed';
|
||||
const line = `[${this.formatLogTime()}] [PHASE] ${action}: ${phase}\n`;
|
||||
|
||||
// Add blank line before phase start for readability
|
||||
if (event === 'start') {
|
||||
await this.logStream.write('\n');
|
||||
}
|
||||
|
||||
await this.logStream.write(line);
|
||||
}
|
||||
|
||||
/**
|
||||
* Log an agent event
|
||||
*/
|
||||
async logAgent(agentName: string, event: 'start' | 'end', details?: AgentLogDetails): Promise<void> {
|
||||
await this.ensureInitialized();
|
||||
|
||||
let message: string;
|
||||
|
||||
if (event === 'start') {
|
||||
const attempt = details?.attemptNumber ?? 1;
|
||||
message = `${agentName}: Starting (attempt ${attempt})`;
|
||||
} else {
|
||||
const parts: string[] = [`${agentName}:`];
|
||||
|
||||
if (details?.success === false) {
|
||||
parts.push('Failed');
|
||||
if (details?.error) {
|
||||
parts.push(`- ${details.error}`);
|
||||
}
|
||||
} else {
|
||||
parts.push('Completed');
|
||||
}
|
||||
|
||||
if (details?.duration_ms !== undefined) {
|
||||
parts.push(`(${formatDuration(details.duration_ms)}`);
|
||||
if (details?.cost_usd !== undefined) {
|
||||
parts.push(`$${details.cost_usd.toFixed(2)})`);
|
||||
} else {
|
||||
parts.push(')');
|
||||
}
|
||||
}
|
||||
|
||||
message = parts.join(' ');
|
||||
}
|
||||
|
||||
const line = `[${this.formatLogTime()}] [AGENT] ${message}\n`;
|
||||
await this.logStream.write(line);
|
||||
}
|
||||
|
||||
/**
|
||||
* Log a general event
|
||||
*/
|
||||
async logEvent(eventType: string, message: string): Promise<void> {
|
||||
await this.ensureInitialized();
|
||||
|
||||
const line = `[${this.formatLogTime()}] [${eventType.toUpperCase()}] ${message}\n`;
|
||||
await this.logStream.write(line);
|
||||
}
|
||||
|
||||
/**
|
||||
* Log an error
|
||||
*/
|
||||
async logError(error: Error, context?: string): Promise<void> {
|
||||
await this.ensureInitialized();
|
||||
|
||||
const contextStr = context ? ` (${context})` : '';
|
||||
const line = `[${this.formatLogTime()}] [ERROR] ${error.message}${contextStr}\n`;
|
||||
await this.logStream.write(line);
|
||||
}
|
||||
|
||||
/**
|
||||
* Truncate string to max length with ellipsis
|
||||
*/
|
||||
private truncate(str: string, maxLen: number): string {
|
||||
if (str.length <= maxLen) return str;
|
||||
return `${str.slice(0, maxLen - 3)}...`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format tool parameters for human-readable display
|
||||
*/
|
||||
private formatToolParams(toolName: string, params: unknown): string {
|
||||
if (!params || typeof params !== 'object') {
|
||||
return '';
|
||||
}
|
||||
|
||||
const p = params as Record<string, unknown>;
|
||||
|
||||
// Tool-specific formatting for common tools
|
||||
switch (toolName) {
|
||||
case 'Bash':
|
||||
if (p.command) {
|
||||
return this.truncate(String(p.command).replace(/\n/g, ' '), 100);
|
||||
}
|
||||
break;
|
||||
case 'Read':
|
||||
if (p.file_path) {
|
||||
return String(p.file_path);
|
||||
}
|
||||
break;
|
||||
case 'Write':
|
||||
if (p.file_path) {
|
||||
return String(p.file_path);
|
||||
}
|
||||
break;
|
||||
case 'Edit':
|
||||
if (p.file_path) {
|
||||
return String(p.file_path);
|
||||
}
|
||||
break;
|
||||
case 'Glob':
|
||||
if (p.pattern) {
|
||||
return String(p.pattern);
|
||||
}
|
||||
break;
|
||||
case 'Grep':
|
||||
if (p.pattern) {
|
||||
const path = p.path ? ` in ${p.path}` : '';
|
||||
return `"${this.truncate(String(p.pattern), 50)}"${path}`;
|
||||
}
|
||||
break;
|
||||
case 'WebFetch':
|
||||
if (p.url) {
|
||||
return String(p.url);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
// Default: show first string-valued param truncated
|
||||
for (const [key, val] of Object.entries(p)) {
|
||||
if (typeof val === 'string' && val.length > 0) {
|
||||
return `${key}=${this.truncate(val, 60)}`;
|
||||
}
|
||||
}
|
||||
|
||||
return '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Log tool start event
|
||||
*/
|
||||
async logToolStart(agentName: string, toolName: string, parameters: unknown): Promise<void> {
|
||||
await this.ensureInitialized();
|
||||
|
||||
const params = this.formatToolParams(toolName, parameters);
|
||||
const paramStr = params ? `: ${params}` : '';
|
||||
const line = `[${this.formatLogTime()}] [${agentName}] [TOOL] ${toolName}${paramStr}\n`;
|
||||
await this.logStream.write(line);
|
||||
}
|
||||
|
||||
/**
|
||||
* Log LLM response
|
||||
*/
|
||||
async logLlmResponse(agentName: string, turn: number, content: string): Promise<void> {
|
||||
await this.ensureInitialized();
|
||||
|
||||
// Show full content, replacing newlines with escaped version for single-line output
|
||||
const escaped = content.replace(/\n/g, '\\n');
|
||||
const line = `[${this.formatLogTime()}] [${agentName}] [LLM] Turn ${turn}: ${escaped}\n`;
|
||||
await this.logStream.write(line);
|
||||
}
|
||||
|
||||
/**
|
||||
* Format a pipe-delimited error string into indented multi-line display.
|
||||
*
|
||||
* Input: "phase context|ErrorType|message|Hint: ..."
|
||||
* Output: "Error: phase context\n ErrorType\n ..."
|
||||
*/
|
||||
private formatErrorBlock(errorString: string): string {
|
||||
const segments = errorString.split('|');
|
||||
const label = 'Error: ';
|
||||
const indent = ' '.repeat(label.length);
|
||||
|
||||
const lines = segments.map((segment, i) => (i === 0 ? `${label}${segment.trim()}` : `${indent}${segment.trim()}`));
|
||||
|
||||
return `${lines.join('\n')}\n`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Log workflow completion with full summary
|
||||
*/
|
||||
async logWorkflowComplete(summary: WorkflowSummary): Promise<void> {
|
||||
await this.ensureInitialized();
|
||||
|
||||
const status = summary.status === 'completed' ? 'COMPLETED' : 'FAILED';
|
||||
|
||||
const lines: string[] = [
|
||||
'',
|
||||
'================================================================================',
|
||||
`Workflow ${status}`,
|
||||
'────────────────────────────────────────',
|
||||
`Workflow ID: ${this.workflowId ?? this.sessionMetadata.id}`,
|
||||
`Status: ${summary.status}`,
|
||||
`Duration: ${formatDuration(summary.totalDurationMs)}`,
|
||||
`Total Cost: $${summary.totalCostUsd.toFixed(4)}`,
|
||||
`Agents: ${summary.completedAgents.length} completed`,
|
||||
];
|
||||
|
||||
if (summary.error) {
|
||||
lines.push(this.formatErrorBlock(summary.error).trimEnd());
|
||||
}
|
||||
|
||||
lines.push('');
|
||||
lines.push('Agent Breakdown:');
|
||||
|
||||
for (const agentName of summary.completedAgents) {
|
||||
const metrics = summary.agentMetrics[agentName];
|
||||
if (metrics) {
|
||||
const duration = formatDuration(metrics.durationMs);
|
||||
const cost = metrics.costUsd !== null ? `$${metrics.costUsd.toFixed(4)}` : 'N/A';
|
||||
lines.push(` - ${agentName} (${duration}, ${cost})`);
|
||||
} else {
|
||||
lines.push(` - ${agentName}`);
|
||||
}
|
||||
}
|
||||
|
||||
lines.push('================================================================================');
|
||||
|
||||
// Single atomic write to prevent interleaved/duplicate output in log tailers
|
||||
await this.logStream.write(`${lines.join('\n')}\n`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure initialized (helper for lazy initialization)
|
||||
*/
|
||||
private async ensureInitialized(): Promise<void> {
|
||||
if (!this.logStream.isOpen) {
|
||||
await this.initialize();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Close the log stream
|
||||
*/
|
||||
async close(): Promise<void> {
|
||||
return this.logStream.close();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user