diff --git a/CLAUDE.md b/CLAUDE.md index a799ae82..f11a2f5c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -87,7 +87,7 @@ pnpm biome:fix # Auto-fix lint, format, and import sorting **Monorepo tooling:** pnpm workspaces, Turborepo for task orchestration, Biome for linting/formatting. TypeScript compiler options shared via `tsconfig.base.json` at the root. All packages extend it, overriding only `rootDir` and `outDir`. Shared devDependencies (`typescript`, `@types/node`, `turbo`, `@biomejs/biome`) are hoisted to the root workspace. -**Options:** `-c ` (YAML config), `--models-config ` (pi `models.json` defining models pi's catalogue lacks), `-o ` (output directory), `-w ` (named workspace; auto-resumes if exists), `--validate-auth` (run preflight and auth validation only, then stop; no pentest or report; requires a fresh workspace and an `authentication` block in the config), `--pipeline-testing` (minimal prompts, 10s retries), `--keep-container` (preserve worker container after exit for log inspection), `--yes`/`-y` (skip the confirmation prompt on `stop`; required for non-interactive use; `reset` requires a typed `confirm` and cannot be skipped) +**Options:** `-c ` (YAML config), `--models-config ` (pi `models.json` defining models pi's catalogue lacks), `-o ` (output directory), `-w ` (named workspace; auto-resumes if exists), `--validate-auth` (run preflight and auth validation only, then stop; no pentest or report; requires a fresh workspace and an `authentication` block in the config), `--validate-model` (run the preflight model checks only — credential/registry resolution plus the exploit-readiness probe for cyber-gated providers — then stop; no pentest or report; requires a fresh workspace; needs no config; mutually exclusive with `--validate-auth`), `--pipeline-testing` (minimal prompts, 10s retries), `--keep-container` (preserve worker container after exit for log inspection), `--yes`/`-y` (skip the confirmation prompt on `stop`; required for non-interactive use; `reset` requires a typed `confirm` and cannot be skipped) ## Architecture diff --git a/apps/cli/src/commands/start.ts b/apps/cli/src/commands/start.ts index 82bbba5d..54bb8aed 100644 --- a/apps/cli/src/commands/start.ts +++ b/apps/cli/src/commands/start.ts @@ -46,6 +46,7 @@ export interface StartArgs { keepContainer: boolean; follow: boolean; authOnly: boolean; + validateModel: boolean; version: string; } @@ -63,6 +64,8 @@ interface LaunchState { readonly customer_output_path?: string; /** True when the workspace was created by an auth-validation run; such a workspace is not a scan. */ readonly auth_only?: boolean; + /** True when the workspace was created by a model-validation run; such a workspace is not a scan. */ + readonly model_only?: boolean; } export interface WorkspaceLaunchDecision { @@ -128,7 +131,7 @@ function readLaunchState(filePath: string): LaunchState { // Unknown keys mean a newer release wrote this workspace; refuse rather than half-read it. const keys = Object.keys(value).sort(); const keysAreValid = keys.every( - (key) => key === 'auth_only' || key === 'customer_output_path' || key === 'schema_version', + (key) => key === 'auth_only' || key === 'model_only' || key === 'customer_output_path' || key === 'schema_version', ); const customerPath = value.customer_output_path; const pathIsValid = @@ -136,13 +139,22 @@ function readLaunchState(filePath: string): LaunchState { (typeof customerPath === 'string' && path.isAbsolute(customerPath) && path.resolve(customerPath) === customerPath); const authOnly = value.auth_only; const authOnlyIsValid = authOnly === undefined || typeof authOnly === 'boolean'; - if (value.schema_version !== LAUNCH_STATE_SCHEMA_VERSION || !keysAreValid || !pathIsValid || !authOnlyIsValid) { + const modelOnly = value.model_only; + const modelOnlyIsValid = modelOnly === undefined || typeof modelOnly === 'boolean'; + if ( + value.schema_version !== LAUNCH_STATE_SCHEMA_VERSION || + !keysAreValid || + !pathIsValid || + !authOnlyIsValid || + !modelOnlyIsValid + ) { fail(NEWER_RELEASE_MESSAGE); } return { schema_version: LAUNCH_STATE_SCHEMA_VERSION, ...(typeof customerPath === 'string' && { customer_output_path: customerPath }), ...(authOnly === true && { auth_only: true }), + ...(modelOnly === true && { model_only: true }), }; } @@ -158,6 +170,7 @@ export function classifyWorkspaceLaunch( expectedUrl: string, requestedOutputDir: string | undefined, requestedAuthOnly: boolean, + requestedModelOnly: boolean, ): WorkspaceLaunchDecision { const sessionPath = resolveRunFile(workspacePath, 'session.json'); const sessionExists = fs.existsSync(sessionPath); @@ -177,6 +190,11 @@ export function classifyWorkspaceLaunch( 'This workspace was created to validate authentication only, so it cannot be run as a scan. Start a new scan with a different -w name.', ); } + if (launch.model_only && !requestedModelOnly) { + fail( + 'This workspace was created to validate the AI model only, so it cannot be run as a scan. Start a new scan with a different -w name.', + ); + } const session = readJsonFile(sessionPath); if (!isRecord(session) || !isRecord(session.session) || session.session.webUrl !== expectedUrl) { fail( @@ -208,6 +226,7 @@ export function writeLaunchStateAtomically( internalPath: string, outputDir: string | undefined, authOnly: boolean, + modelOnly: boolean, ): void { const finalPath = path.join(internalPath, LAUNCH_STATE_FILENAME); const temporaryPath = path.join(internalPath, `${LAUNCH_STATE_FILENAME}.tmp-${process.pid}-${randomSuffix()}`); @@ -215,6 +234,7 @@ export function writeLaunchStateAtomically( schema_version: LAUNCH_STATE_SCHEMA_VERSION, ...(outputDir !== undefined && { customer_output_path: outputDir }), ...(authOnly && { auth_only: true }), + ...(modelOnly && { model_only: true }), }; const descriptor = fs.openSync(temporaryPath, 'wx', 0o600); try { @@ -244,8 +264,9 @@ export function createWorkflowId(workspace: string, isResume: boolean, timestamp } export async function start(args: StartArgs): Promise { - // Auth-only runs are short and have no report to come back for, so they always stream to the end. - if (args.authOnly) args.follow = true; + // Validation-only runs are short and have no report to come back for, so they always stream to the end. + const validationOnly = args.authOnly || args.validateModel; + if (validationOnly) args.follow = true; // 1. Resolve non-mutating inputs and classify the workspace before changing it. initHome(); @@ -262,13 +283,36 @@ export async function start(args: StartArgs): Promise { args.workspace ?? `${new URL(args.url).hostname.replace(/[^a-zA-Z0-9-]/g, '-')}_shannon-${Date.now()}`; const workspacePath = path.join(workspacesDir, workspace); const requestedOutputDir = args.output ? path.resolve(expandHome(args.output)) : undefined; - const launchDecision = classifyWorkspaceLaunch(workspacePath, args.url, requestedOutputDir, args.authOnly); + const launchDecision = classifyWorkspaceLaunch( + workspacePath, + args.url, + requestedOutputDir, + args.authOnly, + args.validateModel, + ); - // Auth-only runs write no resumable state, so they always run fresh; reusing a workspace would resume it. - if (args.authOnly && launchDecision.isResume) { - fail( - 'An auth-validation run needs a fresh workspace. Omit -w to auto-name one, or choose a -w name that is not in use.', - ); + // Validation-only runs write no resumable state, so they always run fresh; reusing a workspace would resume it. + if (validationOnly && launchDecision.isResume) { + const what = args.authOnly ? 'An auth-validation run' : 'A model-validation run'; + fail(`${what} needs a fresh workspace. Omit -w to auto-name one, or choose a -w name that is not in use.`); + } + + // User-facing status wording. Auth-only and model-only are both "validation" runs, but each + // names what it validated. A plain scan keeps its original phrasing. + let startingLabel = 'Starting scan'; + let waitingLabel = 'Waiting for the scan to start'; + let couldNotStartLabel = 'The scan could not start'; + let startedLabel = `Scan started — ${workspace}`; + if (args.authOnly) { + startingLabel = 'Starting authentication validation'; + waitingLabel = 'Waiting for authentication validation to start'; + couldNotStartLabel = 'Authentication validation could not start'; + startedLabel = `Validating authentication — ${workspace}`; + } else if (args.validateModel) { + startingLabel = 'Starting model validation'; + waitingLabel = 'Waiting for model validation to start'; + couldNotStartLabel = 'Model validation could not start'; + startedLabel = `Validating model — ${workspace}`; } // 2. Inputs are valid; identify the run before initializing shared infrastructure. @@ -283,7 +327,7 @@ export async function start(args: StartArgs): Promise { ensureDocker(); ensureImage(args.version); const spinner = p.spinner(); - spinner.start(args.authOnly ? 'Starting authentication validation' : 'Starting scan'); + spinner.start(startingLabel); await ensureInfra(spinner); // 3. Generate the invocation identity. @@ -306,7 +350,7 @@ export async function start(args: StartArgs): Promise { fs.chmodSync(dirPath, 0o777); } if (!launchDecision.isResume) { - writeLaunchStateAtomically(internalPath, launchDecision.outputDir, args.authOnly); + writeLaunchStateAtomically(internalPath, launchDecision.outputDir, args.authOnly, args.validateModel); } // 5. Pre-create overlay mount points (:ro mounts cannot create them). @@ -366,6 +410,7 @@ export async function start(args: StartArgs): Promise { ...(args.pipelineTesting && { pipelineTesting: true }), ...(args.keepContainer && { keepContainer: true }), ...(args.authOnly && { authOnly: true }), + ...(args.validateModel && { validateModel: true }), ...(shouldUsePiAuth() && { piAuthHostPath: resolveHostPiAuthPath() }), }); @@ -416,7 +461,7 @@ export async function start(args: StartArgs): Promise { }); // Poll for the workflow to register in session.json; the spinner resolves once it does. - spinner.message(args.authOnly ? 'Waiting for authentication validation to start' : 'Waiting for the scan to start'); + spinner.message(waitingLabel); for (let attempts = 0; attempts < 60; attempts++) { // A pre-workflow failure leaves its reason here (nothing reached Temporal); surface it // rather than polling out to a generic timeout. @@ -450,15 +495,15 @@ export async function start(args: StartArgs): Promise { spinner.message('Running preflight checks'); const outcome = await awaitPreflightOutcome(workflowId); if (outcome.kind === 'failed') { - spinner.error(args.authOnly ? 'Authentication validation could not start' : 'The scan could not start'); + spinner.error(couldNotStartLabel); printScanStartFailure(outcome.message); process.exit(1); } - spinner.stop(args.authOnly ? `Validating authentication — ${workspace}` : `Scan started — ${workspace}`); + spinner.stop(startedLabel); printInfo(args, workspace, repo.hostPath, workspacesDir); if (args.follow) { - await followScan(workspace, workspacesDir, args.authOnly); + await followScan(workspace, workspacesDir, validationOnly); } return; } @@ -606,7 +651,7 @@ function printUnconfirmedScanHint(workspace: string, taskQueue: string, containe * That tracks whether the pipeline ran, not whether vulnerabilities were found. On failure the * root-cause message is printed so a red CI build says why. */ -async function followScan(workspace: string, workspacesDir: string, authOnly = false): Promise { +async function followScan(workspace: string, workspacesDir: string, validationOnly = false): Promise { const logFile = resolveRunFile(path.join(workspacesDir, workspace), 'workflow.log'); const workflowId = resolveWorkflowId(workspace); @@ -617,7 +662,7 @@ async function followScan(workspace: string, workspacesDir: string, authOnly = f } if (stdoutIsTerminal()) { - const what = authOnly ? 'validation' : 'scan'; + const what = validationOnly ? 'validation' : 'scan'; console.error(`\n Following ${what} log (Ctrl-C to stop watching):\n`); } @@ -706,7 +751,7 @@ function printInfo(args: StartArgs, workspace: string, repoPath: string, workspa console.log(` Progress: ${prefix} status ${workspace}`); } - if (!args.authOnly) { + if (!args.authOnly && !args.validateModel) { console.log(''); console.log(' Report (when the scan finishes):'); console.log(` ${reportDir}${path.sep}`); diff --git a/apps/cli/src/docker.ts b/apps/cli/src/docker.ts index c3c216c8..1cd86a1d 100644 --- a/apps/cli/src/docker.ts +++ b/apps/cli/src/docker.ts @@ -413,6 +413,7 @@ export interface WorkerOptions { pipelineTesting?: boolean; keepContainer?: boolean; authOnly?: boolean; + validateModel?: boolean; piAuthHostPath?: string; } @@ -515,6 +516,9 @@ export function spawnWorker(opts: WorkerOptions): ChildProcess { if (opts.authOnly) { args.push('--validate-auth'); } + if (opts.validateModel) { + args.push('--validate-model'); + } // Inherit stderr so `docker run` daemon errors surface to the user; // ignore stdin/stdout (the container ID is noise). diff --git a/apps/cli/src/help.ts b/apps/cli/src/help.ts index 47301834..cdd64261 100644 --- a/apps/cli/src/help.ts +++ b/apps/cli/src/help.ts @@ -34,6 +34,7 @@ export const START_OPTIONS: readonly (readonly [string, string])[] = [ ['-w, --workspace ', 'Named workspace (auto-resumes if it exists)'], ['-f, --follow', 'Stream the scan log until it finishes'], ['--validate-auth', 'Validate authentication only, then stop (no pentest)'], + ['--validate-model', 'Validate the AI model only, then stop (no pentest)'], ['--pipeline-testing', 'Use minimal prompts for fast testing'], ['--keep-container', 'Preserve the worker container after exit for log inspection'], ]; @@ -47,6 +48,7 @@ const COMMAND_HELP: Readonly> = { 'start -u https://example.com -r /path/to/repo -c config.yaml -w q1-audit', 'start -u https://example.com -r ./my-repo --follow', 'start -u https://example.com -r ./my-repo -c config.yaml --validate-auth', + 'start -u https://example.com -r ./my-repo --validate-model', ], }, stop: { diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts index f74b21db..6a1db43e 100644 --- a/apps/cli/src/index.ts +++ b/apps/cli/src/index.ts @@ -190,6 +190,7 @@ interface ParsedStartArgs { keepContainer: boolean; follow: boolean; authOnly: boolean; + validateModel: boolean; } function parseStartArgs(argv: string[]): ParsedStartArgs { @@ -207,6 +208,7 @@ function parseStartArgs(argv: string[]): ParsedStartArgs { keepContainer: ['--keep-container'], follow: ['-f', '--follow'], authOnly: ['--validate-auth'], + validateModel: ['--validate-model'], }, }); @@ -222,6 +224,10 @@ function parseStartArgs(argv: string[]): ParsedStartArgs { failUsage(`invalid --url: ${url}`); } + if (flags.authOnly && flags.validateModel) { + failUsage('--validate-auth and --validate-model cannot be combined; run one validation at a time'); + } + if (flags.authOnly && !values.config) { failUsage( '--validate-auth needs a config file with an authentication block', @@ -236,6 +242,7 @@ function parseStartArgs(argv: string[]): ParsedStartArgs { keepContainer: !!flags.keepContainer, follow: !!flags.follow, authOnly: !!flags.authOnly, + validateModel: !!flags.validateModel, ...(values.config && { config: values.config }), ...(values.modelsConfig && { modelsConfig: values.modelsConfig }), ...(values.workspace && { workspace: values.workspace }), diff --git a/apps/cli/src/scan/pipeline.ts b/apps/cli/src/scan/pipeline.ts index 41396e5d..ef7b656a 100644 --- a/apps/cli/src/scan/pipeline.ts +++ b/apps/cli/src/scan/pipeline.ts @@ -108,6 +108,7 @@ const MISCELLANEOUS_EXPLOIT_AGENT: AgentSpec = { * available guess. */ export function pipelineForState(state: PipelineState | null): readonly PhaseSpec[] { + if (state?.validateModel === true) return []; if (state?.authOnly === true) return PIPELINE.filter((phase) => phase.key === 'auth-validation'); if (state?.expectedAgents === undefined) return PIPELINE; const expected = new Set(state.expectedAgents); @@ -355,6 +356,7 @@ export type PipelineStatus = 'running' | 'completed' | 'failed' | 'cancelled' | export interface PipelineState { readonly status: PipelineStatus; readonly authOnly?: boolean; + readonly validateModel?: boolean; readonly currentPhase: string | null; readonly currentAgent: string | null; readonly completedAgents: string[]; diff --git a/apps/worker/src/audit/workflow-logger.ts b/apps/worker/src/audit/workflow-logger.ts index d615a40f..e585b4f2 100644 --- a/apps/worker/src/audit/workflow-logger.ts +++ b/apps/worker/src/audit/workflow-logger.ts @@ -120,6 +120,23 @@ function isAuthOnlyRun(): boolean { return process.env.SHANNON_AUTH_ONLY === '1'; } +/** One scan per worker process; the worker sets this flag for a model-validation run (see worker.ts). */ +function isModelOnlyRun(): boolean { + return process.env.SHANNON_VALIDATE_MODEL === '1'; +} + +/** Both validation-only modes share the terminal heading and drop the pentest-only lines. */ +function isValidationOnlyRun(): boolean { + return isAuthOnlyRun() || isModelOnlyRun(); +} + +/** The log header title, framing a validation-only run by what it validated. */ +function validationLogTitle(): string { + if (isAuthOnlyRun()) return 'Shannon - Authentication Validation Log'; + if (isModelOnlyRun()) return 'Shannon - Model Validation Log'; + return 'Shannon Pentest - Scan Log'; +} + function safeAgenticSastStageLabel(label: string | undefined): string | undefined { return label !== undefined && isCapellaTerminalStageLabel(label) ? label : undefined; } @@ -441,7 +458,7 @@ export class WorkflowLogger { try { this.logStream = await LogStream.acquire(this.logPath); const workflowId = safeWorkflowIdentifier(this.workflowId ?? this.sessionMetadata.id); - const title = isAuthOnlyRun() ? 'Shannon - Authentication Validation Log' : 'Shannon Pentest - Scan Log'; + const title = validationLogTitle(); const header = [ '================================================================================', title, @@ -664,8 +681,8 @@ export class WorkflowLogger { failed: 'FAILED', }; const status = statusHeaders[summary.status]; - const authOnly = isAuthOnlyRun(); - const runLabel = authOnly ? 'Validation' : 'Scan'; + const validationOnly = isValidationOnlyRun(); + const runLabel = validationOnly ? 'Validation' : 'Scan'; const completedAgents = summary.completedAgents.filter(isLoggableAgentName); const skippedAgents = (summary.skippedAgents ?? []).filter(isLoggableAgentName); const operationalGroups = summarizeOperationalMetrics(summary.operationalMetrics, summary.operationalStages); @@ -679,7 +696,7 @@ export class WorkflowLogger { `Status: ${summary.status}`, `Duration: ${formatDuration(Math.max(0, summary.totalDurationMs))}`, `Total Cost: $${Math.max(0, summary.totalCostUsd).toFixed(4)}`, - ...(authOnly ? [] : [`Agents: ${completedAgents.length} ran, ${skippedAgents.length} skipped`]), + ...(validationOnly ? [] : [`Agents: ${completedAgents.length} ran, ${skippedAgents.length} skipped`]), ]; if (summary.usageAccountingComplete === false) { lines.push('Cost Note: Cost is incomplete — some background work is not included in this total.'); diff --git a/apps/worker/src/temporal/shared.ts b/apps/worker/src/temporal/shared.ts index 178f66fa..09406a3c 100644 --- a/apps/worker/src/temporal/shared.ts +++ b/apps/worker/src/temporal/shared.ts @@ -109,6 +109,7 @@ export interface PipelineInput { checkpointsEnabled?: boolean; // Enable checkpoint activities (default: false) exploit?: boolean; // false skips the exploitation phase authOnly?: boolean; // true stops the run after auth validation (no pentest, no report) + validateModel?: boolean; // true stops the run after the preflight model checks (no pentest, no report) } /** What `loadResumeState` reconstructs from a prior workspace: independently verified, never assumed from session.json alone. */ @@ -186,6 +187,7 @@ export interface PipelineSummary { export interface PipelineState { status: 'running' | 'completed' | 'failed' | 'cancelled' | 'partial'; authOnly: boolean; + validateModel: boolean; currentPhase: string | null; currentAgent: string | null; /** Agents that actually ran. Mutually exclusive from `skippedAgents`. */ diff --git a/apps/worker/src/temporal/worker.ts b/apps/worker/src/temporal/worker.ts index e10fd9e8..912e41f8 100644 --- a/apps/worker/src/temporal/worker.ts +++ b/apps/worker/src/temporal/worker.ts @@ -251,6 +251,7 @@ interface CliArgs { customerOutputPath?: string; pipelineTestingMode: boolean; authOnly: boolean; + validateModel: boolean; resumeFromWorkspace?: string; } @@ -266,7 +267,8 @@ function showUsage(): void { console.log(' --workspace Resume from existing workspace'); console.log(' --output Stable mounted path for final customer report copies'); console.log(' --pipeline-testing Use minimal prompts for fast testing'); - console.log(' --validate-auth Validate authentication only, then stop\n'); + console.log(' --validate-auth Validate authentication only, then stop'); + console.log(' --validate-model Validate the AI model only, then stop\n'); } function parseCliArgs(argv: string[]): CliArgs { @@ -283,6 +285,7 @@ function parseCliArgs(argv: string[]): CliArgs { let customerOutputPath: string | undefined; let pipelineTestingMode = false; let authOnly = false; + let validateModel = false; let resumeFromWorkspace: string | undefined; for (let i = 0; i < argv.length; i++) { @@ -321,6 +324,8 @@ function parseCliArgs(argv: string[]): CliArgs { pipelineTestingMode = true; } else if (arg === '--validate-auth') { authOnly = true; + } else if (arg === '--validate-model') { + validateModel = true; } else if (arg && !arg.startsWith('-')) { if (!webUrl) { webUrl = arg; @@ -349,6 +354,7 @@ function parseCliArgs(argv: string[]): CliArgs { ...(workflowId && { workflowId }), pipelineTestingMode, authOnly, + validateModel, ...(configPath && { configPath }), ...(customerOutputPath && { customerOutputPath }), ...(resumeFromWorkspace && { resumeFromWorkspace }), @@ -598,6 +604,7 @@ function buildPipelineInput( ...(orchestration.agenticSast !== undefined && { agenticSast: orchestration.agenticSast }), ...(orchestration.exploit !== undefined && { exploit: orchestration.exploit }), ...(args.authOnly && { authOnly: true }), + ...(args.validateModel && { validateModel: true }), }; } @@ -654,6 +661,8 @@ async function waitForWorkflowResult( console.log('\nScan cancelled before it finished.'); } else if (result.authOnly) { console.log('\nAuthentication validated. No pentest was run (--validate-auth).'); + } else if (result.validateModel) { + console.log('\nModel validated. No pentest was run (--validate-model).'); } else { console.log('\nScan completed.'); } @@ -766,9 +775,10 @@ async function run(): Promise { // 1. Parse CLI args const args = parseCliArgs(process.argv.slice(2)); - // One scan per worker process, so an auth-only run is a process-wide fact. The log writers - // read it to frame the log as a validation rather than a pentest. + // One scan per worker process, so an auth-only or model-validation run is a process-wide fact. + // The log writers read these to frame the log as a validation rather than a pentest. if (args.authOnly) process.env.SHANNON_AUTH_ONLY = '1'; + if (args.validateModel) process.env.SHANNON_VALIDATE_MODEL = '1'; // 2. Connect to Temporal server const address = process.env.TEMPORAL_ADDRESS || 'localhost:7233'; diff --git a/apps/worker/src/temporal/workflows.ts b/apps/worker/src/temporal/workflows.ts index aa8409b0..e40efa49 100644 --- a/apps/worker/src/temporal/workflows.ts +++ b/apps/worker/src/temporal/workflows.ts @@ -382,12 +382,14 @@ export async function pentestPipeline(input: PipelineInput): Promise