feat: add --validate-model to run the preflight model checks only

apps/cli/src/commands/start.ts

@@ -46,6 +46,7 @@ export interface StartArgs {
   keepContainer: boolean;
   follow: boolean;
   authOnly: boolean;
+  validateModel: boolean;
   version: string;
 }
 
@@ -63,6 +64,8 @@ interface LaunchState {
   readonly customer_output_path?: string;
   /** True when the workspace was created by an auth-validation run; such a workspace is not a scan. */
   readonly auth_only?: boolean;
+  /** True when the workspace was created by a model-validation run; such a workspace is not a scan. */
+  readonly model_only?: boolean;
 }
 
 export interface WorkspaceLaunchDecision {
@@ -128,7 +131,7 @@ function readLaunchState(filePath: string): LaunchState {
   // Unknown keys mean a newer release wrote this workspace; refuse rather than half-read it.
   const keys = Object.keys(value).sort();
   const keysAreValid = keys.every(
-    (key) => key === 'auth_only' || key === 'customer_output_path' || key === 'schema_version',
+    (key) => key === 'auth_only' || key === 'model_only' || key === 'customer_output_path' || key === 'schema_version',
   );
   const customerPath = value.customer_output_path;
   const pathIsValid =
@@ -136,13 +139,22 @@ function readLaunchState(filePath: string): LaunchState {
     (typeof customerPath === 'string' && path.isAbsolute(customerPath) && path.resolve(customerPath) === customerPath);
   const authOnly = value.auth_only;
   const authOnlyIsValid = authOnly === undefined || typeof authOnly === 'boolean';
-  if (value.schema_version !== LAUNCH_STATE_SCHEMA_VERSION || !keysAreValid || !pathIsValid || !authOnlyIsValid) {
+  const modelOnly = value.model_only;
+  const modelOnlyIsValid = modelOnly === undefined || typeof modelOnly === 'boolean';
+  if (
+    value.schema_version !== LAUNCH_STATE_SCHEMA_VERSION ||
+    !keysAreValid ||
+    !pathIsValid ||
+    !authOnlyIsValid ||
+    !modelOnlyIsValid
+  ) {
     fail(NEWER_RELEASE_MESSAGE);
   }
   return {
     schema_version: LAUNCH_STATE_SCHEMA_VERSION,
     ...(typeof customerPath === 'string' && { customer_output_path: customerPath }),
     ...(authOnly === true && { auth_only: true }),
+    ...(modelOnly === true && { model_only: true }),
   };
 }
 
@@ -158,6 +170,7 @@ export function classifyWorkspaceLaunch(
   expectedUrl: string,
   requestedOutputDir: string | undefined,
   requestedAuthOnly: boolean,
+  requestedModelOnly: boolean,
 ): WorkspaceLaunchDecision {
   const sessionPath = resolveRunFile(workspacePath, 'session.json');
   const sessionExists = fs.existsSync(sessionPath);
@@ -177,6 +190,11 @@ export function classifyWorkspaceLaunch(
       'This workspace was created to validate authentication only, so it cannot be run as a scan. Start a new scan with a different -w name.',
     );
   }
+  if (launch.model_only && !requestedModelOnly) {
+    fail(
+      'This workspace was created to validate the AI model only, so it cannot be run as a scan. Start a new scan with a different -w name.',
+    );
+  }
   const session = readJsonFile(sessionPath);
   if (!isRecord(session) || !isRecord(session.session) || session.session.webUrl !== expectedUrl) {
     fail(
@@ -208,6 +226,7 @@ export function writeLaunchStateAtomically(
   internalPath: string,
   outputDir: string | undefined,
   authOnly: boolean,
+  modelOnly: boolean,
 ): void {
   const finalPath = path.join(internalPath, LAUNCH_STATE_FILENAME);
   const temporaryPath = path.join(internalPath, `${LAUNCH_STATE_FILENAME}.tmp-${process.pid}-${randomSuffix()}`);
@@ -215,6 +234,7 @@ export function writeLaunchStateAtomically(
     schema_version: LAUNCH_STATE_SCHEMA_VERSION,
     ...(outputDir !== undefined && { customer_output_path: outputDir }),
     ...(authOnly && { auth_only: true }),
+    ...(modelOnly && { model_only: true }),
   };
   const descriptor = fs.openSync(temporaryPath, 'wx', 0o600);
   try {
@@ -244,8 +264,9 @@ export function createWorkflowId(workspace: string, isResume: boolean, timestamp
 }
 
 export async function start(args: StartArgs): Promise<void> {
-  // Auth-only runs are short and have no report to come back for, so they always stream to the end.
-  if (args.authOnly) args.follow = true;
+  // Validation-only runs are short and have no report to come back for, so they always stream to the end.
+  const validationOnly = args.authOnly || args.validateModel;
+  if (validationOnly) args.follow = true;
 
   // 1. Resolve non-mutating inputs and classify the workspace before changing it.
   initHome();
@@ -262,13 +283,36 @@ export async function start(args: StartArgs): Promise<void> {
     args.workspace ?? `${new URL(args.url).hostname.replace(/[^a-zA-Z0-9-]/g, '-')}_shannon-${Date.now()}`;
   const workspacePath = path.join(workspacesDir, workspace);
   const requestedOutputDir = args.output ? path.resolve(expandHome(args.output)) : undefined;
-  const launchDecision = classifyWorkspaceLaunch(workspacePath, args.url, requestedOutputDir, args.authOnly);
+  const launchDecision = classifyWorkspaceLaunch(
+    workspacePath,
+    args.url,
+    requestedOutputDir,
+    args.authOnly,
+    args.validateModel,
+  );
 
-  // Auth-only runs write no resumable state, so they always run fresh; reusing a workspace would resume it.
-  if (args.authOnly && launchDecision.isResume) {
-    fail(
-      'An auth-validation run needs a fresh workspace. Omit -w to auto-name one, or choose a -w name that is not in use.',
-    );
+  // Validation-only runs write no resumable state, so they always run fresh; reusing a workspace would resume it.
+  if (validationOnly && launchDecision.isResume) {
+    const what = args.authOnly ? 'An auth-validation run' : 'A model-validation run';
+    fail(`${what} needs a fresh workspace. Omit -w to auto-name one, or choose a -w name that is not in use.`);
+  }
+
+  // User-facing status wording. Auth-only and model-only are both "validation" runs, but each
+  // names what it validated. A plain scan keeps its original phrasing.
+  let startingLabel = 'Starting scan';
+  let waitingLabel = 'Waiting for the scan to start';
+  let couldNotStartLabel = 'The scan could not start';
+  let startedLabel = `Scan started — ${workspace}`;
+  if (args.authOnly) {
+    startingLabel = 'Starting authentication validation';
+    waitingLabel = 'Waiting for authentication validation to start';
+    couldNotStartLabel = 'Authentication validation could not start';
+    startedLabel = `Validating authentication — ${workspace}`;
+  } else if (args.validateModel) {
+    startingLabel = 'Starting model validation';
+    waitingLabel = 'Waiting for model validation to start';
+    couldNotStartLabel = 'Model validation could not start';
+    startedLabel = `Validating model — ${workspace}`;
   }
 
   // 2. Inputs are valid; identify the run before initializing shared infrastructure.
@@ -283,7 +327,7 @@ export async function start(args: StartArgs): Promise<void> {
   ensureDocker();
   ensureImage(args.version);
   const spinner = p.spinner();
-  spinner.start(args.authOnly ? 'Starting authentication validation' : 'Starting scan');
+  spinner.start(startingLabel);
   await ensureInfra(spinner);
 
   // 3. Generate the invocation identity.
@@ -306,7 +350,7 @@ export async function start(args: StartArgs): Promise<void> {
     fs.chmodSync(dirPath, 0o777);
   }
   if (!launchDecision.isResume) {
-    writeLaunchStateAtomically(internalPath, launchDecision.outputDir, args.authOnly);
+    writeLaunchStateAtomically(internalPath, launchDecision.outputDir, args.authOnly, args.validateModel);
   }
 
   // 5. Pre-create overlay mount points (:ro mounts cannot create them).
@@ -366,6 +410,7 @@ export async function start(args: StartArgs): Promise<void> {
     ...(args.pipelineTesting && { pipelineTesting: true }),
     ...(args.keepContainer && { keepContainer: true }),
     ...(args.authOnly && { authOnly: true }),
+    ...(args.validateModel && { validateModel: true }),
     ...(shouldUsePiAuth() && { piAuthHostPath: resolveHostPiAuthPath() }),
   });
 
@@ -416,7 +461,7 @@ export async function start(args: StartArgs): Promise<void> {
   });
 
   // Poll for the workflow to register in session.json; the spinner resolves once it does.
-  spinner.message(args.authOnly ? 'Waiting for authentication validation to start' : 'Waiting for the scan to start');
+  spinner.message(waitingLabel);
   for (let attempts = 0; attempts < 60; attempts++) {
     // A pre-workflow failure leaves its reason here (nothing reached Temporal); surface it
     // rather than polling out to a generic timeout.
@@ -450,15 +495,15 @@ export async function start(args: StartArgs): Promise<void> {
         spinner.message('Running preflight checks');
         const outcome = await awaitPreflightOutcome(workflowId);
         if (outcome.kind === 'failed') {
-          spinner.error(args.authOnly ? 'Authentication validation could not start' : 'The scan could not start');
+          spinner.error(couldNotStartLabel);
           printScanStartFailure(outcome.message);
           process.exit(1);
         }
 
-        spinner.stop(args.authOnly ? `Validating authentication — ${workspace}` : `Scan started — ${workspace}`);
+        spinner.stop(startedLabel);
         printInfo(args, workspace, repo.hostPath, workspacesDir);
         if (args.follow) {
-          await followScan(workspace, workspacesDir, args.authOnly);
+          await followScan(workspace, workspacesDir, validationOnly);
         }
         return;
       }
@@ -606,7 +651,7 @@ function printUnconfirmedScanHint(workspace: string, taskQueue: string, containe
  * That tracks whether the pipeline ran, not whether vulnerabilities were found. On failure the
  * root-cause message is printed so a red CI build says why.
  */
-async function followScan(workspace: string, workspacesDir: string, authOnly = false): Promise<never> {
+async function followScan(workspace: string, workspacesDir: string, validationOnly = false): Promise<never> {
   const logFile = resolveRunFile(path.join(workspacesDir, workspace), 'workflow.log');
   const workflowId = resolveWorkflowId(workspace);
 
@@ -617,7 +662,7 @@ async function followScan(workspace: string, workspacesDir: string, authOnly = f
   }
 
   if (stdoutIsTerminal()) {
-    const what = authOnly ? 'validation' : 'scan';
+    const what = validationOnly ? 'validation' : 'scan';
     console.error(`\n  Following ${what} log (Ctrl-C to stop watching):\n`);
   }
 
@@ -706,7 +751,7 @@ function printInfo(args: StartArgs, workspace: string, repoPath: string, workspa
     console.log(`    Progress:   ${prefix} status ${workspace}`);
   }
 
-  if (!args.authOnly) {
+  if (!args.authOnly && !args.validateModel) {
     console.log('');
     console.log('  Report (when the scan finishes):');
     console.log(`    ${reportDir}${path.sep}`);

apps/cli/src/docker.ts

@@ -413,6 +413,7 @@ export interface WorkerOptions {
   pipelineTesting?: boolean;
   keepContainer?: boolean;
   authOnly?: boolean;
+  validateModel?: boolean;
   piAuthHostPath?: string;
 }
 
@@ -515,6 +516,9 @@ export function spawnWorker(opts: WorkerOptions): ChildProcess {
   if (opts.authOnly) {
     args.push('--validate-auth');
   }
+  if (opts.validateModel) {
+    args.push('--validate-model');
+  }
 
   // Inherit stderr so `docker run` daemon errors surface to the user;
   // ignore stdin/stdout (the container ID is noise).

apps/cli/src/help.ts

@@ -34,6 +34,7 @@ export const START_OPTIONS: readonly (readonly [string, string])[] = [
   ['-w, --workspace <name>', 'Named workspace (auto-resumes if it exists)'],
   ['-f, --follow', 'Stream the scan log until it finishes'],
   ['--validate-auth', 'Validate authentication only, then stop (no pentest)'],
+  ['--validate-model', 'Validate the AI model only, then stop (no pentest)'],
   ['--pipeline-testing', 'Use minimal prompts for fast testing'],
   ['--keep-container', 'Preserve the worker container after exit for log inspection'],
 ];
@@ -47,6 +48,7 @@ const COMMAND_HELP: Readonly<Record<string, CommandHelp>> = {
       'start -u https://example.com -r /path/to/repo -c config.yaml -w q1-audit',
       'start -u https://example.com -r ./my-repo --follow',
       'start -u https://example.com -r ./my-repo -c config.yaml --validate-auth',
+      'start -u https://example.com -r ./my-repo --validate-model',
     ],
   },
   stop: {

apps/cli/src/index.ts

@@ -190,6 +190,7 @@ interface ParsedStartArgs {
   keepContainer: boolean;
   follow: boolean;
   authOnly: boolean;
+  validateModel: boolean;
 }
 
 function parseStartArgs(argv: string[]): ParsedStartArgs {
@@ -207,6 +208,7 @@ function parseStartArgs(argv: string[]): ParsedStartArgs {
       keepContainer: ['--keep-container'],
       follow: ['-f', '--follow'],
       authOnly: ['--validate-auth'],
+      validateModel: ['--validate-model'],
     },
   });
 
@@ -222,6 +224,10 @@ function parseStartArgs(argv: string[]): ParsedStartArgs {
     failUsage(`invalid --url: ${url}`);
   }
 
+  if (flags.authOnly && flags.validateModel) {
+    failUsage('--validate-auth and --validate-model cannot be combined; run one validation at a time');
+  }
+
   if (flags.authOnly && !values.config) {
     failUsage(
       '--validate-auth needs a config file with an authentication block',
@@ -236,6 +242,7 @@ function parseStartArgs(argv: string[]): ParsedStartArgs {
     keepContainer: !!flags.keepContainer,
     follow: !!flags.follow,
     authOnly: !!flags.authOnly,
+    validateModel: !!flags.validateModel,
     ...(values.config && { config: values.config }),
     ...(values.modelsConfig && { modelsConfig: values.modelsConfig }),
     ...(values.workspace && { workspace: values.workspace }),

apps/cli/src/scan/pipeline.ts

@@ -108,6 +108,7 @@ const MISCELLANEOUS_EXPLOIT_AGENT: AgentSpec = {
  * available guess.
  */
 export function pipelineForState(state: PipelineState | null): readonly PhaseSpec[] {
+  if (state?.validateModel === true) return [];
   if (state?.authOnly === true) return PIPELINE.filter((phase) => phase.key === 'auth-validation');
   if (state?.expectedAgents === undefined) return PIPELINE;
   const expected = new Set(state.expectedAgents);
@@ -355,6 +356,7 @@ export type PipelineStatus = 'running' | 'completed' | 'failed' | 'cancelled' |
 export interface PipelineState {
   readonly status: PipelineStatus;
   readonly authOnly?: boolean;
+  readonly validateModel?: boolean;
   readonly currentPhase: string | null;
   readonly currentAgent: string | null;
   readonly completedAgents: string[];