subscription login preflight + Playwright MCP fixes (browser install, codex wiring)

Why runs came back empty / "MCP didn't execute":
- Not logged in: a subscription CLI that isn't authenticated returns empty
  instantly (the Juice Shop symptom — every agent 0 candidates, no tool activity).
  Added models::cli_login_status + subscription_preflight(): before a run we check
  the primary provider's CLI is installed AND logged in and warn clearly if not
  (CLI run_mode + REPL start_background).
- Missing browser: ensure_playwright_mcp now also runs `npx playwright install
  chromium` (best-effort; NEUROSPLOIT_SKIP_BROWSER_INSTALL=1 to skip) so the first
  browser action doesn't fail/hang.
- Codex MCP was mis-wired (`--config mcp_config_file=` is not a codex key). Now
  injects our .mcp.json servers via `-c mcp_servers.<name>.command/.args` TOML
  overrides — MCP works on Codex, not only Claude. gemini/grok remain built-in-tools
  only (no MCP flag).
- REPL diagnostic: subscription+MCP run with zero tool/browser events warns the
  CLI likely isn't logged in / MCP didn't start.
This commit is contained in:
CyberSecurityUP
2026-07-05 16:09:15 -03:00
parent 3ca04498a9
commit 4ac4faec32
4 changed files with 144 additions and 4 deletions
+18
View File
@@ -102,6 +102,24 @@ interactive line-editing.
- **Rate-limit testing** is a first-class control check (small non-disruptive
burst → look for 429/lockout/Retry-After), never a DoS.
## Subscription login check & Playwright MCP fixes
- **Subscription login preflight.** Before a `--subscription` run, the harness
checks that the local CLI (claude/codex/…) is **installed and logged in** and
prints a clear warning if not — instead of the run silently coming back with
0 findings. (Not logged in → the CLI returns empty instantly, which was the
usual cause of "it found nothing / MCP didn't execute".)
- **Playwright MCP now installs the browser.** `ensure_playwright_mcp` also runs
`npx playwright install chromium` (best-effort; skip with
`NEUROSPLOIT_SKIP_BROWSER_INSTALL=1`) so the first browser action doesn't
fail/hang with a missing Chromium.
- **Codex MCP wiring fixed.** Codex takes MCP servers as `-c mcp_servers.*` TOML
overrides (not a config-file path); the harness now injects our Playwright
server correctly, so MCP works on Codex too — not just Claude.
- **"No tool activity" diagnostic.** If a subscription+MCP run performs zero
browser/tool actions, the REPL warns that the CLI likely isn't logged in or the
MCP didn't start.
## Multi-role auth & access-control testing
- **Named identities in `creds.yaml`** for IDOR / BOLA / BFLA / privilege-escalation
+27
View File
@@ -552,6 +552,32 @@ pub(crate) struct Spawned {
pub workdir: PathBuf,
}
/// When running in subscription mode, verify the local CLI is installed AND
/// logged in before the engagement starts — otherwise every agent comes back
/// empty and it looks like "0 findings" when the real cause is auth. Checks the
/// primary model's provider; prints a clear warning (non-fatal).
pub(crate) async fn subscription_preflight(cfg: &RunConfig) {
if !cfg.subscription || cfg.offline { return; }
let Some(primary) = cfg.models.first() else { return };
let provider = ModelRef::parse(primary).provider;
if harness::models::cli_binary_for(&provider).is_none() { return; }
print!(" [*] checking {provider} subscription login… ");
use std::io::Write; let _ = std::io::stdout().flush();
match harness::models::cli_login_status(&provider).await {
harness::models::LoginStatus::LoggedIn => println!("\r [*] {provider} subscription: logged in ✓ "),
harness::models::LoginStatus::NotLoggedIn => {
let cli = harness::models::cli_binary_for(&provider).unwrap_or("the CLI");
println!("\r \x1b[1;33m[!] {provider} subscription NOT logged in\x1b[0m — run `{cli}` and log in (e.g. `claude` → /login), then retry.");
println!(" \x1b[2m(without login every agent returns empty — this is usually why a run finds 0.)\x1b[0m");
}
harness::models::LoginStatus::NotInstalled => {
let cli = harness::models::cli_binary_for(&provider).unwrap_or("?");
println!("\r \x1b[1;33m[!] subscription CLI `{cli}` for {provider} is not installed\x1b[0m — install it or use an API key (drop --subscription).");
}
harness::models::LoginStatus::Unknown => println!("\r [*] {provider} subscription: login state unknown (continuing) "),
}
}
/// Set up + start an engagement (synchronous setup; the work runs in the task).
pub(crate) fn spawn_engagement(base: &Path, mut cfg: RunConfig, mcp: bool, mode: Mode) -> Spawned {
let lib = agents::load(base);
@@ -666,6 +692,7 @@ pub(crate) fn finalize_run(mut out: RunOutput, workdir: &Path) -> RunOutput {
}
async fn run_mode(base: &Path, cfg: RunConfig, mcp: bool, mode: Mode) -> anyhow::Result<RunOutput> {
subscription_preflight(&cfg).await;
let Spawned { mut task, mut rx, cancel, workdir, .. } = spawn_engagement(base, cfg, mcp, mode);
let printer = tokio::spawn(async move {
while let Some(line) = rx.recv().await { render_line(&line); }
+11
View File
@@ -830,6 +830,7 @@ async fn start_background(base: &Path, s: &Session, reader: &mut Reader,
cfg.auth = s.auth.clone();
if matches!(mode_e, crate::Mode::Grey) { cfg.repo = s.repo.clone(); }
crate::apply_creds(&mut cfg, s.creds.as_deref()).await;
crate::subscription_preflight(&cfg).await; // warn early if the CLI isn't logged in
let mut printer = reader.external_printer()?; // None on piped stdin → blocking fallback
let sp = crate::spawn_engagement(base, cfg, mcp, mode_e);
@@ -848,6 +849,7 @@ async fn start_background(base: &Path, s: &Session, reader: &mut Reader,
let choice = Arc::new(Mutex::new(StopMode::Run));
let soft_task = soft.clone(); // idle guardrail triggers a soft-stop (validate)
let cancel_task = cancel.clone();
let sub_mcp = s.subscription && mcp; // for the "browser/tools never engaged" diagnostic
let (live2, done2, hist2, choice2) = (live.clone(), done.clone(), history, choice.clone());
tokio::spawn(async move {
@@ -855,6 +857,7 @@ async fn start_background(base: &Path, s: &Session, reader: &mut Reader,
let mut last_saved = 0usize;
let mut last_find = Instant::now(); // time of the last NEW finding
let mut idle_fired = false;
let mut tool_events = 0usize; // exec/net/read/browser activity seen
let mut ticker = tokio::time::interval(std::time::Duration::from_secs(15));
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
loop {
@@ -862,6 +865,7 @@ async fn start_background(base: &Path, s: &Session, reader: &mut Reader,
maybe = rx.recv() => {
let Some(line) = maybe else { break };
live2.lock().unwrap().ingest(&line);
if line.contains("exec:") || line.contains("net:") || line.contains("read:") || line.contains("browser") { tool_events += 1; }
if let Some(out) = crate::render_compact(&line) { let _ = printer.print(out); }
// Checkpoint on each new finding; also resets the idle clock.
let snap = {
@@ -896,6 +900,13 @@ async fn start_background(base: &Path, s: &Session, reader: &mut Reader,
let task_out = task.await.unwrap_or_default();
let mode_choice = *choice2.lock().unwrap();
// Diagnostic: subscription + MCP but the agents never ran a single tool/
// browser action → the CLI almost certainly isn't logged in (or the MCP
// didn't engage), which is why nothing was found.
if sub_mcp && tool_events == 0 {
let _ = printer.print("\x1b[1;33m[!] no browser/tool activity was observed this run — the subscription CLI is likely NOT logged in (run `claude` → /login) or the Playwright MCP didn't start. That's usually why a run finds 0.\x1b[0m".to_string());
}
if mode_choice == StopMode::Discard {
std::fs::remove_dir_all(&workdir).ok();
clear_checkpoint();
+88 -4
View File
@@ -201,8 +201,14 @@ impl ChatClient {
"codex" => {
cmd.arg("exec").arg("--model").arg(model)
.arg("--dangerously-bypass-approvals-and-sandbox");
// Codex takes MCP servers as `-c mcp_servers.<name>....` TOML
// overrides, NOT a config-file path. Read our .mcp.json and inject
// each server's command/args so the browser MCP actually loads.
if let Some(mcp) = mcp_config {
cmd.arg("--config").arg(format!("mcp_config_file={mcp}"));
for (name, cmdline, args) in mcp_servers_from(mcp) {
cmd.arg("-c").arg(format!("mcp_servers.{name}.command={cmdline}"));
cmd.arg("-c").arg(format!("mcp_servers.{name}.args={args}"));
}
}
cmd.arg("-");
}
@@ -372,6 +378,25 @@ fn tool_event(name: &str, input: Option<&serde_json::Value>) -> String {
}
}
/// Parse an `.mcp.json` (`{ "mcpServers": { name: { command, args } } }`) into
/// `(name, command, args_json)` tuples — used to inject servers into Codex's
/// `-c mcp_servers.*` TOML overrides (Codex has no config-file flag).
fn mcp_servers_from(path: &str) -> Vec<(String, String, String)> {
let mut out = Vec::new();
let Ok(txt) = std::fs::read_to_string(path) else { return out };
let Ok(v) = serde_json::from_str::<serde_json::Value>(&txt) else { return out };
let servers = v.get("mcpServers").cloned().unwrap_or(v);
if let Some(obj) = servers.as_object() {
for (name, s) in obj {
let command = s.get("command").and_then(|c| c.as_str()).unwrap_or("").to_string();
if command.is_empty() { continue; }
let args = s.get("args").cloned().unwrap_or(serde_json::json!([]));
out.push((name.clone(), command, args.to_string()));
}
}
out
}
/// Map a provider to its local agentic CLI binary (subscription backend).
pub fn cli_binary_for(provider: &str) -> Option<&'static str> {
match provider {
@@ -395,6 +420,54 @@ pub fn installed_cli_backends() -> Vec<&'static str> {
["claude", "codex", "grok", "gemini"].into_iter().filter(|b| binary_in_path(b)).collect()
}
/// Login state of a subscription CLI backend.
#[derive(Debug, PartialEq, Eq)]
pub enum LoginStatus {
NotInstalled,
LoggedIn,
NotLoggedIn,
Unknown, // installed but couldn't determine (timeout / weird output)
}
/// Check whether a subscription CLI is installed AND logged in, by sending it a
/// trivial prompt and inspecting the reply. Cheap (a few tokens) and bounded by a
/// short timeout. Detects the common "not authenticated / please login / no
/// credit" errors so the operator is warned before a whole run comes back empty.
pub async fn cli_login_status(provider: &str) -> LoginStatus {
let Some(bin) = cli_binary_for(provider) else { return LoginStatus::NotInstalled };
if !binary_in_path(bin) { return LoginStatus::NotInstalled; }
let mut cmd = Command::new(bin);
match bin {
"claude" => { cmd.arg("-p").arg("--output-format").arg("text").arg("--dangerously-skip-permissions"); }
"codex" => { cmd.arg("exec").arg("--dangerously-bypass-approvals-and-sandbox").arg("-"); }
_ => { cmd.arg("-p"); } // grok / gemini: prompt on stdin
}
cmd.stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::piped()).kill_on_drop(true);
let mut child = match cmd.spawn() { Ok(c) => c, Err(_) => return LoginStatus::Unknown };
if let Some(mut stdin) = child.stdin.take() {
let _ = stdin.write_all(b"Reply with exactly: OK").await;
}
let out = match tokio::time::timeout(Duration::from_secs(45), child.wait_with_output()).await {
Ok(Ok(o)) => o,
_ => return LoginStatus::Unknown,
};
let text = format!("{}\n{}", String::from_utf8_lossy(&out.stdout), String::from_utf8_lossy(&out.stderr)).to_lowercase();
let auth_err = ["not logged in", "please log in", "please login", "run /login", "authenticate",
"authentication", "unauthorized", "not authenticated", "no credit", "credit balance",
"invalid api key", "no api key", "sign in", "session expired", "logged out"];
if auth_err.iter().any(|k| text.contains(k)) {
return LoginStatus::NotLoggedIn;
}
if out.status.success() && text.contains("ok") {
return LoginStatus::LoggedIn;
}
// Produced *some* non-auth output → almost certainly usable.
if out.status.success() && !text.trim().is_empty() {
return LoginStatus::LoggedIn;
}
LoginStatus::Unknown
}
/// Does this provider's agentic CLI accept a Playwright MCP config?
/// Claude Code and Codex do; Gemini/Grok CLIs don't take an MCP-config flag, so
/// they fall back to their own built-in tools.
@@ -418,10 +491,21 @@ pub fn ensure_playwright_mcp() -> Result<()> {
.stdout(Stdio::null())
.stderr(Stdio::null())
.status();
match out {
Ok(_) => Ok(()),
Err(e) => Err(anyhow!("could not provision @playwright/mcp via npx: {e}")),
if let Err(e) = out {
return Err(anyhow!("could not provision @playwright/mcp via npx: {e}"));
}
// Ensure the Chromium browser the MCP server drives is actually installed —
// otherwise the FIRST browser action fails/hangs and the agent gives up with
// no findings (a very common "MCP doesn't execute" cause). Best-effort,
// skippable via NEUROSPLOIT_SKIP_BROWSER_INSTALL=1; non-fatal on failure.
if std::env::var("NEUROSPLOIT_SKIP_BROWSER_INSTALL").ok().as_deref() != Some("1") {
let _ = std::process::Command::new("npx")
.args(["-y", "playwright", "install", "chromium"])
.stdout(Stdio::null())
.stderr(Stdio::null())
.status();
}
Ok(())
}
/// Write an `.mcp.json` into `dir` (Playwright by default) and return its path,