fix(ios-qa): recover device tunnels safely after relaunch

ios-qa/daemon/src/devicectl.ts

@@ -13,7 +13,10 @@ export interface DeviceEntry {
   identifier: string;
   name: string;
   model: string;
+  platform: string; // "iOS" | "iPadOS" | "visionOS" | ...
+  deviceType: string; // "iPhone" | "iPad" | "realityDevice" | ...
   state: string; // "connected" | "available" | "available (paired)" | ...
+  transport: string; // "wired" on USB devices; empty for stale/unavailable entries
   paired: boolean;
 }
 
@@ -83,7 +86,10 @@ export function listDevices(spawn: SpawnImpl = defaultSpawn): DeviceEntry[] {
         identifier: String(d.identifier ?? ''),
         name: String(props?.name ?? 'unknown'),
         model: String(hw?.productType ?? 'unknown'),
+        platform: String(hw?.platform ?? ''),
+        deviceType: String(hw?.deviceType ?? ''),
         state: String(conn?.tunnelState ?? 'unknown'),
+        transport: String(conn?.transportType ?? ''),
         paired: pairingState === 'paired',
       };
     });

ios-qa/daemon/src/index.ts

@@ -62,16 +62,36 @@ export async function startDaemon(opts: DaemonOptions): Promise<RunningDaemon |
 
   const tokenStore = new SessionTokenStore();
   let tunnel: DeviceTunnel | null = null;
-  let cachedTunnelAt = 0;
+  let tunnelInFlight: Promise<DeviceTunnel | null> | null = null;
 
   const getTunnel = async (): Promise<DeviceTunnel | null> => {
-    // Cache the tunnel for 30s; refresh on demand.
-    if (tunnel && Date.now() - cachedTunnelAt < 30_000) return tunnel;
-    if (opts.tunnelProvider) {
-      tunnel = await opts.tunnelProvider();
-      cachedTunnelAt = Date.now();
+    // A successful bootstrap consumes and deletes the app's one-shot boot
+    // token. Keep that rotated tunnel for this daemon's lifetime instead of
+    // trying to bootstrap it again on a timer. Failed attempts are not cached.
+    if (tunnel) return tunnel;
+    if (!opts.tunnelProvider) return null;
+
+    // Multiple first requests can arrive before bootstrap completes. Share
+    // one provider call so they cannot race through independent rotations.
+    if (!tunnelInFlight) {
+      tunnelInFlight = Promise.resolve()
+        .then(() => opts.tunnelProvider!())
+        .then((candidate) => {
+          if (candidate) tunnel = candidate;
+          return candidate;
+        })
+        .finally(() => {
+          tunnelInFlight = null;
+        });
     }
-    return tunnel;
+    return tunnelInFlight;
+  };
+
+  const invalidateTunnel = (failedTunnel: DeviceTunnel): void => {
+    // A late response from the old app must not evict a tunnel that another
+    // request has already refreshed. Object identity gives each bootstrap a
+    // cheap generation token without exposing generation state elsewhere.
+    if (tunnel === failedTunnel) tunnel = null;
   };
 
   // 2. Tailnet probe (fail-closed).
@@ -86,7 +106,7 @@ export async function startDaemon(opts: DaemonOptions): Promise<RunningDaemon |
 
   // 3. Loopback listener (full surface).
   const loopbackServer = createServer(async (req, res) => {
-    await handleLoopback({ req, res, tokenStore, getTunnel });
+    await handleLoopback({ req, res, tokenStore, getTunnel, invalidateTunnel });
   });
   // Use port 0 for OS-assigned port when test/random port collisions are a risk.
   const requestedPort = opts.loopbackPort;
@@ -97,7 +117,7 @@ export async function startDaemon(opts: DaemonOptions): Promise<RunningDaemon |
   // mode this can collide; we try the actualPort first and skip ipv6 if it
   // fails (tests don't exercise ::1 explicitly).
   const loopbackServerV6 = createServer(async (req, res) => {
-    await handleLoopback({ req, res, tokenStore, getTunnel });
+    await handleLoopback({ req, res, tokenStore, getTunnel, invalidateTunnel });
   });
   let v6Bound = false;
   try {
@@ -118,6 +138,7 @@ export async function startDaemon(opts: DaemonOptions): Promise<RunningDaemon |
         res,
         tokenStore,
         getTunnel,
+        invalidateTunnel,
         auditPath: opts.auditPath,
         attemptsPath: opts.attemptsPath,
         allowlistPath: opts.allowlistPath,
@@ -181,12 +202,113 @@ interface HandlerCtx {
   res: ServerResponse;
   tokenStore: SessionTokenStore;
   getTunnel: () => Promise<DeviceTunnel | null>;
+  invalidateTunnel: (failedTunnel: DeviceTunnel) => void;
   // Explicit security-log + allowlist paths (default to env-derived when undefined).
   auditPath?: string;
   attemptsPath?: string;
   allowlistPath?: string;
 }
 
+type DeviceProxyResponse = Awaited<ReturnType<typeof proxyToDevice>>;
+const RECOVERABLE_SOCKET_ERRORS = new Set([
+  'ECONNABORTED',
+  'ECONNREFUSED',
+  'ECONNRESET',
+  'EHOSTUNREACH',
+  'ENETUNREACH',
+  'EPIPE',
+  'ETIMEDOUT',
+]);
+
+function localProxyError(status: number, error: string): DeviceProxyResponse {
+  const body = Buffer.from(JSON.stringify({ error }, sanitizeReplacer));
+  return {
+    status,
+    headers: { 'content-type': 'application/json', 'content-length': String(body.length) },
+    body,
+  };
+}
+
+async function proxyAttempt(opts: Parameters<typeof proxyToDevice>[0]): Promise<DeviceProxyResponse> {
+  try {
+    return await proxyToDevice(opts);
+  } catch (err) {
+    const code = (err as { code?: string }).code;
+    // CoreDevice can surface the same stale route as several different socket
+    // failures while an app is being relaunched or replaced. Normalize those
+    // failures so the cache recovery path below can handle all of them.
+    if (code && RECOVERABLE_SOCKET_ERRORS.has(code)) {
+      return localProxyError(503, 'device_disconnected');
+    }
+    throw err;
+  }
+}
+
+function shouldRefreshTunnel(upstream: DeviceProxyResponse): boolean {
+  // A relaunched app has a new in-memory bearer and rejects the daemon's old
+  // rotated token. A redeploy can instead leave the old CoreDevice route
+  // refusing connections or timing out. Both cases require a fresh bootstrap.
+  if (upstream.status === 401) return true;
+  if (upstream.status !== 503 && upstream.status !== 504) return false;
+  try {
+    const body = JSON.parse(upstream.body.toString('utf-8')) as { error?: string };
+    return body.error === 'device_disconnected' || body.error === 'upstream_timeout';
+  } catch {
+    return false;
+  }
+}
+
+function canReplayAfterRefresh(inbound: IncomingMessage, upstream: DeviceProxyResponse): boolean {
+  // A 401 proves the stale bearer was rejected before StateServer dispatched
+  // the operation, so retrying is safe even for a mutation. Connection loss
+  // is ambiguous: the app may have applied a tap/write before its response was
+  // lost. Replay only read-only requests in that case to prevent double taps.
+  if (upstream.status === 401) return true;
+  const method = inbound.method ?? 'GET';
+  return method === 'GET' || method === 'HEAD' || method === 'OPTIONS';
+}
+
+async function proxyWithTunnelRecovery(opts: {
+  inbound: IncomingMessage;
+  body: Buffer;
+  sessionId: string | null;
+  agentIdentity?: string;
+  getTunnel: HandlerCtx['getTunnel'];
+  invalidateTunnel: HandlerCtx['invalidateTunnel'];
+}): Promise<{ tunnel: DeviceTunnel; upstream: DeviceProxyResponse } | null> {
+  let tunnel = await opts.getTunnel();
+  if (!tunnel) return null;
+
+  const makeAttempt = (candidate: DeviceTunnel) => proxyAttempt({
+    inbound: opts.inbound,
+    body: opts.body,
+    tunnel: candidate,
+    sessionId: opts.sessionId,
+    agentIdentity: opts.agentIdentity,
+  });
+
+  let upstream = await makeAttempt(tunnel);
+  if (!shouldRefreshTunnel(upstream)) return { tunnel, upstream };
+
+  const failedTunnel = tunnel;
+  const replaySafe = canReplayAfterRefresh(opts.inbound, upstream);
+  opts.invalidateTunnel(tunnel);
+  const refreshed = await opts.getTunnel();
+  if (!refreshed) return replaySafe ? null : { tunnel: failedTunnel, upstream };
+
+  // The replacement is now cached for the next request, but never replay an
+  // ambiguous mutation whose response was lost: doing so could double-tap or
+  // apply a state transition twice.
+  if (!replaySafe) return { tunnel: failedTunnel, upstream };
+
+  tunnel = refreshed;
+  upstream = await makeAttempt(tunnel);
+  // Do not loop forever if the replacement app is itself unavailable. Leave
+  // the cache empty so the next independent request can bootstrap again.
+  if (shouldRefreshTunnel(upstream)) opts.invalidateTunnel(tunnel);
+  return { tunnel, upstream };
+}
+
 function readBody(req: IncomingMessage, maxBytes = 1_048_576): Promise<Buffer | { error: 'body_too_large' }> {
   return new Promise((resolve, reject) => {
     const chunks: Buffer[] = [];
@@ -228,7 +350,7 @@ function sendJson(res: ServerResponse, status: number, body: unknown): void {
  * loopback bind itself is the boundary).
  */
 async function handleLoopback(ctx: HandlerCtx): Promise<void> {
-  const { req, res, tokenStore, getTunnel } = ctx;
+  const { req, res, tokenStore, getTunnel, invalidateTunnel } = ctx;
   const url = parseUrl(req.url ?? '/');
   const path = url.pathname ?? '/';
   const method = req.method ?? 'GET';
@@ -262,16 +384,23 @@ async function handleLoopback(ctx: HandlerCtx): Promise<void> {
     }
 
     // Other endpoints — proxy to the device.
-    const tunnel = await getTunnel();
-    if (!tunnel) {
-      sendJson(res, 503, { error: 'device_not_connected' });
-      return;
-    }
     const body = await readBody(req);
     if ('error' in body) { sendJson(res, 413, body); return; }
     const sessionId = (req.headers['x-session-id'] as string | undefined) ?? null;
     const agentIdentity = (req.headers['x-agent-identity'] as string | undefined) ?? undefined;
-    const upstream = await proxyToDevice({ inbound: req, body, tunnel, sessionId, agentIdentity });
+    const proxied = await proxyWithTunnelRecovery({
+      inbound: req,
+      body,
+      sessionId,
+      agentIdentity,
+      getTunnel,
+      invalidateTunnel,
+    });
+    if (!proxied) {
+      sendJson(res, 503, { error: 'device_not_connected' });
+      return;
+    }
+    const { upstream } = proxied;
     res.writeHead(upstream.status, upstream.headers);
     res.end(upstream.body);
   } catch (err) {
@@ -287,7 +416,7 @@ interface TailnetCtx extends HandlerCtx {
  * Tailnet handler — locked allowlist + capability tiers.
  */
 async function handleTailnet(ctx: TailnetCtx): Promise<void> {
-  const { req, res, tokenStore, getTunnel, whoIsImpl, auditPath, attemptsPath, allowlistPath } = ctx;
+  const { req, res, tokenStore, getTunnel, invalidateTunnel, whoIsImpl, auditPath, attemptsPath, allowlistPath } = ctx;
   const url = parseUrl(req.url ?? '/');
   const path = url.pathname ?? '/';
   const method = req.method ?? 'GET';
@@ -375,19 +504,20 @@ async function handleTailnet(ctx: TailnetCtx): Promise<void> {
     }
 
     // Proxy to device.
-    const tunnel = await getTunnel();
-    if (!tunnel) {
+    const sessionId = (req.headers['x-session-id'] as string | undefined) ?? null;
+    const proxied = await proxyWithTunnelRecovery({
+      inbound: req,
+      body,
+      sessionId,
+      agentIdentity: session.identity,
+      getTunnel,
+      invalidateTunnel,
+    });
+    if (!proxied) {
       sendJson(res, 503, { error: 'device_not_connected' });
       return;
     }
-    const sessionId = (req.headers['x-session-id'] as string | undefined) ?? null;
-    const upstream = await proxyToDevice({
-      inbound: req,
-      body,
-      tunnel,
-      sessionId,
-      agentIdentity: session.identity,
-    });
+    const { tunnel, upstream } = proxied;
 
     // Audit the action (mutating endpoints only).
     if (requiredCapability !== 'observe') {

ios-qa/daemon/src/tunnel-bootstrap.ts

@@ -5,6 +5,8 @@
 //   2. launch the app on it (no-op if already running)
 //   3. wait briefly for the in-app StateServer to start
 //   4. copy the boot token from the app's sandbox via devicectl copy from
+//      If an earlier daemon already consumed it, relaunch the app once to mint
+//      a fresh boot token, then verify the relaunched StateServer again.
 //   5. POST /auth/rotate to swap boot token → fresh in-memory token
 //   6. return a DeviceTunnel pointing at the device's IPv6 with the rotated
 //      bearer that subsequent proxied requests carry
@@ -14,6 +16,7 @@
 // live token, which it scopes per-tailnet-session via /auth/mint.
 
 import { randomBytes } from 'crypto';
+import { spawnSync } from 'child_process';
 import type { DeviceTunnel } from './proxy';
 import {
   listDevices,
@@ -21,12 +24,13 @@ import {
   isAppRunning,
   launchApp,
   copyFileFromAppContainer,
+  type DeviceEntry,
   type SpawnImpl,
   type ResolveImpl,
 } from './devicectl';
 
 export interface BootstrapOptions {
-  /** Target device UDID. If null, picks the first connected paired device. */
+  /** Target iPhone UDID. If null, picks the best connected paired iPhone. */
   udid?: string;
   /** Bundle ID of the iOS app hosting the StateServer. */
   bundleId: string;
@@ -53,10 +57,85 @@ export type BootstrapErrorReason =
   | 'launch_failed'
   | 'device_locked'
   | 'state_server_unreachable'
+  | 'wrong_app'
   | 'boot_token_unavailable'
   | 'rotate_failed'
   | 'resolve_failed';
 
+function isIPhoneDevice(device: DeviceEntry): boolean {
+  const platform = device.platform.trim().toLowerCase();
+  const deviceType = device.deviceType.trim().toLowerCase();
+  const model = device.model.trim().toLowerCase();
+
+  // productType is present even on older CoreDevice versions. Prefer the
+  // explicit platform/type fields when available, but retain productType as
+  // a compatibility fallback. An explicit non-iOS platform always loses.
+  if (platform && platform !== 'ios') return false;
+  return deviceType === 'iphone' || model.startsWith('iphone');
+}
+
+function isAvailableDevice(device: Pick<DeviceEntry, 'state' | 'transport'>): boolean {
+  const state = device.state.trim().toLowerCase();
+  const transport = device.transport.trim().toLowerCase();
+  // Xcode 26.6 / iOS 27 beta can report a USB-reachable iPhone as
+  // tunnelState=disconnected until the next devicectl command establishes
+  // the CoreDevice tunnel. The wired transport is the authoritative signal
+  // in that transitional state. Stale devices have no wired transport.
+  return state === 'connected'
+    || state.startsWith('available')
+    || (state === 'disconnected' && transport === 'wired');
+}
+
+function defaultDeviceRank(device: DeviceEntry): number {
+  if (!device.paired || !isIPhoneDevice(device) || !isAvailableDevice(device)) return -1;
+
+  const state = device.state.trim().toLowerCase();
+  const transport = device.transport.trim().toLowerCase();
+  // Prefer the USB-connected phone the user is actively working with. Then
+  // prefer an established CoreDevice tunnel over a merely available device.
+  return (transport === 'wired' ? 100 : 0)
+    + (state === 'connected' ? 10 : 0)
+    + (state.startsWith('available') ? 1 : 0);
+}
+
+function pickDefaultDevice(devices: DeviceEntry[]): DeviceEntry | undefined {
+  let best: DeviceEntry | undefined;
+  let bestRank = -1;
+  for (const device of devices) {
+    const rank = defaultDeviceRank(device);
+    if (rank > bestRank) {
+      best = device;
+      bestRank = rank;
+    }
+  }
+  return best;
+}
+
+const defaultSpawn: SpawnImpl = (cmd, args) => spawnSync(cmd, args, {
+  stdio: 'pipe',
+  timeout: 60_000,
+});
+
+function relaunchApp(
+  udid: string,
+  bundleId: string,
+  spawn: SpawnImpl = defaultSpawn,
+): { ok: true } | { ok: false; error: 'device_locked' | 'launch_failed'; detail?: string } {
+  const r = spawn('xcrun', [
+    'devicectl', 'device', 'process', 'launch',
+    '--device', udid,
+    '--terminate-existing',
+    bundleId,
+  ]);
+  if (r.status === 0) return { ok: true };
+
+  const detail = `${r.stderr?.toString() ?? ''}${r.stdout?.toString() ?? ''}`.trim();
+  if (detail.includes('was not, or could not be, unlocked')) {
+    return { ok: false, error: 'device_locked', detail };
+  }
+  return { ok: false, error: 'launch_failed', detail };
+}
+
 /**
  * Bootstrap a real CoreDevice tunnel to an iOS app's StateServer. Used by
  * the daemon's default tunnelProvider when GSTACK_IOS_TARGET_UDID is set
@@ -77,9 +156,39 @@ export async function bootstrapTunnel(opts: BootstrapOptions): Promise<Bootstrap
   }
   const target = opts.udid
     ? devices.find((d) => d.identifier === opts.udid)
-    : devices.find((d) => d.paired) ?? devices[0];
+    : pickDefaultDevice(devices);
   if (!target) {
-    return { ok: false, error: 'device_not_found', detail: opts.udid };
+    if (opts.udid) {
+      return { ok: false, error: 'device_not_found', detail: opts.udid };
+    }
+    const pairedIPhone = devices.find((d) => d.paired && isIPhoneDevice(d));
+    if (pairedIPhone) {
+      return {
+        ok: false,
+        error: 'device_not_found',
+        detail: `paired iPhone ${pairedIPhone.name} (${pairedIPhone.identifier}) is ${pairedIPhone.state}; connect it over USB and unlock it`,
+      };
+    }
+    const firstIPhone = devices.find(isIPhoneDevice);
+    if (!firstIPhone) {
+      return {
+        ok: false,
+        error: 'device_not_found',
+        detail: 'no iPhone is connected; non-iOS devices are not eligible for iOS QA',
+      };
+    }
+    return {
+      ok: false,
+      error: 'no_paired_device',
+      detail: `device ${firstIPhone.name} (${firstIPhone.identifier}) is ${firstIPhone.state}; run \`xcrun devicectl manage pair --device ${firstIPhone.identifier}\` and tap Trust on the iPhone`,
+    };
+  }
+  if (!isIPhoneDevice(target)) {
+    return {
+      ok: false,
+      error: 'device_not_found',
+      detail: `device ${target.name} (${target.identifier}) is ${target.platform || target.model}, not an iPhone`,
+    };
   }
   if (!target.paired) {
     return {
@@ -88,6 +197,13 @@ export async function bootstrapTunnel(opts: BootstrapOptions): Promise<Bootstrap
       detail: `device ${target.name} (${target.identifier}) is ${target.state}; run \`xcrun devicectl manage pair --device ${target.identifier}\` and tap Trust on the iPhone`,
     };
   }
+  if (!isAvailableDevice(target)) {
+    return {
+      ok: false,
+      error: 'device_not_found',
+      detail: `device ${target.name} (${target.identifier}) is ${target.state}; connect it over USB and unlock it`,
+    };
+  }
 
   // Step 2: launch app (idempotent — devicectl returns success if already running)
   if (!isAppRunning(target.identifier, opts.bundleId, spawn)) {
@@ -118,29 +234,77 @@ export async function bootstrapTunnel(opts: BootstrapOptions): Promise<Bootstrap
 
   // Step 4: wait for StateServer to become reachable, then scrape boot token.
   // Probe /healthz with retries (the listener can take a moment to bind).
-  const deadline = Date.now() + startupTimeoutMs;
-  let healthOK = false;
-  while (Date.now() < deadline) {
-    try {
-      const r = await fetchFn(`http://[${ipv6}]:${port}/healthz`, {
-        signal: AbortSignal.timeout(2_000),
-      });
-      if (r.ok) { healthOK = true; break; }
-    } catch { /* retry */ }
-    await new Promise((res) => setTimeout(res, 250));
-  }
-  if (!healthOK) {
-    return { ok: false, error: 'state_server_unreachable', detail: `no /healthz response from [${ipv6}]:${port} within ${startupTimeoutMs}ms` };
-  }
+  const waitForStateServer = async (): Promise<BootstrapResult | null> => {
+    const deadline = Date.now() + startupTimeoutMs;
+    while (Date.now() < deadline) {
+      try {
+        const r = await fetchFn(`http://[${ipv6}]:${port}/healthz`, {
+          signal: AbortSignal.timeout(2_000),
+        });
+        if (r.ok) {
+          const health = await r.json().catch(() => null) as { bundle_id?: string } | null;
+          // Older bridges did not identify their bundle. Preserve compatibility,
+          // but reject an explicit mismatch from current bridges: another debug
+          // app already owns the fixed StateServer port on this device.
+          if (health?.bundle_id && health.bundle_id !== opts.bundleId) {
+            return {
+              ok: false,
+              error: 'wrong_app',
+              detail: `expected ${opts.bundleId} but StateServer port ${port} belongs to ${health.bundle_id}; terminate the other debug app`,
+            };
+          }
+          return null;
+        }
+      } catch { /* retry */ }
+      await new Promise((res) => setTimeout(res, 250));
+    }
+    return {
+      ok: false,
+      error: 'state_server_unreachable',
+      detail: `no /healthz response from [${ipv6}]:${port} within ${startupTimeoutMs}ms`,
+    };
+  };
 
-  const bootToken = copyFileFromAppContainer({
+  const healthFailure = await waitForStateServer();
+  if (healthFailure) return healthFailure;
+
+  const readBootToken = () => copyFileFromAppContainer({
     udid: target.identifier,
     bundleId: opts.bundleId,
     sourceRelativePath: tokenPath,
     spawn,
   });
+
+  let bootToken = readBootToken();
   if (!bootToken) {
-    return { ok: false, error: 'boot_token_unavailable', detail: `couldn't read ${tokenPath} from ${opts.bundleId}` };
+    // A healthy running app can lack a boot token when an earlier daemon
+    // already rotated it. A new daemon has no way to recover that in-memory
+    // bearer, so restart exactly once to make StateServer mint a fresh one.
+    // The explicit bundle check above prevents disrupting an unrelated app
+    // that happens to own the fixed StateServer port.
+    const relaunched = relaunchApp(target.identifier, opts.bundleId, spawn);
+    if (!relaunched.ok) {
+      return { ok: false, error: relaunched.error, detail: relaunched.detail };
+    }
+
+    // The token is written before StateServer opens its listener. Waiting for
+    // it first prevents a stale response from the terminating process from
+    // being mistaken for readiness of the replacement process.
+    const tokenDeadline = Date.now() + startupTimeoutMs;
+    while (!bootToken && Date.now() < tokenDeadline) {
+      bootToken = readBootToken();
+      if (!bootToken) await new Promise((res) => setTimeout(res, 250));
+    }
+    if (!bootToken) {
+      return {
+        ok: false,
+        error: 'boot_token_unavailable',
+        detail: `couldn't read ${tokenPath} from ${opts.bundleId} after relaunch`,
+      };
+    }
+
+    const relaunchedHealthFailure = await waitForStateServer();
+    if (relaunchedHealthFailure) return relaunchedHealthFailure;
   }
 
   // Step 5: rotate the boot token to a fresh in-memory-only one.