fix(ios-qa): recover device tunnels safely after relaunch

2026-09-22 04:40:44 +02:00 · 2026-07-14 16:35:30 -07:00
parent 145fe2d4d1
commit c378074ab7
5 changed files with 972 additions and 46 deletions
@@ -5,6 +5,8 @@
 //   2. launch the app on it (no-op if already running)
 //   3. wait briefly for the in-app StateServer to start
 //   4. copy the boot token from the app's sandbox via devicectl copy from
+//      If an earlier daemon already consumed it, relaunch the app once to mint
+//      a fresh boot token, then verify the relaunched StateServer again.
 //   5. POST /auth/rotate to swap boot token → fresh in-memory token
 //   6. return a DeviceTunnel pointing at the device's IPv6 with the rotated
 //      bearer that subsequent proxied requests carry
@@ -14,6 +16,7 @@
 // live token, which it scopes per-tailnet-session via /auth/mint.

 import { randomBytes } from 'crypto';
+import { spawnSync } from 'child_process';
 import type { DeviceTunnel } from './proxy';
 import {
  listDevices,
@@ -21,12 +24,13 @@ import {
  isAppRunning,
  launchApp,
  copyFileFromAppContainer,
+  type DeviceEntry,
  type SpawnImpl,
  type ResolveImpl,
 } from './devicectl';

 export interface BootstrapOptions {
-  /** Target device UDID. If null, picks the first connected paired device. */
+  /** Target iPhone UDID. If null, picks the best connected paired iPhone. */
  udid?: string;
  /** Bundle ID of the iOS app hosting the StateServer. */
  bundleId: string;
@@ -53,10 +57,85 @@ export type BootstrapErrorReason =
  | 'launch_failed'
  | 'device_locked'
  | 'state_server_unreachable'
+  | 'wrong_app'
  | 'boot_token_unavailable'
  | 'rotate_failed'
  | 'resolve_failed';

+function isIPhoneDevice(device: DeviceEntry): boolean {
+  const platform = device.platform.trim().toLowerCase();
+  const deviceType = device.deviceType.trim().toLowerCase();
+  const model = device.model.trim().toLowerCase();
+
+  // productType is present even on older CoreDevice versions. Prefer the
+  // explicit platform/type fields when available, but retain productType as
+  // a compatibility fallback. An explicit non-iOS platform always loses.
+  if (platform && platform !== 'ios') return false;
+  return deviceType === 'iphone' || model.startsWith('iphone');
+}
+
+function isAvailableDevice(device: Pick<DeviceEntry, 'state' | 'transport'>): boolean {
+  const state = device.state.trim().toLowerCase();
+  const transport = device.transport.trim().toLowerCase();
+  // Xcode 26.6 / iOS 27 beta can report a USB-reachable iPhone as
+  // tunnelState=disconnected until the next devicectl command establishes
+  // the CoreDevice tunnel. The wired transport is the authoritative signal
+  // in that transitional state. Stale devices have no wired transport.
+  return state === 'connected'
+    || state.startsWith('available')
+    || (state === 'disconnected' && transport === 'wired');
+}
+
+function defaultDeviceRank(device: DeviceEntry): number {
+  if (!device.paired || !isIPhoneDevice(device) || !isAvailableDevice(device)) return -1;
+
+  const state = device.state.trim().toLowerCase();
+  const transport = device.transport.trim().toLowerCase();
+  // Prefer the USB-connected phone the user is actively working with. Then
+  // prefer an established CoreDevice tunnel over a merely available device.
+  return (transport === 'wired' ? 100 : 0)
+    + (state === 'connected' ? 10 : 0)
+    + (state.startsWith('available') ? 1 : 0);
+}
+
+function pickDefaultDevice(devices: DeviceEntry[]): DeviceEntry | undefined {
+  let best: DeviceEntry | undefined;
+  let bestRank = -1;
+  for (const device of devices) {
+    const rank = defaultDeviceRank(device);
+    if (rank > bestRank) {
+      best = device;
+      bestRank = rank;
+    }
+  }
+  return best;
+}
+
+const defaultSpawn: SpawnImpl = (cmd, args) => spawnSync(cmd, args, {
+  stdio: 'pipe',
+  timeout: 60_000,
+});
+
+function relaunchApp(
+  udid: string,
+  bundleId: string,
+  spawn: SpawnImpl = defaultSpawn,
+): { ok: true } | { ok: false; error: 'device_locked' | 'launch_failed'; detail?: string } {
+  const r = spawn('xcrun', [
+    'devicectl', 'device', 'process', 'launch',
+    '--device', udid,
+    '--terminate-existing',
+    bundleId,
+  ]);
+  if (r.status === 0) return { ok: true };
+
+  const detail = `${r.stderr?.toString() ?? ''}${r.stdout?.toString() ?? ''}`.trim();
+  if (detail.includes('was not, or could not be, unlocked')) {
+    return { ok: false, error: 'device_locked', detail };
+  }
+  return { ok: false, error: 'launch_failed', detail };
+}
+
 /**
 * Bootstrap a real CoreDevice tunnel to an iOS app's StateServer. Used by
 * the daemon's default tunnelProvider when GSTACK_IOS_TARGET_UDID is set
@@ -77,9 +156,39 @@ export async function bootstrapTunnel(opts: BootstrapOptions): Promise<Bootstrap
  }
  const target = opts.udid
    ? devices.find((d) => d.identifier === opts.udid)
-    : devices.find((d) => d.paired) ?? devices[0];
+    : pickDefaultDevice(devices);
  if (!target) {
-    return { ok: false, error: 'device_not_found', detail: opts.udid };
+    if (opts.udid) {
+      return { ok: false, error: 'device_not_found', detail: opts.udid };
+    }
+    const pairedIPhone = devices.find((d) => d.paired && isIPhoneDevice(d));
+    if (pairedIPhone) {
+      return {
+        ok: false,
+        error: 'device_not_found',
+        detail: `paired iPhone ${pairedIPhone.name} (${pairedIPhone.identifier}) is ${pairedIPhone.state}; connect it over USB and unlock it`,
+      };
+    }
+    const firstIPhone = devices.find(isIPhoneDevice);
+    if (!firstIPhone) {
+      return {
+        ok: false,
+        error: 'device_not_found',
+        detail: 'no iPhone is connected; non-iOS devices are not eligible for iOS QA',
+      };
+    }
+    return {
+      ok: false,
+      error: 'no_paired_device',
+      detail: `device ${firstIPhone.name} (${firstIPhone.identifier}) is ${firstIPhone.state}; run \`xcrun devicectl manage pair --device ${firstIPhone.identifier}\` and tap Trust on the iPhone`,
+    };
+  }
+  if (!isIPhoneDevice(target)) {
+    return {
+      ok: false,
+      error: 'device_not_found',
+      detail: `device ${target.name} (${target.identifier}) is ${target.platform || target.model}, not an iPhone`,
+    };
  }
  if (!target.paired) {
    return {
@@ -88,6 +197,13 @@ export async function bootstrapTunnel(opts: BootstrapOptions): Promise<Bootstrap
      detail: `device ${target.name} (${target.identifier}) is ${target.state}; run \`xcrun devicectl manage pair --device ${target.identifier}\` and tap Trust on the iPhone`,
    };
  }
+  if (!isAvailableDevice(target)) {
+    return {
+      ok: false,
+      error: 'device_not_found',
+      detail: `device ${target.name} (${target.identifier}) is ${target.state}; connect it over USB and unlock it`,
+    };
+  }

  // Step 2: launch app (idempotent — devicectl returns success if already running)
  if (!isAppRunning(target.identifier, opts.bundleId, spawn)) {
@@ -118,29 +234,77 @@ export async function bootstrapTunnel(opts: BootstrapOptions): Promise<Bootstrap

  // Step 4: wait for StateServer to become reachable, then scrape boot token.
  // Probe /healthz with retries (the listener can take a moment to bind).
-  const deadline = Date.now() + startupTimeoutMs;
-  let healthOK = false;
-  while (Date.now() < deadline) {
-    try {
-      const r = await fetchFn(`http://[${ipv6}]:${port}/healthz`, {
-        signal: AbortSignal.timeout(2_000),
-      });
-      if (r.ok) { healthOK = true; break; }
-    } catch { /* retry */ }
-    await new Promise((res) => setTimeout(res, 250));
-  }
-  if (!healthOK) {
-    return { ok: false, error: 'state_server_unreachable', detail: `no /healthz response from [${ipv6}]:${port} within ${startupTimeoutMs}ms` };
-  }
+  const waitForStateServer = async (): Promise<BootstrapResult | null> => {
+    const deadline = Date.now() + startupTimeoutMs;
+    while (Date.now() < deadline) {
+      try {
+        const r = await fetchFn(`http://[${ipv6}]:${port}/healthz`, {
+          signal: AbortSignal.timeout(2_000),
+        });
+        if (r.ok) {
+          const health = await r.json().catch(() => null) as { bundle_id?: string } | null;
+          // Older bridges did not identify their bundle. Preserve compatibility,
+          // but reject an explicit mismatch from current bridges: another debug
+          // app already owns the fixed StateServer port on this device.
+          if (health?.bundle_id && health.bundle_id !== opts.bundleId) {
+            return {
+              ok: false,
+              error: 'wrong_app',
+              detail: `expected ${opts.bundleId} but StateServer port ${port} belongs to ${health.bundle_id}; terminate the other debug app`,
+            };
+          }
+          return null;
+        }
+      } catch { /* retry */ }
+      await new Promise((res) => setTimeout(res, 250));
+    }
+    return {
+      ok: false,
+      error: 'state_server_unreachable',
+      detail: `no /healthz response from [${ipv6}]:${port} within ${startupTimeoutMs}ms`,
+    };
+  };

-  const bootToken = copyFileFromAppContainer({
+  const healthFailure = await waitForStateServer();
+  if (healthFailure) return healthFailure;
+
+  const readBootToken = () => copyFileFromAppContainer({
    udid: target.identifier,
    bundleId: opts.bundleId,
    sourceRelativePath: tokenPath,
    spawn,
  });
+
+  let bootToken = readBootToken();
  if (!bootToken) {
-    return { ok: false, error: 'boot_token_unavailable', detail: `couldn't read ${tokenPath} from ${opts.bundleId}` };
+    // A healthy running app can lack a boot token when an earlier daemon
+    // already rotated it. A new daemon has no way to recover that in-memory
+    // bearer, so restart exactly once to make StateServer mint a fresh one.
+    // The explicit bundle check above prevents disrupting an unrelated app
+    // that happens to own the fixed StateServer port.
+    const relaunched = relaunchApp(target.identifier, opts.bundleId, spawn);
+    if (!relaunched.ok) {
+      return { ok: false, error: relaunched.error, detail: relaunched.detail };
+    }
+
+    // The token is written before StateServer opens its listener. Waiting for
+    // it first prevents a stale response from the terminating process from
+    // being mistaken for readiness of the replacement process.
+    const tokenDeadline = Date.now() + startupTimeoutMs;
+    while (!bootToken && Date.now() < tokenDeadline) {
+      bootToken = readBootToken();
+      if (!bootToken) await new Promise((res) => setTimeout(res, 250));
+    }
+    if (!bootToken) {
+      return {
+        ok: false,
+        error: 'boot_token_unavailable',
+        detail: `couldn't read ${tokenPath} from ${opts.bundleId} after relaunch`,
+      };
+    }
+
+    const relaunchedHealthFailure = await waitForStateServer();
+    if (relaunchedHealthFailure) return relaunchedHealthFailure;
  }

  // Step 5: rotate the boot token to a fresh in-memory-only one.