From c378074ab767555edafc6a70409d56b6f30e332b Mon Sep 17 00:00:00 2001 From: t Date: Tue, 14 Jul 2026 16:35:30 -0700 Subject: [PATCH] fix(ios-qa): recover device tunnels safely after relaunch --- ios-qa/daemon/src/devicectl.ts | 6 + ios-qa/daemon/src/index.ts | 184 +++++++-- ios-qa/daemon/src/tunnel-bootstrap.ts | 202 +++++++++- ios-qa/daemon/test/daemon-integration.test.ts | 252 ++++++++++++ ios-qa/daemon/test/tunnel-bootstrap.test.ts | 374 ++++++++++++++++++ 5 files changed, 972 insertions(+), 46 deletions(-) diff --git a/ios-qa/daemon/src/devicectl.ts b/ios-qa/daemon/src/devicectl.ts index ee1696eb9..c17aec751 100644 --- a/ios-qa/daemon/src/devicectl.ts +++ b/ios-qa/daemon/src/devicectl.ts @@ -13,7 +13,10 @@ export interface DeviceEntry { identifier: string; name: string; model: string; + platform: string; // "iOS" | "iPadOS" | "visionOS" | ... + deviceType: string; // "iPhone" | "iPad" | "realityDevice" | ... state: string; // "connected" | "available" | "available (paired)" | ... + transport: string; // "wired" on USB devices; empty for stale/unavailable entries paired: boolean; } @@ -83,7 +86,10 @@ export function listDevices(spawn: SpawnImpl = defaultSpawn): DeviceEntry[] { identifier: String(d.identifier ?? ''), name: String(props?.name ?? 'unknown'), model: String(hw?.productType ?? 'unknown'), + platform: String(hw?.platform ?? ''), + deviceType: String(hw?.deviceType ?? ''), state: String(conn?.tunnelState ?? 'unknown'), + transport: String(conn?.transportType ?? ''), paired: pairingState === 'paired', }; }); diff --git a/ios-qa/daemon/src/index.ts b/ios-qa/daemon/src/index.ts index 82628b136..f3a70c89f 100644 --- a/ios-qa/daemon/src/index.ts +++ b/ios-qa/daemon/src/index.ts @@ -62,16 +62,36 @@ export async function startDaemon(opts: DaemonOptions): Promise | null = null; const getTunnel = async (): Promise => { - // Cache the tunnel for 30s; refresh on demand. - if (tunnel && Date.now() - cachedTunnelAt < 30_000) return tunnel; - if (opts.tunnelProvider) { - tunnel = await opts.tunnelProvider(); - cachedTunnelAt = Date.now(); + // A successful bootstrap consumes and deletes the app's one-shot boot + // token. Keep that rotated tunnel for this daemon's lifetime instead of + // trying to bootstrap it again on a timer. Failed attempts are not cached. + if (tunnel) return tunnel; + if (!opts.tunnelProvider) return null; + + // Multiple first requests can arrive before bootstrap completes. Share + // one provider call so they cannot race through independent rotations. + if (!tunnelInFlight) { + tunnelInFlight = Promise.resolve() + .then(() => opts.tunnelProvider!()) + .then((candidate) => { + if (candidate) tunnel = candidate; + return candidate; + }) + .finally(() => { + tunnelInFlight = null; + }); } - return tunnel; + return tunnelInFlight; + }; + + const invalidateTunnel = (failedTunnel: DeviceTunnel): void => { + // A late response from the old app must not evict a tunnel that another + // request has already refreshed. Object identity gives each bootstrap a + // cheap generation token without exposing generation state elsewhere. + if (tunnel === failedTunnel) tunnel = null; }; // 2. Tailnet probe (fail-closed). @@ -86,7 +106,7 @@ export async function startDaemon(opts: DaemonOptions): Promise { - await handleLoopback({ req, res, tokenStore, getTunnel }); + await handleLoopback({ req, res, tokenStore, getTunnel, invalidateTunnel }); }); // Use port 0 for OS-assigned port when test/random port collisions are a risk. const requestedPort = opts.loopbackPort; @@ -97,7 +117,7 @@ export async function startDaemon(opts: DaemonOptions): Promise { - await handleLoopback({ req, res, tokenStore, getTunnel }); + await handleLoopback({ req, res, tokenStore, getTunnel, invalidateTunnel }); }); let v6Bound = false; try { @@ -118,6 +138,7 @@ export async function startDaemon(opts: DaemonOptions): Promise Promise; + invalidateTunnel: (failedTunnel: DeviceTunnel) => void; // Explicit security-log + allowlist paths (default to env-derived when undefined). auditPath?: string; attemptsPath?: string; allowlistPath?: string; } +type DeviceProxyResponse = Awaited>; +const RECOVERABLE_SOCKET_ERRORS = new Set([ + 'ECONNABORTED', + 'ECONNREFUSED', + 'ECONNRESET', + 'EHOSTUNREACH', + 'ENETUNREACH', + 'EPIPE', + 'ETIMEDOUT', +]); + +function localProxyError(status: number, error: string): DeviceProxyResponse { + const body = Buffer.from(JSON.stringify({ error }, sanitizeReplacer)); + return { + status, + headers: { 'content-type': 'application/json', 'content-length': String(body.length) }, + body, + }; +} + +async function proxyAttempt(opts: Parameters[0]): Promise { + try { + return await proxyToDevice(opts); + } catch (err) { + const code = (err as { code?: string }).code; + // CoreDevice can surface the same stale route as several different socket + // failures while an app is being relaunched or replaced. Normalize those + // failures so the cache recovery path below can handle all of them. + if (code && RECOVERABLE_SOCKET_ERRORS.has(code)) { + return localProxyError(503, 'device_disconnected'); + } + throw err; + } +} + +function shouldRefreshTunnel(upstream: DeviceProxyResponse): boolean { + // A relaunched app has a new in-memory bearer and rejects the daemon's old + // rotated token. A redeploy can instead leave the old CoreDevice route + // refusing connections or timing out. Both cases require a fresh bootstrap. + if (upstream.status === 401) return true; + if (upstream.status !== 503 && upstream.status !== 504) return false; + try { + const body = JSON.parse(upstream.body.toString('utf-8')) as { error?: string }; + return body.error === 'device_disconnected' || body.error === 'upstream_timeout'; + } catch { + return false; + } +} + +function canReplayAfterRefresh(inbound: IncomingMessage, upstream: DeviceProxyResponse): boolean { + // A 401 proves the stale bearer was rejected before StateServer dispatched + // the operation, so retrying is safe even for a mutation. Connection loss + // is ambiguous: the app may have applied a tap/write before its response was + // lost. Replay only read-only requests in that case to prevent double taps. + if (upstream.status === 401) return true; + const method = inbound.method ?? 'GET'; + return method === 'GET' || method === 'HEAD' || method === 'OPTIONS'; +} + +async function proxyWithTunnelRecovery(opts: { + inbound: IncomingMessage; + body: Buffer; + sessionId: string | null; + agentIdentity?: string; + getTunnel: HandlerCtx['getTunnel']; + invalidateTunnel: HandlerCtx['invalidateTunnel']; +}): Promise<{ tunnel: DeviceTunnel; upstream: DeviceProxyResponse } | null> { + let tunnel = await opts.getTunnel(); + if (!tunnel) return null; + + const makeAttempt = (candidate: DeviceTunnel) => proxyAttempt({ + inbound: opts.inbound, + body: opts.body, + tunnel: candidate, + sessionId: opts.sessionId, + agentIdentity: opts.agentIdentity, + }); + + let upstream = await makeAttempt(tunnel); + if (!shouldRefreshTunnel(upstream)) return { tunnel, upstream }; + + const failedTunnel = tunnel; + const replaySafe = canReplayAfterRefresh(opts.inbound, upstream); + opts.invalidateTunnel(tunnel); + const refreshed = await opts.getTunnel(); + if (!refreshed) return replaySafe ? null : { tunnel: failedTunnel, upstream }; + + // The replacement is now cached for the next request, but never replay an + // ambiguous mutation whose response was lost: doing so could double-tap or + // apply a state transition twice. + if (!replaySafe) return { tunnel: failedTunnel, upstream }; + + tunnel = refreshed; + upstream = await makeAttempt(tunnel); + // Do not loop forever if the replacement app is itself unavailable. Leave + // the cache empty so the next independent request can bootstrap again. + if (shouldRefreshTunnel(upstream)) opts.invalidateTunnel(tunnel); + return { tunnel, upstream }; +} + function readBody(req: IncomingMessage, maxBytes = 1_048_576): Promise { return new Promise((resolve, reject) => { const chunks: Buffer[] = []; @@ -228,7 +350,7 @@ function sendJson(res: ServerResponse, status: number, body: unknown): void { * loopback bind itself is the boundary). */ async function handleLoopback(ctx: HandlerCtx): Promise { - const { req, res, tokenStore, getTunnel } = ctx; + const { req, res, tokenStore, getTunnel, invalidateTunnel } = ctx; const url = parseUrl(req.url ?? '/'); const path = url.pathname ?? '/'; const method = req.method ?? 'GET'; @@ -262,16 +384,23 @@ async function handleLoopback(ctx: HandlerCtx): Promise { } // Other endpoints — proxy to the device. - const tunnel = await getTunnel(); - if (!tunnel) { - sendJson(res, 503, { error: 'device_not_connected' }); - return; - } const body = await readBody(req); if ('error' in body) { sendJson(res, 413, body); return; } const sessionId = (req.headers['x-session-id'] as string | undefined) ?? null; const agentIdentity = (req.headers['x-agent-identity'] as string | undefined) ?? undefined; - const upstream = await proxyToDevice({ inbound: req, body, tunnel, sessionId, agentIdentity }); + const proxied = await proxyWithTunnelRecovery({ + inbound: req, + body, + sessionId, + agentIdentity, + getTunnel, + invalidateTunnel, + }); + if (!proxied) { + sendJson(res, 503, { error: 'device_not_connected' }); + return; + } + const { upstream } = proxied; res.writeHead(upstream.status, upstream.headers); res.end(upstream.body); } catch (err) { @@ -287,7 +416,7 @@ interface TailnetCtx extends HandlerCtx { * Tailnet handler — locked allowlist + capability tiers. */ async function handleTailnet(ctx: TailnetCtx): Promise { - const { req, res, tokenStore, getTunnel, whoIsImpl, auditPath, attemptsPath, allowlistPath } = ctx; + const { req, res, tokenStore, getTunnel, invalidateTunnel, whoIsImpl, auditPath, attemptsPath, allowlistPath } = ctx; const url = parseUrl(req.url ?? '/'); const path = url.pathname ?? '/'; const method = req.method ?? 'GET'; @@ -375,19 +504,20 @@ async function handleTailnet(ctx: TailnetCtx): Promise { } // Proxy to device. - const tunnel = await getTunnel(); - if (!tunnel) { + const sessionId = (req.headers['x-session-id'] as string | undefined) ?? null; + const proxied = await proxyWithTunnelRecovery({ + inbound: req, + body, + sessionId, + agentIdentity: session.identity, + getTunnel, + invalidateTunnel, + }); + if (!proxied) { sendJson(res, 503, { error: 'device_not_connected' }); return; } - const sessionId = (req.headers['x-session-id'] as string | undefined) ?? null; - const upstream = await proxyToDevice({ - inbound: req, - body, - tunnel, - sessionId, - agentIdentity: session.identity, - }); + const { tunnel, upstream } = proxied; // Audit the action (mutating endpoints only). if (requiredCapability !== 'observe') { diff --git a/ios-qa/daemon/src/tunnel-bootstrap.ts b/ios-qa/daemon/src/tunnel-bootstrap.ts index aa6636938..f1be1ca48 100644 --- a/ios-qa/daemon/src/tunnel-bootstrap.ts +++ b/ios-qa/daemon/src/tunnel-bootstrap.ts @@ -5,6 +5,8 @@ // 2. launch the app on it (no-op if already running) // 3. wait briefly for the in-app StateServer to start // 4. copy the boot token from the app's sandbox via devicectl copy from +// If an earlier daemon already consumed it, relaunch the app once to mint +// a fresh boot token, then verify the relaunched StateServer again. // 5. POST /auth/rotate to swap boot token → fresh in-memory token // 6. return a DeviceTunnel pointing at the device's IPv6 with the rotated // bearer that subsequent proxied requests carry @@ -14,6 +16,7 @@ // live token, which it scopes per-tailnet-session via /auth/mint. import { randomBytes } from 'crypto'; +import { spawnSync } from 'child_process'; import type { DeviceTunnel } from './proxy'; import { listDevices, @@ -21,12 +24,13 @@ import { isAppRunning, launchApp, copyFileFromAppContainer, + type DeviceEntry, type SpawnImpl, type ResolveImpl, } from './devicectl'; export interface BootstrapOptions { - /** Target device UDID. If null, picks the first connected paired device. */ + /** Target iPhone UDID. If null, picks the best connected paired iPhone. */ udid?: string; /** Bundle ID of the iOS app hosting the StateServer. */ bundleId: string; @@ -53,10 +57,85 @@ export type BootstrapErrorReason = | 'launch_failed' | 'device_locked' | 'state_server_unreachable' + | 'wrong_app' | 'boot_token_unavailable' | 'rotate_failed' | 'resolve_failed'; +function isIPhoneDevice(device: DeviceEntry): boolean { + const platform = device.platform.trim().toLowerCase(); + const deviceType = device.deviceType.trim().toLowerCase(); + const model = device.model.trim().toLowerCase(); + + // productType is present even on older CoreDevice versions. Prefer the + // explicit platform/type fields when available, but retain productType as + // a compatibility fallback. An explicit non-iOS platform always loses. + if (platform && platform !== 'ios') return false; + return deviceType === 'iphone' || model.startsWith('iphone'); +} + +function isAvailableDevice(device: Pick): boolean { + const state = device.state.trim().toLowerCase(); + const transport = device.transport.trim().toLowerCase(); + // Xcode 26.6 / iOS 27 beta can report a USB-reachable iPhone as + // tunnelState=disconnected until the next devicectl command establishes + // the CoreDevice tunnel. The wired transport is the authoritative signal + // in that transitional state. Stale devices have no wired transport. + return state === 'connected' + || state.startsWith('available') + || (state === 'disconnected' && transport === 'wired'); +} + +function defaultDeviceRank(device: DeviceEntry): number { + if (!device.paired || !isIPhoneDevice(device) || !isAvailableDevice(device)) return -1; + + const state = device.state.trim().toLowerCase(); + const transport = device.transport.trim().toLowerCase(); + // Prefer the USB-connected phone the user is actively working with. Then + // prefer an established CoreDevice tunnel over a merely available device. + return (transport === 'wired' ? 100 : 0) + + (state === 'connected' ? 10 : 0) + + (state.startsWith('available') ? 1 : 0); +} + +function pickDefaultDevice(devices: DeviceEntry[]): DeviceEntry | undefined { + let best: DeviceEntry | undefined; + let bestRank = -1; + for (const device of devices) { + const rank = defaultDeviceRank(device); + if (rank > bestRank) { + best = device; + bestRank = rank; + } + } + return best; +} + +const defaultSpawn: SpawnImpl = (cmd, args) => spawnSync(cmd, args, { + stdio: 'pipe', + timeout: 60_000, +}); + +function relaunchApp( + udid: string, + bundleId: string, + spawn: SpawnImpl = defaultSpawn, +): { ok: true } | { ok: false; error: 'device_locked' | 'launch_failed'; detail?: string } { + const r = spawn('xcrun', [ + 'devicectl', 'device', 'process', 'launch', + '--device', udid, + '--terminate-existing', + bundleId, + ]); + if (r.status === 0) return { ok: true }; + + const detail = `${r.stderr?.toString() ?? ''}${r.stdout?.toString() ?? ''}`.trim(); + if (detail.includes('was not, or could not be, unlocked')) { + return { ok: false, error: 'device_locked', detail }; + } + return { ok: false, error: 'launch_failed', detail }; +} + /** * Bootstrap a real CoreDevice tunnel to an iOS app's StateServer. Used by * the daemon's default tunnelProvider when GSTACK_IOS_TARGET_UDID is set @@ -77,9 +156,39 @@ export async function bootstrapTunnel(opts: BootstrapOptions): Promise d.identifier === opts.udid) - : devices.find((d) => d.paired) ?? devices[0]; + : pickDefaultDevice(devices); if (!target) { - return { ok: false, error: 'device_not_found', detail: opts.udid }; + if (opts.udid) { + return { ok: false, error: 'device_not_found', detail: opts.udid }; + } + const pairedIPhone = devices.find((d) => d.paired && isIPhoneDevice(d)); + if (pairedIPhone) { + return { + ok: false, + error: 'device_not_found', + detail: `paired iPhone ${pairedIPhone.name} (${pairedIPhone.identifier}) is ${pairedIPhone.state}; connect it over USB and unlock it`, + }; + } + const firstIPhone = devices.find(isIPhoneDevice); + if (!firstIPhone) { + return { + ok: false, + error: 'device_not_found', + detail: 'no iPhone is connected; non-iOS devices are not eligible for iOS QA', + }; + } + return { + ok: false, + error: 'no_paired_device', + detail: `device ${firstIPhone.name} (${firstIPhone.identifier}) is ${firstIPhone.state}; run \`xcrun devicectl manage pair --device ${firstIPhone.identifier}\` and tap Trust on the iPhone`, + }; + } + if (!isIPhoneDevice(target)) { + return { + ok: false, + error: 'device_not_found', + detail: `device ${target.name} (${target.identifier}) is ${target.platform || target.model}, not an iPhone`, + }; } if (!target.paired) { return { @@ -88,6 +197,13 @@ export async function bootstrapTunnel(opts: BootstrapOptions): Promise setTimeout(res, 250)); - } - if (!healthOK) { - return { ok: false, error: 'state_server_unreachable', detail: `no /healthz response from [${ipv6}]:${port} within ${startupTimeoutMs}ms` }; - } + const waitForStateServer = async (): Promise => { + const deadline = Date.now() + startupTimeoutMs; + while (Date.now() < deadline) { + try { + const r = await fetchFn(`http://[${ipv6}]:${port}/healthz`, { + signal: AbortSignal.timeout(2_000), + }); + if (r.ok) { + const health = await r.json().catch(() => null) as { bundle_id?: string } | null; + // Older bridges did not identify their bundle. Preserve compatibility, + // but reject an explicit mismatch from current bridges: another debug + // app already owns the fixed StateServer port on this device. + if (health?.bundle_id && health.bundle_id !== opts.bundleId) { + return { + ok: false, + error: 'wrong_app', + detail: `expected ${opts.bundleId} but StateServer port ${port} belongs to ${health.bundle_id}; terminate the other debug app`, + }; + } + return null; + } + } catch { /* retry */ } + await new Promise((res) => setTimeout(res, 250)); + } + return { + ok: false, + error: 'state_server_unreachable', + detail: `no /healthz response from [${ipv6}]:${port} within ${startupTimeoutMs}ms`, + }; + }; - const bootToken = copyFileFromAppContainer({ + const healthFailure = await waitForStateServer(); + if (healthFailure) return healthFailure; + + const readBootToken = () => copyFileFromAppContainer({ udid: target.identifier, bundleId: opts.bundleId, sourceRelativePath: tokenPath, spawn, }); + + let bootToken = readBootToken(); if (!bootToken) { - return { ok: false, error: 'boot_token_unavailable', detail: `couldn't read ${tokenPath} from ${opts.bundleId}` }; + // A healthy running app can lack a boot token when an earlier daemon + // already rotated it. A new daemon has no way to recover that in-memory + // bearer, so restart exactly once to make StateServer mint a fresh one. + // The explicit bundle check above prevents disrupting an unrelated app + // that happens to own the fixed StateServer port. + const relaunched = relaunchApp(target.identifier, opts.bundleId, spawn); + if (!relaunched.ok) { + return { ok: false, error: relaunched.error, detail: relaunched.detail }; + } + + // The token is written before StateServer opens its listener. Waiting for + // it first prevents a stale response from the terminating process from + // being mistaken for readiness of the replacement process. + const tokenDeadline = Date.now() + startupTimeoutMs; + while (!bootToken && Date.now() < tokenDeadline) { + bootToken = readBootToken(); + if (!bootToken) await new Promise((res) => setTimeout(res, 250)); + } + if (!bootToken) { + return { + ok: false, + error: 'boot_token_unavailable', + detail: `couldn't read ${tokenPath} from ${opts.bundleId} after relaunch`, + }; + } + + const relaunchedHealthFailure = await waitForStateServer(); + if (relaunchedHealthFailure) return relaunchedHealthFailure; } // Step 5: rotate the boot token to a fresh in-memory-only one. diff --git a/ios-qa/daemon/test/daemon-integration.test.ts b/ios-qa/daemon/test/daemon-integration.test.ts index b131cc3ff..b41144ecd 100644 --- a/ios-qa/daemon/test/daemon-integration.test.ts +++ b/ios-qa/daemon/test/daemon-integration.test.ts @@ -132,6 +132,258 @@ describe('daemon — loopback listener', () => { expect(lastReq?.headers['x-session-id']).toBe('sess-loopback-1'); }); + test('concurrent first requests share one tunnel bootstrap', async () => { + let bootstraps = 0; + let releaseBootstrap!: () => void; + let markBootstrapStarted!: () => void; + const bootstrapStarted = new Promise((resolve) => { markBootstrapStarted = resolve; }); + const bootstrapGate = new Promise((resolve) => { releaseBootstrap = resolve; }); + const tunnel: DeviceTunnel = { + udid: 'STUB-UDID', + ipv6Addr: '127.0.0.1', + port: stub.port, + bootTokenRotated: STATE_SERVER_TOKEN, + }; + const d = await startDaemon({ + loopbackPort: 0, + tailnetEnabled: false, + pidfilePath: join(workDir, 'daemon-concurrent-bootstrap.pid'), + tunnelProvider: async () => { + bootstraps++; + markBootstrapStarted(); + await bootstrapGate; + return tunnel; + }, + }); + if ('error' in d) throw new Error(d.error); + + try { + const base = `http://127.0.0.1:${d.loopbackPort}`; + const requests = [ + fetchWith('GET', `${base}/screenshot`), + fetchWith('GET', `${base}/screenshot`), + fetchWith('GET', `${base}/screenshot`), + ]; + await bootstrapStarted; + await new Promise((resolve) => setTimeout(resolve, 25)); + releaseBootstrap(); + const responses = await Promise.all(requests); + expect(responses.map((response) => response.status)).toEqual([200, 200, 200]); + expect(bootstraps).toBe(1); + } finally { + releaseBootstrap(); + await d.close(); + } + }); + + test('reuses a healthy rotated tunnel beyond the old 30-second boundary', async () => { + let bootstraps = 0; + const tunnel: DeviceTunnel = { + udid: 'STUB-UDID', + ipv6Addr: '127.0.0.1', + port: stub.port, + bootTokenRotated: STATE_SERVER_TOKEN, + }; + const d = await startDaemon({ + loopbackPort: 0, + tailnetEnabled: false, + pidfilePath: join(workDir, 'daemon-one-shot-bootstrap.pid'), + tunnelProvider: async () => { + bootstraps++; + if (bootstraps > 1) throw new Error('one-shot boot token was already consumed'); + return tunnel; + }, + }); + if ('error' in d) throw new Error(d.error); + + const realNow = Date.now; + const firstRequestAt = realNow(); + try { + const base = `http://127.0.0.1:${d.loopbackPort}`; + const first = await fetchWith('GET', `${base}/screenshot`); + expect(first.status).toBe(200); + + Date.now = () => firstRequestAt + 30_001; + const later = await fetchWith('GET', `${base}/screenshot`); + expect(later.status).toBe(200); + expect(bootstraps).toBe(1); + } finally { + Date.now = realNow; + await d.close(); + } + }); + + test('401 after app relaunch invalidates the token and concurrent requests share one rebootstrap', async () => { + let bootstraps = 0; + let markRefreshStarted!: () => void; + let releaseRefresh!: () => void; + const refreshStarted = new Promise((resolve) => { markRefreshStarted = resolve; }); + const refreshGate = new Promise((resolve) => { releaseRefresh = resolve; }); + const staleTunnel: DeviceTunnel = { + udid: 'STUB-UDID', + ipv6Addr: '127.0.0.1', + port: stub.port, + bootTokenRotated: 'expired-after-relaunch', + }; + const refreshedTunnel: DeviceTunnel = { + ...staleTunnel, + bootTokenRotated: STATE_SERVER_TOKEN, + }; + const d = await startDaemon({ + loopbackPort: 0, + tailnetEnabled: false, + pidfilePath: join(workDir, 'daemon-relaunch-refresh.pid'), + tunnelProvider: async () => { + bootstraps++; + if (bootstraps === 1) return staleTunnel; + if (bootstraps === 2) { + markRefreshStarted(); + await refreshGate; + return refreshedTunnel; + } + throw new Error('concurrent 401s caused duplicate bootstraps'); + }, + }); + if ('error' in d) throw new Error(d.error); + + const requestStart = stub.receivedRequests.length; + try { + const base = `http://127.0.0.1:${d.loopbackPort}`; + const requests = [ + fetchWith('GET', `${base}/screenshot`), + fetchWith('GET', `${base}/screenshot`), + fetchWith('GET', `${base}/screenshot`), + ]; + await refreshStarted; + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(bootstraps).toBe(2); + releaseRefresh(); + + const responses = await Promise.all(requests); + expect(responses.map((response) => response.status)).toEqual([200, 200, 200]); + const attempts = stub.receivedRequests.slice(requestStart); + expect(attempts.filter((request) => request.headers.authorization === 'Bearer expired-after-relaunch')).toHaveLength(3); + expect(attempts.filter((request) => request.headers.authorization === `Bearer ${STATE_SERVER_TOKEN}`)).toHaveLength(3); + + const healthyReuse = await fetchWith('GET', `${base}/screenshot`); + expect(healthyReuse.status).toBe(200); + expect(bootstraps).toBe(2); + } finally { + releaseRefresh(); + await d.close(); + } + }); + + test('connection failure after redeploy reboots the tunnel once and keeps the replacement cached', async () => { + const deadPort = await new Promise((resolve, reject) => { + const reservation = createServer(); + reservation.once('error', reject); + reservation.listen(0, '127.0.0.1', () => { + const address = reservation.address(); + const port = typeof address === 'object' && address ? address.port : 0; + reservation.close((err) => err ? reject(err) : resolve(port)); + }); + }); + let bootstraps = 0; + const d = await startDaemon({ + loopbackPort: 0, + tailnetEnabled: false, + pidfilePath: join(workDir, 'daemon-redeploy-refresh.pid'), + tunnelProvider: async () => { + bootstraps++; + if (bootstraps === 1) { + return { + udid: 'STUB-UDID', + ipv6Addr: '127.0.0.1', + port: deadPort, + bootTokenRotated: 'old-deploy-token', + }; + } + if (bootstraps === 2) { + return { + udid: 'STUB-UDID', + ipv6Addr: '127.0.0.1', + port: stub.port, + bootTokenRotated: STATE_SERVER_TOKEN, + }; + } + throw new Error('healthy replacement tunnel was not reused'); + }, + }); + if ('error' in d) throw new Error(d.error); + + try { + const base = `http://127.0.0.1:${d.loopbackPort}`; + const recovered = await fetchWith('GET', `${base}/screenshot`); + expect(recovered.status).toBe(200); + expect(JSON.parse(recovered.bodyText)).toEqual({ png_base64: 'abc=' }); + expect(bootstraps).toBe(2); + + const healthyReuse = await fetchWith('GET', `${base}/screenshot`); + expect(healthyReuse.status).toBe(200); + expect(bootstraps).toBe(2); + } finally { + await d.close(); + } + }); + + test('connection failure refreshes but never replays an ambiguous tap', async () => { + const deadPort = await new Promise((resolve, reject) => { + const reservation = createServer(); + reservation.once('error', reject); + reservation.listen(0, '127.0.0.1', () => { + const address = reservation.address(); + const port = typeof address === 'object' && address ? address.port : 0; + reservation.close((err) => err ? reject(err) : resolve(port)); + }); + }); + let bootstraps = 0; + const d = await startDaemon({ + loopbackPort: 0, + tailnetEnabled: false, + pidfilePath: join(workDir, 'daemon-mutation-no-replay.pid'), + tunnelProvider: async () => { + bootstraps++; + return bootstraps === 1 + ? { + udid: 'STUB-UDID', + ipv6Addr: '127.0.0.1', + port: deadPort, + bootTokenRotated: 'old-deploy-token', + } + : { + udid: 'STUB-UDID', + ipv6Addr: '127.0.0.1', + port: stub.port, + bootTokenRotated: STATE_SERVER_TOKEN, + }; + }, + }); + if ('error' in d) throw new Error(d.error); + + const beforeTaps = stub.receivedRequests.filter((request) => request.path === '/tap').length; + try { + const base = `http://127.0.0.1:${d.loopbackPort}`; + const ambiguous = await fetchWith('POST', `${base}/tap`, { + headers: { 'x-session-id': 'old-session', 'content-type': 'application/json' }, + body: JSON.stringify({ x: 10, y: 20 }), + }); + expect(ambiguous.status).toBe(503); + expect(bootstraps).toBe(2); + expect(stub.receivedRequests.filter((request) => request.path === '/tap')).toHaveLength(beforeTaps); + + const explicitRetry = await fetchWith('POST', `${base}/tap`, { + headers: { 'x-session-id': 'new-session', 'content-type': 'application/json' }, + body: JSON.stringify({ x: 10, y: 20 }), + }); + expect(explicitRetry.status).toBe(200); + expect(stub.receivedRequests.filter((request) => request.path === '/tap')).toHaveLength(beforeTaps + 1); + expect(bootstraps).toBe(2); + } finally { + await d.close(); + } + }); + test('returns 503 when no device tunnel is provided', async () => { // Force tunnel provider to return null by closing + restarting with null provider. await daemon.close(); diff --git a/ios-qa/daemon/test/tunnel-bootstrap.test.ts b/ios-qa/daemon/test/tunnel-bootstrap.test.ts index 188659b78..37af481be 100644 --- a/ios-qa/daemon/test/tunnel-bootstrap.test.ts +++ b/ios-qa/daemon/test/tunnel-bootstrap.test.ts @@ -105,6 +105,219 @@ describe('bootstrapTunnel', () => { } }); + test('never auto-selects a connected paired Vision Pro ahead of an iPhone', async () => { + const spawn = makeSpawn([ + { + argsMatch: /devicectl list devices/, + jsonOutput: { + result: { devices: [ + { + identifier: 'VISION', + connectionProperties: { tunnelState: 'connected', pairingState: 'paired' }, + deviceProperties: { name: 'Vision Pro' }, + hardwareProperties: { + productType: 'RealityDevice17,1', + platform: 'visionOS', + deviceType: 'realityDevice', + }, + }, + { + identifier: 'IPHONE', + connectionProperties: { tunnelState: 'connected', pairingState: 'paired', transportType: 'wired' }, + deviceProperties: { name: 'USB iPhone' }, + hardwareProperties: { + productType: 'iPhone17,1', + platform: 'iOS', + deviceType: 'iPhone', + }, + }, + ] }, + }, + }, + { + argsMatch: /devicectl device info processes -d IPHONE/, + jsonOutput: { result: { runningProcesses: [{ executable: 'file:///var/containers/Bundle/Application/X/com.test.app/com.test' }] } }, + }, + { + argsMatch: /devicectl device info details --device IPHONE/, + jsonOutput: { result: { connectionProperties: { tunnelIPAddress: 'fd00::17' } } }, + }, + { + argsMatch: /devicectl device copy from --device IPHONE/, + destOutput: 'TOKEN\n', + }, + ]); + + const r = await bootstrapTunnel({ + bundleId: 'com.test', + spawnImpl: spawn, + fetchImpl: (async () => new Response('{"ok":true}', { status: 200 })) as typeof fetch, + }); + + expect(r.ok).toBe(true); + if (r.ok) expect(r.tunnel.udid).toBe('IPHONE'); + }); + + test('fails clearly when only non-iPhone platforms are connected', async () => { + const spawn = makeSpawn([ + { + argsMatch: /devicectl list devices/, + jsonOutput: { + result: { devices: [ + { + identifier: 'VISION', + connectionProperties: { tunnelState: 'connected', pairingState: 'paired' }, + deviceProperties: { name: 'Vision Pro' }, + hardwareProperties: { + productType: 'RealityDevice17,1', + platform: 'visionOS', + deviceType: 'realityDevice', + }, + }, + { + identifier: 'IPAD', + connectionProperties: { tunnelState: 'connected', pairingState: 'paired' }, + deviceProperties: { name: 'iPad' }, + hardwareProperties: { + productType: 'iPad16,6', + platform: 'iOS', + deviceType: 'iPad', + }, + }, + ] }, + }, + }, + ]); + + const r = await bootstrapTunnel({ bundleId: 'com.test', spawnImpl: spawn }); + expect(r.ok).toBe(false); + if (!r.ok) { + expect(r.error).toBe('device_not_found'); + expect(r.detail).toContain('no iPhone'); + } + }); + + test('rejects an explicitly targeted non-iPhone device', async () => { + const spawn = makeSpawn([ + { + argsMatch: /devicectl list devices/, + jsonOutput: { + result: { devices: [{ + identifier: 'VISION', + connectionProperties: { tunnelState: 'connected', pairingState: 'paired' }, + deviceProperties: { name: 'Vision Pro' }, + hardwareProperties: { + productType: 'RealityDevice17,1', + platform: 'visionOS', + deviceType: 'realityDevice', + }, + }] }, + }, + }, + ]); + + const r = await bootstrapTunnel({ bundleId: 'com.test', udid: 'VISION', spawnImpl: spawn }); + expect(r.ok).toBe(false); + if (!r.ok) { + expect(r.error).toBe('device_not_found'); + expect(r.detail).toContain('not an iPhone'); + } + }); + + test('skips an unavailable paired device for a connected or available paired device', async () => { + const spawn = makeSpawn([ + { + argsMatch: /devicectl list devices/, + jsonOutput: { + result: { devices: [ + { + identifier: 'STALE', + connectionProperties: { tunnelState: 'unavailable', pairingState: 'paired' }, + deviceProperties: { name: 'Stale iPhone' }, + hardwareProperties: { productType: 'iPhone17,1' }, + }, + { + identifier: 'WIRED', + connectionProperties: { tunnelState: 'available', pairingState: 'paired', transportType: 'wired' }, + deviceProperties: { name: 'Wired iPhone' }, + hardwareProperties: { productType: 'iPhone18,2' }, + }, + ] }, + }, + }, + { + argsMatch: /devicectl device info processes -d WIRED/, + jsonOutput: { result: { runningProcesses: [{ executable: 'file:///var/containers/Bundle/Application/X/com.test.app/com.test' }] } }, + }, + { + argsMatch: /devicectl device info details --device WIRED/, + jsonOutput: { result: { connectionProperties: { tunnelIPAddress: 'fd00::2' } } }, + }, + { + argsMatch: /devicectl device copy from --device WIRED/, + destOutput: 'TOKEN\n', + }, + ]); + + const r = await bootstrapTunnel({ + bundleId: 'com.test', + spawnImpl: spawn, + fetchImpl: (async () => new Response('{"ok":true}', { status: 200 })) as typeof fetch, + }); + + expect(r.ok).toBe(true); + if (r.ok) expect(r.tunnel.udid).toBe('WIRED'); + }); + + test('accepts a wired iPhone whose tunnel is disconnected until devicectl wakes it', async () => { + const spawn = makeSpawn([ + { + argsMatch: /devicectl list devices/, + jsonOutput: { + result: { devices: [ + { + identifier: 'STALE-VISION', + connectionProperties: { tunnelState: 'unavailable', pairingState: 'paired' }, + deviceProperties: { name: 'Stale Vision Pro' }, + hardwareProperties: { productType: 'RealityDevice14,1' }, + }, + { + identifier: 'WIRED-DISCONNECTED', + connectionProperties: { + tunnelState: 'disconnected', + pairingState: 'paired', + transportType: 'wired', + }, + deviceProperties: { name: 'USB iPhone' }, + hardwareProperties: { productType: 'iPhone18,2' }, + }, + ] }, + }, + }, + { + argsMatch: /devicectl device info processes -d WIRED-DISCONNECTED/, + jsonOutput: { result: { runningProcesses: [{ executable: 'file:\/\/\/var\/containers\/Bundle\/Application\/X\/com.test.app\/com.test' }] } }, + }, + { + argsMatch: /devicectl device info details --device WIRED-DISCONNECTED/, + jsonOutput: { result: { connectionProperties: { tunnelIPAddress: 'fd00::27' } } }, + }, + { + argsMatch: /devicectl device copy from --device WIRED-DISCONNECTED/, + destOutput: 'TOKEN\n', + }, + ]); + + const r = await bootstrapTunnel({ + bundleId: 'com.test', + spawnImpl: spawn, + fetchImpl: (async () => new Response('{"ok":true}', { status: 200 })) as typeof fetch, + }); + + expect(r.ok).toBe(true); + if (r.ok) expect(r.tunnel.udid).toBe('WIRED-DISCONNECTED'); + }); + test('returns device_locked when launchApp errors due to lock', async () => { const spawn = makeSpawn([ { @@ -166,6 +379,48 @@ describe('bootstrapTunnel', () => { if (!r.ok) expect(r.error).toBe('state_server_unreachable'); }); + test('rejects a StateServer owned by a different app bundle', async () => { + const spawn = makeSpawn([ + { + argsMatch: /devicectl list devices/, + jsonOutput: { + result: { devices: [{ + identifier: 'TEST', + connectionProperties: { tunnelState: 'connected', pairingState: 'paired' }, + deviceProperties: { name: 'Test' }, + hardwareProperties: { productType: 'iPhone18,2' }, + }] }, + }, + }, + { + argsMatch: /devicectl device info processes/, + jsonOutput: { result: { runningProcesses: [] } }, + }, + { + argsMatch: /devicectl device process launch/, + }, + { + argsMatch: /devicectl device info details/, + jsonOutput: { result: { connectionProperties: { tunnelIPAddress: 'fd00::9' } } }, + }, + ]); + + const r = await bootstrapTunnel({ + bundleId: 'com.expected.app', + spawnImpl: spawn, + fetchImpl: (async () => new Response( + JSON.stringify({ bundle_id: 'com.other.debug-app' }), + { status: 200, headers: { 'content-type': 'application/json' } }, + )) as typeof fetch, + }); + + expect(r.ok).toBe(false); + if (!r.ok) { + expect(r.error).toBe('wrong_app'); + expect(r.detail).toContain('com.other.debug-app'); + } + }); + test('happy path: returns DeviceTunnel with rotated token', async () => { const spawn = makeSpawn([ { @@ -234,6 +489,125 @@ describe('bootstrapTunnel', () => { expect(fetchCalls[fetchCalls.length - 1]?.url).toContain('/auth/rotate'); }); + test('relaunches once when a prior daemon consumed the running app boot token', async () => { + const spawn = makeSpawn([ + { + argsMatch: /devicectl list devices/, + jsonOutput: { + result: { devices: [{ + identifier: 'TEST-UDID', + connectionProperties: { tunnelState: 'connected', pairingState: 'paired' }, + deviceProperties: { name: 'Test Device' }, + hardwareProperties: { productType: 'iPhone18,2' }, + }] }, + }, + }, + { + argsMatch: /devicectl device info processes/, + jsonOutput: { result: { runningProcesses: [{ executable: 'file:\/\/\/var\/containers\/Bundle\/Application\/X\/com.test.app\/com.test' }] } }, + }, + { + argsMatch: /devicectl device info details/, + jsonOutput: { result: { connectionProperties: { tunnelIPAddress: 'fd99::beef' } } }, + }, + { + // A prior daemon rotated the one-use token, so StateServer deleted it. + argsMatch: /devicectl device copy from/, + exitCode: 1, + stderr: 'source does not exist', + }, + { + argsMatch: /devicectl device process launch --device TEST-UDID --terminate-existing com\.test/, + }, + { + argsMatch: /devicectl device copy from/, + destOutput: 'FRESH-BOOT-TOKEN\n', + }, + ]); + const fetchCalls: Array<{ url: string; authorization?: string }> = []; + + const r = await bootstrapTunnel({ + bundleId: 'com.test', + spawnImpl: spawn, + fetchImpl: (async (url, init) => { + const u = String(url); + const headers = init?.headers as Record | undefined; + fetchCalls.push({ url: u, authorization: headers?.Authorization }); + if (u.endsWith('/healthz')) { + return new Response(JSON.stringify({ bundle_id: 'com.test' }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + if (u.endsWith('/auth/rotate') && headers?.Authorization === 'Bearer FRESH-BOOT-TOKEN') { + return new Response('{"ok":true}', { status: 200 }); + } + return new Response('unauthorized', { status: 401 }); + }) as typeof fetch, + startupTimeoutMs: 1_000, + }); + + expect(r.ok).toBe(true); + expect(fetchCalls.filter((call) => call.url.endsWith('/healthz'))).toHaveLength(2); + expect(fetchCalls.at(-1)?.authorization).toBe('Bearer FRESH-BOOT-TOKEN'); + }); + + test('rechecks bundle ownership after recovering a consumed boot token', async () => { + const spawn = makeSpawn([ + { + argsMatch: /devicectl list devices/, + jsonOutput: { + result: { devices: [{ + identifier: 'TEST-UDID', + connectionProperties: { tunnelState: 'connected', pairingState: 'paired' }, + deviceProperties: { name: 'Test Device' }, + hardwareProperties: { productType: 'iPhone18,2' }, + }] }, + }, + }, + { + argsMatch: /devicectl device info processes/, + jsonOutput: { result: { runningProcesses: [{ executable: 'file:\/\/\/var\/containers\/Bundle\/Application\/X\/com.test.app\/com.test' }] } }, + }, + { + argsMatch: /devicectl device info details/, + jsonOutput: { result: { connectionProperties: { tunnelIPAddress: 'fd99::beef' } } }, + }, + { argsMatch: /devicectl device copy from/, exitCode: 1 }, + { + argsMatch: /devicectl device process launch --device TEST-UDID --terminate-existing com\.test/, + }, + { argsMatch: /devicectl device copy from/, destOutput: 'FRESH-BOOT-TOKEN\n' }, + ]); + let healthChecks = 0; + let rotateCalls = 0; + + const r = await bootstrapTunnel({ + bundleId: 'com.test', + spawnImpl: spawn, + fetchImpl: (async (url) => { + const u = String(url); + if (u.endsWith('/healthz')) { + healthChecks++; + return new Response(JSON.stringify({ + bundle_id: healthChecks === 1 ? 'com.test' : 'com.other.debug-app', + }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + rotateCalls++; + return new Response('{"ok":true}', { status: 200 }); + }) as typeof fetch, + startupTimeoutMs: 1_000, + }); + + expect(r.ok).toBe(false); + if (!r.ok) expect(r.error).toBe('wrong_app'); + expect(healthChecks).toBe(2); + expect(rotateCalls).toBe(0); + }); + test('resolve_failed when hostname cant be resolved to an IPv6', async () => { const spawn = makeSpawn([ {