fix(ios-qa): recover device tunnels safely after relaunch

This commit is contained in:
t
2026-07-14 16:35:30 -07:00
parent 145fe2d4d1
commit c378074ab7
5 changed files with 972 additions and 46 deletions
+6
View File
@@ -13,7 +13,10 @@ export interface DeviceEntry {
identifier: string;
name: string;
model: string;
platform: string; // "iOS" | "iPadOS" | "visionOS" | ...
deviceType: string; // "iPhone" | "iPad" | "realityDevice" | ...
state: string; // "connected" | "available" | "available (paired)" | ...
transport: string; // "wired" on USB devices; empty for stale/unavailable entries
paired: boolean;
}
@@ -83,7 +86,10 @@ export function listDevices(spawn: SpawnImpl = defaultSpawn): DeviceEntry[] {
identifier: String(d.identifier ?? ''),
name: String(props?.name ?? 'unknown'),
model: String(hw?.productType ?? 'unknown'),
platform: String(hw?.platform ?? ''),
deviceType: String(hw?.deviceType ?? ''),
state: String(conn?.tunnelState ?? 'unknown'),
transport: String(conn?.transportType ?? ''),
paired: pairingState === 'paired',
};
});
+157 -27
View File
@@ -62,16 +62,36 @@ export async function startDaemon(opts: DaemonOptions): Promise<RunningDaemon |
const tokenStore = new SessionTokenStore();
let tunnel: DeviceTunnel | null = null;
let cachedTunnelAt = 0;
let tunnelInFlight: Promise<DeviceTunnel | null> | null = null;
const getTunnel = async (): Promise<DeviceTunnel | null> => {
// Cache the tunnel for 30s; refresh on demand.
if (tunnel && Date.now() - cachedTunnelAt < 30_000) return tunnel;
if (opts.tunnelProvider) {
tunnel = await opts.tunnelProvider();
cachedTunnelAt = Date.now();
// A successful bootstrap consumes and deletes the app's one-shot boot
// token. Keep that rotated tunnel for this daemon's lifetime instead of
// trying to bootstrap it again on a timer. Failed attempts are not cached.
if (tunnel) return tunnel;
if (!opts.tunnelProvider) return null;
// Multiple first requests can arrive before bootstrap completes. Share
// one provider call so they cannot race through independent rotations.
if (!tunnelInFlight) {
tunnelInFlight = Promise.resolve()
.then(() => opts.tunnelProvider!())
.then((candidate) => {
if (candidate) tunnel = candidate;
return candidate;
})
.finally(() => {
tunnelInFlight = null;
});
}
return tunnel;
return tunnelInFlight;
};
const invalidateTunnel = (failedTunnel: DeviceTunnel): void => {
// A late response from the old app must not evict a tunnel that another
// request has already refreshed. Object identity gives each bootstrap a
// cheap generation token without exposing generation state elsewhere.
if (tunnel === failedTunnel) tunnel = null;
};
// 2. Tailnet probe (fail-closed).
@@ -86,7 +106,7 @@ export async function startDaemon(opts: DaemonOptions): Promise<RunningDaemon |
// 3. Loopback listener (full surface).
const loopbackServer = createServer(async (req, res) => {
await handleLoopback({ req, res, tokenStore, getTunnel });
await handleLoopback({ req, res, tokenStore, getTunnel, invalidateTunnel });
});
// Use port 0 for OS-assigned port when test/random port collisions are a risk.
const requestedPort = opts.loopbackPort;
@@ -97,7 +117,7 @@ export async function startDaemon(opts: DaemonOptions): Promise<RunningDaemon |
// mode this can collide; we try the actualPort first and skip ipv6 if it
// fails (tests don't exercise ::1 explicitly).
const loopbackServerV6 = createServer(async (req, res) => {
await handleLoopback({ req, res, tokenStore, getTunnel });
await handleLoopback({ req, res, tokenStore, getTunnel, invalidateTunnel });
});
let v6Bound = false;
try {
@@ -118,6 +138,7 @@ export async function startDaemon(opts: DaemonOptions): Promise<RunningDaemon |
res,
tokenStore,
getTunnel,
invalidateTunnel,
auditPath: opts.auditPath,
attemptsPath: opts.attemptsPath,
allowlistPath: opts.allowlistPath,
@@ -181,12 +202,113 @@ interface HandlerCtx {
res: ServerResponse;
tokenStore: SessionTokenStore;
getTunnel: () => Promise<DeviceTunnel | null>;
invalidateTunnel: (failedTunnel: DeviceTunnel) => void;
// Explicit security-log + allowlist paths (default to env-derived when undefined).
auditPath?: string;
attemptsPath?: string;
allowlistPath?: string;
}
type DeviceProxyResponse = Awaited<ReturnType<typeof proxyToDevice>>;
const RECOVERABLE_SOCKET_ERRORS = new Set([
'ECONNABORTED',
'ECONNREFUSED',
'ECONNRESET',
'EHOSTUNREACH',
'ENETUNREACH',
'EPIPE',
'ETIMEDOUT',
]);
function localProxyError(status: number, error: string): DeviceProxyResponse {
const body = Buffer.from(JSON.stringify({ error }, sanitizeReplacer));
return {
status,
headers: { 'content-type': 'application/json', 'content-length': String(body.length) },
body,
};
}
async function proxyAttempt(opts: Parameters<typeof proxyToDevice>[0]): Promise<DeviceProxyResponse> {
try {
return await proxyToDevice(opts);
} catch (err) {
const code = (err as { code?: string }).code;
// CoreDevice can surface the same stale route as several different socket
// failures while an app is being relaunched or replaced. Normalize those
// failures so the cache recovery path below can handle all of them.
if (code && RECOVERABLE_SOCKET_ERRORS.has(code)) {
return localProxyError(503, 'device_disconnected');
}
throw err;
}
}
function shouldRefreshTunnel(upstream: DeviceProxyResponse): boolean {
// A relaunched app has a new in-memory bearer and rejects the daemon's old
// rotated token. A redeploy can instead leave the old CoreDevice route
// refusing connections or timing out. Both cases require a fresh bootstrap.
if (upstream.status === 401) return true;
if (upstream.status !== 503 && upstream.status !== 504) return false;
try {
const body = JSON.parse(upstream.body.toString('utf-8')) as { error?: string };
return body.error === 'device_disconnected' || body.error === 'upstream_timeout';
} catch {
return false;
}
}
function canReplayAfterRefresh(inbound: IncomingMessage, upstream: DeviceProxyResponse): boolean {
// A 401 proves the stale bearer was rejected before StateServer dispatched
// the operation, so retrying is safe even for a mutation. Connection loss
// is ambiguous: the app may have applied a tap/write before its response was
// lost. Replay only read-only requests in that case to prevent double taps.
if (upstream.status === 401) return true;
const method = inbound.method ?? 'GET';
return method === 'GET' || method === 'HEAD' || method === 'OPTIONS';
}
async function proxyWithTunnelRecovery(opts: {
inbound: IncomingMessage;
body: Buffer;
sessionId: string | null;
agentIdentity?: string;
getTunnel: HandlerCtx['getTunnel'];
invalidateTunnel: HandlerCtx['invalidateTunnel'];
}): Promise<{ tunnel: DeviceTunnel; upstream: DeviceProxyResponse } | null> {
let tunnel = await opts.getTunnel();
if (!tunnel) return null;
const makeAttempt = (candidate: DeviceTunnel) => proxyAttempt({
inbound: opts.inbound,
body: opts.body,
tunnel: candidate,
sessionId: opts.sessionId,
agentIdentity: opts.agentIdentity,
});
let upstream = await makeAttempt(tunnel);
if (!shouldRefreshTunnel(upstream)) return { tunnel, upstream };
const failedTunnel = tunnel;
const replaySafe = canReplayAfterRefresh(opts.inbound, upstream);
opts.invalidateTunnel(tunnel);
const refreshed = await opts.getTunnel();
if (!refreshed) return replaySafe ? null : { tunnel: failedTunnel, upstream };
// The replacement is now cached for the next request, but never replay an
// ambiguous mutation whose response was lost: doing so could double-tap or
// apply a state transition twice.
if (!replaySafe) return { tunnel: failedTunnel, upstream };
tunnel = refreshed;
upstream = await makeAttempt(tunnel);
// Do not loop forever if the replacement app is itself unavailable. Leave
// the cache empty so the next independent request can bootstrap again.
if (shouldRefreshTunnel(upstream)) opts.invalidateTunnel(tunnel);
return { tunnel, upstream };
}
function readBody(req: IncomingMessage, maxBytes = 1_048_576): Promise<Buffer | { error: 'body_too_large' }> {
return new Promise((resolve, reject) => {
const chunks: Buffer[] = [];
@@ -228,7 +350,7 @@ function sendJson(res: ServerResponse, status: number, body: unknown): void {
* loopback bind itself is the boundary).
*/
async function handleLoopback(ctx: HandlerCtx): Promise<void> {
const { req, res, tokenStore, getTunnel } = ctx;
const { req, res, tokenStore, getTunnel, invalidateTunnel } = ctx;
const url = parseUrl(req.url ?? '/');
const path = url.pathname ?? '/';
const method = req.method ?? 'GET';
@@ -262,16 +384,23 @@ async function handleLoopback(ctx: HandlerCtx): Promise<void> {
}
// Other endpoints — proxy to the device.
const tunnel = await getTunnel();
if (!tunnel) {
sendJson(res, 503, { error: 'device_not_connected' });
return;
}
const body = await readBody(req);
if ('error' in body) { sendJson(res, 413, body); return; }
const sessionId = (req.headers['x-session-id'] as string | undefined) ?? null;
const agentIdentity = (req.headers['x-agent-identity'] as string | undefined) ?? undefined;
const upstream = await proxyToDevice({ inbound: req, body, tunnel, sessionId, agentIdentity });
const proxied = await proxyWithTunnelRecovery({
inbound: req,
body,
sessionId,
agentIdentity,
getTunnel,
invalidateTunnel,
});
if (!proxied) {
sendJson(res, 503, { error: 'device_not_connected' });
return;
}
const { upstream } = proxied;
res.writeHead(upstream.status, upstream.headers);
res.end(upstream.body);
} catch (err) {
@@ -287,7 +416,7 @@ interface TailnetCtx extends HandlerCtx {
* Tailnet handler — locked allowlist + capability tiers.
*/
async function handleTailnet(ctx: TailnetCtx): Promise<void> {
const { req, res, tokenStore, getTunnel, whoIsImpl, auditPath, attemptsPath, allowlistPath } = ctx;
const { req, res, tokenStore, getTunnel, invalidateTunnel, whoIsImpl, auditPath, attemptsPath, allowlistPath } = ctx;
const url = parseUrl(req.url ?? '/');
const path = url.pathname ?? '/';
const method = req.method ?? 'GET';
@@ -375,19 +504,20 @@ async function handleTailnet(ctx: TailnetCtx): Promise<void> {
}
// Proxy to device.
const tunnel = await getTunnel();
if (!tunnel) {
const sessionId = (req.headers['x-session-id'] as string | undefined) ?? null;
const proxied = await proxyWithTunnelRecovery({
inbound: req,
body,
sessionId,
agentIdentity: session.identity,
getTunnel,
invalidateTunnel,
});
if (!proxied) {
sendJson(res, 503, { error: 'device_not_connected' });
return;
}
const sessionId = (req.headers['x-session-id'] as string | undefined) ?? null;
const upstream = await proxyToDevice({
inbound: req,
body,
tunnel,
sessionId,
agentIdentity: session.identity,
});
const { tunnel, upstream } = proxied;
// Audit the action (mutating endpoints only).
if (requiredCapability !== 'observe') {
+183 -19
View File
@@ -5,6 +5,8 @@
// 2. launch the app on it (no-op if already running)
// 3. wait briefly for the in-app StateServer to start
// 4. copy the boot token from the app's sandbox via devicectl copy from
// If an earlier daemon already consumed it, relaunch the app once to mint
// a fresh boot token, then verify the relaunched StateServer again.
// 5. POST /auth/rotate to swap boot token → fresh in-memory token
// 6. return a DeviceTunnel pointing at the device's IPv6 with the rotated
// bearer that subsequent proxied requests carry
@@ -14,6 +16,7 @@
// live token, which it scopes per-tailnet-session via /auth/mint.
import { randomBytes } from 'crypto';
import { spawnSync } from 'child_process';
import type { DeviceTunnel } from './proxy';
import {
listDevices,
@@ -21,12 +24,13 @@ import {
isAppRunning,
launchApp,
copyFileFromAppContainer,
type DeviceEntry,
type SpawnImpl,
type ResolveImpl,
} from './devicectl';
export interface BootstrapOptions {
/** Target device UDID. If null, picks the first connected paired device. */
/** Target iPhone UDID. If null, picks the best connected paired iPhone. */
udid?: string;
/** Bundle ID of the iOS app hosting the StateServer. */
bundleId: string;
@@ -53,10 +57,85 @@ export type BootstrapErrorReason =
| 'launch_failed'
| 'device_locked'
| 'state_server_unreachable'
| 'wrong_app'
| 'boot_token_unavailable'
| 'rotate_failed'
| 'resolve_failed';
function isIPhoneDevice(device: DeviceEntry): boolean {
const platform = device.platform.trim().toLowerCase();
const deviceType = device.deviceType.trim().toLowerCase();
const model = device.model.trim().toLowerCase();
// productType is present even on older CoreDevice versions. Prefer the
// explicit platform/type fields when available, but retain productType as
// a compatibility fallback. An explicit non-iOS platform always loses.
if (platform && platform !== 'ios') return false;
return deviceType === 'iphone' || model.startsWith('iphone');
}
function isAvailableDevice(device: Pick<DeviceEntry, 'state' | 'transport'>): boolean {
const state = device.state.trim().toLowerCase();
const transport = device.transport.trim().toLowerCase();
// Xcode 26.6 / iOS 27 beta can report a USB-reachable iPhone as
// tunnelState=disconnected until the next devicectl command establishes
// the CoreDevice tunnel. The wired transport is the authoritative signal
// in that transitional state. Stale devices have no wired transport.
return state === 'connected'
|| state.startsWith('available')
|| (state === 'disconnected' && transport === 'wired');
}
function defaultDeviceRank(device: DeviceEntry): number {
if (!device.paired || !isIPhoneDevice(device) || !isAvailableDevice(device)) return -1;
const state = device.state.trim().toLowerCase();
const transport = device.transport.trim().toLowerCase();
// Prefer the USB-connected phone the user is actively working with. Then
// prefer an established CoreDevice tunnel over a merely available device.
return (transport === 'wired' ? 100 : 0)
+ (state === 'connected' ? 10 : 0)
+ (state.startsWith('available') ? 1 : 0);
}
function pickDefaultDevice(devices: DeviceEntry[]): DeviceEntry | undefined {
let best: DeviceEntry | undefined;
let bestRank = -1;
for (const device of devices) {
const rank = defaultDeviceRank(device);
if (rank > bestRank) {
best = device;
bestRank = rank;
}
}
return best;
}
const defaultSpawn: SpawnImpl = (cmd, args) => spawnSync(cmd, args, {
stdio: 'pipe',
timeout: 60_000,
});
function relaunchApp(
udid: string,
bundleId: string,
spawn: SpawnImpl = defaultSpawn,
): { ok: true } | { ok: false; error: 'device_locked' | 'launch_failed'; detail?: string } {
const r = spawn('xcrun', [
'devicectl', 'device', 'process', 'launch',
'--device', udid,
'--terminate-existing',
bundleId,
]);
if (r.status === 0) return { ok: true };
const detail = `${r.stderr?.toString() ?? ''}${r.stdout?.toString() ?? ''}`.trim();
if (detail.includes('was not, or could not be, unlocked')) {
return { ok: false, error: 'device_locked', detail };
}
return { ok: false, error: 'launch_failed', detail };
}
/**
* Bootstrap a real CoreDevice tunnel to an iOS app's StateServer. Used by
* the daemon's default tunnelProvider when GSTACK_IOS_TARGET_UDID is set
@@ -77,9 +156,39 @@ export async function bootstrapTunnel(opts: BootstrapOptions): Promise<Bootstrap
}
const target = opts.udid
? devices.find((d) => d.identifier === opts.udid)
: devices.find((d) => d.paired) ?? devices[0];
: pickDefaultDevice(devices);
if (!target) {
return { ok: false, error: 'device_not_found', detail: opts.udid };
if (opts.udid) {
return { ok: false, error: 'device_not_found', detail: opts.udid };
}
const pairedIPhone = devices.find((d) => d.paired && isIPhoneDevice(d));
if (pairedIPhone) {
return {
ok: false,
error: 'device_not_found',
detail: `paired iPhone ${pairedIPhone.name} (${pairedIPhone.identifier}) is ${pairedIPhone.state}; connect it over USB and unlock it`,
};
}
const firstIPhone = devices.find(isIPhoneDevice);
if (!firstIPhone) {
return {
ok: false,
error: 'device_not_found',
detail: 'no iPhone is connected; non-iOS devices are not eligible for iOS QA',
};
}
return {
ok: false,
error: 'no_paired_device',
detail: `device ${firstIPhone.name} (${firstIPhone.identifier}) is ${firstIPhone.state}; run \`xcrun devicectl manage pair --device ${firstIPhone.identifier}\` and tap Trust on the iPhone`,
};
}
if (!isIPhoneDevice(target)) {
return {
ok: false,
error: 'device_not_found',
detail: `device ${target.name} (${target.identifier}) is ${target.platform || target.model}, not an iPhone`,
};
}
if (!target.paired) {
return {
@@ -88,6 +197,13 @@ export async function bootstrapTunnel(opts: BootstrapOptions): Promise<Bootstrap
detail: `device ${target.name} (${target.identifier}) is ${target.state}; run \`xcrun devicectl manage pair --device ${target.identifier}\` and tap Trust on the iPhone`,
};
}
if (!isAvailableDevice(target)) {
return {
ok: false,
error: 'device_not_found',
detail: `device ${target.name} (${target.identifier}) is ${target.state}; connect it over USB and unlock it`,
};
}
// Step 2: launch app (idempotent — devicectl returns success if already running)
if (!isAppRunning(target.identifier, opts.bundleId, spawn)) {
@@ -118,29 +234,77 @@ export async function bootstrapTunnel(opts: BootstrapOptions): Promise<Bootstrap
// Step 4: wait for StateServer to become reachable, then scrape boot token.
// Probe /healthz with retries (the listener can take a moment to bind).
const deadline = Date.now() + startupTimeoutMs;
let healthOK = false;
while (Date.now() < deadline) {
try {
const r = await fetchFn(`http://[${ipv6}]:${port}/healthz`, {
signal: AbortSignal.timeout(2_000),
});
if (r.ok) { healthOK = true; break; }
} catch { /* retry */ }
await new Promise((res) => setTimeout(res, 250));
}
if (!healthOK) {
return { ok: false, error: 'state_server_unreachable', detail: `no /healthz response from [${ipv6}]:${port} within ${startupTimeoutMs}ms` };
}
const waitForStateServer = async (): Promise<BootstrapResult | null> => {
const deadline = Date.now() + startupTimeoutMs;
while (Date.now() < deadline) {
try {
const r = await fetchFn(`http://[${ipv6}]:${port}/healthz`, {
signal: AbortSignal.timeout(2_000),
});
if (r.ok) {
const health = await r.json().catch(() => null) as { bundle_id?: string } | null;
// Older bridges did not identify their bundle. Preserve compatibility,
// but reject an explicit mismatch from current bridges: another debug
// app already owns the fixed StateServer port on this device.
if (health?.bundle_id && health.bundle_id !== opts.bundleId) {
return {
ok: false,
error: 'wrong_app',
detail: `expected ${opts.bundleId} but StateServer port ${port} belongs to ${health.bundle_id}; terminate the other debug app`,
};
}
return null;
}
} catch { /* retry */ }
await new Promise((res) => setTimeout(res, 250));
}
return {
ok: false,
error: 'state_server_unreachable',
detail: `no /healthz response from [${ipv6}]:${port} within ${startupTimeoutMs}ms`,
};
};
const bootToken = copyFileFromAppContainer({
const healthFailure = await waitForStateServer();
if (healthFailure) return healthFailure;
const readBootToken = () => copyFileFromAppContainer({
udid: target.identifier,
bundleId: opts.bundleId,
sourceRelativePath: tokenPath,
spawn,
});
let bootToken = readBootToken();
if (!bootToken) {
return { ok: false, error: 'boot_token_unavailable', detail: `couldn't read ${tokenPath} from ${opts.bundleId}` };
// A healthy running app can lack a boot token when an earlier daemon
// already rotated it. A new daemon has no way to recover that in-memory
// bearer, so restart exactly once to make StateServer mint a fresh one.
// The explicit bundle check above prevents disrupting an unrelated app
// that happens to own the fixed StateServer port.
const relaunched = relaunchApp(target.identifier, opts.bundleId, spawn);
if (!relaunched.ok) {
return { ok: false, error: relaunched.error, detail: relaunched.detail };
}
// The token is written before StateServer opens its listener. Waiting for
// it first prevents a stale response from the terminating process from
// being mistaken for readiness of the replacement process.
const tokenDeadline = Date.now() + startupTimeoutMs;
while (!bootToken && Date.now() < tokenDeadline) {
bootToken = readBootToken();
if (!bootToken) await new Promise((res) => setTimeout(res, 250));
}
if (!bootToken) {
return {
ok: false,
error: 'boot_token_unavailable',
detail: `couldn't read ${tokenPath} from ${opts.bundleId} after relaunch`,
};
}
const relaunchedHealthFailure = await waitForStateServer();
if (relaunchedHealthFailure) return relaunchedHealthFailure;
}
// Step 5: rotate the boot token to a fresh in-memory-only one.
@@ -132,6 +132,258 @@ describe('daemon — loopback listener', () => {
expect(lastReq?.headers['x-session-id']).toBe('sess-loopback-1');
});
test('concurrent first requests share one tunnel bootstrap', async () => {
let bootstraps = 0;
let releaseBootstrap!: () => void;
let markBootstrapStarted!: () => void;
const bootstrapStarted = new Promise<void>((resolve) => { markBootstrapStarted = resolve; });
const bootstrapGate = new Promise<void>((resolve) => { releaseBootstrap = resolve; });
const tunnel: DeviceTunnel = {
udid: 'STUB-UDID',
ipv6Addr: '127.0.0.1',
port: stub.port,
bootTokenRotated: STATE_SERVER_TOKEN,
};
const d = await startDaemon({
loopbackPort: 0,
tailnetEnabled: false,
pidfilePath: join(workDir, 'daemon-concurrent-bootstrap.pid'),
tunnelProvider: async () => {
bootstraps++;
markBootstrapStarted();
await bootstrapGate;
return tunnel;
},
});
if ('error' in d) throw new Error(d.error);
try {
const base = `http://127.0.0.1:${d.loopbackPort}`;
const requests = [
fetchWith('GET', `${base}/screenshot`),
fetchWith('GET', `${base}/screenshot`),
fetchWith('GET', `${base}/screenshot`),
];
await bootstrapStarted;
await new Promise((resolve) => setTimeout(resolve, 25));
releaseBootstrap();
const responses = await Promise.all(requests);
expect(responses.map((response) => response.status)).toEqual([200, 200, 200]);
expect(bootstraps).toBe(1);
} finally {
releaseBootstrap();
await d.close();
}
});
test('reuses a healthy rotated tunnel beyond the old 30-second boundary', async () => {
let bootstraps = 0;
const tunnel: DeviceTunnel = {
udid: 'STUB-UDID',
ipv6Addr: '127.0.0.1',
port: stub.port,
bootTokenRotated: STATE_SERVER_TOKEN,
};
const d = await startDaemon({
loopbackPort: 0,
tailnetEnabled: false,
pidfilePath: join(workDir, 'daemon-one-shot-bootstrap.pid'),
tunnelProvider: async () => {
bootstraps++;
if (bootstraps > 1) throw new Error('one-shot boot token was already consumed');
return tunnel;
},
});
if ('error' in d) throw new Error(d.error);
const realNow = Date.now;
const firstRequestAt = realNow();
try {
const base = `http://127.0.0.1:${d.loopbackPort}`;
const first = await fetchWith('GET', `${base}/screenshot`);
expect(first.status).toBe(200);
Date.now = () => firstRequestAt + 30_001;
const later = await fetchWith('GET', `${base}/screenshot`);
expect(later.status).toBe(200);
expect(bootstraps).toBe(1);
} finally {
Date.now = realNow;
await d.close();
}
});
test('401 after app relaunch invalidates the token and concurrent requests share one rebootstrap', async () => {
let bootstraps = 0;
let markRefreshStarted!: () => void;
let releaseRefresh!: () => void;
const refreshStarted = new Promise<void>((resolve) => { markRefreshStarted = resolve; });
const refreshGate = new Promise<void>((resolve) => { releaseRefresh = resolve; });
const staleTunnel: DeviceTunnel = {
udid: 'STUB-UDID',
ipv6Addr: '127.0.0.1',
port: stub.port,
bootTokenRotated: 'expired-after-relaunch',
};
const refreshedTunnel: DeviceTunnel = {
...staleTunnel,
bootTokenRotated: STATE_SERVER_TOKEN,
};
const d = await startDaemon({
loopbackPort: 0,
tailnetEnabled: false,
pidfilePath: join(workDir, 'daemon-relaunch-refresh.pid'),
tunnelProvider: async () => {
bootstraps++;
if (bootstraps === 1) return staleTunnel;
if (bootstraps === 2) {
markRefreshStarted();
await refreshGate;
return refreshedTunnel;
}
throw new Error('concurrent 401s caused duplicate bootstraps');
},
});
if ('error' in d) throw new Error(d.error);
const requestStart = stub.receivedRequests.length;
try {
const base = `http://127.0.0.1:${d.loopbackPort}`;
const requests = [
fetchWith('GET', `${base}/screenshot`),
fetchWith('GET', `${base}/screenshot`),
fetchWith('GET', `${base}/screenshot`),
];
await refreshStarted;
await new Promise((resolve) => setTimeout(resolve, 25));
expect(bootstraps).toBe(2);
releaseRefresh();
const responses = await Promise.all(requests);
expect(responses.map((response) => response.status)).toEqual([200, 200, 200]);
const attempts = stub.receivedRequests.slice(requestStart);
expect(attempts.filter((request) => request.headers.authorization === 'Bearer expired-after-relaunch')).toHaveLength(3);
expect(attempts.filter((request) => request.headers.authorization === `Bearer ${STATE_SERVER_TOKEN}`)).toHaveLength(3);
const healthyReuse = await fetchWith('GET', `${base}/screenshot`);
expect(healthyReuse.status).toBe(200);
expect(bootstraps).toBe(2);
} finally {
releaseRefresh();
await d.close();
}
});
test('connection failure after redeploy reboots the tunnel once and keeps the replacement cached', async () => {
const deadPort = await new Promise<number>((resolve, reject) => {
const reservation = createServer();
reservation.once('error', reject);
reservation.listen(0, '127.0.0.1', () => {
const address = reservation.address();
const port = typeof address === 'object' && address ? address.port : 0;
reservation.close((err) => err ? reject(err) : resolve(port));
});
});
let bootstraps = 0;
const d = await startDaemon({
loopbackPort: 0,
tailnetEnabled: false,
pidfilePath: join(workDir, 'daemon-redeploy-refresh.pid'),
tunnelProvider: async () => {
bootstraps++;
if (bootstraps === 1) {
return {
udid: 'STUB-UDID',
ipv6Addr: '127.0.0.1',
port: deadPort,
bootTokenRotated: 'old-deploy-token',
};
}
if (bootstraps === 2) {
return {
udid: 'STUB-UDID',
ipv6Addr: '127.0.0.1',
port: stub.port,
bootTokenRotated: STATE_SERVER_TOKEN,
};
}
throw new Error('healthy replacement tunnel was not reused');
},
});
if ('error' in d) throw new Error(d.error);
try {
const base = `http://127.0.0.1:${d.loopbackPort}`;
const recovered = await fetchWith('GET', `${base}/screenshot`);
expect(recovered.status).toBe(200);
expect(JSON.parse(recovered.bodyText)).toEqual({ png_base64: 'abc=' });
expect(bootstraps).toBe(2);
const healthyReuse = await fetchWith('GET', `${base}/screenshot`);
expect(healthyReuse.status).toBe(200);
expect(bootstraps).toBe(2);
} finally {
await d.close();
}
});
test('connection failure refreshes but never replays an ambiguous tap', async () => {
const deadPort = await new Promise<number>((resolve, reject) => {
const reservation = createServer();
reservation.once('error', reject);
reservation.listen(0, '127.0.0.1', () => {
const address = reservation.address();
const port = typeof address === 'object' && address ? address.port : 0;
reservation.close((err) => err ? reject(err) : resolve(port));
});
});
let bootstraps = 0;
const d = await startDaemon({
loopbackPort: 0,
tailnetEnabled: false,
pidfilePath: join(workDir, 'daemon-mutation-no-replay.pid'),
tunnelProvider: async () => {
bootstraps++;
return bootstraps === 1
? {
udid: 'STUB-UDID',
ipv6Addr: '127.0.0.1',
port: deadPort,
bootTokenRotated: 'old-deploy-token',
}
: {
udid: 'STUB-UDID',
ipv6Addr: '127.0.0.1',
port: stub.port,
bootTokenRotated: STATE_SERVER_TOKEN,
};
},
});
if ('error' in d) throw new Error(d.error);
const beforeTaps = stub.receivedRequests.filter((request) => request.path === '/tap').length;
try {
const base = `http://127.0.0.1:${d.loopbackPort}`;
const ambiguous = await fetchWith('POST', `${base}/tap`, {
headers: { 'x-session-id': 'old-session', 'content-type': 'application/json' },
body: JSON.stringify({ x: 10, y: 20 }),
});
expect(ambiguous.status).toBe(503);
expect(bootstraps).toBe(2);
expect(stub.receivedRequests.filter((request) => request.path === '/tap')).toHaveLength(beforeTaps);
const explicitRetry = await fetchWith('POST', `${base}/tap`, {
headers: { 'x-session-id': 'new-session', 'content-type': 'application/json' },
body: JSON.stringify({ x: 10, y: 20 }),
});
expect(explicitRetry.status).toBe(200);
expect(stub.receivedRequests.filter((request) => request.path === '/tap')).toHaveLength(beforeTaps + 1);
expect(bootstraps).toBe(2);
} finally {
await d.close();
}
});
test('returns 503 when no device tunnel is provided', async () => {
// Force tunnel provider to return null by closing + restarting with null provider.
await daemon.close();
+374
View File
@@ -105,6 +105,219 @@ describe('bootstrapTunnel', () => {
}
});
test('never auto-selects a connected paired Vision Pro ahead of an iPhone', async () => {
const spawn = makeSpawn([
{
argsMatch: /devicectl list devices/,
jsonOutput: {
result: { devices: [
{
identifier: 'VISION',
connectionProperties: { tunnelState: 'connected', pairingState: 'paired' },
deviceProperties: { name: 'Vision Pro' },
hardwareProperties: {
productType: 'RealityDevice17,1',
platform: 'visionOS',
deviceType: 'realityDevice',
},
},
{
identifier: 'IPHONE',
connectionProperties: { tunnelState: 'connected', pairingState: 'paired', transportType: 'wired' },
deviceProperties: { name: 'USB iPhone' },
hardwareProperties: {
productType: 'iPhone17,1',
platform: 'iOS',
deviceType: 'iPhone',
},
},
] },
},
},
{
argsMatch: /devicectl device info processes -d IPHONE/,
jsonOutput: { result: { runningProcesses: [{ executable: 'file:///var/containers/Bundle/Application/X/com.test.app/com.test' }] } },
},
{
argsMatch: /devicectl device info details --device IPHONE/,
jsonOutput: { result: { connectionProperties: { tunnelIPAddress: 'fd00::17' } } },
},
{
argsMatch: /devicectl device copy from --device IPHONE/,
destOutput: 'TOKEN\n',
},
]);
const r = await bootstrapTunnel({
bundleId: 'com.test',
spawnImpl: spawn,
fetchImpl: (async () => new Response('{"ok":true}', { status: 200 })) as typeof fetch,
});
expect(r.ok).toBe(true);
if (r.ok) expect(r.tunnel.udid).toBe('IPHONE');
});
test('fails clearly when only non-iPhone platforms are connected', async () => {
const spawn = makeSpawn([
{
argsMatch: /devicectl list devices/,
jsonOutput: {
result: { devices: [
{
identifier: 'VISION',
connectionProperties: { tunnelState: 'connected', pairingState: 'paired' },
deviceProperties: { name: 'Vision Pro' },
hardwareProperties: {
productType: 'RealityDevice17,1',
platform: 'visionOS',
deviceType: 'realityDevice',
},
},
{
identifier: 'IPAD',
connectionProperties: { tunnelState: 'connected', pairingState: 'paired' },
deviceProperties: { name: 'iPad' },
hardwareProperties: {
productType: 'iPad16,6',
platform: 'iOS',
deviceType: 'iPad',
},
},
] },
},
},
]);
const r = await bootstrapTunnel({ bundleId: 'com.test', spawnImpl: spawn });
expect(r.ok).toBe(false);
if (!r.ok) {
expect(r.error).toBe('device_not_found');
expect(r.detail).toContain('no iPhone');
}
});
test('rejects an explicitly targeted non-iPhone device', async () => {
const spawn = makeSpawn([
{
argsMatch: /devicectl list devices/,
jsonOutput: {
result: { devices: [{
identifier: 'VISION',
connectionProperties: { tunnelState: 'connected', pairingState: 'paired' },
deviceProperties: { name: 'Vision Pro' },
hardwareProperties: {
productType: 'RealityDevice17,1',
platform: 'visionOS',
deviceType: 'realityDevice',
},
}] },
},
},
]);
const r = await bootstrapTunnel({ bundleId: 'com.test', udid: 'VISION', spawnImpl: spawn });
expect(r.ok).toBe(false);
if (!r.ok) {
expect(r.error).toBe('device_not_found');
expect(r.detail).toContain('not an iPhone');
}
});
test('skips an unavailable paired device for a connected or available paired device', async () => {
const spawn = makeSpawn([
{
argsMatch: /devicectl list devices/,
jsonOutput: {
result: { devices: [
{
identifier: 'STALE',
connectionProperties: { tunnelState: 'unavailable', pairingState: 'paired' },
deviceProperties: { name: 'Stale iPhone' },
hardwareProperties: { productType: 'iPhone17,1' },
},
{
identifier: 'WIRED',
connectionProperties: { tunnelState: 'available', pairingState: 'paired', transportType: 'wired' },
deviceProperties: { name: 'Wired iPhone' },
hardwareProperties: { productType: 'iPhone18,2' },
},
] },
},
},
{
argsMatch: /devicectl device info processes -d WIRED/,
jsonOutput: { result: { runningProcesses: [{ executable: 'file:///var/containers/Bundle/Application/X/com.test.app/com.test' }] } },
},
{
argsMatch: /devicectl device info details --device WIRED/,
jsonOutput: { result: { connectionProperties: { tunnelIPAddress: 'fd00::2' } } },
},
{
argsMatch: /devicectl device copy from --device WIRED/,
destOutput: 'TOKEN\n',
},
]);
const r = await bootstrapTunnel({
bundleId: 'com.test',
spawnImpl: spawn,
fetchImpl: (async () => new Response('{"ok":true}', { status: 200 })) as typeof fetch,
});
expect(r.ok).toBe(true);
if (r.ok) expect(r.tunnel.udid).toBe('WIRED');
});
test('accepts a wired iPhone whose tunnel is disconnected until devicectl wakes it', async () => {
const spawn = makeSpawn([
{
argsMatch: /devicectl list devices/,
jsonOutput: {
result: { devices: [
{
identifier: 'STALE-VISION',
connectionProperties: { tunnelState: 'unavailable', pairingState: 'paired' },
deviceProperties: { name: 'Stale Vision Pro' },
hardwareProperties: { productType: 'RealityDevice14,1' },
},
{
identifier: 'WIRED-DISCONNECTED',
connectionProperties: {
tunnelState: 'disconnected',
pairingState: 'paired',
transportType: 'wired',
},
deviceProperties: { name: 'USB iPhone' },
hardwareProperties: { productType: 'iPhone18,2' },
},
] },
},
},
{
argsMatch: /devicectl device info processes -d WIRED-DISCONNECTED/,
jsonOutput: { result: { runningProcesses: [{ executable: 'file:\/\/\/var\/containers\/Bundle\/Application\/X\/com.test.app\/com.test' }] } },
},
{
argsMatch: /devicectl device info details --device WIRED-DISCONNECTED/,
jsonOutput: { result: { connectionProperties: { tunnelIPAddress: 'fd00::27' } } },
},
{
argsMatch: /devicectl device copy from --device WIRED-DISCONNECTED/,
destOutput: 'TOKEN\n',
},
]);
const r = await bootstrapTunnel({
bundleId: 'com.test',
spawnImpl: spawn,
fetchImpl: (async () => new Response('{"ok":true}', { status: 200 })) as typeof fetch,
});
expect(r.ok).toBe(true);
if (r.ok) expect(r.tunnel.udid).toBe('WIRED-DISCONNECTED');
});
test('returns device_locked when launchApp errors due to lock', async () => {
const spawn = makeSpawn([
{
@@ -166,6 +379,48 @@ describe('bootstrapTunnel', () => {
if (!r.ok) expect(r.error).toBe('state_server_unreachable');
});
test('rejects a StateServer owned by a different app bundle', async () => {
const spawn = makeSpawn([
{
argsMatch: /devicectl list devices/,
jsonOutput: {
result: { devices: [{
identifier: 'TEST',
connectionProperties: { tunnelState: 'connected', pairingState: 'paired' },
deviceProperties: { name: 'Test' },
hardwareProperties: { productType: 'iPhone18,2' },
}] },
},
},
{
argsMatch: /devicectl device info processes/,
jsonOutput: { result: { runningProcesses: [] } },
},
{
argsMatch: /devicectl device process launch/,
},
{
argsMatch: /devicectl device info details/,
jsonOutput: { result: { connectionProperties: { tunnelIPAddress: 'fd00::9' } } },
},
]);
const r = await bootstrapTunnel({
bundleId: 'com.expected.app',
spawnImpl: spawn,
fetchImpl: (async () => new Response(
JSON.stringify({ bundle_id: 'com.other.debug-app' }),
{ status: 200, headers: { 'content-type': 'application/json' } },
)) as typeof fetch,
});
expect(r.ok).toBe(false);
if (!r.ok) {
expect(r.error).toBe('wrong_app');
expect(r.detail).toContain('com.other.debug-app');
}
});
test('happy path: returns DeviceTunnel with rotated token', async () => {
const spawn = makeSpawn([
{
@@ -234,6 +489,125 @@ describe('bootstrapTunnel', () => {
expect(fetchCalls[fetchCalls.length - 1]?.url).toContain('/auth/rotate');
});
test('relaunches once when a prior daemon consumed the running app boot token', async () => {
const spawn = makeSpawn([
{
argsMatch: /devicectl list devices/,
jsonOutput: {
result: { devices: [{
identifier: 'TEST-UDID',
connectionProperties: { tunnelState: 'connected', pairingState: 'paired' },
deviceProperties: { name: 'Test Device' },
hardwareProperties: { productType: 'iPhone18,2' },
}] },
},
},
{
argsMatch: /devicectl device info processes/,
jsonOutput: { result: { runningProcesses: [{ executable: 'file:\/\/\/var\/containers\/Bundle\/Application\/X\/com.test.app\/com.test' }] } },
},
{
argsMatch: /devicectl device info details/,
jsonOutput: { result: { connectionProperties: { tunnelIPAddress: 'fd99::beef' } } },
},
{
// A prior daemon rotated the one-use token, so StateServer deleted it.
argsMatch: /devicectl device copy from/,
exitCode: 1,
stderr: 'source does not exist',
},
{
argsMatch: /devicectl device process launch --device TEST-UDID --terminate-existing com\.test/,
},
{
argsMatch: /devicectl device copy from/,
destOutput: 'FRESH-BOOT-TOKEN\n',
},
]);
const fetchCalls: Array<{ url: string; authorization?: string }> = [];
const r = await bootstrapTunnel({
bundleId: 'com.test',
spawnImpl: spawn,
fetchImpl: (async (url, init) => {
const u = String(url);
const headers = init?.headers as Record<string, string> | undefined;
fetchCalls.push({ url: u, authorization: headers?.Authorization });
if (u.endsWith('/healthz')) {
return new Response(JSON.stringify({ bundle_id: 'com.test' }), {
status: 200,
headers: { 'content-type': 'application/json' },
});
}
if (u.endsWith('/auth/rotate') && headers?.Authorization === 'Bearer FRESH-BOOT-TOKEN') {
return new Response('{"ok":true}', { status: 200 });
}
return new Response('unauthorized', { status: 401 });
}) as typeof fetch,
startupTimeoutMs: 1_000,
});
expect(r.ok).toBe(true);
expect(fetchCalls.filter((call) => call.url.endsWith('/healthz'))).toHaveLength(2);
expect(fetchCalls.at(-1)?.authorization).toBe('Bearer FRESH-BOOT-TOKEN');
});
test('rechecks bundle ownership after recovering a consumed boot token', async () => {
const spawn = makeSpawn([
{
argsMatch: /devicectl list devices/,
jsonOutput: {
result: { devices: [{
identifier: 'TEST-UDID',
connectionProperties: { tunnelState: 'connected', pairingState: 'paired' },
deviceProperties: { name: 'Test Device' },
hardwareProperties: { productType: 'iPhone18,2' },
}] },
},
},
{
argsMatch: /devicectl device info processes/,
jsonOutput: { result: { runningProcesses: [{ executable: 'file:\/\/\/var\/containers\/Bundle\/Application\/X\/com.test.app\/com.test' }] } },
},
{
argsMatch: /devicectl device info details/,
jsonOutput: { result: { connectionProperties: { tunnelIPAddress: 'fd99::beef' } } },
},
{ argsMatch: /devicectl device copy from/, exitCode: 1 },
{
argsMatch: /devicectl device process launch --device TEST-UDID --terminate-existing com\.test/,
},
{ argsMatch: /devicectl device copy from/, destOutput: 'FRESH-BOOT-TOKEN\n' },
]);
let healthChecks = 0;
let rotateCalls = 0;
const r = await bootstrapTunnel({
bundleId: 'com.test',
spawnImpl: spawn,
fetchImpl: (async (url) => {
const u = String(url);
if (u.endsWith('/healthz')) {
healthChecks++;
return new Response(JSON.stringify({
bundle_id: healthChecks === 1 ? 'com.test' : 'com.other.debug-app',
}), {
status: 200,
headers: { 'content-type': 'application/json' },
});
}
rotateCalls++;
return new Response('{"ok":true}', { status: 200 });
}) as typeof fetch,
startupTimeoutMs: 1_000,
});
expect(r.ok).toBe(false);
if (!r.ok) expect(r.error).toBe('wrong_app');
expect(healthChecks).toBe(2);
expect(rotateCalls).toBe(0);
});
test('resolve_failed when hostname cant be resolved to an IPv6', async () => {
const spawn = makeSpawn([
{