fix(ios-qa): recover device tunnels safely after relaunch

This commit is contained in:
t
2026-07-14 16:35:30 -07:00
parent 145fe2d4d1
commit c378074ab7
5 changed files with 972 additions and 46 deletions
@@ -132,6 +132,258 @@ describe('daemon — loopback listener', () => {
expect(lastReq?.headers['x-session-id']).toBe('sess-loopback-1');
});
test('concurrent first requests share one tunnel bootstrap', async () => {
let bootstraps = 0;
let releaseBootstrap!: () => void;
let markBootstrapStarted!: () => void;
const bootstrapStarted = new Promise<void>((resolve) => { markBootstrapStarted = resolve; });
const bootstrapGate = new Promise<void>((resolve) => { releaseBootstrap = resolve; });
const tunnel: DeviceTunnel = {
udid: 'STUB-UDID',
ipv6Addr: '127.0.0.1',
port: stub.port,
bootTokenRotated: STATE_SERVER_TOKEN,
};
const d = await startDaemon({
loopbackPort: 0,
tailnetEnabled: false,
pidfilePath: join(workDir, 'daemon-concurrent-bootstrap.pid'),
tunnelProvider: async () => {
bootstraps++;
markBootstrapStarted();
await bootstrapGate;
return tunnel;
},
});
if ('error' in d) throw new Error(d.error);
try {
const base = `http://127.0.0.1:${d.loopbackPort}`;
const requests = [
fetchWith('GET', `${base}/screenshot`),
fetchWith('GET', `${base}/screenshot`),
fetchWith('GET', `${base}/screenshot`),
];
await bootstrapStarted;
await new Promise((resolve) => setTimeout(resolve, 25));
releaseBootstrap();
const responses = await Promise.all(requests);
expect(responses.map((response) => response.status)).toEqual([200, 200, 200]);
expect(bootstraps).toBe(1);
} finally {
releaseBootstrap();
await d.close();
}
});
test('reuses a healthy rotated tunnel beyond the old 30-second boundary', async () => {
let bootstraps = 0;
const tunnel: DeviceTunnel = {
udid: 'STUB-UDID',
ipv6Addr: '127.0.0.1',
port: stub.port,
bootTokenRotated: STATE_SERVER_TOKEN,
};
const d = await startDaemon({
loopbackPort: 0,
tailnetEnabled: false,
pidfilePath: join(workDir, 'daemon-one-shot-bootstrap.pid'),
tunnelProvider: async () => {
bootstraps++;
if (bootstraps > 1) throw new Error('one-shot boot token was already consumed');
return tunnel;
},
});
if ('error' in d) throw new Error(d.error);
const realNow = Date.now;
const firstRequestAt = realNow();
try {
const base = `http://127.0.0.1:${d.loopbackPort}`;
const first = await fetchWith('GET', `${base}/screenshot`);
expect(first.status).toBe(200);
Date.now = () => firstRequestAt + 30_001;
const later = await fetchWith('GET', `${base}/screenshot`);
expect(later.status).toBe(200);
expect(bootstraps).toBe(1);
} finally {
Date.now = realNow;
await d.close();
}
});
test('401 after app relaunch invalidates the token and concurrent requests share one rebootstrap', async () => {
let bootstraps = 0;
let markRefreshStarted!: () => void;
let releaseRefresh!: () => void;
const refreshStarted = new Promise<void>((resolve) => { markRefreshStarted = resolve; });
const refreshGate = new Promise<void>((resolve) => { releaseRefresh = resolve; });
const staleTunnel: DeviceTunnel = {
udid: 'STUB-UDID',
ipv6Addr: '127.0.0.1',
port: stub.port,
bootTokenRotated: 'expired-after-relaunch',
};
const refreshedTunnel: DeviceTunnel = {
...staleTunnel,
bootTokenRotated: STATE_SERVER_TOKEN,
};
const d = await startDaemon({
loopbackPort: 0,
tailnetEnabled: false,
pidfilePath: join(workDir, 'daemon-relaunch-refresh.pid'),
tunnelProvider: async () => {
bootstraps++;
if (bootstraps === 1) return staleTunnel;
if (bootstraps === 2) {
markRefreshStarted();
await refreshGate;
return refreshedTunnel;
}
throw new Error('concurrent 401s caused duplicate bootstraps');
},
});
if ('error' in d) throw new Error(d.error);
const requestStart = stub.receivedRequests.length;
try {
const base = `http://127.0.0.1:${d.loopbackPort}`;
const requests = [
fetchWith('GET', `${base}/screenshot`),
fetchWith('GET', `${base}/screenshot`),
fetchWith('GET', `${base}/screenshot`),
];
await refreshStarted;
await new Promise((resolve) => setTimeout(resolve, 25));
expect(bootstraps).toBe(2);
releaseRefresh();
const responses = await Promise.all(requests);
expect(responses.map((response) => response.status)).toEqual([200, 200, 200]);
const attempts = stub.receivedRequests.slice(requestStart);
expect(attempts.filter((request) => request.headers.authorization === 'Bearer expired-after-relaunch')).toHaveLength(3);
expect(attempts.filter((request) => request.headers.authorization === `Bearer ${STATE_SERVER_TOKEN}`)).toHaveLength(3);
const healthyReuse = await fetchWith('GET', `${base}/screenshot`);
expect(healthyReuse.status).toBe(200);
expect(bootstraps).toBe(2);
} finally {
releaseRefresh();
await d.close();
}
});
test('connection failure after redeploy reboots the tunnel once and keeps the replacement cached', async () => {
const deadPort = await new Promise<number>((resolve, reject) => {
const reservation = createServer();
reservation.once('error', reject);
reservation.listen(0, '127.0.0.1', () => {
const address = reservation.address();
const port = typeof address === 'object' && address ? address.port : 0;
reservation.close((err) => err ? reject(err) : resolve(port));
});
});
let bootstraps = 0;
const d = await startDaemon({
loopbackPort: 0,
tailnetEnabled: false,
pidfilePath: join(workDir, 'daemon-redeploy-refresh.pid'),
tunnelProvider: async () => {
bootstraps++;
if (bootstraps === 1) {
return {
udid: 'STUB-UDID',
ipv6Addr: '127.0.0.1',
port: deadPort,
bootTokenRotated: 'old-deploy-token',
};
}
if (bootstraps === 2) {
return {
udid: 'STUB-UDID',
ipv6Addr: '127.0.0.1',
port: stub.port,
bootTokenRotated: STATE_SERVER_TOKEN,
};
}
throw new Error('healthy replacement tunnel was not reused');
},
});
if ('error' in d) throw new Error(d.error);
try {
const base = `http://127.0.0.1:${d.loopbackPort}`;
const recovered = await fetchWith('GET', `${base}/screenshot`);
expect(recovered.status).toBe(200);
expect(JSON.parse(recovered.bodyText)).toEqual({ png_base64: 'abc=' });
expect(bootstraps).toBe(2);
const healthyReuse = await fetchWith('GET', `${base}/screenshot`);
expect(healthyReuse.status).toBe(200);
expect(bootstraps).toBe(2);
} finally {
await d.close();
}
});
test('connection failure refreshes but never replays an ambiguous tap', async () => {
const deadPort = await new Promise<number>((resolve, reject) => {
const reservation = createServer();
reservation.once('error', reject);
reservation.listen(0, '127.0.0.1', () => {
const address = reservation.address();
const port = typeof address === 'object' && address ? address.port : 0;
reservation.close((err) => err ? reject(err) : resolve(port));
});
});
let bootstraps = 0;
const d = await startDaemon({
loopbackPort: 0,
tailnetEnabled: false,
pidfilePath: join(workDir, 'daemon-mutation-no-replay.pid'),
tunnelProvider: async () => {
bootstraps++;
return bootstraps === 1
? {
udid: 'STUB-UDID',
ipv6Addr: '127.0.0.1',
port: deadPort,
bootTokenRotated: 'old-deploy-token',
}
: {
udid: 'STUB-UDID',
ipv6Addr: '127.0.0.1',
port: stub.port,
bootTokenRotated: STATE_SERVER_TOKEN,
};
},
});
if ('error' in d) throw new Error(d.error);
const beforeTaps = stub.receivedRequests.filter((request) => request.path === '/tap').length;
try {
const base = `http://127.0.0.1:${d.loopbackPort}`;
const ambiguous = await fetchWith('POST', `${base}/tap`, {
headers: { 'x-session-id': 'old-session', 'content-type': 'application/json' },
body: JSON.stringify({ x: 10, y: 20 }),
});
expect(ambiguous.status).toBe(503);
expect(bootstraps).toBe(2);
expect(stub.receivedRequests.filter((request) => request.path === '/tap')).toHaveLength(beforeTaps);
const explicitRetry = await fetchWith('POST', `${base}/tap`, {
headers: { 'x-session-id': 'new-session', 'content-type': 'application/json' },
body: JSON.stringify({ x: 10, y: 20 }),
});
expect(explicitRetry.status).toBe(200);
expect(stub.receivedRequests.filter((request) => request.path === '/tap')).toHaveLength(beforeTaps + 1);
expect(bootstraps).toBe(2);
} finally {
await d.close();
}
});
test('returns 503 when no device tunnel is provided', async () => {
// Force tunnel provider to return null by closing + restarting with null provider.
await daemon.close();
+374
View File
@@ -105,6 +105,219 @@ describe('bootstrapTunnel', () => {
}
});
test('never auto-selects a connected paired Vision Pro ahead of an iPhone', async () => {
const spawn = makeSpawn([
{
argsMatch: /devicectl list devices/,
jsonOutput: {
result: { devices: [
{
identifier: 'VISION',
connectionProperties: { tunnelState: 'connected', pairingState: 'paired' },
deviceProperties: { name: 'Vision Pro' },
hardwareProperties: {
productType: 'RealityDevice17,1',
platform: 'visionOS',
deviceType: 'realityDevice',
},
},
{
identifier: 'IPHONE',
connectionProperties: { tunnelState: 'connected', pairingState: 'paired', transportType: 'wired' },
deviceProperties: { name: 'USB iPhone' },
hardwareProperties: {
productType: 'iPhone17,1',
platform: 'iOS',
deviceType: 'iPhone',
},
},
] },
},
},
{
argsMatch: /devicectl device info processes -d IPHONE/,
jsonOutput: { result: { runningProcesses: [{ executable: 'file:///var/containers/Bundle/Application/X/com.test.app/com.test' }] } },
},
{
argsMatch: /devicectl device info details --device IPHONE/,
jsonOutput: { result: { connectionProperties: { tunnelIPAddress: 'fd00::17' } } },
},
{
argsMatch: /devicectl device copy from --device IPHONE/,
destOutput: 'TOKEN\n',
},
]);
const r = await bootstrapTunnel({
bundleId: 'com.test',
spawnImpl: spawn,
fetchImpl: (async () => new Response('{"ok":true}', { status: 200 })) as typeof fetch,
});
expect(r.ok).toBe(true);
if (r.ok) expect(r.tunnel.udid).toBe('IPHONE');
});
test('fails clearly when only non-iPhone platforms are connected', async () => {
const spawn = makeSpawn([
{
argsMatch: /devicectl list devices/,
jsonOutput: {
result: { devices: [
{
identifier: 'VISION',
connectionProperties: { tunnelState: 'connected', pairingState: 'paired' },
deviceProperties: { name: 'Vision Pro' },
hardwareProperties: {
productType: 'RealityDevice17,1',
platform: 'visionOS',
deviceType: 'realityDevice',
},
},
{
identifier: 'IPAD',
connectionProperties: { tunnelState: 'connected', pairingState: 'paired' },
deviceProperties: { name: 'iPad' },
hardwareProperties: {
productType: 'iPad16,6',
platform: 'iOS',
deviceType: 'iPad',
},
},
] },
},
},
]);
const r = await bootstrapTunnel({ bundleId: 'com.test', spawnImpl: spawn });
expect(r.ok).toBe(false);
if (!r.ok) {
expect(r.error).toBe('device_not_found');
expect(r.detail).toContain('no iPhone');
}
});
test('rejects an explicitly targeted non-iPhone device', async () => {
const spawn = makeSpawn([
{
argsMatch: /devicectl list devices/,
jsonOutput: {
result: { devices: [{
identifier: 'VISION',
connectionProperties: { tunnelState: 'connected', pairingState: 'paired' },
deviceProperties: { name: 'Vision Pro' },
hardwareProperties: {
productType: 'RealityDevice17,1',
platform: 'visionOS',
deviceType: 'realityDevice',
},
}] },
},
},
]);
const r = await bootstrapTunnel({ bundleId: 'com.test', udid: 'VISION', spawnImpl: spawn });
expect(r.ok).toBe(false);
if (!r.ok) {
expect(r.error).toBe('device_not_found');
expect(r.detail).toContain('not an iPhone');
}
});
test('skips an unavailable paired device for a connected or available paired device', async () => {
const spawn = makeSpawn([
{
argsMatch: /devicectl list devices/,
jsonOutput: {
result: { devices: [
{
identifier: 'STALE',
connectionProperties: { tunnelState: 'unavailable', pairingState: 'paired' },
deviceProperties: { name: 'Stale iPhone' },
hardwareProperties: { productType: 'iPhone17,1' },
},
{
identifier: 'WIRED',
connectionProperties: { tunnelState: 'available', pairingState: 'paired', transportType: 'wired' },
deviceProperties: { name: 'Wired iPhone' },
hardwareProperties: { productType: 'iPhone18,2' },
},
] },
},
},
{
argsMatch: /devicectl device info processes -d WIRED/,
jsonOutput: { result: { runningProcesses: [{ executable: 'file:///var/containers/Bundle/Application/X/com.test.app/com.test' }] } },
},
{
argsMatch: /devicectl device info details --device WIRED/,
jsonOutput: { result: { connectionProperties: { tunnelIPAddress: 'fd00::2' } } },
},
{
argsMatch: /devicectl device copy from --device WIRED/,
destOutput: 'TOKEN\n',
},
]);
const r = await bootstrapTunnel({
bundleId: 'com.test',
spawnImpl: spawn,
fetchImpl: (async () => new Response('{"ok":true}', { status: 200 })) as typeof fetch,
});
expect(r.ok).toBe(true);
if (r.ok) expect(r.tunnel.udid).toBe('WIRED');
});
test('accepts a wired iPhone whose tunnel is disconnected until devicectl wakes it', async () => {
const spawn = makeSpawn([
{
argsMatch: /devicectl list devices/,
jsonOutput: {
result: { devices: [
{
identifier: 'STALE-VISION',
connectionProperties: { tunnelState: 'unavailable', pairingState: 'paired' },
deviceProperties: { name: 'Stale Vision Pro' },
hardwareProperties: { productType: 'RealityDevice14,1' },
},
{
identifier: 'WIRED-DISCONNECTED',
connectionProperties: {
tunnelState: 'disconnected',
pairingState: 'paired',
transportType: 'wired',
},
deviceProperties: { name: 'USB iPhone' },
hardwareProperties: { productType: 'iPhone18,2' },
},
] },
},
},
{
argsMatch: /devicectl device info processes -d WIRED-DISCONNECTED/,
jsonOutput: { result: { runningProcesses: [{ executable: 'file:\/\/\/var\/containers\/Bundle\/Application\/X\/com.test.app\/com.test' }] } },
},
{
argsMatch: /devicectl device info details --device WIRED-DISCONNECTED/,
jsonOutput: { result: { connectionProperties: { tunnelIPAddress: 'fd00::27' } } },
},
{
argsMatch: /devicectl device copy from --device WIRED-DISCONNECTED/,
destOutput: 'TOKEN\n',
},
]);
const r = await bootstrapTunnel({
bundleId: 'com.test',
spawnImpl: spawn,
fetchImpl: (async () => new Response('{"ok":true}', { status: 200 })) as typeof fetch,
});
expect(r.ok).toBe(true);
if (r.ok) expect(r.tunnel.udid).toBe('WIRED-DISCONNECTED');
});
test('returns device_locked when launchApp errors due to lock', async () => {
const spawn = makeSpawn([
{
@@ -166,6 +379,48 @@ describe('bootstrapTunnel', () => {
if (!r.ok) expect(r.error).toBe('state_server_unreachable');
});
test('rejects a StateServer owned by a different app bundle', async () => {
const spawn = makeSpawn([
{
argsMatch: /devicectl list devices/,
jsonOutput: {
result: { devices: [{
identifier: 'TEST',
connectionProperties: { tunnelState: 'connected', pairingState: 'paired' },
deviceProperties: { name: 'Test' },
hardwareProperties: { productType: 'iPhone18,2' },
}] },
},
},
{
argsMatch: /devicectl device info processes/,
jsonOutput: { result: { runningProcesses: [] } },
},
{
argsMatch: /devicectl device process launch/,
},
{
argsMatch: /devicectl device info details/,
jsonOutput: { result: { connectionProperties: { tunnelIPAddress: 'fd00::9' } } },
},
]);
const r = await bootstrapTunnel({
bundleId: 'com.expected.app',
spawnImpl: spawn,
fetchImpl: (async () => new Response(
JSON.stringify({ bundle_id: 'com.other.debug-app' }),
{ status: 200, headers: { 'content-type': 'application/json' } },
)) as typeof fetch,
});
expect(r.ok).toBe(false);
if (!r.ok) {
expect(r.error).toBe('wrong_app');
expect(r.detail).toContain('com.other.debug-app');
}
});
test('happy path: returns DeviceTunnel with rotated token', async () => {
const spawn = makeSpawn([
{
@@ -234,6 +489,125 @@ describe('bootstrapTunnel', () => {
expect(fetchCalls[fetchCalls.length - 1]?.url).toContain('/auth/rotate');
});
test('relaunches once when a prior daemon consumed the running app boot token', async () => {
const spawn = makeSpawn([
{
argsMatch: /devicectl list devices/,
jsonOutput: {
result: { devices: [{
identifier: 'TEST-UDID',
connectionProperties: { tunnelState: 'connected', pairingState: 'paired' },
deviceProperties: { name: 'Test Device' },
hardwareProperties: { productType: 'iPhone18,2' },
}] },
},
},
{
argsMatch: /devicectl device info processes/,
jsonOutput: { result: { runningProcesses: [{ executable: 'file:\/\/\/var\/containers\/Bundle\/Application\/X\/com.test.app\/com.test' }] } },
},
{
argsMatch: /devicectl device info details/,
jsonOutput: { result: { connectionProperties: { tunnelIPAddress: 'fd99::beef' } } },
},
{
// A prior daemon rotated the one-use token, so StateServer deleted it.
argsMatch: /devicectl device copy from/,
exitCode: 1,
stderr: 'source does not exist',
},
{
argsMatch: /devicectl device process launch --device TEST-UDID --terminate-existing com\.test/,
},
{
argsMatch: /devicectl device copy from/,
destOutput: 'FRESH-BOOT-TOKEN\n',
},
]);
const fetchCalls: Array<{ url: string; authorization?: string }> = [];
const r = await bootstrapTunnel({
bundleId: 'com.test',
spawnImpl: spawn,
fetchImpl: (async (url, init) => {
const u = String(url);
const headers = init?.headers as Record<string, string> | undefined;
fetchCalls.push({ url: u, authorization: headers?.Authorization });
if (u.endsWith('/healthz')) {
return new Response(JSON.stringify({ bundle_id: 'com.test' }), {
status: 200,
headers: { 'content-type': 'application/json' },
});
}
if (u.endsWith('/auth/rotate') && headers?.Authorization === 'Bearer FRESH-BOOT-TOKEN') {
return new Response('{"ok":true}', { status: 200 });
}
return new Response('unauthorized', { status: 401 });
}) as typeof fetch,
startupTimeoutMs: 1_000,
});
expect(r.ok).toBe(true);
expect(fetchCalls.filter((call) => call.url.endsWith('/healthz'))).toHaveLength(2);
expect(fetchCalls.at(-1)?.authorization).toBe('Bearer FRESH-BOOT-TOKEN');
});
test('rechecks bundle ownership after recovering a consumed boot token', async () => {
const spawn = makeSpawn([
{
argsMatch: /devicectl list devices/,
jsonOutput: {
result: { devices: [{
identifier: 'TEST-UDID',
connectionProperties: { tunnelState: 'connected', pairingState: 'paired' },
deviceProperties: { name: 'Test Device' },
hardwareProperties: { productType: 'iPhone18,2' },
}] },
},
},
{
argsMatch: /devicectl device info processes/,
jsonOutput: { result: { runningProcesses: [{ executable: 'file:\/\/\/var\/containers\/Bundle\/Application\/X\/com.test.app\/com.test' }] } },
},
{
argsMatch: /devicectl device info details/,
jsonOutput: { result: { connectionProperties: { tunnelIPAddress: 'fd99::beef' } } },
},
{ argsMatch: /devicectl device copy from/, exitCode: 1 },
{
argsMatch: /devicectl device process launch --device TEST-UDID --terminate-existing com\.test/,
},
{ argsMatch: /devicectl device copy from/, destOutput: 'FRESH-BOOT-TOKEN\n' },
]);
let healthChecks = 0;
let rotateCalls = 0;
const r = await bootstrapTunnel({
bundleId: 'com.test',
spawnImpl: spawn,
fetchImpl: (async (url) => {
const u = String(url);
if (u.endsWith('/healthz')) {
healthChecks++;
return new Response(JSON.stringify({
bundle_id: healthChecks === 1 ? 'com.test' : 'com.other.debug-app',
}), {
status: 200,
headers: { 'content-type': 'application/json' },
});
}
rotateCalls++;
return new Response('{"ok":true}', { status: 200 });
}) as typeof fetch,
startupTimeoutMs: 1_000,
});
expect(r.ok).toBe(false);
if (!r.ok) expect(r.error).toBe('wrong_app');
expect(healthChecks).toBe(2);
expect(rotateCalls).toBe(0);
});
test('resolve_failed when hostname cant be resolved to an IPv6', async () => {
const spawn = makeSpawn([
{