fix(desktop): bundle portable Python runtime

desktop-shell/tauri-skeleton/scripts/build-backend-runtime.cjs

@@ -1,6 +1,8 @@
 #!/usr/bin/env node
 
+const crypto = require('node:crypto');
 const fs = require('node:fs');
+const os = require('node:os');
 const path = require('node:path');
 const { spawnSync } = require('node:child_process');
 
@@ -17,6 +19,15 @@ const stagedReleaseAttestationPath = path.join(
   'data',
   'release_attestation.json',
 );
+const runtimeLayoutVersion = 2;
+const windowsEmbeddedPython = Object.freeze({
+  version: '3.11.9',
+  major: 3,
+  minor: 11,
+  arch: 'x64',
+  archiveName: 'python-3.11.9-embed-amd64.zip',
+  sha256: '009d6bf7e3b2ddca3d784fa09f90fe54336d5b60f0e0f305c37f400bf83cfd3b',
+});
 
 const excludedNames = new Set([
   '.env',
@@ -32,6 +43,7 @@ const excludedNames = new Set([
 
 const excludedFiles = new Set([
   '.env.example',
+  '.venv-dir',
   'ais_cache.json',
   'carrier_cache.json',
   'cctv.db',
@@ -39,7 +51,9 @@ const excludedFiles = new Set([
   'pytest.ini',
 ]);
 
-function backendPythonPath() {
+const conventionalVenvNames = new Set(['venv', '.venv', 'venv-repair', '.venv-repair']);
+
+function selectedVenvDirName() {
   let venvDir = 'venv';
   try {
     const persisted = fs.readFileSync(venvMarkerPath, 'utf8').trim();
@@ -48,17 +62,39 @@ function backendPythonPath() {
     }
   } catch {}
 
+  if (path.isAbsolute(venvDir) || path.basename(venvDir) !== venvDir) {
+    throw new Error(`Invalid backend venv directory marker: ${venvDir}`);
+  }
+  return venvDir;
+}
+
+function backendPythonPath() {
+  if (process.env.SHADOWBROKER_BACKEND_PYTHON) {
+    return path.resolve(process.env.SHADOWBROKER_BACKEND_PYTHON);
+  }
+  const venvDir = selectedVenvDirName();
   if (process.platform === 'win32') {
     return path.join(backendDir, venvDir, 'Scripts', 'python.exe');
   }
   return path.join(backendDir, venvDir, 'bin', 'python3');
 }
 
-function shouldCopy(srcPath) {
+function shouldCopyBackendPath(
+  srcPath,
+  platform = process.platform,
+  venvDirName = selectedVenvDirName(),
+) {
   const relativePath = path.relative(backendDir, srcPath);
   if (!relativePath) return true;
 
   const parts = relativePath.split(path.sep);
+  if (
+    platform === 'win32' &&
+    (parts[0] === venvDirName || conventionalVenvNames.has(parts[0]))
+  ) {
+    return false;
+  }
+
   return parts.every((part, index) => {
     const isLeaf = index === parts.length - 1;
     if (excludedNames.has(part)) return false;
@@ -68,13 +104,26 @@ function shouldCopy(srcPath) {
   });
 }
 
+function shouldCopySitePackagePath(sitePackagesRoot, srcPath) {
+  const relativePath = path.relative(sitePackagesRoot, srcPath);
+  if (!relativePath) return true;
+
+  const parts = relativePath.split(path.sep);
+  if (parts.includes('__pycache__')) return false;
+  if (parts.some((part) => /^backend-.*\.dist-info$/i.test(part))) return false;
+  const leaf = parts.at(-1);
+  if (/\.(?:egg-link|pth|pyc)$/i.test(leaf)) return false;
+  if (leaf.toLowerCase() === 'direct_url.json') return false;
+  return true;
+}
+
 function ensureRuntimePrereqs() {
   if (!fs.existsSync(path.join(backendDir, 'main.py'))) {
     throw new Error(`Missing backend/main.py at ${backendDir}`);
   }
   if (!fs.existsSync(backendPythonPath())) {
     throw new Error(
-      `Missing bundled backend Python runtime at ${backendPythonPath()}. ` +
+      `Missing backend build interpreter at ${backendPythonPath()}. ` +
       'Create the backend venv before packaging the desktop app.',
     );
   }
@@ -86,6 +135,25 @@ function ensureRuntimePrereqs() {
   }
 }
 
+function readBuildPythonInfo() {
+  const python = backendPythonPath();
+  const code = [
+    'import json, platform, sys, sysconfig',
+    "print(json.dumps({'major': sys.version_info.major, 'minor': sys.version_info.minor, " +
+      "'micro': sys.version_info.micro, 'machine': platform.machine(), " +
+      "'prefix': sys.prefix, 'base_prefix': sys.base_prefix, " +
+      "'purelib': sysconfig.get_paths()['purelib']}))",
+  ].join('; ');
+  const result = spawnSync(python, ['-I', '-c', code], {
+    cwd: backendDir,
+    encoding: 'utf8',
+  });
+  if (result.error || result.status !== 0) {
+    throw new Error(`Failed to inspect backend build interpreter: ${result.stderr || result.error}`);
+  }
+  return JSON.parse(result.stdout.trim());
+}
+
 function privacyCoreArtifactName() {
   if (process.platform === 'win32') return 'privacy_core.dll';
   if (process.platform === 'darwin') return 'libprivacy_core.dylib';
@@ -122,34 +190,240 @@ function ensurePrivacyCoreArtifact() {
   return artifact;
 }
 
-function stageBackendRuntime() {
+function sha256File(filePath) {
+  const hash = crypto.createHash('sha256');
+  hash.update(fs.readFileSync(filePath));
+  return hash.digest('hex');
+}
+
+async function downloadFile(url, destination) {
+  const response = await fetch(url, { redirect: 'follow' });
+  if (!response.ok) {
+    throw new Error(`Failed to download ${url}: HTTP ${response.status}`);
+  }
+  const temporary = `${destination}.partial-${process.pid}`;
+  fs.writeFileSync(temporary, Buffer.from(await response.arrayBuffer()));
+  fs.renameSync(temporary, destination);
+}
+
+async function windowsEmbeddedPythonArchive() {
+  const configured = process.env.SHADOWBROKER_PYTHON_EMBED_ZIP;
+  const cacheDir = path.join(os.tmpdir(), 'shadowbroker-python-embed');
+  const archivePath = configured
+    ? path.resolve(configured)
+    : path.join(cacheDir, windowsEmbeddedPython.archiveName);
+
+  if (!fs.existsSync(archivePath)) {
+    if (configured) {
+      throw new Error(`SHADOWBROKER_PYTHON_EMBED_ZIP does not exist: ${archivePath}`);
+    }
+    fs.mkdirSync(cacheDir, { recursive: true });
+    const url =
+      `https://www.python.org/ftp/python/${windowsEmbeddedPython.version}/` +
+      windowsEmbeddedPython.archiveName;
+    console.log(`Downloading verified embedded Python ${windowsEmbeddedPython.version}...`);
+    await downloadFile(url, archivePath);
+  }
+
+  const actualHash = sha256File(archivePath);
+  if (actualHash !== windowsEmbeddedPython.sha256) {
+    throw new Error(
+      `Embedded Python archive SHA-256 mismatch for ${archivePath}: ` +
+      `expected ${windowsEmbeddedPython.sha256}, got ${actualHash}`,
+    );
+  }
+  return archivePath;
+}
+
+function extractZipWithPowerShell(archivePath, destination) {
+  fs.mkdirSync(destination, { recursive: true });
+  const env = {
+    ...process.env,
+    SHADOWBROKER_EMBED_ARCHIVE: archivePath,
+    SHADOWBROKER_EMBED_DESTINATION: destination,
+  };
+  const result = spawnSync(
+    'powershell.exe',
+    [
+      '-NoProfile',
+      '-NonInteractive',
+      '-Command',
+      "$ErrorActionPreference='Stop'; Expand-Archive -LiteralPath $env:SHADOWBROKER_EMBED_ARCHIVE -DestinationPath $env:SHADOWBROKER_EMBED_DESTINATION -Force",
+    ],
+    { env, encoding: 'utf8' },
+  );
+  if (result.error || result.status !== 0) {
+    throw new Error(`Failed to extract embedded Python: ${result.stderr || result.error}`);
+  }
+}
+
+function configureEmbeddedPythonPath(pythonRoot) {
+  const pthPath = path.join(
+    pythonRoot,
+    `python${windowsEmbeddedPython.major}${windowsEmbeddedPython.minor}._pth`,
+  );
+  if (!fs.existsSync(pthPath)) {
+    throw new Error(`Embedded Python path file is missing: ${pthPath}`);
+  }
+  fs.writeFileSync(
+    pthPath,
+    [
+      `python${windowsEmbeddedPython.major}${windowsEmbeddedPython.minor}.zip`,
+      '.',
+      'Lib',
+      'Lib\\site-packages',
+      '..',
+      'import site',
+      '',
+    ].join('\n'),
+    'utf8',
+  );
+}
+
+async function stageWindowsEmbeddedPython() {
+  if (process.arch !== windowsEmbeddedPython.arch) {
+    throw new Error(
+      `Windows desktop packaging currently supports ${windowsEmbeddedPython.arch}, not ${process.arch}`,
+    );
+  }
+
+  const buildInfo = readBuildPythonInfo();
+  if (
+    buildInfo.major !== windowsEmbeddedPython.major ||
+    buildInfo.minor !== windowsEmbeddedPython.minor
+  ) {
+    throw new Error(
+      `Backend venv uses Python ${buildInfo.major}.${buildInfo.minor}; ` +
+      `desktop packaging requires Python ${windowsEmbeddedPython.major}.${windowsEmbeddedPython.minor}.x ` +
+      'so compiled extension modules match the embedded runtime.',
+    );
+  }
+  if (!fs.existsSync(buildInfo.purelib)) {
+    throw new Error(`Backend site-packages directory is missing: ${buildInfo.purelib}`);
+  }
+
+  const pythonRoot = path.join(outputDir, 'python');
+  const archivePath = await windowsEmbeddedPythonArchive();
+  extractZipWithPowerShell(archivePath, pythonRoot);
+  configureEmbeddedPythonPath(pythonRoot);
+
+  const stagedSitePackages = path.join(pythonRoot, 'Lib', 'site-packages');
+  fs.mkdirSync(stagedSitePackages, { recursive: true });
+  fs.cpSync(buildInfo.purelib, stagedSitePackages, {
+    recursive: true,
+    filter: (srcPath) => shouldCopySitePackagePath(buildInfo.purelib, srcPath),
+  });
+
+  fs.writeFileSync(
+    path.join(outputDir, '.runtime-layout.json'),
+    `${JSON.stringify(
+      {
+        layout: 'embedded-python',
+        layoutVersion: runtimeLayoutVersion,
+        python: 'python/python.exe',
+        pythonVersion: windowsEmbeddedPython.version,
+      },
+      null,
+      2,
+    )}\n`,
+    'utf8',
+  );
+  return buildInfo;
+}
+
+function walkFiles(root) {
+  const files = [];
+  for (const entry of fs.readdirSync(root, { withFileTypes: true })) {
+    const entryPath = path.join(root, entry.name);
+    if (entry.isDirectory()) {
+      files.push(...walkFiles(entryPath));
+    } else {
+      files.push(entryPath);
+    }
+  }
+  return files;
+}
+
+function assertPortableWindowsBundle(root, buildInfo) {
+  const forbiddenPaths = [repoRoot, backendDir, buildInfo.prefix, buildInfo.base_prefix]
+    .filter(Boolean)
+    .map((value) => path.resolve(value).toLowerCase());
+
+  for (const filePath of walkFiles(root)) {
+    const leaf = path.basename(filePath).toLowerCase();
+    if (leaf === 'pyvenv.cfg' || /\.(?:egg-link|pth)$/i.test(leaf)) {
+      throw new Error(`Non-portable virtualenv metadata was staged: ${filePath}`);
+    }
+
+    const stat = fs.statSync(filePath);
+    if (stat.size > 2 * 1024 * 1024) continue;
+    const bytes = fs.readFileSync(filePath);
+    if (bytes.includes(0)) continue;
+    const text = bytes.toString('utf8').toLowerCase();
+    const leakedPath = forbiddenPaths.find((value) => text.includes(value));
+    if (leakedPath) {
+      throw new Error(`Build-machine path leaked into staged runtime: ${filePath}`);
+    }
+  }
+}
+
+function verifyRelocatableWindowsRuntime() {
+  const probeDir = `${outputDir}-portability-probe-${process.pid}`;
+  if (fs.existsSync(probeDir)) {
+    throw new Error(`Portability probe path already exists: ${probeDir}`);
+  }
+
+  fs.renameSync(outputDir, probeDir);
+  let result;
+  try {
+    const env = { ...process.env };
+    delete env.PYTHONHOME;
+    delete env.PYTHONPATH;
+    result = spawnSync(
+      path.join(probeDir, 'python', 'python.exe'),
+      [
+        '-I',
+        '-c',
+        [
+          'import sys',
+          'import fastapi, uvicorn, cryptography, numpy, orjson, pydantic',
+          'assert sys.prefix == sys.base_prefix',
+          'print(sys.executable)',
+        ].join('; '),
+      ],
+      { cwd: probeDir, env, encoding: 'utf8' },
+    );
+  } finally {
+    fs.renameSync(probeDir, outputDir);
+  }
+
+  if (result.error || result.status !== 0) {
+    throw new Error(`Relocated embedded Python smoke test failed: ${result.stderr || result.error}`);
+  }
+}
+
+async function stageBackendRuntime() {
   fs.rmSync(outputDir, { recursive: true, force: true });
   fs.cpSync(backendDir, outputDir, {
     recursive: true,
-    filter: shouldCopy,
+    filter: shouldCopyBackendPath,
   });
+
+  let buildInfo = null;
+  if (process.platform === 'win32') {
+    buildInfo = await stageWindowsEmbeddedPython();
+  }
+
   stagePrivacyCoreArtifact();
   stageReleaseAttestation();
   stageStartScripts();
+
+  if (process.platform === 'win32') {
+    assertPortableWindowsBundle(outputDir, buildInfo);
+    verifyRelocatableWindowsRuntime();
+  }
 }
 
-/**
- * Copy ``start.bat`` and ``start.sh`` from the repo root into the
- * staged backend-runtime/ so they sit next to ``privacy_core.dll``.
- *
- * Why: an MSI/EXE/AppImage user who wants to launch via the dev-style
- * scripts (because the desktop shell is failing, or they prefer the
- * browser frontend at localhost:3000) shouldn't have to clone the
- * source repo just to get the scripts. Having them inside the install
- * directory also means the bundled ``privacy_core.dll`` fallback in
- * those scripts resolves to the SAME directory as the script, which
- * is exactly the layout the v0.9.81 script update is looking for.
- *
- * Tracked from issue #319: users who fell back to start.bat from
- * their MSI install dir had to go fetch it from GitHub, then saw a
- * scary "install Rust" warning because the script didn't know where
- * the bundled DLL was. Bundling the script removes both problems.
- */
 function stageStartScripts() {
   const scripts = ['start.bat', 'start.sh'];
   for (const name of scripts) {
@@ -160,7 +434,6 @@ function stageStartScripts() {
     }
     const dst = path.join(outputDir, name);
     fs.copyFileSync(src, dst);
-    // Preserve executable bit on POSIX systems for the .sh script.
     if (name.endsWith('.sh') && process.platform !== 'win32') {
       try {
         fs.chmodSync(dst, 0o755);
@@ -191,7 +464,11 @@ function writeBundleVersion() {
   const pkg = JSON.parse(
     fs.readFileSync(path.join(repoRoot, 'desktop-shell', 'package.json'), 'utf8'),
   );
-  fs.writeFileSync(versionPath, `${pkg.version || '0.0.0'}\n`, 'utf8');
+  fs.writeFileSync(
+    versionPath,
+    `${pkg.version || '0.0.0'}-runtime-${runtimeLayoutVersion}\n`,
+    'utf8',
+  );
 }
 
 function fileCount(root) {
@@ -207,7 +484,24 @@ function fileCount(root) {
   return count;
 }
 
-ensureRuntimePrereqs();
-stageBackendRuntime();
-writeBundleVersion();
-console.log(`backend-runtime staged: ${fileCount(outputDir)} files`);
+async function main() {
+  ensureRuntimePrereqs();
+  await stageBackendRuntime();
+  writeBundleVersion();
+  console.log(`backend-runtime staged: ${fileCount(outputDir)} files`);
+}
+
+if (require.main === module) {
+  main().catch((error) => {
+    console.error(error instanceof Error ? error.message : error);
+    process.exitCode = 1;
+  });
+}
+
+module.exports = {
+  assertPortableWindowsBundle,
+  configureEmbeddedPythonPath,
+  shouldCopyBackendPath,
+  shouldCopySitePackagePath,
+  windowsEmbeddedPython,
+};

desktop-shell/tauri-skeleton/scripts/build-backend-runtime.test.cjs

@@ -0,0 +1,108 @@
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const test = require('node:test');
+
+const {
+  assertPortableWindowsBundle,
+  configureEmbeddedPythonPath,
+  shouldCopyBackendPath,
+  shouldCopySitePackagePath,
+} = require('./build-backend-runtime.cjs');
+
+const repoRoot = path.resolve(__dirname, '..', '..', '..');
+const backendDir = path.join(repoRoot, 'backend');
+
+test('Windows staging never copies a virtualenv or its marker', () => {
+  assert.equal(
+    shouldCopyBackendPath(path.join(backendDir, 'venv', 'pyvenv.cfg'), 'win32', 'venv'),
+    false,
+  );
+  assert.equal(
+    shouldCopyBackendPath(path.join(backendDir, '.venv', 'Scripts', 'python.exe'), 'win32'),
+    false,
+  );
+  assert.equal(
+    shouldCopyBackendPath(path.join(backendDir, '.venv-dir'), 'win32', 'custom-venv'),
+    false,
+  );
+  assert.equal(
+    shouldCopyBackendPath(path.join(backendDir, 'services', 'config.py'), 'win32', 'venv'),
+    true,
+  );
+});
+
+test('site-packages staging strips host-bound path files and editable metadata', () => {
+  const sitePackages = path.join('C:', 'build', 'site-packages');
+  assert.equal(
+    shouldCopySitePackagePath(sitePackages, path.join(sitePackages, 'host-path.pth')),
+    false,
+  );
+  assert.equal(
+    shouldCopySitePackagePath(
+      sitePackages,
+      path.join(sitePackages, 'backend-0.9.84.dist-info', 'METADATA'),
+    ),
+    false,
+  );
+  assert.equal(
+    shouldCopySitePackagePath(sitePackages, path.join(sitePackages, 'fastapi', '__init__.py')),
+    true,
+  );
+});
+
+test('embedded Python search paths are relative to the packaged runtime', () => {
+  const temp = fs.mkdtempSync(path.join(os.tmpdir(), 'sb-python-pth-test-'));
+  try {
+    const pthPath = path.join(temp, 'python311._pth');
+    fs.writeFileSync(pthPath, 'C:\\Users\\developer\\Python311\\python311.zip\n', 'utf8');
+
+    configureEmbeddedPythonPath(temp);
+
+    const configured = fs.readFileSync(pthPath, 'utf8');
+    assert.doesNotMatch(configured, /[A-Za-z]:\\/);
+    assert.match(configured, /^python311\.zip$/m);
+    assert.match(configured, /^Lib\\site-packages$/m);
+    assert.match(configured, /^\.\.$/m);
+    assert.match(configured, /^import site$/m);
+  } finally {
+    fs.rmSync(temp, { recursive: true, force: true });
+  }
+});
+
+test('bundle validation rejects pyvenv.cfg and developer home paths', () => {
+  const temp = fs.mkdtempSync(path.join(os.tmpdir(), 'sb-python-portability-test-'));
+  const buildInfo = {
+    prefix: 'C:\\Users\\developer\\Shadowbroker\\backend\\venv',
+    base_prefix: 'C:\\Users\\developer\\Python311',
+  };
+
+  try {
+    fs.writeFileSync(path.join(temp, 'safe.txt'), 'portable runtime\n', 'utf8');
+    assert.doesNotThrow(() => assertPortableWindowsBundle(temp, buildInfo));
+
+    fs.writeFileSync(
+      path.join(temp, 'pyvenv.cfg'),
+      'home = C:\\Users\\developer\\Python311\n',
+      'utf8',
+    );
+    assert.throws(
+      () => assertPortableWindowsBundle(temp, buildInfo),
+      /Non-portable virtualenv metadata/,
+    );
+
+    fs.rmSync(path.join(temp, 'pyvenv.cfg'));
+    fs.writeFileSync(
+      path.join(temp, 'leak.txt'),
+      `executable = ${buildInfo.base_prefix}\\python.exe\n`,
+      'utf8',
+    );
+    assert.throws(
+      () => assertPortableWindowsBundle(temp, buildInfo),
+      /Build-machine path leaked/,
+    );
+  } finally {
+    fs.rmSync(temp, { recursive: true, force: true });
+  }
+});