mirror of
https://github.com/BigBodyCobain/Shadowbroker.git
synced 2026-09-14 05:08:59 +02:00
fix(desktop): bundle portable Python runtime
This commit is contained in:
@@ -5,8 +5,9 @@
|
||||
"description": "ShadowBroker desktop shell packaging, runtime bridge, and release tooling",
|
||||
"scripts": {
|
||||
"typecheck": "tsc --noEmit",
|
||||
"build:desktop": "node ./scripts/run-desktop-build.cjs",
|
||||
"build:desktop:clean": "node ./scripts/run-desktop-build.cjs --clean"
|
||||
"test:backend-runtime": "node --test ./tauri-skeleton/scripts/build-backend-runtime.test.cjs",
|
||||
"build:desktop": "npm run test:backend-runtime && node ./scripts/run-desktop-build.cjs",
|
||||
"build:desktop:clean": "npm run test:backend-runtime && node ./scripts/run-desktop-build.cjs --clean"
|
||||
},
|
||||
"devDependencies": {
|
||||
"typescript": "^5.6.0"
|
||||
|
||||
@@ -77,7 +77,9 @@ The release build now does the full packaging pipeline:
|
||||
2. Stages a desktop-only frontend export tree that omits Next server-only
|
||||
routes/proxy (`src/app/api`, `src/proxy.ts`)
|
||||
3. Stages a managed backend runtime bundle from `backend/` into
|
||||
`src-tauri/backend-runtime/`
|
||||
`src-tauri/backend-runtime/`. On Windows this contains a relocatable,
|
||||
checksum-verified embedded Python runtime; the developer venv is never
|
||||
included.
|
||||
4. Builds the frontend export with `NEXT_OUTPUT=export`
|
||||
5. Copies `frontend/out` to `src-tauri/companion-www/`
|
||||
6. Runs `cargo tauri build`
|
||||
@@ -87,6 +89,10 @@ The release build now does the full packaging pipeline:
|
||||
If `cargo tauri` is not installed, the build now fails immediately with the
|
||||
required install command instead of failing after the frontend export.
|
||||
|
||||
Windows packaging also runs a relocation smoke test before creating the
|
||||
installer and fails closed if `pyvenv.cfg`, `.pth`/`.egg-link` files, or the
|
||||
build machine's Python paths leak into the staged runtime.
|
||||
|
||||
See [RELEASE.md](./RELEASE.md) for the release-oriented checklist.
|
||||
See [RELEASE_INPUTS.md](./RELEASE_INPUTS.md) for the future credentials/secrets
|
||||
that only matter once you want signed/notarized public distribution.
|
||||
|
||||
@@ -28,6 +28,8 @@ Prerequisites:
|
||||
- Rust toolchain
|
||||
- `cargo tauri` available via `cargo install tauri-cli@^2`
|
||||
- Node.js / npm with the frontend dependencies already installed
|
||||
- On Windows, an x64 Python 3.11 backend venv with the production dependencies
|
||||
installed. The venv is used only as a package source and is never shipped.
|
||||
|
||||
## CI / GitHub Actions
|
||||
|
||||
@@ -53,7 +55,10 @@ See [RELEASE_INPUTS.md](./RELEASE_INPUTS.md) for the plain-language answer to
|
||||
1. Generates the desktop icon set in `src-tauri/icons/`
|
||||
2. Stages a desktop-only frontend export tree that omits Next server-only
|
||||
routes/proxy (`src/app/api`, `src/proxy.ts`)
|
||||
3. Stages a managed backend runtime bundle into `src-tauri/backend-runtime/`
|
||||
3. Stages a managed backend runtime bundle into `src-tauri/backend-runtime/`.
|
||||
Windows builds use the checksum-pinned official Python 3.11.9 embeddable
|
||||
distribution plus the backend venv's installed packages; they never copy
|
||||
`pyvenv.cfg` or a host-bound venv launcher.
|
||||
4. Builds the frontend export with `NEXT_OUTPUT=export`
|
||||
5. Copies `frontend/out` into `src-tauri/companion-www/`
|
||||
6. Runs `cargo tauri build`
|
||||
@@ -68,6 +73,18 @@ the managed backend bundle at `backend-runtime/data/release_attestation.json`,
|
||||
and the managed-backend updater refreshes that file on version sync without
|
||||
overwriting the rest of the runtime `data/` directory.
|
||||
|
||||
The Windows staging step fails the build if virtualenv metadata or an absolute
|
||||
build-machine Python path reaches the bundle. It also renames the completed
|
||||
runtime and launches the embedded interpreter from the relocated path before
|
||||
Tauri packaging begins. Set `SHADOWBROKER_PYTHON_EMBED_ZIP` to an offline copy
|
||||
of `python-3.11.9-embed-amd64.zip`; the pinned SHA-256 is still enforced. Set
|
||||
`SHADOWBROKER_BACKEND_PYTHON` only when the production backend interpreter is
|
||||
outside the normal `backend/venv` location.
|
||||
|
||||
When an installed runtime layout changes, the desktop updater removes legacy
|
||||
virtualenv directories and their marker after copying the new bundle. The
|
||||
operator's `.env`, `data/`, and unrelated runtime files remain preserved.
|
||||
|
||||
## Release artifacts
|
||||
|
||||
Artifacts are emitted under:
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
const crypto = require('node:crypto');
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { spawnSync } = require('node:child_process');
|
||||
|
||||
@@ -17,6 +19,15 @@ const stagedReleaseAttestationPath = path.join(
|
||||
'data',
|
||||
'release_attestation.json',
|
||||
);
|
||||
const runtimeLayoutVersion = 2;
|
||||
const windowsEmbeddedPython = Object.freeze({
|
||||
version: '3.11.9',
|
||||
major: 3,
|
||||
minor: 11,
|
||||
arch: 'x64',
|
||||
archiveName: 'python-3.11.9-embed-amd64.zip',
|
||||
sha256: '009d6bf7e3b2ddca3d784fa09f90fe54336d5b60f0e0f305c37f400bf83cfd3b',
|
||||
});
|
||||
|
||||
const excludedNames = new Set([
|
||||
'.env',
|
||||
@@ -32,6 +43,7 @@ const excludedNames = new Set([
|
||||
|
||||
const excludedFiles = new Set([
|
||||
'.env.example',
|
||||
'.venv-dir',
|
||||
'ais_cache.json',
|
||||
'carrier_cache.json',
|
||||
'cctv.db',
|
||||
@@ -39,7 +51,9 @@ const excludedFiles = new Set([
|
||||
'pytest.ini',
|
||||
]);
|
||||
|
||||
function backendPythonPath() {
|
||||
const conventionalVenvNames = new Set(['venv', '.venv', 'venv-repair', '.venv-repair']);
|
||||
|
||||
function selectedVenvDirName() {
|
||||
let venvDir = 'venv';
|
||||
try {
|
||||
const persisted = fs.readFileSync(venvMarkerPath, 'utf8').trim();
|
||||
@@ -48,17 +62,39 @@ function backendPythonPath() {
|
||||
}
|
||||
} catch {}
|
||||
|
||||
if (path.isAbsolute(venvDir) || path.basename(venvDir) !== venvDir) {
|
||||
throw new Error(`Invalid backend venv directory marker: ${venvDir}`);
|
||||
}
|
||||
return venvDir;
|
||||
}
|
||||
|
||||
function backendPythonPath() {
|
||||
if (process.env.SHADOWBROKER_BACKEND_PYTHON) {
|
||||
return path.resolve(process.env.SHADOWBROKER_BACKEND_PYTHON);
|
||||
}
|
||||
const venvDir = selectedVenvDirName();
|
||||
if (process.platform === 'win32') {
|
||||
return path.join(backendDir, venvDir, 'Scripts', 'python.exe');
|
||||
}
|
||||
return path.join(backendDir, venvDir, 'bin', 'python3');
|
||||
}
|
||||
|
||||
function shouldCopy(srcPath) {
|
||||
function shouldCopyBackendPath(
|
||||
srcPath,
|
||||
platform = process.platform,
|
||||
venvDirName = selectedVenvDirName(),
|
||||
) {
|
||||
const relativePath = path.relative(backendDir, srcPath);
|
||||
if (!relativePath) return true;
|
||||
|
||||
const parts = relativePath.split(path.sep);
|
||||
if (
|
||||
platform === 'win32' &&
|
||||
(parts[0] === venvDirName || conventionalVenvNames.has(parts[0]))
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return parts.every((part, index) => {
|
||||
const isLeaf = index === parts.length - 1;
|
||||
if (excludedNames.has(part)) return false;
|
||||
@@ -68,13 +104,26 @@ function shouldCopy(srcPath) {
|
||||
});
|
||||
}
|
||||
|
||||
function shouldCopySitePackagePath(sitePackagesRoot, srcPath) {
|
||||
const relativePath = path.relative(sitePackagesRoot, srcPath);
|
||||
if (!relativePath) return true;
|
||||
|
||||
const parts = relativePath.split(path.sep);
|
||||
if (parts.includes('__pycache__')) return false;
|
||||
if (parts.some((part) => /^backend-.*\.dist-info$/i.test(part))) return false;
|
||||
const leaf = parts.at(-1);
|
||||
if (/\.(?:egg-link|pth|pyc)$/i.test(leaf)) return false;
|
||||
if (leaf.toLowerCase() === 'direct_url.json') return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
function ensureRuntimePrereqs() {
|
||||
if (!fs.existsSync(path.join(backendDir, 'main.py'))) {
|
||||
throw new Error(`Missing backend/main.py at ${backendDir}`);
|
||||
}
|
||||
if (!fs.existsSync(backendPythonPath())) {
|
||||
throw new Error(
|
||||
`Missing bundled backend Python runtime at ${backendPythonPath()}. ` +
|
||||
`Missing backend build interpreter at ${backendPythonPath()}. ` +
|
||||
'Create the backend venv before packaging the desktop app.',
|
||||
);
|
||||
}
|
||||
@@ -86,6 +135,25 @@ function ensureRuntimePrereqs() {
|
||||
}
|
||||
}
|
||||
|
||||
function readBuildPythonInfo() {
|
||||
const python = backendPythonPath();
|
||||
const code = [
|
||||
'import json, platform, sys, sysconfig',
|
||||
"print(json.dumps({'major': sys.version_info.major, 'minor': sys.version_info.minor, " +
|
||||
"'micro': sys.version_info.micro, 'machine': platform.machine(), " +
|
||||
"'prefix': sys.prefix, 'base_prefix': sys.base_prefix, " +
|
||||
"'purelib': sysconfig.get_paths()['purelib']}))",
|
||||
].join('; ');
|
||||
const result = spawnSync(python, ['-I', '-c', code], {
|
||||
cwd: backendDir,
|
||||
encoding: 'utf8',
|
||||
});
|
||||
if (result.error || result.status !== 0) {
|
||||
throw new Error(`Failed to inspect backend build interpreter: ${result.stderr || result.error}`);
|
||||
}
|
||||
return JSON.parse(result.stdout.trim());
|
||||
}
|
||||
|
||||
function privacyCoreArtifactName() {
|
||||
if (process.platform === 'win32') return 'privacy_core.dll';
|
||||
if (process.platform === 'darwin') return 'libprivacy_core.dylib';
|
||||
@@ -122,34 +190,240 @@ function ensurePrivacyCoreArtifact() {
|
||||
return artifact;
|
||||
}
|
||||
|
||||
function stageBackendRuntime() {
|
||||
function sha256File(filePath) {
|
||||
const hash = crypto.createHash('sha256');
|
||||
hash.update(fs.readFileSync(filePath));
|
||||
return hash.digest('hex');
|
||||
}
|
||||
|
||||
async function downloadFile(url, destination) {
|
||||
const response = await fetch(url, { redirect: 'follow' });
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to download ${url}: HTTP ${response.status}`);
|
||||
}
|
||||
const temporary = `${destination}.partial-${process.pid}`;
|
||||
fs.writeFileSync(temporary, Buffer.from(await response.arrayBuffer()));
|
||||
fs.renameSync(temporary, destination);
|
||||
}
|
||||
|
||||
async function windowsEmbeddedPythonArchive() {
|
||||
const configured = process.env.SHADOWBROKER_PYTHON_EMBED_ZIP;
|
||||
const cacheDir = path.join(os.tmpdir(), 'shadowbroker-python-embed');
|
||||
const archivePath = configured
|
||||
? path.resolve(configured)
|
||||
: path.join(cacheDir, windowsEmbeddedPython.archiveName);
|
||||
|
||||
if (!fs.existsSync(archivePath)) {
|
||||
if (configured) {
|
||||
throw new Error(`SHADOWBROKER_PYTHON_EMBED_ZIP does not exist: ${archivePath}`);
|
||||
}
|
||||
fs.mkdirSync(cacheDir, { recursive: true });
|
||||
const url =
|
||||
`https://www.python.org/ftp/python/${windowsEmbeddedPython.version}/` +
|
||||
windowsEmbeddedPython.archiveName;
|
||||
console.log(`Downloading verified embedded Python ${windowsEmbeddedPython.version}...`);
|
||||
await downloadFile(url, archivePath);
|
||||
}
|
||||
|
||||
const actualHash = sha256File(archivePath);
|
||||
if (actualHash !== windowsEmbeddedPython.sha256) {
|
||||
throw new Error(
|
||||
`Embedded Python archive SHA-256 mismatch for ${archivePath}: ` +
|
||||
`expected ${windowsEmbeddedPython.sha256}, got ${actualHash}`,
|
||||
);
|
||||
}
|
||||
return archivePath;
|
||||
}
|
||||
|
||||
function extractZipWithPowerShell(archivePath, destination) {
|
||||
fs.mkdirSync(destination, { recursive: true });
|
||||
const env = {
|
||||
...process.env,
|
||||
SHADOWBROKER_EMBED_ARCHIVE: archivePath,
|
||||
SHADOWBROKER_EMBED_DESTINATION: destination,
|
||||
};
|
||||
const result = spawnSync(
|
||||
'powershell.exe',
|
||||
[
|
||||
'-NoProfile',
|
||||
'-NonInteractive',
|
||||
'-Command',
|
||||
"$ErrorActionPreference='Stop'; Expand-Archive -LiteralPath $env:SHADOWBROKER_EMBED_ARCHIVE -DestinationPath $env:SHADOWBROKER_EMBED_DESTINATION -Force",
|
||||
],
|
||||
{ env, encoding: 'utf8' },
|
||||
);
|
||||
if (result.error || result.status !== 0) {
|
||||
throw new Error(`Failed to extract embedded Python: ${result.stderr || result.error}`);
|
||||
}
|
||||
}
|
||||
|
||||
function configureEmbeddedPythonPath(pythonRoot) {
|
||||
const pthPath = path.join(
|
||||
pythonRoot,
|
||||
`python${windowsEmbeddedPython.major}${windowsEmbeddedPython.minor}._pth`,
|
||||
);
|
||||
if (!fs.existsSync(pthPath)) {
|
||||
throw new Error(`Embedded Python path file is missing: ${pthPath}`);
|
||||
}
|
||||
fs.writeFileSync(
|
||||
pthPath,
|
||||
[
|
||||
`python${windowsEmbeddedPython.major}${windowsEmbeddedPython.minor}.zip`,
|
||||
'.',
|
||||
'Lib',
|
||||
'Lib\\site-packages',
|
||||
'..',
|
||||
'import site',
|
||||
'',
|
||||
].join('\n'),
|
||||
'utf8',
|
||||
);
|
||||
}
|
||||
|
||||
async function stageWindowsEmbeddedPython() {
|
||||
if (process.arch !== windowsEmbeddedPython.arch) {
|
||||
throw new Error(
|
||||
`Windows desktop packaging currently supports ${windowsEmbeddedPython.arch}, not ${process.arch}`,
|
||||
);
|
||||
}
|
||||
|
||||
const buildInfo = readBuildPythonInfo();
|
||||
if (
|
||||
buildInfo.major !== windowsEmbeddedPython.major ||
|
||||
buildInfo.minor !== windowsEmbeddedPython.minor
|
||||
) {
|
||||
throw new Error(
|
||||
`Backend venv uses Python ${buildInfo.major}.${buildInfo.minor}; ` +
|
||||
`desktop packaging requires Python ${windowsEmbeddedPython.major}.${windowsEmbeddedPython.minor}.x ` +
|
||||
'so compiled extension modules match the embedded runtime.',
|
||||
);
|
||||
}
|
||||
if (!fs.existsSync(buildInfo.purelib)) {
|
||||
throw new Error(`Backend site-packages directory is missing: ${buildInfo.purelib}`);
|
||||
}
|
||||
|
||||
const pythonRoot = path.join(outputDir, 'python');
|
||||
const archivePath = await windowsEmbeddedPythonArchive();
|
||||
extractZipWithPowerShell(archivePath, pythonRoot);
|
||||
configureEmbeddedPythonPath(pythonRoot);
|
||||
|
||||
const stagedSitePackages = path.join(pythonRoot, 'Lib', 'site-packages');
|
||||
fs.mkdirSync(stagedSitePackages, { recursive: true });
|
||||
fs.cpSync(buildInfo.purelib, stagedSitePackages, {
|
||||
recursive: true,
|
||||
filter: (srcPath) => shouldCopySitePackagePath(buildInfo.purelib, srcPath),
|
||||
});
|
||||
|
||||
fs.writeFileSync(
|
||||
path.join(outputDir, '.runtime-layout.json'),
|
||||
`${JSON.stringify(
|
||||
{
|
||||
layout: 'embedded-python',
|
||||
layoutVersion: runtimeLayoutVersion,
|
||||
python: 'python/python.exe',
|
||||
pythonVersion: windowsEmbeddedPython.version,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
'utf8',
|
||||
);
|
||||
return buildInfo;
|
||||
}
|
||||
|
||||
function walkFiles(root) {
|
||||
const files = [];
|
||||
for (const entry of fs.readdirSync(root, { withFileTypes: true })) {
|
||||
const entryPath = path.join(root, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
files.push(...walkFiles(entryPath));
|
||||
} else {
|
||||
files.push(entryPath);
|
||||
}
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
function assertPortableWindowsBundle(root, buildInfo) {
|
||||
const forbiddenPaths = [repoRoot, backendDir, buildInfo.prefix, buildInfo.base_prefix]
|
||||
.filter(Boolean)
|
||||
.map((value) => path.resolve(value).toLowerCase());
|
||||
|
||||
for (const filePath of walkFiles(root)) {
|
||||
const leaf = path.basename(filePath).toLowerCase();
|
||||
if (leaf === 'pyvenv.cfg' || /\.(?:egg-link|pth)$/i.test(leaf)) {
|
||||
throw new Error(`Non-portable virtualenv metadata was staged: ${filePath}`);
|
||||
}
|
||||
|
||||
const stat = fs.statSync(filePath);
|
||||
if (stat.size > 2 * 1024 * 1024) continue;
|
||||
const bytes = fs.readFileSync(filePath);
|
||||
if (bytes.includes(0)) continue;
|
||||
const text = bytes.toString('utf8').toLowerCase();
|
||||
const leakedPath = forbiddenPaths.find((value) => text.includes(value));
|
||||
if (leakedPath) {
|
||||
throw new Error(`Build-machine path leaked into staged runtime: ${filePath}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function verifyRelocatableWindowsRuntime() {
|
||||
const probeDir = `${outputDir}-portability-probe-${process.pid}`;
|
||||
if (fs.existsSync(probeDir)) {
|
||||
throw new Error(`Portability probe path already exists: ${probeDir}`);
|
||||
}
|
||||
|
||||
fs.renameSync(outputDir, probeDir);
|
||||
let result;
|
||||
try {
|
||||
const env = { ...process.env };
|
||||
delete env.PYTHONHOME;
|
||||
delete env.PYTHONPATH;
|
||||
result = spawnSync(
|
||||
path.join(probeDir, 'python', 'python.exe'),
|
||||
[
|
||||
'-I',
|
||||
'-c',
|
||||
[
|
||||
'import sys',
|
||||
'import fastapi, uvicorn, cryptography, numpy, orjson, pydantic',
|
||||
'assert sys.prefix == sys.base_prefix',
|
||||
'print(sys.executable)',
|
||||
].join('; '),
|
||||
],
|
||||
{ cwd: probeDir, env, encoding: 'utf8' },
|
||||
);
|
||||
} finally {
|
||||
fs.renameSync(probeDir, outputDir);
|
||||
}
|
||||
|
||||
if (result.error || result.status !== 0) {
|
||||
throw new Error(`Relocated embedded Python smoke test failed: ${result.stderr || result.error}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function stageBackendRuntime() {
|
||||
fs.rmSync(outputDir, { recursive: true, force: true });
|
||||
fs.cpSync(backendDir, outputDir, {
|
||||
recursive: true,
|
||||
filter: shouldCopy,
|
||||
filter: shouldCopyBackendPath,
|
||||
});
|
||||
|
||||
let buildInfo = null;
|
||||
if (process.platform === 'win32') {
|
||||
buildInfo = await stageWindowsEmbeddedPython();
|
||||
}
|
||||
|
||||
stagePrivacyCoreArtifact();
|
||||
stageReleaseAttestation();
|
||||
stageStartScripts();
|
||||
|
||||
if (process.platform === 'win32') {
|
||||
assertPortableWindowsBundle(outputDir, buildInfo);
|
||||
verifyRelocatableWindowsRuntime();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Copy ``start.bat`` and ``start.sh`` from the repo root into the
|
||||
* staged backend-runtime/ so they sit next to ``privacy_core.dll``.
|
||||
*
|
||||
* Why: an MSI/EXE/AppImage user who wants to launch via the dev-style
|
||||
* scripts (because the desktop shell is failing, or they prefer the
|
||||
* browser frontend at localhost:3000) shouldn't have to clone the
|
||||
* source repo just to get the scripts. Having them inside the install
|
||||
* directory also means the bundled ``privacy_core.dll`` fallback in
|
||||
* those scripts resolves to the SAME directory as the script, which
|
||||
* is exactly the layout the v0.9.81 script update is looking for.
|
||||
*
|
||||
* Tracked from issue #319: users who fell back to start.bat from
|
||||
* their MSI install dir had to go fetch it from GitHub, then saw a
|
||||
* scary "install Rust" warning because the script didn't know where
|
||||
* the bundled DLL was. Bundling the script removes both problems.
|
||||
*/
|
||||
function stageStartScripts() {
|
||||
const scripts = ['start.bat', 'start.sh'];
|
||||
for (const name of scripts) {
|
||||
@@ -160,7 +434,6 @@ function stageStartScripts() {
|
||||
}
|
||||
const dst = path.join(outputDir, name);
|
||||
fs.copyFileSync(src, dst);
|
||||
// Preserve executable bit on POSIX systems for the .sh script.
|
||||
if (name.endsWith('.sh') && process.platform !== 'win32') {
|
||||
try {
|
||||
fs.chmodSync(dst, 0o755);
|
||||
@@ -191,7 +464,11 @@ function writeBundleVersion() {
|
||||
const pkg = JSON.parse(
|
||||
fs.readFileSync(path.join(repoRoot, 'desktop-shell', 'package.json'), 'utf8'),
|
||||
);
|
||||
fs.writeFileSync(versionPath, `${pkg.version || '0.0.0'}\n`, 'utf8');
|
||||
fs.writeFileSync(
|
||||
versionPath,
|
||||
`${pkg.version || '0.0.0'}-runtime-${runtimeLayoutVersion}\n`,
|
||||
'utf8',
|
||||
);
|
||||
}
|
||||
|
||||
function fileCount(root) {
|
||||
@@ -207,7 +484,24 @@ function fileCount(root) {
|
||||
return count;
|
||||
}
|
||||
|
||||
ensureRuntimePrereqs();
|
||||
stageBackendRuntime();
|
||||
writeBundleVersion();
|
||||
console.log(`backend-runtime staged: ${fileCount(outputDir)} files`);
|
||||
async function main() {
|
||||
ensureRuntimePrereqs();
|
||||
await stageBackendRuntime();
|
||||
writeBundleVersion();
|
||||
console.log(`backend-runtime staged: ${fileCount(outputDir)} files`);
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
main().catch((error) => {
|
||||
console.error(error instanceof Error ? error.message : error);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
assertPortableWindowsBundle,
|
||||
configureEmbeddedPythonPath,
|
||||
shouldCopyBackendPath,
|
||||
shouldCopySitePackagePath,
|
||||
windowsEmbeddedPython,
|
||||
};
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const test = require('node:test');
|
||||
|
||||
const {
|
||||
assertPortableWindowsBundle,
|
||||
configureEmbeddedPythonPath,
|
||||
shouldCopyBackendPath,
|
||||
shouldCopySitePackagePath,
|
||||
} = require('./build-backend-runtime.cjs');
|
||||
|
||||
const repoRoot = path.resolve(__dirname, '..', '..', '..');
|
||||
const backendDir = path.join(repoRoot, 'backend');
|
||||
|
||||
test('Windows staging never copies a virtualenv or its marker', () => {
|
||||
assert.equal(
|
||||
shouldCopyBackendPath(path.join(backendDir, 'venv', 'pyvenv.cfg'), 'win32', 'venv'),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
shouldCopyBackendPath(path.join(backendDir, '.venv', 'Scripts', 'python.exe'), 'win32'),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
shouldCopyBackendPath(path.join(backendDir, '.venv-dir'), 'win32', 'custom-venv'),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
shouldCopyBackendPath(path.join(backendDir, 'services', 'config.py'), 'win32', 'venv'),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('site-packages staging strips host-bound path files and editable metadata', () => {
|
||||
const sitePackages = path.join('C:', 'build', 'site-packages');
|
||||
assert.equal(
|
||||
shouldCopySitePackagePath(sitePackages, path.join(sitePackages, 'host-path.pth')),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
shouldCopySitePackagePath(
|
||||
sitePackages,
|
||||
path.join(sitePackages, 'backend-0.9.84.dist-info', 'METADATA'),
|
||||
),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
shouldCopySitePackagePath(sitePackages, path.join(sitePackages, 'fastapi', '__init__.py')),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('embedded Python search paths are relative to the packaged runtime', () => {
|
||||
const temp = fs.mkdtempSync(path.join(os.tmpdir(), 'sb-python-pth-test-'));
|
||||
try {
|
||||
const pthPath = path.join(temp, 'python311._pth');
|
||||
fs.writeFileSync(pthPath, 'C:\\Users\\developer\\Python311\\python311.zip\n', 'utf8');
|
||||
|
||||
configureEmbeddedPythonPath(temp);
|
||||
|
||||
const configured = fs.readFileSync(pthPath, 'utf8');
|
||||
assert.doesNotMatch(configured, /[A-Za-z]:\\/);
|
||||
assert.match(configured, /^python311\.zip$/m);
|
||||
assert.match(configured, /^Lib\\site-packages$/m);
|
||||
assert.match(configured, /^\.\.$/m);
|
||||
assert.match(configured, /^import site$/m);
|
||||
} finally {
|
||||
fs.rmSync(temp, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('bundle validation rejects pyvenv.cfg and developer home paths', () => {
|
||||
const temp = fs.mkdtempSync(path.join(os.tmpdir(), 'sb-python-portability-test-'));
|
||||
const buildInfo = {
|
||||
prefix: 'C:\\Users\\developer\\Shadowbroker\\backend\\venv',
|
||||
base_prefix: 'C:\\Users\\developer\\Python311',
|
||||
};
|
||||
|
||||
try {
|
||||
fs.writeFileSync(path.join(temp, 'safe.txt'), 'portable runtime\n', 'utf8');
|
||||
assert.doesNotThrow(() => assertPortableWindowsBundle(temp, buildInfo));
|
||||
|
||||
fs.writeFileSync(
|
||||
path.join(temp, 'pyvenv.cfg'),
|
||||
'home = C:\\Users\\developer\\Python311\n',
|
||||
'utf8',
|
||||
);
|
||||
assert.throws(
|
||||
() => assertPortableWindowsBundle(temp, buildInfo),
|
||||
/Non-portable virtualenv metadata/,
|
||||
);
|
||||
|
||||
fs.rmSync(path.join(temp, 'pyvenv.cfg'));
|
||||
fs.writeFileSync(
|
||||
path.join(temp, 'leak.txt'),
|
||||
`executable = ${buildInfo.base_prefix}\\python.exe\n`,
|
||||
'utf8',
|
||||
);
|
||||
assert.throws(
|
||||
() => assertPortableWindowsBundle(temp, buildInfo),
|
||||
/Build-machine path leaked/,
|
||||
);
|
||||
} finally {
|
||||
fs.rmSync(temp, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -139,6 +139,7 @@ fn install_bundled_backend(
|
||||
fs::create_dir_all(&install_root)
|
||||
.map_err(|e| format!("managed_backend_install_dir_failed:{e}"))?;
|
||||
sync_runtime_tree(bundled_root, &install_root)?;
|
||||
remove_legacy_virtualenv_entries(&install_root)?;
|
||||
fs::write(
|
||||
install_root.join(BUNDLE_VERSION_FILE),
|
||||
format!("{bundled_version}\n"),
|
||||
@@ -152,6 +153,39 @@ fn install_bundled_backend(
|
||||
Ok(install_root)
|
||||
}
|
||||
|
||||
fn remove_legacy_virtualenv_entries(install_root: &Path) -> Result<(), String> {
|
||||
let selected_venv = read_trimmed_file_optional(&install_root.join(".venv-dir"));
|
||||
for entry in fs::read_dir(install_root)
|
||||
.map_err(|e| format!("managed_backend_cleanup_read_dir_failed:{e}"))?
|
||||
{
|
||||
let entry = entry.map_err(|e| format!("managed_backend_cleanup_entry_failed:{e}"))?;
|
||||
let file_name = entry.file_name();
|
||||
let file_name_str = file_name.to_string_lossy();
|
||||
let is_legacy_venv = matches!(
|
||||
file_name_str.as_ref(),
|
||||
"venv" | ".venv" | "venv-repair" | ".venv-repair" | ".venv-dir"
|
||||
) || file_name_str.starts_with("venv-repair-")
|
||||
|| file_name_str.starts_with(".venv-repair-")
|
||||
|| selected_venv.as_deref() == Some(file_name_str.as_ref());
|
||||
if !is_legacy_venv {
|
||||
continue;
|
||||
}
|
||||
|
||||
let entry_path = entry.path();
|
||||
let file_type = entry
|
||||
.file_type()
|
||||
.map_err(|e| format!("managed_backend_cleanup_file_type_failed:{e}"))?;
|
||||
if file_type.is_dir() {
|
||||
fs::remove_dir_all(&entry_path)
|
||||
.map_err(|e| format!("managed_backend_legacy_venv_cleanup_failed:{e}"))?;
|
||||
} else {
|
||||
fs::remove_file(&entry_path)
|
||||
.map_err(|e| format!("managed_backend_legacy_venv_marker_cleanup_failed:{e}"))?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn sync_runtime_tree(src: &Path, dst: &Path) -> Result<(), String> {
|
||||
for entry in fs::read_dir(src).map_err(|e| format!("managed_backend_read_dir_failed:{e}"))? {
|
||||
let entry = entry.map_err(|e| format!("managed_backend_dir_entry_failed:{e}"))?;
|
||||
@@ -449,10 +483,13 @@ fn resolve_python_bin(runtime_root: &Path) -> Result<PathBuf, String> {
|
||||
}
|
||||
|
||||
let candidates = if cfg!(target_os = "windows") {
|
||||
candidate_roots
|
||||
.into_iter()
|
||||
.map(|root| root.join("Scripts").join("python.exe"))
|
||||
.collect::<Vec<_>>()
|
||||
let mut candidates = vec![runtime_root.join("python").join("python.exe")];
|
||||
candidates.extend(
|
||||
candidate_roots
|
||||
.into_iter()
|
||||
.map(|root| root.join("Scripts").join("python.exe")),
|
||||
);
|
||||
candidates
|
||||
} else {
|
||||
candidate_roots
|
||||
.into_iter()
|
||||
@@ -571,6 +608,67 @@ mod tests {
|
||||
let _ = fs::remove_dir_all(temp);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_refresh_removes_stale_venv_only() {
|
||||
let temp = std::env::temp_dir().join(format!(
|
||||
"sb_backend_cleanup_test_{}",
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos()
|
||||
));
|
||||
fs::create_dir_all(temp.join("data")).unwrap();
|
||||
fs::create_dir_all(temp.join("custom-venv")).unwrap();
|
||||
fs::write(temp.join(".env"), "ADMIN_KEY=preserve\n").unwrap();
|
||||
fs::write(temp.join(".venv-dir"), "custom-venv\n").unwrap();
|
||||
fs::write(temp.join("data").join("keep.txt"), "keep").unwrap();
|
||||
fs::write(
|
||||
temp.join("custom-venv").join("pyvenv.cfg"),
|
||||
"home=C:\\Users\\builder",
|
||||
)
|
||||
.unwrap();
|
||||
fs::write(temp.join("old_runtime.py"), "stale").unwrap();
|
||||
|
||||
remove_legacy_virtualenv_entries(&temp).unwrap();
|
||||
|
||||
assert!(temp.join(".env").exists());
|
||||
assert!(temp.join("data").join("keep.txt").exists());
|
||||
assert!(!temp.join("custom-venv").exists());
|
||||
assert!(!temp.join(".venv-dir").exists());
|
||||
assert!(temp.join("old_runtime.py").exists());
|
||||
|
||||
let _ = fs::remove_dir_all(temp);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn portable_python_is_preferred_over_legacy_venv() {
|
||||
let temp = std::env::temp_dir().join(format!(
|
||||
"sb_backend_python_resolver_test_{}",
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos()
|
||||
));
|
||||
let portable = if cfg!(target_os = "windows") {
|
||||
temp.join("python").join("python.exe")
|
||||
} else {
|
||||
temp.join("venv").join("bin").join("python3")
|
||||
};
|
||||
let legacy = if cfg!(target_os = "windows") {
|
||||
temp.join("venv").join("Scripts").join("python.exe")
|
||||
} else {
|
||||
temp.join("venv").join("bin").join("python")
|
||||
};
|
||||
fs::create_dir_all(portable.parent().unwrap()).unwrap();
|
||||
fs::create_dir_all(legacy.parent().unwrap()).unwrap();
|
||||
fs::write(&portable, "portable").unwrap();
|
||||
fs::write(&legacy, "legacy").unwrap();
|
||||
|
||||
assert_eq!(resolve_python_bin(&temp).unwrap(), portable);
|
||||
|
||||
let _ = fs::remove_dir_all(temp);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sync_release_attestation_updates_only_attestation_file() {
|
||||
let temp = std::env::temp_dir().join(format!(
|
||||
|
||||
Reference in New Issue
Block a user