mirror of
https://github.com/BigBodyCobain/Shadowbroker.git
synced 2026-08-18 16:37:24 +02:00
563 lines
21 KiB
Python
563 lines
21 KiB
Python
"""Self-update module — downloads latest GitHub release, backs up current files,
|
|
extracts the update over the project, and restarts the app.
|
|
|
|
Public API:
|
|
perform_update(project_root) -> dict (download + backup + extract)
|
|
schedule_restart(project_root) (spawn detached start script, then exit)
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
import logging
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import tempfile
|
|
import time
|
|
import zipfile
|
|
import hashlib
|
|
from urllib.parse import urlparse
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
import requests
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
GITHUB_RELEASES_URL = "https://api.github.com/repos/BigBodyCobain/Shadowbroker/releases/latest"
|
|
GITHUB_RELEASES_PAGE_URL = "https://github.com/BigBodyCobain/Shadowbroker/releases/latest"
|
|
DOCKER_UPDATE_COMMANDS = (
|
|
"docker compose pull && docker compose up -d"
|
|
)
|
|
|
|
# Issue #231: baked-in release digests. Loaded lazily and treated as an
|
|
# independent trust source because they ship with the already-installed code.
|
|
_RELEASE_DIGESTS_FILE = (
|
|
Path(__file__).resolve().parent.parent / "data" / "release_digests.json"
|
|
)
|
|
# Pattern for the maintainer's source-archive release asset. The matching
|
|
# SHA256SUMS.txt is still useful as a consistency check, but because both are
|
|
# delivered by the same release channel it is not sufficient by itself to
|
|
# authorize executable source replacement.
|
|
_SOURCE_ASSET_PATTERN = re.compile(r"^ShadowBroker_v\d", re.IGNORECASE)
|
|
_SHA256SUMS_ASSET_NAME = "SHA256SUMS.txt"
|
|
|
|
|
|
def _is_docker() -> bool:
|
|
"""Detect if we're running inside a Docker container."""
|
|
if os.path.isfile("/.dockerenv"):
|
|
return True
|
|
try:
|
|
with open("/proc/1/cgroup", "r") as f:
|
|
return "docker" in f.read()
|
|
except (FileNotFoundError, PermissionError):
|
|
pass
|
|
return os.environ.get("container") == "docker"
|
|
_ALLOWED_UPDATE_HOSTS = {
|
|
"api.github.com",
|
|
"codeload.github.com",
|
|
"github.com",
|
|
"objects.githubusercontent.com",
|
|
"release-assets.githubusercontent.com",
|
|
"github-releases.githubusercontent.com",
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Protected patterns — files/dirs that must NEVER be overwritten during update
|
|
# ---------------------------------------------------------------------------
|
|
_PROTECTED_DIRS = {
|
|
"venv", "node_modules", ".next", "__pycache__", ".git", ".github", ".claude",
|
|
"_domain_keys", "node-local", "gate_persona", "gate_session", "dm_alias",
|
|
"root", "transport", "reputation",
|
|
}
|
|
_PROTECTED_EXTENSIONS = {".db", ".sqlite", ".key", ".pem", ".bin"}
|
|
_PROTECTED_NAMES = {
|
|
".env",
|
|
"ais_cache.json",
|
|
"carrier_cache.json",
|
|
"geocode_cache.json",
|
|
"infonet.json",
|
|
"infonet.json.bak",
|
|
"peer_store.json",
|
|
"node.json",
|
|
"wormhole.json",
|
|
"wormhole_status.json",
|
|
"wormhole_secure_store.key",
|
|
"dm_token_pepper.key",
|
|
"voter_blind_salt.bin",
|
|
"reputation_ledger.json",
|
|
"gates.json",
|
|
}
|
|
|
|
|
|
def _is_protected(rel_path: str) -> bool:
|
|
"""Return True if *rel_path* (forward-slash separated) should be skipped."""
|
|
parts = rel_path.replace("\\", "/").split("/")
|
|
name = parts[-1]
|
|
|
|
# Check directory components
|
|
for part in parts[:-1]:
|
|
if part in _PROTECTED_DIRS:
|
|
return True
|
|
|
|
# Check filename
|
|
if name in _PROTECTED_NAMES:
|
|
return True
|
|
_, ext = os.path.splitext(name)
|
|
if ext.lower() in _PROTECTED_EXTENSIONS:
|
|
return True
|
|
|
|
return False
|
|
|
|
|
|
def _validate_update_url(url: str, *, allow_release_page: bool = False) -> str:
|
|
parsed = urlparse(str(url or "").strip())
|
|
host = (parsed.hostname or "").strip().lower()
|
|
if parsed.scheme != "https":
|
|
raise RuntimeError("Updater refused a non-HTTPS release URL")
|
|
if parsed.username or parsed.password:
|
|
raise RuntimeError("Updater refused a credentialed release URL")
|
|
if not host or host not in _ALLOWED_UPDATE_HOSTS:
|
|
raise RuntimeError(f"Updater refused an untrusted release host: {host or 'unknown'}")
|
|
if parsed.port not in (None, 443):
|
|
raise RuntimeError("Updater refused a non-standard release port")
|
|
if not allow_release_page and host == "github.com" and "/releases/" not in parsed.path:
|
|
raise RuntimeError("Updater refused a non-release GitHub URL")
|
|
return parsed.geturl()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Download
|
|
# ---------------------------------------------------------------------------
|
|
def _download_release(temp_dir: str) -> tuple:
|
|
"""Fetch latest release info and download the source zip archive.
|
|
|
|
Prefer the maintainer's named release asset (matching
|
|
``ShadowBroker_v*.zip``) over the auto-generated ``zipball_url`` so a
|
|
stable filename can be matched against a pre-pinned digest.
|
|
|
|
Returns (zip_path, version_tag, download_url, release_url, asset_name,
|
|
sha256sums_url) — the last two are empty strings when the release does not
|
|
publish a named asset.
|
|
"""
|
|
logger.info("Fetching latest release info from GitHub...")
|
|
_validate_update_url(GITHUB_RELEASES_URL)
|
|
resp = requests.get(GITHUB_RELEASES_URL, timeout=15)
|
|
resp.raise_for_status()
|
|
_validate_update_url(resp.url)
|
|
release = resp.json()
|
|
|
|
tag = release.get("tag_name", "unknown")
|
|
release_url = str(release.get("html_url") or GITHUB_RELEASES_PAGE_URL).strip()
|
|
_validate_update_url(release_url, allow_release_page=True)
|
|
|
|
assets = release.get("assets") or []
|
|
asset_name = ""
|
|
asset_url = ""
|
|
sha256sums_url = ""
|
|
for a in assets:
|
|
name = str(a.get("name") or "").strip()
|
|
download = str(a.get("browser_download_url") or "").strip()
|
|
if not name or not download:
|
|
continue
|
|
if _SOURCE_ASSET_PATTERN.match(name) and name.lower().endswith(".zip"):
|
|
asset_name = name
|
|
asset_url = download
|
|
elif name == _SHA256SUMS_ASSET_NAME:
|
|
sha256sums_url = download
|
|
|
|
if asset_url:
|
|
zip_url = asset_url
|
|
logger.info(
|
|
"Using release asset %s (sha256sums=%s)",
|
|
asset_name,
|
|
"yes" if sha256sums_url else "no",
|
|
)
|
|
else:
|
|
zip_url = str(release.get("zipball_url") or "").strip()
|
|
if not zip_url:
|
|
raise RuntimeError("Latest release is missing a source archive URL")
|
|
logger.warning(
|
|
"Release does not publish a ShadowBroker_v*.zip asset — falling "
|
|
"back to auto-generated zipball_url. In-place installation will "
|
|
"still require an explicit MESH_UPDATE_SHA256 pin or a matching "
|
|
"baked-in digest."
|
|
)
|
|
|
|
_validate_update_url(zip_url)
|
|
|
|
logger.info(f"Downloading {zip_url} ...")
|
|
zip_path = os.path.join(temp_dir, "update.zip")
|
|
with requests.get(zip_url, stream=True, timeout=120) as dl:
|
|
dl.raise_for_status()
|
|
_validate_update_url(dl.url)
|
|
with open(zip_path, "wb") as f:
|
|
for chunk in dl.iter_content(chunk_size=1024 * 64):
|
|
f.write(chunk)
|
|
|
|
if not zipfile.is_zipfile(zip_path):
|
|
raise RuntimeError("Downloaded file is not a valid ZIP archive")
|
|
|
|
size_mb = os.path.getsize(zip_path) / (1024 * 1024)
|
|
logger.info(f"Downloaded {size_mb:.1f} MB — ZIP validated OK")
|
|
return zip_path, tag, zip_url, release_url, asset_name, sha256sums_url
|
|
|
|
|
|
def _compute_sha256(zip_path: str) -> str:
|
|
"""Return the hex SHA-256 of the file at ``zip_path`` (lowercase)."""
|
|
h = hashlib.sha256()
|
|
with open(zip_path, "rb") as f:
|
|
for chunk in iter(lambda: f.read(1024 * 128), b""):
|
|
h.update(chunk)
|
|
return h.hexdigest().lower()
|
|
|
|
|
|
def _load_baked_in_release_digests() -> dict:
|
|
"""Return the ``release_digests.json`` mapping, or an empty dict."""
|
|
try:
|
|
raw = _RELEASE_DIGESTS_FILE.read_text(encoding="utf-8")
|
|
parsed = json.loads(raw)
|
|
except (OSError, ValueError) as exc:
|
|
logger.debug("Release digest file unreadable: %s", exc)
|
|
return {}
|
|
if not isinstance(parsed, dict):
|
|
return {}
|
|
cleaned: dict[str, dict[str, str]] = {}
|
|
for k, v in parsed.items():
|
|
if not isinstance(k, str) or k.startswith("_"):
|
|
continue
|
|
if isinstance(v, dict):
|
|
entries = {
|
|
fname: digest.strip().lower()
|
|
for fname, digest in v.items()
|
|
if isinstance(fname, str) and isinstance(digest, str)
|
|
}
|
|
if entries:
|
|
cleaned[k] = entries
|
|
return cleaned
|
|
|
|
|
|
def _fetch_sha256sums(sha256sums_url: str) -> dict[str, str]:
|
|
"""Download a SHA256SUMS.txt and return {filename: digest_hex_lower}."""
|
|
try:
|
|
_validate_update_url(sha256sums_url)
|
|
except RuntimeError as exc:
|
|
logger.warning("SHA256SUMS URL rejected: %s", exc)
|
|
return {}
|
|
try:
|
|
resp = requests.get(sha256sums_url, timeout=15)
|
|
resp.raise_for_status()
|
|
_validate_update_url(resp.url)
|
|
except requests.RequestException as exc:
|
|
logger.info("SHA256SUMS fetch failed: %s", exc)
|
|
return {}
|
|
except RuntimeError as exc:
|
|
logger.warning("SHA256SUMS redirect rejected: %s", exc)
|
|
return {}
|
|
out: dict[str, str] = {}
|
|
for line in resp.text.splitlines():
|
|
line = line.strip()
|
|
if not line or line.startswith("#"):
|
|
continue
|
|
parts = line.split(None, 1)
|
|
if len(parts) != 2:
|
|
continue
|
|
digest, fname = parts
|
|
fname = fname.lstrip("*").strip()
|
|
digest = digest.strip().lower()
|
|
if len(digest) == 64 and all(c in "0123456789abcdef" for c in digest) and fname:
|
|
out[fname] = digest
|
|
return out
|
|
|
|
|
|
def _validate_zip_hash(
|
|
zip_path: str,
|
|
*,
|
|
asset_name: str = "",
|
|
sha256sums_url: str = "",
|
|
release_tag: str = "",
|
|
) -> str:
|
|
"""Verify an executable source update against an independent trust source.
|
|
|
|
Authorization order:
|
|
|
|
1. ``MESH_UPDATE_SHA256`` — explicit operator pin.
|
|
2. ``backend/data/release_digests.json`` — a digest already shipped in
|
|
the installed application.
|
|
|
|
A release ``SHA256SUMS.txt`` is fetched only as a consistency diagnostic.
|
|
Because it is controlled by the same GitHub release channel as the ZIP, a
|
|
matching value cannot independently authenticate a compromised release.
|
|
If neither independent source is available, the updater fails closed and
|
|
leaves the existing installation untouched.
|
|
"""
|
|
actual = _compute_sha256(zip_path)
|
|
|
|
override = os.environ.get("MESH_UPDATE_SHA256", "").strip().lower()
|
|
if override:
|
|
if len(override) != 64 or any(c not in "0123456789abcdef" for c in override):
|
|
raise RuntimeError("MESH_UPDATE_SHA256 must be a 64-character hexadecimal SHA-256 digest")
|
|
if actual == override:
|
|
return f"verified via MESH_UPDATE_SHA256 ({actual[:16]}...)"
|
|
raise RuntimeError(
|
|
f"Update SHA-256 mismatch vs MESH_UPDATE_SHA256: archive={actual[:16]}..., "
|
|
f"expected={override[:16]}..."
|
|
)
|
|
|
|
baked = _load_baked_in_release_digests()
|
|
baked_expected = ""
|
|
if release_tag and asset_name:
|
|
baked_expected = baked.get(release_tag, {}).get(asset_name, "")
|
|
if baked_expected:
|
|
if actual == baked_expected:
|
|
return f"verified via baked-in digest list ({actual[:16]}...)"
|
|
raise RuntimeError(
|
|
f"Update SHA-256 mismatch vs baked-in digest list: "
|
|
f"archive={actual[:16]}..., expected={baked_expected[:16]}..."
|
|
)
|
|
|
|
release_checksum_note = ""
|
|
if sha256sums_url and asset_name:
|
|
sums_expected = _fetch_sha256sums(sha256sums_url).get(asset_name)
|
|
if sums_expected:
|
|
if actual != sums_expected:
|
|
raise RuntimeError(
|
|
f"Update SHA-256 mismatch vs release SHA256SUMS.txt: "
|
|
f"archive={actual[:16]}..., expected={sums_expected[:16]}..."
|
|
)
|
|
release_checksum_note = " The same-release SHA256SUMS.txt matched, but is not an independent trust root."
|
|
|
|
raise RuntimeError(
|
|
"Update refused: no independent archive digest is trusted for "
|
|
f"release={release_tag or 'unknown'} asset={asset_name or 'unknown'}."
|
|
f"{release_checksum_note} Pin MESH_UPDATE_SHA256 or ship a matching "
|
|
"backend/data/release_digests.json entry before enabling in-place installation."
|
|
)
|
|
|
|
|
|
def _is_source_checkout(project_root: str) -> bool:
|
|
root = Path(project_root)
|
|
return (root / "frontend").is_dir() and (root / "backend").is_dir()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Backup
|
|
# ---------------------------------------------------------------------------
|
|
def _backup_current(project_root: str, temp_dir: str) -> str:
|
|
"""Create a backup zip of backend/ and frontend/ in temp_dir."""
|
|
stamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
|
backup_path = os.path.join(temp_dir, f"backup_{stamp}.zip")
|
|
logger.info(f"Backing up current files to {backup_path} ...")
|
|
|
|
dirs_to_backup = ["backend", "frontend"]
|
|
count = 0
|
|
|
|
with zipfile.ZipFile(backup_path, "w", zipfile.ZIP_DEFLATED) as zf:
|
|
for dir_name in dirs_to_backup:
|
|
dir_path = os.path.join(project_root, dir_name)
|
|
if not os.path.isdir(dir_path):
|
|
continue
|
|
for root, dirs, files in os.walk(dir_path):
|
|
dirs[:] = [d for d in dirs if d not in _PROTECTED_DIRS]
|
|
for fname in files:
|
|
full = os.path.join(root, fname)
|
|
rel = os.path.relpath(full, project_root)
|
|
if _is_protected(rel):
|
|
continue
|
|
try:
|
|
zf.write(full, rel)
|
|
count += 1
|
|
except (PermissionError, OSError) as e:
|
|
logger.warning(f"Backup skip (locked): {rel} — {e}")
|
|
|
|
logger.info(f"Backup complete: {count} files archived")
|
|
return backup_path
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Extract & Copy
|
|
# ---------------------------------------------------------------------------
|
|
def _extract_and_copy(zip_path: str, project_root: str, temp_dir: str) -> int:
|
|
"""Extract the update zip and copy files over the project, skipping protected files.
|
|
Returns count of files copied.
|
|
"""
|
|
extract_dir = os.path.join(temp_dir, "extracted")
|
|
logger.info("Extracting update zip...")
|
|
with zipfile.ZipFile(zip_path, "r") as zf:
|
|
extract_root = Path(extract_dir).resolve()
|
|
for member in zf.infolist():
|
|
try:
|
|
target = (extract_root / member.filename).resolve()
|
|
except OSError as exc:
|
|
raise RuntimeError(f"Updater refused archive entry {member.filename}: {exc}") from exc
|
|
try:
|
|
target.relative_to(extract_root)
|
|
except ValueError:
|
|
raise RuntimeError(f"Updater refused archive path traversal entry: {member.filename}")
|
|
zf.extractall(extract_dir)
|
|
|
|
base = extract_dir
|
|
entries = [e for e in os.listdir(base) if not e.startswith(".")]
|
|
if len(entries) == 1:
|
|
candidate = os.path.join(base, entries[0])
|
|
if os.path.isdir(candidate):
|
|
sub = os.listdir(candidate)
|
|
if "frontend" in sub or "backend" in sub:
|
|
base = candidate
|
|
logger.info(f"Detected wrapper folder: {entries[0]}")
|
|
|
|
copied = 0
|
|
skipped = 0
|
|
|
|
for root, dirs, files in os.walk(base):
|
|
dirs[:] = [d for d in dirs if d not in _PROTECTED_DIRS]
|
|
|
|
for fname in files:
|
|
src = os.path.join(root, fname)
|
|
rel = os.path.relpath(src, base).replace("\\", "/")
|
|
|
|
if _is_protected(rel):
|
|
skipped += 1
|
|
continue
|
|
|
|
dst = os.path.abspath(os.path.join(project_root, rel))
|
|
if not dst.startswith(os.path.abspath(project_root)):
|
|
logger.warning(f"Safety skip (path traversal): {rel}")
|
|
skipped += 1
|
|
continue
|
|
try:
|
|
os.makedirs(os.path.dirname(dst), exist_ok=True)
|
|
shutil.copy2(src, dst)
|
|
copied += 1
|
|
except (PermissionError, OSError) as e:
|
|
logger.warning(f"Copy failed (skipping): {rel} — {e}")
|
|
skipped += 1
|
|
|
|
logger.info(f"Update applied: {copied} files copied, {skipped} skipped/protected")
|
|
return copied
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Restart
|
|
# ---------------------------------------------------------------------------
|
|
def schedule_restart(project_root: str):
|
|
"""Spawn a detached process that re-runs start.bat / start.sh after a short
|
|
delay, then forcefully exit the current Python process."""
|
|
tmp = tempfile.mkdtemp(prefix="sb_restart_")
|
|
|
|
if sys.platform == "win32":
|
|
script = os.path.join(tmp, "restart.bat")
|
|
with open(script, "w") as f:
|
|
f.write("@echo off\n")
|
|
f.write("timeout /t 3 /nobreak >nul\n")
|
|
f.write(f'cd /d "{project_root}"\n')
|
|
f.write("call start.bat\n")
|
|
|
|
CREATE_NEW_PROCESS_GROUP = 0x00000200
|
|
DETACHED_PROCESS = 0x00000008
|
|
subprocess.Popen(
|
|
["cmd", "/c", script],
|
|
creationflags=DETACHED_PROCESS | CREATE_NEW_PROCESS_GROUP,
|
|
close_fds=True,
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
)
|
|
else:
|
|
script = os.path.join(tmp, "restart.sh")
|
|
with open(script, "w") as f:
|
|
f.write("#!/bin/bash\n")
|
|
f.write("sleep 3\n")
|
|
f.write(f'cd "{project_root}"\n')
|
|
f.write("bash start.sh\n")
|
|
os.chmod(script, 0o755)
|
|
subprocess.Popen(
|
|
["bash", script],
|
|
start_new_session=True,
|
|
close_fds=True,
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
)
|
|
|
|
logger.info("Restart script spawned — exiting current process")
|
|
os._exit(0)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Public entry point
|
|
# ---------------------------------------------------------------------------
|
|
def perform_update(project_root: str) -> dict:
|
|
"""Download the latest release, back up current files, and extract the update.
|
|
|
|
Returns a dict with status info on success, or {"status": "error", "message": ...}
|
|
on failure. Does NOT trigger restart — caller should call schedule_restart()
|
|
separately after the HTTP response has been sent.
|
|
|
|
In Docker, file extraction is skipped because containers run from immutable
|
|
images. Instead the response tells the frontend to show pull instructions.
|
|
"""
|
|
in_docker = _is_docker()
|
|
temp_dir = tempfile.mkdtemp(prefix="sb_update_")
|
|
manual_url = GITHUB_RELEASES_PAGE_URL
|
|
try:
|
|
zip_path, version, url, release_url, asset_name, sha256sums_url = _download_release(temp_dir)
|
|
manual_url = release_url or manual_url
|
|
|
|
if in_docker:
|
|
logger.info("Docker detected — skipping file extraction")
|
|
return {
|
|
"status": "docker",
|
|
"version": version,
|
|
"manual_url": manual_url,
|
|
"release_url": release_url,
|
|
"download_url": url,
|
|
"docker_commands": DOCKER_UPDATE_COMMANDS,
|
|
"message": (
|
|
f"Version {version} is available. "
|
|
"Docker containers must be updated by pulling the new images."
|
|
),
|
|
}
|
|
|
|
if not _is_source_checkout(project_root):
|
|
logger.info("Non-source runtime detected — refusing in-place source update")
|
|
return {
|
|
"status": "manual",
|
|
"version": version,
|
|
"manual_url": manual_url,
|
|
"release_url": release_url,
|
|
"download_url": url,
|
|
"message": (
|
|
"This runtime does not support in-place source updates. "
|
|
"Download the latest release package manually."
|
|
),
|
|
}
|
|
|
|
verification_note = _validate_zip_hash(
|
|
zip_path,
|
|
asset_name=asset_name,
|
|
sha256sums_url=sha256sums_url,
|
|
release_tag=version,
|
|
)
|
|
logger.info("Update archive %s", verification_note)
|
|
backup_path = _backup_current(project_root, temp_dir)
|
|
copied = _extract_and_copy(zip_path, project_root, temp_dir)
|
|
|
|
return {
|
|
"status": "ok",
|
|
"version": version,
|
|
"files_updated": copied,
|
|
"backup_path": backup_path,
|
|
"manual_url": manual_url,
|
|
"release_url": release_url,
|
|
"download_url": url,
|
|
"integrity": verification_note,
|
|
"message": f"Updated to {version} — {copied} files replaced. Restarting...",
|
|
}
|
|
except Exception as e:
|
|
logger.error(f"Update failed: {e}", exc_info=True)
|
|
return {
|
|
"status": "error",
|
|
"message": str(e),
|
|
"manual_url": manual_url,
|
|
}
|