CalvinBackup/banderole
1
0
Fork 0
mirror of https://github.com/zhom/banderole.git synced 2026-08-09 11:16:01 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat: better compression

Browse Source
This commit is contained in:
zhom
2025-08-10 02:01:31 +04:00
parent 61785dc8f5
commit d7b711c0ea
11 changed files with 312 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/bundler.rs
+16 -5
View File
@@ -103,11 +103,22 @@ pub async fn bundle_project(
let node_executable = node_downloader
.ensure_node_binary_with_progress(Some(&pb_prepare))
.await?;
let node_root = node_executable
.parent()
.expect("node executable must have a parent")
.parent()
.unwrap_or_else(|| panic!("Unexpected node layout for {}", node_executable.display()));
let node_root_buf = if Platform::current().is_windows() {
// On Windows, node.exe lives directly under the platform directory
node_executable
.parent()
.expect("node executable must have a parent")
.to_path_buf()
} else {
// On Unix, node is under <platform>/bin/node
node_executable
.parent()
.expect("node executable must have a parent")
.parent()
.unwrap_or_else(|| panic!("Unexpected node layout for {}", node_executable.display()))
.to_path_buf()
};
let node_root: &Path = &node_root_buf;
pb_prepare.finish_and_clear();
// Stage 2: Bundle application into archive
src/executable.rs
+11 -2
View File
@@ -31,8 +31,17 @@ pub fn create_self_extracting_executable_with_progress(
copy_template_to_build_dir(build_dir)?;
let zip_path = build_dir.join("embedded_data.zip");
fs::write(&zip_path, &zip_data).context("Failed to write embedded zip data")?;
// For improved compression ratio, store an xz-compressed stream of the zip payload.
// The template executable will decompress XZ first, then read the inner zip.
let xz_path = build_dir.join("embedded_data.xz");
{
use std::io::Cursor;
let mut xz_bytes: Vec<u8> = Vec::new();
let mut reader = Cursor::new(&zip_data);
lzma_rs::xz_compress(&mut reader, &mut xz_bytes)
.context("Failed to XZ-compress embedded payload")?;
fs::write(&xz_path, &xz_bytes).context("Failed to write embedded xz data")?;
}
let build_id_path = build_dir.join("build_id.txt");
fs::write(&build_id_path, &build_id).context("Failed to write build ID")?;
src/node_downloader.rs
+72 -58
View File
@@ -213,10 +213,9 @@ impl NodeDownloader {
pb.set_position(0);
}
if self.platform.is_windows() {
self.extract_zip(&archive_path, target_dir, progress)
.await?;
self.extract_7z(&archive_path, target_dir, progress).await?;
} else {
self.extract_tar_gz(&archive_path, target_dir, progress)
self.extract_tar_xz(&archive_path, target_dir, progress)
.await?;
}
@@ -239,7 +238,7 @@ impl NodeDownloader {
Ok(())
}
async fn extract_zip(
async fn extract_7z(
&self,
archive_path: &Path,
target_dir: &Path,
@@ -249,56 +248,59 @@ impl NodeDownloader {
let target_dir = target_dir.to_path_buf();
let progress = progress.cloned();
tokio::task::spawn_blocking(move || -> Result<()> {
let file = std::fs::File::open(&archive_path).context("Failed to open zip archive")?;
let mut archive = zip::ZipArchive::new(file).context("Failed to read zip archive")?;
if let Some(pb) = &progress {
pb.set_length(archive.len() as u64);
pb.set_position(0);
pb.set_message("Extracting 7z archive");
}
sevenz_rust::decompress_file(&archive_path, &target_dir)
.context("Failed to extract 7z archive")?;
for i in 0..archive.len() {
let mut file = archive.by_index(i).context("Failed to read zip entry")?;
let outpath = match file.enclosed_name() {
Some(path) => {
let components: Vec<_> = path.components().collect();
if components.len() > 1 {
target_dir.join(components[1..].iter().collect::<PathBuf>())
} else {
if let Some(pb) = &progress {
pb.inc(1);
// Post-process: many Node archives have a single top-level folder. Flatten it.
let entries = std::fs::read_dir(&target_dir)
.context("Failed to read extraction directory")?
.filter_map(|e| e.ok())
.collect::<Vec<_>>();
let top_dirs: Vec<_> = entries
.iter()
.filter(|e| e.file_type().map(|t| t.is_dir()).unwrap_or(false))
.collect();
let top_files_exist = entries
.iter()
.any(|e| e.file_type().map(|t| t.is_file()).unwrap_or(false));
if top_dirs.len() == 1 && !top_files_exist {
let inner = top_dirs[0].path();
for inner_entry in std::fs::read_dir(&inner)? {
let inner_entry = inner_entry?;
let from = inner_entry.path();
let to = target_dir.join(inner_entry.file_name());
std::fs::rename(&from, &to)
.or_else(|_| {
if inner_entry.file_type()?.is_dir() {
std::fs::create_dir_all(&to)?;
for sub in walkdir::WalkDir::new(&from).into_iter().flatten() {
let p = sub.path();
let rel = p.strip_prefix(&from).unwrap();
let dest = to.join(rel);
if sub.file_type().is_dir() {
std::fs::create_dir_all(&dest)?;
} else if sub.file_type().is_file() {
if let Some(parent) = dest.parent() {
std::fs::create_dir_all(parent)?;
}
std::fs::copy(p, &dest).map(|_| ())?;
}
}
Ok(())
} else {
std::fs::copy(&from, &to).map(|_| ())
}
continue;
}
}
None => {
if let Some(pb) = &progress {
pb.inc(1);
}
continue;
}
};
if file.is_dir() {
std::fs::create_dir_all(&outpath).context("Failed to create directory")?;
} else {
if let Some(p) = outpath.parent() {
std::fs::create_dir_all(p).context("Failed to create parent directory")?;
}
let mut outfile =
std::fs::File::create(&outpath).context("Failed to create output file")?;
std::io::copy(&mut file, &mut outfile)
.context("Failed to extract zip entry")?;
}
if let Some(pb) = &progress {
pb.inc(1);
})
.context("Failed to move extracted files")?;
}
let _ = std::fs::remove_dir_all(&inner);
}
if let Some(pb) = &progress {
pb.finish_and_clear();
}
Ok(())
})
.await??;
@@ -306,7 +308,7 @@ impl NodeDownloader {
Ok(())
}
async fn extract_tar_gz(
async fn extract_tar_xz(
&self,
archive_path: &Path,
target_dir: &Path,
@@ -317,14 +319,28 @@ impl NodeDownloader {
let progress = progress.cloned();
tokio::task::spawn_blocking(move || -> Result<()> {
use flate2::read::GzDecoder;
use std::io::Cursor;
use tar::Archive;
// First pass: count entries
let file_for_count =
std::fs::File::open(&archive_path).context("Failed to open tar.gz for counting")?;
let decoder_for_count = GzDecoder::new(file_for_count);
let mut archive_for_count = Archive::new(decoder_for_count);
// Read entire .xz into memory (Node archives are moderate size) and decode
let mut raw = Vec::new();
std::fs::File::open(&archive_path)
.and_then(|mut f| {
use std::io::Read;
f.read_to_end(&mut raw)
})
.context("Failed to read .xz archive")?;
// Decompress xz -> tar bytes
let mut tar_bytes: Vec<u8> = Vec::new();
{
let mut reader = Cursor::new(&raw);
lzma_rs::xz_decompress(&mut reader, &mut tar_bytes)
.context("Failed to decompress .xz archive")?;
}
// First pass: count tar entries
let mut archive_for_count = Archive::new(Cursor::new(&tar_bytes));
let mut total_entries: u64 = 0;
for _ in archive_for_count
.entries()
@@ -339,9 +355,7 @@ impl NodeDownloader {
}
// Second pass: extract
let file = std::fs::File::open(&archive_path).context("Failed to open tar.gz")?;
let decoder = GzDecoder::new(file);
let mut archive = Archive::new(decoder);
let mut archive = Archive::new(Cursor::new(&tar_bytes));
for entry in archive.entries().context("Failed to iterate tar entries")? {
let mut entry = entry.context("Failed to read tar entry")?;
src/platform.rs
+6 -6
View File
@@ -29,12 +29,12 @@ impl Platform {
pub fn node_archive_name(&self, version: &str) -> String {
match self {
Platform::LinuxX64 => format!("node-v{version}-linux-x64.tar.gz"),
Platform::LinuxArm64 => format!("node-v{version}-linux-arm64.tar.gz"),
Platform::MacosX64 => format!("node-v{version}-darwin-x64.tar.gz"),
Platform::MacosArm64 => format!("node-v{version}-darwin-arm64.tar.gz"),
Platform::WindowsX64 => format!("node-v{version}-win-x64.zip"),
Platform::WindowsArm64 => format!("node-v{version}-win-arm64.zip"),
Platform::LinuxX64 => format!("node-v{version}-linux-x64.tar.xz"),
Platform::LinuxArm64 => format!("node-v{version}-linux-arm64.tar.xz"),
Platform::MacosX64 => format!("node-v{version}-darwin-x64.tar.xz"),
Platform::MacosArm64 => format!("node-v{version}-darwin-arm64.tar.xz"),
Platform::WindowsX64 => format!("node-v{version}-win-x64.7z"),
Platform::WindowsArm64 => format!("node-v{version}-win-arm64.7z"),
}
}
src/template/build.rs
+9 -9
View File
@@ -7,24 +7,24 @@ fn main() {
let dest_path = Path::new(&out_dir).join("data.rs");
// Check if we have embedded data files
let zip_data_path = Path::new("embedded_data.zip");
let xz_data_path = Path::new("embedded_data.xz");
let build_id_path = Path::new("build_id.txt");
if zip_data_path.exists() && build_id_path.exists() {
if xz_data_path.exists() && build_id_path.exists() {
// Read the build ID
let build_id = fs::read_to_string(build_id_path)
.expect("Failed to read build ID");
// Copy the zip file to the OUT_DIR so include_bytes! can find it
let out_zip_path = Path::new(&out_dir).join("embedded_data.zip");
fs::copy(zip_data_path, &out_zip_path)
// Copy the xz file to the OUT_DIR so include_bytes! can find it
let out_xz_path = Path::new(&out_dir).join("embedded_data.xz");
fs::copy(xz_data_path, &out_xz_path)
.expect("Failed to copy embedded data to OUT_DIR");
// Generate the data.rs file with embedded data
let data_rs_content = format!(
r#"
// Generated at build time - contains embedded application data
const ZIP_DATA: &[u8] = include_bytes!("embedded_data.zip");
// Generated at build time - contains embedded application data (xz-compressed zip)
const XZ_DATA: &[u8] = include_bytes!("embedded_data.xz");
const BUILD_ID: &str = "{}";
"#,
build_id.trim()
@@ -36,7 +36,7 @@ const BUILD_ID: &str = "{}";
// Generate placeholder data for template compilation
let data_rs_content = r#"
// Placeholder data for template compilation
const ZIP_DATA: &[u8] = &[];
const XZ_DATA: &[u8] = &[];
const BUILD_ID: &str = "template";
"#;
@@ -45,6 +45,6 @@ const BUILD_ID: &str = "template";
}
// Tell Cargo to rerun this script if the embedded data changes
println!("cargo:rerun-if-changed=embedded_data.zip");
println!("cargo:rerun-if-changed=embedded_data.xz");
println!("cargo:rerun-if-changed=build_id.txt");
}
src/template/crg.toml
+2
View File
@@ -9,6 +9,8 @@ directories = "6"
zip = "4"
serde_json = "1.0"
fs2 = "0.4"
lzma-rs = "0.3"
walkdir = "2.4"
[build-dependencies]
# No build dependencies needed - data is embedded at compile time
src/template/src/main.rs
+71 -20
View File
@@ -4,6 +4,7 @@ use std::fs;
use std::io::Cursor;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::ffi::OsString;
use zip::ZipArchive;
use directories::BaseDirs;
use fs2::FileExt;
@@ -65,29 +66,59 @@ fn get_cache_dir() -> Result<PathBuf> {
}
fn get_node_executable_path(app_dir: &Path) -> PathBuf {
let node_dir = app_dir.join("node");
if cfg!(windows) {
// On Windows, Node.js is extracted to node/{platform}/node.exe
// Try to find the platform-specific subdirectory
let node_dir = app_dir.join("node");
// Look for platform-specific subdirectories
if let Ok(entries) = fs::read_dir(&node_dir) {
for entry in entries.flatten() {
let path = entry.path();
if path.is_dir() {
let node_exe = path.join("node.exe");
if node_exe.exists() {
return node_exe;
// Prefer common locations first
let candidates = [
node_dir.join("node.exe"),
];
for c in candidates {
if c.exists() {
return c;
}
}
// Recursively search for node.exe under node/
if node_dir.exists() {
for entry in walkdir::WalkDir::new(&node_dir).follow_links(true) {
if let Ok(e) = entry {
let p = e.path();
if p.is_file() {
if let Some(name) = p.file_name().and_then(|n| n.to_str()) {
if name.eq_ignore_ascii_case("node.exe") {
return p.to_path_buf();
}
}
}
}
}
}
// Fallback to direct path (shouldn't happen with new extraction logic)
// Fallback: default where Windows Node is usually at after extraction
node_dir.join("node.exe")
} else {
// On Unix systems, Node.js is in node/bin/node
app_dir.join("node").join("bin").join("node")
let candidate = node_dir.join("bin").join("node");
if candidate.exists() {
candidate
} else {
// As a last resort, search recursively
if node_dir.exists() {
for entry in walkdir::WalkDir::new(&node_dir).follow_links(true) {
if let Ok(e) = entry {
let p = e.path();
if p.is_file() {
if let Some(name) = p.file_name().and_then(|n| n.to_str()) {
if name == "node" {
return p.to_path_buf();
}
}
}
}
}
}
candidate
}
}
}
@@ -136,8 +167,14 @@ fn extract_application(app_dir: &Path) -> Result<()> {
// Create app directory
fs::create_dir_all(app_dir).context("Failed to create app directory")?;
// Extract embedded zip data
let cursor = Cursor::new(ZIP_DATA);
// Decompress embedded XZ data to get inner ZIP, then extract
let mut tar_buf: Vec<u8> = Vec::new();
{
let mut reader = Cursor::new(XZ_DATA);
lzma_rs::xz_decompress(&mut reader, &mut tar_buf)
.context("Failed to decompress embedded xz data")?;
}
let cursor = Cursor::new(tar_buf);
let mut archive = ZipArchive::new(cursor).context("Failed to open embedded zip archive")?;
for i in 0..archive.len() {
@@ -296,12 +333,26 @@ fn run_app(app_dir: &Path, args: &[String]) -> Result<()> {
let mut cmd_args = vec![main_script.clone()];
cmd_args.extend(args.iter().cloned());
// Execute Node.js application with a few retries to tolerate transient Windows issues
let mut status = None;
let mut last_err: Option<anyhow::Error> = None;
let max_attempts: u32 = 8;
let mut status: Option<std::process::ExitStatus> = None;
for attempt in 1..=max_attempts {
match Command::new(&node_executable)
// Prepend Node's directory to PATH and launch via program name to avoid path parsing quirks
let node_bin_dir = node_executable
.parent()
.ok_or_else(|| anyhow::anyhow!("Invalid node executable path: {}", node_executable.display()))?
.to_path_buf();
let program_name = if cfg!(windows) { "node.exe" } else { "node" };
let mut cmd = Command::new(program_name);
// Ensure PATH includes the Node directory first
let mut new_path = std::env::var_os("PATH").unwrap_or_default();
let sep = if cfg!(windows) { ";" } else { ":" };
let mut prefixed: OsString = OsString::new();
prefixed.push(node_bin_dir.as_os_str());
prefixed.push(sep);
prefixed.push(&new_path);
cmd.env("PATH", prefixed);
match cmd
.args(&cmd_args)
.stdin(Stdio::inherit())
.stdout(Stdio::inherit())