CalvinBackup/banderole
1
0
Fork 0
mirror of https://github.com/zhom/banderole.git synced 2026-06-06 06:23:53 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat: better compression

Browse Source
This commit is contained in:
zhom
2025-08-10 02:01:31 +04:00
parent 61785dc8f5
commit d7b711c0ea
11 changed files with 312 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.vscode/settings.json
Vendored
+3
View File
@@ -2,6 +2,7 @@
"cSpell.words": [
"chrono",
"Clippy",
"codegen",
"dtolnay",
"enabledelayedexpansion",
"ERRORLEVEL",
@@ -10,6 +11,7 @@
"indicatif",
"LOCALAPPDATA",
"lockfiles",
"lzma",
"mktemp",
"msrv",
"msvc",
@@ -21,6 +23,7 @@
"rustc",
"serde",
"setlocal",
"sevenz",
"sindresorhus",
"Swatinem",
"taskkill",
Cargo.lock
Generated
+106
View File
@@ -161,10 +161,12 @@ dependencies = [
"indicatif-log-bridge",
"lazy_static",
"log",
"lzma-rs",
"reqwest",
"serde",
"serde_json",
"serial_test",
"sevenz-rust",
"sha2",
"tar",
"tempfile",
@@ -180,6 +182,21 @@ version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
[[package]]
name = "bit-set"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0481a0e032742109b1133a095184ee93d88f3dc9e0d28a5d033dc77a073f44f"
dependencies = [
"bit-vec",
]
[[package]]
name = "bit-vec"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2c54ff287cfc0a34f38a6b832ea1bd8e448a330b3e40a50859e6488bee07f22"
[[package]]
name = "bitflags"
version = "2.9.1"
@@ -201,6 +218,12 @@ version = "3.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43"
[[package]]
name = "byteorder"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
[[package]]
name = "bytes"
version = "1.10.1"
@@ -348,6 +371,21 @@ dependencies = [
"libc",
]
[[package]]
name = "crc"
version = "3.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675"
dependencies = [
"crc-catalog",
]
[[package]]
name = "crc-catalog"
version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5"
[[package]]
name = "crc32fast"
version = "1.5.0"
@@ -508,6 +546,17 @@ dependencies = [
"windows-sys 0.59.0",
]
[[package]]
name = "filetime_creation"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c25b5d475550e559de5b0c0084761c65325444e3b6c9e298af9cefe7a9ef3a5f"
dependencies = [
"cfg-if",
"filetime",
"windows-sys 0.52.0",
]
[[package]]
name = "flate2"
version = "1.1.2"
@@ -1177,6 +1226,25 @@ version = "0.4.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94"
[[package]]
name = "lzma-rs"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e"
dependencies = [
"byteorder",
"crc",
]
[[package]]
name = "lzma-rust"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5baab2bbbd7d75a144d671e9ff79270e903957d92fb7386fd39034c709bd2661"
dependencies = [
"byteorder",
]
[[package]]
name = "memchr"
version = "2.7.5"
@@ -1226,6 +1294,16 @@ dependencies = [
"tempfile",
]
[[package]]
name = "nt-time"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2de419e64947cd8830e66beb584acc3fb42ed411d103e3c794dda355d1b374b5"
dependencies = [
"chrono",
"time",
]
[[package]]
name = "num-conv"
version = "0.1.0"
@@ -1731,6 +1809,23 @@ dependencies = [
"syn",
]
[[package]]
name = "sevenz-rust"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26482cf1ecce4540dc782fc70019eba89ffc4d87b3717eb5ec524b5db6fdefef"
dependencies = [
"bit-set",
"byteorder",
"crc",
"filetime_creation",
"js-sys",
"lzma-rust",
"nt-time",
"sha2",
"wasm-bindgen",
]
[[package]]
name = "sha1"
version = "0.10.6"
@@ -1921,6 +2016,7 @@ dependencies = [
"powerfmt",
"serde",
"time-core",
"time-macros",
]
[[package]]
@@ -1929,6 +2025,16 @@ version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c"
[[package]]
name = "time-macros"
version = "0.2.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49"
dependencies = [
"num-conv",
"time-core",
]
[[package]]
name = "tinystr"
version = "0.8.1"
Cargo.toml
+2
View File
@@ -32,6 +32,8 @@ indicatif-log-bridge = "0.2"
console = "0.16"
flate2 = "1.0"
tar = "0.4"
sevenz-rust = "0.6"
lzma-rs = "0.3"
[build-dependencies]
reqwest = { version = "0.12", features = ["blocking"] }
src/bundler.rs
+16 -5
View File
@@ -103,11 +103,22 @@ pub async fn bundle_project(
let node_executable = node_downloader
.ensure_node_binary_with_progress(Some(&pb_prepare))
.await?;
let node_root = node_executable
.parent()
.expect("node executable must have a parent")
.parent()
.unwrap_or_else(|| panic!("Unexpected node layout for {}", node_executable.display()));
let node_root_buf = if Platform::current().is_windows() {
// On Windows, node.exe lives directly under the platform directory
node_executable
.parent()
.expect("node executable must have a parent")
.to_path_buf()
} else {
// On Unix, node is under <platform>/bin/node
node_executable
.parent()
.expect("node executable must have a parent")
.parent()
.unwrap_or_else(|| panic!("Unexpected node layout for {}", node_executable.display()))
.to_path_buf()
};
let node_root: &Path = &node_root_buf;
pb_prepare.finish_and_clear();
// Stage 2: Bundle application into archive
src/executable.rs
+11 -2
View File
@@ -31,8 +31,17 @@ pub fn create_self_extracting_executable_with_progress(
copy_template_to_build_dir(build_dir)?;
let zip_path = build_dir.join("embedded_data.zip");
fs::write(&zip_path, &zip_data).context("Failed to write embedded zip data")?;
// For improved compression ratio, store an xz-compressed stream of the zip payload.
// The template executable will decompress XZ first, then read the inner zip.
let xz_path = build_dir.join("embedded_data.xz");
{
use std::io::Cursor;
let mut xz_bytes: Vec<u8> = Vec::new();
let mut reader = Cursor::new(&zip_data);
lzma_rs::xz_compress(&mut reader, &mut xz_bytes)
.context("Failed to XZ-compress embedded payload")?;
fs::write(&xz_path, &xz_bytes).context("Failed to write embedded xz data")?;
}
let build_id_path = build_dir.join("build_id.txt");
fs::write(&build_id_path, &build_id).context("Failed to write build ID")?;
src/node_downloader.rs
+72 -58
View File
@@ -213,10 +213,9 @@ impl NodeDownloader {
pb.set_position(0);
}
if self.platform.is_windows() {
self.extract_zip(&archive_path, target_dir, progress)
.await?;
self.extract_7z(&archive_path, target_dir, progress).await?;
} else {
self.extract_tar_gz(&archive_path, target_dir, progress)
self.extract_tar_xz(&archive_path, target_dir, progress)
.await?;
}
@@ -239,7 +238,7 @@ impl NodeDownloader {
Ok(())
}
async fn extract_zip(
async fn extract_7z(
&self,
archive_path: &Path,
target_dir: &Path,
@@ -249,56 +248,59 @@ impl NodeDownloader {
let target_dir = target_dir.to_path_buf();
let progress = progress.cloned();
tokio::task::spawn_blocking(move || -> Result<()> {
let file = std::fs::File::open(&archive_path).context("Failed to open zip archive")?;
let mut archive = zip::ZipArchive::new(file).context("Failed to read zip archive")?;
if let Some(pb) = &progress {
pb.set_length(archive.len() as u64);
pb.set_position(0);
pb.set_message("Extracting 7z archive");
}
sevenz_rust::decompress_file(&archive_path, &target_dir)
.context("Failed to extract 7z archive")?;
for i in 0..archive.len() {
let mut file = archive.by_index(i).context("Failed to read zip entry")?;
let outpath = match file.enclosed_name() {
Some(path) => {
let components: Vec<_> = path.components().collect();
if components.len() > 1 {
target_dir.join(components[1..].iter().collect::<PathBuf>())
} else {
if let Some(pb) = &progress {
pb.inc(1);
// Post-process: many Node archives have a single top-level folder. Flatten it.
let entries = std::fs::read_dir(&target_dir)
.context("Failed to read extraction directory")?
.filter_map(|e| e.ok())
.collect::<Vec<_>>();
let top_dirs: Vec<_> = entries
.iter()
.filter(|e| e.file_type().map(|t| t.is_dir()).unwrap_or(false))
.collect();
let top_files_exist = entries
.iter()
.any(|e| e.file_type().map(|t| t.is_file()).unwrap_or(false));
if top_dirs.len() == 1 && !top_files_exist {
let inner = top_dirs[0].path();
for inner_entry in std::fs::read_dir(&inner)? {
let inner_entry = inner_entry?;
let from = inner_entry.path();
let to = target_dir.join(inner_entry.file_name());
std::fs::rename(&from, &to)
.or_else(|_| {
if inner_entry.file_type()?.is_dir() {
std::fs::create_dir_all(&to)?;
for sub in walkdir::WalkDir::new(&from).into_iter().flatten() {
let p = sub.path();
let rel = p.strip_prefix(&from).unwrap();
let dest = to.join(rel);
if sub.file_type().is_dir() {
std::fs::create_dir_all(&dest)?;
} else if sub.file_type().is_file() {
if let Some(parent) = dest.parent() {
std::fs::create_dir_all(parent)?;
}
std::fs::copy(p, &dest).map(|_| ())?;
}
}
Ok(())
} else {
std::fs::copy(&from, &to).map(|_| ())
}
continue;
}
}
None => {
if let Some(pb) = &progress {
pb.inc(1);
}
continue;
}
};
if file.is_dir() {
std::fs::create_dir_all(&outpath).context("Failed to create directory")?;
} else {
if let Some(p) = outpath.parent() {
std::fs::create_dir_all(p).context("Failed to create parent directory")?;
}
let mut outfile =
std::fs::File::create(&outpath).context("Failed to create output file")?;
std::io::copy(&mut file, &mut outfile)
.context("Failed to extract zip entry")?;
}
if let Some(pb) = &progress {
pb.inc(1);
})
.context("Failed to move extracted files")?;
}
let _ = std::fs::remove_dir_all(&inner);
}
if let Some(pb) = &progress {
pb.finish_and_clear();
}
Ok(())
})
.await??;
@@ -306,7 +308,7 @@ impl NodeDownloader {
Ok(())
}
async fn extract_tar_gz(
async fn extract_tar_xz(
&self,
archive_path: &Path,
target_dir: &Path,
@@ -317,14 +319,28 @@ impl NodeDownloader {
let progress = progress.cloned();
tokio::task::spawn_blocking(move || -> Result<()> {
use flate2::read::GzDecoder;
use std::io::Cursor;
use tar::Archive;
// First pass: count entries
let file_for_count =
std::fs::File::open(&archive_path).context("Failed to open tar.gz for counting")?;
let decoder_for_count = GzDecoder::new(file_for_count);
let mut archive_for_count = Archive::new(decoder_for_count);
// Read entire .xz into memory (Node archives are moderate size) and decode
let mut raw = Vec::new();
std::fs::File::open(&archive_path)
.and_then(|mut f| {
use std::io::Read;
f.read_to_end(&mut raw)
})
.context("Failed to read .xz archive")?;
// Decompress xz -> tar bytes
let mut tar_bytes: Vec<u8> = Vec::new();
{
let mut reader = Cursor::new(&raw);
lzma_rs::xz_decompress(&mut reader, &mut tar_bytes)
.context("Failed to decompress .xz archive")?;
}
// First pass: count tar entries
let mut archive_for_count = Archive::new(Cursor::new(&tar_bytes));
let mut total_entries: u64 = 0;
for _ in archive_for_count
.entries()
@@ -339,9 +355,7 @@ impl NodeDownloader {
}
// Second pass: extract
let file = std::fs::File::open(&archive_path).context("Failed to open tar.gz")?;
let decoder = GzDecoder::new(file);
let mut archive = Archive::new(decoder);
let mut archive = Archive::new(Cursor::new(&tar_bytes));
for entry in archive.entries().context("Failed to iterate tar entries")? {
let mut entry = entry.context("Failed to read tar entry")?;
src/platform.rs
+6 -6
View File
@@ -29,12 +29,12 @@ impl Platform {
pub fn node_archive_name(&self, version: &str) -> String {
match self {
Platform::LinuxX64 => format!("node-v{version}-linux-x64.tar.gz"),
Platform::LinuxArm64 => format!("node-v{version}-linux-arm64.tar.gz"),
Platform::MacosX64 => format!("node-v{version}-darwin-x64.tar.gz"),
Platform::MacosArm64 => format!("node-v{version}-darwin-arm64.tar.gz"),
Platform::WindowsX64 => format!("node-v{version}-win-x64.zip"),
Platform::WindowsArm64 => format!("node-v{version}-win-arm64.zip"),
Platform::LinuxX64 => format!("node-v{version}-linux-x64.tar.xz"),
Platform::LinuxArm64 => format!("node-v{version}-linux-arm64.tar.xz"),
Platform::MacosX64 => format!("node-v{version}-darwin-x64.tar.xz"),
Platform::MacosArm64 => format!("node-v{version}-darwin-arm64.tar.xz"),
Platform::WindowsX64 => format!("node-v{version}-win-x64.7z"),
Platform::WindowsArm64 => format!("node-v{version}-win-arm64.7z"),
}
}
src/template/build.rs
+9 -9
View File
@@ -7,24 +7,24 @@ fn main() {
let dest_path = Path::new(&out_dir).join("data.rs");
// Check if we have embedded data files
let zip_data_path = Path::new("embedded_data.zip");
let xz_data_path = Path::new("embedded_data.xz");
let build_id_path = Path::new("build_id.txt");
if zip_data_path.exists() && build_id_path.exists() {
if xz_data_path.exists() && build_id_path.exists() {
// Read the build ID
let build_id = fs::read_to_string(build_id_path)
.expect("Failed to read build ID");
// Copy the zip file to the OUT_DIR so include_bytes! can find it
let out_zip_path = Path::new(&out_dir).join("embedded_data.zip");
fs::copy(zip_data_path, &out_zip_path)
// Copy the xz file to the OUT_DIR so include_bytes! can find it
let out_xz_path = Path::new(&out_dir).join("embedded_data.xz");
fs::copy(xz_data_path, &out_xz_path)
.expect("Failed to copy embedded data to OUT_DIR");
// Generate the data.rs file with embedded data
let data_rs_content = format!(
r#"
// Generated at build time - contains embedded application data
const ZIP_DATA: &[u8] = include_bytes!("embedded_data.zip");
// Generated at build time - contains embedded application data (xz-compressed zip)
const XZ_DATA: &[u8] = include_bytes!("embedded_data.xz");
const BUILD_ID: &str = "{}";
"#,
build_id.trim()
@@ -36,7 +36,7 @@ const BUILD_ID: &str = "{}";
// Generate placeholder data for template compilation
let data_rs_content = r#"
// Placeholder data for template compilation
const ZIP_DATA: &[u8] = &[];
const XZ_DATA: &[u8] = &[];
const BUILD_ID: &str = "template";
"#;
@@ -45,6 +45,6 @@ const BUILD_ID: &str = "template";
}
// Tell Cargo to rerun this script if the embedded data changes
println!("cargo:rerun-if-changed=embedded_data.zip");
println!("cargo:rerun-if-changed=embedded_data.xz");
println!("cargo:rerun-if-changed=build_id.txt");
}
src/template/crg.toml
+2
View File
@@ -9,6 +9,8 @@ directories = "6"
zip = "4"
serde_json = "1.0"
fs2 = "0.4"
lzma-rs = "0.3"
walkdir = "2.4"
[build-dependencies]
# No build dependencies needed - data is embedded at compile time
src/template/src/main.rs
+71 -20
View File
@@ -4,6 +4,7 @@ use std::fs;
use std::io::Cursor;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::ffi::OsString;
use zip::ZipArchive;
use directories::BaseDirs;
use fs2::FileExt;
@@ -65,29 +66,59 @@ fn get_cache_dir() -> Result<PathBuf> {
}
fn get_node_executable_path(app_dir: &Path) -> PathBuf {
let node_dir = app_dir.join("node");
if cfg!(windows) {
// On Windows, Node.js is extracted to node/{platform}/node.exe
// Try to find the platform-specific subdirectory
let node_dir = app_dir.join("node");
// Look for platform-specific subdirectories
if let Ok(entries) = fs::read_dir(&node_dir) {
for entry in entries.flatten() {
let path = entry.path();
if path.is_dir() {
let node_exe = path.join("node.exe");
if node_exe.exists() {
return node_exe;
// Prefer common locations first
let candidates = [
node_dir.join("node.exe"),
];
for c in candidates {
if c.exists() {
return c;
}
}
// Recursively search for node.exe under node/
if node_dir.exists() {
for entry in walkdir::WalkDir::new(&node_dir).follow_links(true) {
if let Ok(e) = entry {
let p = e.path();
if p.is_file() {
if let Some(name) = p.file_name().and_then(|n| n.to_str()) {
if name.eq_ignore_ascii_case("node.exe") {
return p.to_path_buf();
}
}
}
}
}
}
// Fallback to direct path (shouldn't happen with new extraction logic)
// Fallback: default where Windows Node is usually at after extraction
node_dir.join("node.exe")
} else {
// On Unix systems, Node.js is in node/bin/node
app_dir.join("node").join("bin").join("node")
let candidate = node_dir.join("bin").join("node");
if candidate.exists() {
candidate
} else {
// As a last resort, search recursively
if node_dir.exists() {
for entry in walkdir::WalkDir::new(&node_dir).follow_links(true) {
if let Ok(e) = entry {
let p = e.path();
if p.is_file() {
if let Some(name) = p.file_name().and_then(|n| n.to_str()) {
if name == "node" {
return p.to_path_buf();
}
}
}
}
}
}
candidate
}
}
}
@@ -136,8 +167,14 @@ fn extract_application(app_dir: &Path) -> Result<()> {
// Create app directory
fs::create_dir_all(app_dir).context("Failed to create app directory")?;
// Extract embedded zip data
let cursor = Cursor::new(ZIP_DATA);
// Decompress embedded XZ data to get inner ZIP, then extract
let mut tar_buf: Vec<u8> = Vec::new();
{
let mut reader = Cursor::new(XZ_DATA);
lzma_rs::xz_decompress(&mut reader, &mut tar_buf)
.context("Failed to decompress embedded xz data")?;
}
let cursor = Cursor::new(tar_buf);
let mut archive = ZipArchive::new(cursor).context("Failed to open embedded zip archive")?;
for i in 0..archive.len() {
@@ -296,12 +333,26 @@ fn run_app(app_dir: &Path, args: &[String]) -> Result<()> {
let mut cmd_args = vec![main_script.clone()];
cmd_args.extend(args.iter().cloned());
// Execute Node.js application with a few retries to tolerate transient Windows issues
let mut status = None;
let mut last_err: Option<anyhow::Error> = None;
let max_attempts: u32 = 8;
let mut status: Option<std::process::ExitStatus> = None;
for attempt in 1..=max_attempts {
match Command::new(&node_executable)
// Prepend Node's directory to PATH and launch via program name to avoid path parsing quirks
let node_bin_dir = node_executable
.parent()
.ok_or_else(|| anyhow::anyhow!("Invalid node executable path: {}", node_executable.display()))?
.to_path_buf();
let program_name = if cfg!(windows) { "node.exe" } else { "node" };
let mut cmd = Command::new(program_name);
// Ensure PATH includes the Node directory first
let mut new_path = std::env::var_os("PATH").unwrap_or_default();
let sep = if cfg!(windows) { ";" } else { ":" };
let mut prefixed: OsString = OsString::new();
prefixed.push(node_bin_dir.as_os_str());
prefixed.push(sep);
prefixed.push(&new_path);
cmd.env("PATH", prefixed);
match cmd
.args(&cmd_args)
.stdin(Stdio::inherit())
.stdout(Stdio::inherit())
tests/common/mod.rs
+14 -21
View File
@@ -674,31 +674,24 @@ impl BundlerTestHelper {
fs::set_permissions(executable_path, perms)?;
}
// Build command to run the executable. On Windows, copy to a temp dir to avoid
// path locking/races under concurrent CreateProcess, then run from there.
// Build command to run the executable.
let exec_path_owned = executable_path.to_path_buf();
// Use verbatim long-path prefix on Windows to avoid MAX_PATH issues,
// otherwise use the original path.
#[cfg(windows)]
let (exec_path_owned, _tmp_guard) = {
let tmp = TempDir::new().context("Failed to create temp dir for executable copy")?;
let file_name = executable_path.file_name().ok_or_else(|| {
anyhow::anyhow!(
"Executable path missing file name: {}",
executable_path.display()
)
})?;
let dest = tmp.path().join(file_name);
std::fs::copy(executable_path, &dest).with_context(|| {
format!(
"Failed to copy executable to temporary directory: {} -> {}",
executable_path.display(),
dest.display()
)
})?;
(dest, tmp)
let exec_cmd = {
use std::ffi::OsString;
let abs = exec_path_owned
.canonicalize()
.unwrap_or_else(|_| exec_path_owned.clone());
let mut s: OsString = OsString::from(r"\\?\");
s.push(&abs);
s
};
#[cfg(not(windows))]
let exec_path_owned = executable_path.to_path_buf();
let exec_cmd = exec_path_owned.as_os_str().to_os_string();
let mut cmd = Command::new(&exec_path_owned);
let mut cmd = Command::new(&exec_cmd);
if let Some(parent) = exec_path_owned.parent() {
cmd.current_dir(parent);
}