fix: stream tar.xz extraction

This commit is contained in:
zhom committed 2026-10-15 11:08:34 +04:00
1 parent 108d7310cb
commit c897983bf0
4 files changed
+21 -76

No files matched your search

+6 -22
View File
@@ -1291,21 +1291,6 @@ dependencies = [
"libc",
]
[[package]]
name = "crc"
version = "3.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d"
dependencies = [
"crc-catalog",
]
[[package]]
name = "crc-catalog"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853"
[[package]]
name = "crc32c"
version = "0.6.8"
@@ -1772,7 +1757,7 @@ dependencies = [
"jsonc-parser",
"libc",
"log",
"lzma-rs",
"lzma-rust2",
"maxminddb",
"mime_guess",
"msi-extract",
@@ -3764,13 +3749,12 @@ dependencies = [
]
[[package]]
name = "lzma-rs"
version = "0.3.0"
name = "lzma-rust2"
version = "0.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e"
checksum = "7277c9742229dc25243236d578b6471a798c554512c6f6a04365ec2e87c2a3bb"
dependencies = [
"byteorder",
"crc",
"sha2 0.11.0",
]
[[package]]
@@ -6980,7 +6964,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
dependencies = [
"fastrand",
"getrandom 0.3.4",
"getrandom 0.4.3",
"once_cell",
"rustix",
"windows-sys 0.61.2",
+2 -1
View File
@@ -74,7 +74,7 @@ zip = { version = "8", default-features = false, features = ["deflate-flate2"] }
tar = "0"
bzip2 = "0"
flate2 = "1"
lzma-rs = "0"
lzma-rust2 = { version = "0.22", default-features = false, features = ["std", "optimization", "xz"] }
msi-extract = "0"
uuid = { version = "1.27", features = ["v4", "serde"] }
@@ -192,6 +192,7 @@ tower = "0.5"
tower-http = { version = "0.7", features = ["fs", "trace"] }
futures-util = "0.3"
serial_test = "4"
lzma-rust2 = { version = "0.22", features = ["encoder"] }
# Integration test configuration
[[test]]
+11 -43
View File
@@ -97,12 +97,6 @@ impl ExtractionReporter {
self.report((done as f64 / total as f64) * 100.0, done, Some(total));
}
/// Report a pre-computed percentage (multi-phase extractions where byte
/// counts don't map linearly onto overall progress).
pub fn report_percentage(&self, percentage: f64) {
self.report(percentage, 0, None);
}
/// Force a final 100% event so the bar lands full before the next stage.
pub fn finish(&self) {
self.emit(100.0, 0, None);
@@ -871,39 +865,10 @@ impl Extractor {
std::fs::create_dir_all(dest_dir)?;
let file = File::open(tar_path)?;
let mut buf_reader = BufReader::new(file);
// Read the entire file into memory for lzma-rs
let mut compressed_data = Vec::new();
buf_reader.read_to_end(&mut compressed_data)?;
let compressed_total = compressed_data.len() as u64;
// Two phases with no shared byte scale: CPU-bound LZMA decompression
// dominates, so compressed bytes consumed map to 0–80%, and the tar
// unpack of the decompressed data to 80–100%.
let mut decompressed_data = Vec::new();
let counted_input = ProgressReader::new(std::io::Cursor::new(compressed_data), |read| {
if let Some(reporter) = progress {
if compressed_total > 0 {
reporter
.report_percentage(80.0 * read.min(compressed_total) as f64 / compressed_total as f64);
}
}
});
lzma_rs::xz_decompress(&mut BufReader::new(counted_input), &mut decompressed_data)?;
// Create tar archive from decompressed data
let decompressed_total = decompressed_data.len() as u64;
let counted_tar = ProgressReader::new(std::io::Cursor::new(decompressed_data), |read| {
if let Some(reporter) = progress {
if decompressed_total > 0 {
reporter.report_percentage(
80.0 + 20.0 * read.min(decompressed_total) as f64 / decompressed_total as f64,
);
}
}
});
let mut archive = tar::Archive::new(counted_tar);
let counted = progress_file_reader(file, progress)?;
// Streams: memory stays near the xz dictionary size, not the unpacked browser size.
let xz_decoder = lzma_rust2::XzReader::new(BufReader::new(counted), true);
let mut archive = tar::Archive::new(xz_decoder);
archive.unpack(dest_dir)?;
@@ -1911,10 +1876,13 @@ mod tests {
// Then compress with xz
let tar_xz_file =
std::fs::File::create(&tar_xz_path).expect("Failed to create test tar.xz file");
let mut compressed_data = Vec::new();
lzma_rs::xz_compress(&mut std::io::Cursor::new(tar_data), &mut compressed_data)
.expect("Failed to compress with xz");
std::io::Write::write_all(&mut std::io::BufWriter::new(tar_xz_file), &compressed_data)
let mut writer = lzma_rust2::XzWriter::new(
std::io::BufWriter::new(tar_xz_file),
lzma_rust2::XzOptions::default(),
)
.expect("Failed to create xz writer");
std::io::Write::write_all(&mut writer, &tar_data).expect("Failed to compress with xz");
std::io::Write::flush(&mut writer.finish().expect("Failed to finish xz stream"))
.expect("Failed to write compressed data");
}
+2 -10
View File
@@ -831,14 +831,6 @@
"name": "cpufeatures",
"license": "Apache-2.0 OR MIT"
},
{
"name": "crc",
"license": "Apache-2.0 OR MIT"
},
{
"name": "crc-catalog",
"license": "Apache-2.0 OR MIT"
},
{
"name": "crc32c",
"license": "Apache-2.0 OR MIT"
@@ -1740,8 +1732,8 @@
"license": "ISC"
},
{
"name": "lzma-rs",
"license": "MIT"
"name": "lzma-rust2",
"license": "Apache-2.0"
},
{
"name": "lzxd",