mirror of
https://github.com/zhom/donutbrowser.git
synced 2026-10-19 22:17:28 +02:00
fix: stream tar.xz extraction
This commit is contained in:
1 parent
108d7310cb
commit
c897983bf0
4 files changed
+21
-76
No files matched your search
Generated
+6
-22
@@ -1291,21 +1291,6 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crc"
|
||||
version = "3.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d"
|
||||
dependencies = [
|
||||
"crc-catalog",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crc-catalog"
|
||||
version = "2.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853"
|
||||
|
||||
[[package]]
|
||||
name = "crc32c"
|
||||
version = "0.6.8"
|
||||
@@ -1772,7 +1757,7 @@ dependencies = [
|
||||
"jsonc-parser",
|
||||
"libc",
|
||||
"log",
|
||||
"lzma-rs",
|
||||
"lzma-rust2",
|
||||
"maxminddb",
|
||||
"mime_guess",
|
||||
"msi-extract",
|
||||
@@ -3764,13 +3749,12 @@ dependencies = [
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lzma-rs"
|
||||
version = "0.3.0"
|
||||
name = "lzma-rust2"
|
||||
version = "0.22.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e"
|
||||
checksum = "7277c9742229dc25243236d578b6471a798c554512c6f6a04365ec2e87c2a3bb"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"crc",
|
||||
"sha2 0.11.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -6980,7 +6964,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"getrandom 0.3.4",
|
||||
"getrandom 0.4.3",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.61.2",
|
||||
|
||||
@@ -74,7 +74,7 @@ zip = { version = "8", default-features = false, features = ["deflate-flate2"] }
|
||||
tar = "0"
|
||||
bzip2 = "0"
|
||||
flate2 = "1"
|
||||
lzma-rs = "0"
|
||||
lzma-rust2 = { version = "0.22", default-features = false, features = ["std", "optimization", "xz"] }
|
||||
msi-extract = "0"
|
||||
|
||||
uuid = { version = "1.27", features = ["v4", "serde"] }
|
||||
@@ -192,6 +192,7 @@ tower = "0.5"
|
||||
tower-http = { version = "0.7", features = ["fs", "trace"] }
|
||||
futures-util = "0.3"
|
||||
serial_test = "4"
|
||||
lzma-rust2 = { version = "0.22", features = ["encoder"] }
|
||||
|
||||
# Integration test configuration
|
||||
[[test]]
|
||||
|
||||
+11
-43
@@ -97,12 +97,6 @@ impl ExtractionReporter {
|
||||
self.report((done as f64 / total as f64) * 100.0, done, Some(total));
|
||||
}
|
||||
|
||||
/// Report a pre-computed percentage (multi-phase extractions where byte
|
||||
/// counts don't map linearly onto overall progress).
|
||||
pub fn report_percentage(&self, percentage: f64) {
|
||||
self.report(percentage, 0, None);
|
||||
}
|
||||
|
||||
/// Force a final 100% event so the bar lands full before the next stage.
|
||||
pub fn finish(&self) {
|
||||
self.emit(100.0, 0, None);
|
||||
@@ -871,39 +865,10 @@ impl Extractor {
|
||||
std::fs::create_dir_all(dest_dir)?;
|
||||
|
||||
let file = File::open(tar_path)?;
|
||||
let mut buf_reader = BufReader::new(file);
|
||||
|
||||
// Read the entire file into memory for lzma-rs
|
||||
let mut compressed_data = Vec::new();
|
||||
buf_reader.read_to_end(&mut compressed_data)?;
|
||||
let compressed_total = compressed_data.len() as u64;
|
||||
|
||||
// Two phases with no shared byte scale: CPU-bound LZMA decompression
|
||||
// dominates, so compressed bytes consumed map to 0–80%, and the tar
|
||||
// unpack of the decompressed data to 80–100%.
|
||||
let mut decompressed_data = Vec::new();
|
||||
let counted_input = ProgressReader::new(std::io::Cursor::new(compressed_data), |read| {
|
||||
if let Some(reporter) = progress {
|
||||
if compressed_total > 0 {
|
||||
reporter
|
||||
.report_percentage(80.0 * read.min(compressed_total) as f64 / compressed_total as f64);
|
||||
}
|
||||
}
|
||||
});
|
||||
lzma_rs::xz_decompress(&mut BufReader::new(counted_input), &mut decompressed_data)?;
|
||||
|
||||
// Create tar archive from decompressed data
|
||||
let decompressed_total = decompressed_data.len() as u64;
|
||||
let counted_tar = ProgressReader::new(std::io::Cursor::new(decompressed_data), |read| {
|
||||
if let Some(reporter) = progress {
|
||||
if decompressed_total > 0 {
|
||||
reporter.report_percentage(
|
||||
80.0 + 20.0 * read.min(decompressed_total) as f64 / decompressed_total as f64,
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
let mut archive = tar::Archive::new(counted_tar);
|
||||
let counted = progress_file_reader(file, progress)?;
|
||||
// Streams: memory stays near the xz dictionary size, not the unpacked browser size.
|
||||
let xz_decoder = lzma_rust2::XzReader::new(BufReader::new(counted), true);
|
||||
let mut archive = tar::Archive::new(xz_decoder);
|
||||
|
||||
archive.unpack(dest_dir)?;
|
||||
|
||||
@@ -1911,10 +1876,13 @@ mod tests {
|
||||
// Then compress with xz
|
||||
let tar_xz_file =
|
||||
std::fs::File::create(&tar_xz_path).expect("Failed to create test tar.xz file");
|
||||
let mut compressed_data = Vec::new();
|
||||
lzma_rs::xz_compress(&mut std::io::Cursor::new(tar_data), &mut compressed_data)
|
||||
.expect("Failed to compress with xz");
|
||||
std::io::Write::write_all(&mut std::io::BufWriter::new(tar_xz_file), &compressed_data)
|
||||
let mut writer = lzma_rust2::XzWriter::new(
|
||||
std::io::BufWriter::new(tar_xz_file),
|
||||
lzma_rust2::XzOptions::default(),
|
||||
)
|
||||
.expect("Failed to create xz writer");
|
||||
std::io::Write::write_all(&mut writer, &tar_data).expect("Failed to compress with xz");
|
||||
std::io::Write::flush(&mut writer.finish().expect("Failed to finish xz stream"))
|
||||
.expect("Failed to write compressed data");
|
||||
}
|
||||
|
||||
|
||||
@@ -831,14 +831,6 @@
|
||||
"name": "cpufeatures",
|
||||
"license": "Apache-2.0 OR MIT"
|
||||
},
|
||||
{
|
||||
"name": "crc",
|
||||
"license": "Apache-2.0 OR MIT"
|
||||
},
|
||||
{
|
||||
"name": "crc-catalog",
|
||||
"license": "Apache-2.0 OR MIT"
|
||||
},
|
||||
{
|
||||
"name": "crc32c",
|
||||
"license": "Apache-2.0 OR MIT"
|
||||
@@ -1740,8 +1732,8 @@
|
||||
"license": "ISC"
|
||||
},
|
||||
{
|
||||
"name": "lzma-rs",
|
||||
"license": "MIT"
|
||||
"name": "lzma-rust2",
|
||||
"license": "Apache-2.0"
|
||||
},
|
||||
{
|
||||
"name": "lzxd",
|
||||
|
||||
Reference in new issue
Block a user