fix: bound ZIP directory memory for slow downloads

This commit is contained in:
Codex committed 2026-11-09 08:54:26 +00:00
1 parent 4263a87399
commit ac28c64c6c
8 files changed
+263 -7

No files matched your search

+96
View File
@@ -0,0 +1,96 @@
const { expect } = require("chai");
const { setImmediate, setTimeout } = require("timers");
const { once } = require("events");
const { Writable } = require("stream");
const archiver = require("archiver");
const fs = require("fs");
const os = require("os");
const { execFileSync } = require("child_process");
require("ts-node/register/transpile-only");
const { boundedZip } = require("../src/core/bounded-zip");
async function bytes(archive, entries) {
const chunks = [];
archive.on("data", chunk => chunks.push(chunk));
const end = once(archive, "end");
for (const entry of entries) archive.append(entry.body, { date: new Date("2020-01-01T00:00:00Z"), ...entry.options });
await archive.finalize(); await end;
return Buffer.concat(chunks);
}
describe("bounded ZIP central directory", function () {
this.timeout(20000);
it("preserves archive bytes, compression, Unicode names, modes, and empty files", async () => {
const entries = [
{ body: "private text\n".repeat(100), options: { name: "folder/é中.txt", mode: 0o100755 } },
{ body: Buffer.from([0, 1, 255]), options: { name: "binary.dat", store: true } },
{ body: "", options: { name: "empty.txt" } },
];
expect(await bytes(boundedZip(), entries)).to.deep.equal(await bytes(archiver("zip"), entries));
expect(await bytes(boundedZip(), [])).to.deep.equal(await bytes(archiver("zip"), []));
});
it("preserves ZIP64 size and offset encoding beyond 4 GiB", async () => {
const Entry = require("compress-commons/lib/archivers/zip/zip-archive-entry");
async function directory(archive) {
const entry = new Entry("large.bin");
entry.setMethod(0); entry.setTime(new Date("2020-01-01"));
entry.setSize(2 ** 32 + 123); entry.setCompressedSize(2 ** 32 + 123);
entry._offsets = { file: 2 ** 32 + 456 };
archive._module.engine._entries.push(entry);
archive._module.engine.offset = 2 ** 33;
return bytes(archive, []);
}
expect(await directory(boundedZip())).to.deep.equal(await directory(archiver("zip")));
});
it("preserves ZIP64 counts and drains 180,636 records under a 128 MiB heap", function () {
this.timeout(60000);
const output = execFileSync(process.execPath, ["--max-old-space-size=128", "test/fixtures/zip-directory-memory.js"],
{ cwd: process.cwd(), timeout: 55000, encoding: "utf8" });
const result = JSON.parse(output.trim().split("\n").pop());
expect(result.entries).to.equal(180636);
expect(result.zip64Count).to.equal(180636);
expect(result.centralRecords).to.equal(180636);
expect(result.heapMiB).to.be.lessThan(128);
});
it("reports spool write failures through the stream error handler", () => {
const archive = boundedZip();
let failure;
archive.on("error", error => { failure = error; });
const Entry = require("compress-commons/lib/archivers/zip/zip-archive-entry");
const entry = new Entry("file.txt"); entry.setMethod(0); entry.setTime(new Date()); entry._offsets = { file: 0 };
const write = fs.writeSync;
try {
fs.writeSync = () => { throw Object.assign(new Error("disk full"), { code: "ENOSPC" }); };
expect(() => archive._module.engine._entries.push(entry)).not.to.throw();
expect(failure?.code).to.equal("ENOSPC");
} finally { fs.writeSync = write; archive.abort(); }
});
it("removes its spool when aborted while waiting for a slow consumer", async () => {
const existing = new Set(fs.readdirSync(os.tmpdir()));
const archive = boundedZip();
archive.on("error", () => {});
const sink = new Writable({ highWaterMark: 1, write(_chunk, _encoding, _callback) {} });
archive.pipe(sink);
archive.append("small file", { name: "test.txt" });
await once(archive, "entry");
const created = fs.readdirSync(os.tmpdir()).filter(n => n.startsWith("anonymous-zip-directory-") && !existing.has(n));
expect(created).to.have.length(1);
const engine = archive._module.engine;
const Entry = require("compress-commons/lib/archivers/zip/zip-archive-entry");
for (let i = 0; i < 50000; i++) {
const entry = new Entry(`${"directory/".repeat(32)}file-${i}.txt`); entry.setMethod(0); entry.setTime(new Date()); entry._offsets = { file: 0 };
engine._entries.push(entry);
}
try {
void archive.finalize().catch(() => {});
await new Promise(resolve => setTimeout(resolve, 50));
expect(engine._archive.finished).to.equal(false);
} finally { archive.abort(); sink.destroy(); }
await new Promise(resolve => setImmediate(resolve));
expect(fs.existsSync(`${os.tmpdir()}/${created[0]}`)).to.equal(false);
});
});
+43
View File
@@ -0,0 +1,43 @@
// Exercise real library directory encoding without spending minutes compressing
// file bodies: these are the same completed entries held at finalization.
const { setTimeout } = require("timers");
const { Writable } = require("stream");
const { once } = require("events");
const Entry = require("compress-commons/lib/archivers/zip/zip-archive-entry");
const { boundedZip } = require("../../build/core/bounded-zip");
(async () => {
const archive = boundedZip();
const engine = archive._module.engine;
let centralRecords = 0, zip64Count = 0, pending = Buffer.alloc(0);
const sink = new Writable({ highWaterMark: 1024, write(chunk, _encoding, done) {
pending = Buffer.concat([pending, chunk]);
while (pending.length >= 4) {
const signature = pending.readUInt32LE(0);
let length;
if (signature === 0x02014b50) {
if (pending.length < 46) break;
length = 46 + pending.readUInt16LE(28) + pending.readUInt16LE(30) + pending.readUInt16LE(32);
if (pending.length < length) break;
centralRecords++;
} else if (signature === 0x06064b50) {
if (pending.length < 56) break;
length = 56; zip64Count = Number(pending.readBigUInt64LE(32));
} else if (signature === 0x07064b50) length = 20;
else if (signature === 0x06054b50) length = 22;
else throw new Error("invalid ZIP directory signature");
if (pending.length < length) break;
pending = pending.subarray(length);
}
setTimeout(done, 1);
} });
archive.pipe(sink);
const finished = once(sink, "finish");
const entries = 180636;
for (let i = 0; i < entries; i++) {
const entry = new Entry(`file-${i}.txt`); entry.setMethod(0); entry.setTime(new Date("2020-01-01")); entry._offsets = { file: 0 };
engine._entries.push(entry);
}
await archive.finalize(); await finished;
if (pending.length) throw new Error("incomplete directory");
console.log(JSON.stringify({ entries, centralRecords, zip64Count, heapMiB: process.memoryUsage().heapUsed / 1048576 }));
})().catch(error => { console.error(error); process.exitCode = 1; });
+6 -1
View File
@@ -104,8 +104,13 @@ describe("large anonymization resources", function () {
await Promise.all([old, current, other].map(path => fs.mkdir(path)));
await fs.writeFile(join(old, "input"), "private");
await fs.utimes(old, new Date(0), new Date(0));
const oldZip = join(root, "anonymous-zip-directory-old"), liveZip = join(root, "anonymous-zip-directory-live");
await Promise.all([oldZip, liveZip].map(path => fs.mkdir(path)));
await Promise.all([oldZip, liveZip].map(path => fs.writeFile(join(path, "central"), "directory records")));
await fs.utimes(join(oldZip, "central"), new Date(0), new Date(0));
await fs.utimes(liveZip, new Date(0), new Date(0));
await removeStaleTextSpools(root);
expect((await fs.readdir(root)).sort()).to.deep.equal(["anonymous-text-current", "other"]);
expect((await fs.readdir(root)).sort()).to.deep.equal(["anonymous-text-current", "anonymous-zip-directory-live", "other"]);
} finally { await fs.rm(root, { force: true, recursive: true }); }
});
it("bounds an unresponsive Redis command and disconnects its socket", async () => {