fix: respect backpressure when resolving Git LFS pointers

src/core/source/GitHubStream.ts

@@ -177,57 +177,43 @@ export default class GitHubStream extends GitHubBase {
     filePath: string
   ): stream.Readable {
     const out = new stream.PassThrough();
-    let probe = Buffer.alloc(0);
+    let active = blobStream;
-    let decided = false;
+    out.once("close", () => active.destroy());
-    const PROBE_BYTES = 150;
+    // The generator only reads upstream when the output has capacity.
-    const LFS_PREFIX = "version https://git-lfs.github.com/spec/";
+    const downloadRaw = () => this.downloadFileViaRaw(token, filePath);
-
+    async function* resolve() {
-    const decide = (extra?: Buffer, sourceEnded = false) => {
+      let probe = Buffer.alloc(0);
-      if (decided) return;
+      let decided = false;
-      decided = true;
+      for await (const chunk of blobStream) {
-      const head = probe.toString(
+        const bytes = Buffer.from(chunk);
-        "utf8",
+        if (decided) {
-        0,
+          yield bytes;
-        Math.min(probe.length, LFS_PREFIX.length)
+          continue;
-      );
+        }
-      if (head === LFS_PREFIX) {
+        probe = Buffer.concat([probe, bytes]);
-        blobStream.destroy();
+        if (probe.length < 150) continue;
-        const lfsStream = this.downloadFileViaRaw(token, filePath);
+        decided = true;
-        lfsStream.on("error", (err) => out.destroy(err));
+        if (probe.toString("utf8", 0, 39) === "version https://git-lfs.github.com/spec/") {
-        lfsStream.pipe(out);
+          active = downloadRaw();
-        return;
+          for await (const raw of active) yield raw;
+          return;
+        }
+        yield probe;
+        probe = Buffer.alloc(0);
       }
-      out.write(probe);
+      if (!decided) {
-      if (extra && extra.length) out.write(extra);
+        if (probe.toString("utf8").startsWith("version https://git-lfs.github.com/spec/")) {
-      if (sourceEnded) {
+          active = downloadRaw();
-        out.end();
+          for await (const raw of active) yield raw;
-        return;
+        } else if (probe.length) {
+          yield probe;
+        }
       }
-      blobStream.on("data", (c) => out.write(c));
+    }
-      blobStream.on("end", () => out.end());
+    const resolved = stream.Readable.from(resolve());
-      blobStream.on("error", (err) => out.destroy(err));
+    out.once("close", () => resolved.destroy());
-    };
+    resolved.on("error", (error) => out.destroy(error));
-
+    resolved.pipe(out);
-    blobStream.on("data", (chunk: Buffer) => {
-      if (decided) return;
-      const remaining = PROBE_BYTES - probe.length;
-      if (chunk.length <= remaining) {
-        probe = Buffer.concat([probe, chunk]);
-        if (probe.length >= PROBE_BYTES) decide();
-      } else {
-        probe = Buffer.concat([probe, chunk.slice(0, remaining)]);
-        decide(chunk.slice(remaining));
-      }
-    });
-    blobStream.on("end", () => decide(undefined, true));
-    blobStream.on("error", (err) => {
-      // Always propagate — pre-decision this is the only listener; once a
-      // non-LFS decision is made, the inner branch attaches its own
-      // listener that will also fire, but we shouldn't rely on that being
-      // there if the code is later refactored.
-      decided = true;
-      out.destroy(err);
-    });
 
     return out;
   }

test/production-regressions.test.js

@@ -55,6 +55,30 @@ describe("production regressions", function () {
     expect(emitted).to.equal(false);
   });
 
+  it("propagates backpressure and cancellation through the blob probe", async function () {
+    let generated = 0;
+    const input = new Readable({ read() { generated++; this.push(Buffer.alloc(65536, 65)); } });
+    const source = Object.create(GitHubStream.prototype);
+    const output = source.resolveLfsPointer(input, "token", "file.bin");
+    await new Promise(resolve => setTimeout(resolve, 20));
+    expect(generated).to.be.lessThan(10);
+    expect(output.readableLength).to.be.lessThan(300000);
+    output.destroy();
+    await new Promise(resolve => setImmediate(resolve));
+    expect(input.destroyed).to.equal(true);
+  });
+
+  for (const chunkSize of [1, 200]) {
+    it(`resolves LFS pointers using ${chunkSize}-byte chunks`, async function () {
+      const source = Object.create(GitHubStream.prototype);
+      source.downloadFileViaRaw = () => Readable.from(["complete LFS file"]);
+      const pointer = "version https://git-lfs.github.com/spec/v1\noid sha256:" + "a".repeat(64) + "\nsize 1024\n";
+      const chunks = [];
+      for (let i = 0; i < pointer.length; i += chunkSize) chunks.push(Buffer.from(pointer.slice(i, i + chunkSize)));
+      expect(await collect(source.resolveLfsPointer(Readable.from(chunks), "token", "file"))).to.equal("complete LFS file");
+    });
+  }
+
   for (const status of ["removing", "removed", "expiring", "expired"]) {
     it(`ignores delayed downloads for ${status} repositories`, async function () {
       stub(db, "connect", async () => {});