mirror of
https://github.com/tdurieux/anonymous_github.git
synced 2026-11-17 17:44:09 +01:00
fix: isolate streamer headers and reject download redirects
This commit is contained in:
1 parent
511dcd7067
commit
c227bf0421
5 files changed
+105
-14
No files matched your search
@@ -1,3 +1,4 @@
|
||||
import { rejectStreamerRedirects } from "./streamer-response";
|
||||
import { isResponseCancellation } from "./response-cancellation";
|
||||
import { canReadCachedRepository } from "./cached-repository-access";
|
||||
import { probeStreamerCache } from "./streamer-cache-probe";
|
||||
@@ -412,7 +413,8 @@ export default class AnonymizedFile {
|
||||
}
|
||||
|
||||
// use the streamer service
|
||||
const content = got.stream(join(config.STREAMER_ENTRYPOINT, "api"), {
|
||||
const content = rejectStreamerRedirects(got.stream(join(config.STREAMER_ENTRYPOINT, "api"), {
|
||||
followRedirect: false,
|
||||
method: "POST",
|
||||
headers: requestHeaders(),
|
||||
json: {
|
||||
@@ -429,7 +431,7 @@ export default class AnonymizedFile {
|
||||
size: await this.size(),
|
||||
anonymizerOptions: anonymizer.opt,
|
||||
},
|
||||
});
|
||||
}));
|
||||
content.on("error", err => {
|
||||
const { error } = streamerErrorToAnonymous(err, { repoId: this.repository.repoId, filePath: this.anonymizedPath });
|
||||
void notifyOwnerAccessProblem(this.repository.owner.id, error, { kind: "repository", id: String(this.repository.model._id) });
|
||||
@@ -504,17 +506,17 @@ export default class AnonymizedFile {
|
||||
} catch (error) { cached.stream.destroy(); throw error; }
|
||||
}
|
||||
if (res.destroyed || res.writableEnded) { cached?.stream.destroy(); headersDone(); resolve(); return; }
|
||||
const resStream = cached?.stream || got.stream(url, {
|
||||
const resStream = cached?.stream || rejectStreamerRedirects(got.stream(url, {
|
||||
followRedirect: false,
|
||||
method: "POST", headers, decompress: false,
|
||||
json: { ...payload, token: await githubTokenForStreamer(await this.repository.getToken(), this.repository.model.source.repositoryName) },
|
||||
});
|
||||
}));
|
||||
// Forward Content-Type from the streamer's upstream response.
|
||||
// got.stream(...).pipe(res) forwards body bytes only — without
|
||||
// this, the parent response has no Content-Type and the browser
|
||||
// guesses (text renders as download, images as octet-stream).
|
||||
const forwardHeaders = (upstream: { headers: Record<string, string | string[] | undefined> }) => {
|
||||
// streamResponse isolates the body from Got's implicit header forwarding.
|
||||
// Only approved content headers may reach the public response.
|
||||
const forwardHeaders = (upstream: { statusCode?: number; headers: Record<string, string | string[] | undefined> }) => {
|
||||
headersDone();
|
||||
if (res.headersSent) return;
|
||||
if (res.headersSent || (upstream.statusCode !== undefined && upstream.statusCode !== 200)) return;
|
||||
for (const header of ["content-encoding", "content-length", "vary", "accept-ranges"]) {
|
||||
const value = upstream.headers[header];
|
||||
if (typeof value === "string") res.setHeader(header, value);
|
||||
|
||||
@@ -1,15 +1,20 @@
|
||||
import { Readable, Writable } from "stream";
|
||||
import { PassThrough, Readable, Writable } from "stream";
|
||||
|
||||
/** Cancel upstream on every response terminal event, including a pre-header close. */
|
||||
export function streamResponse(source: Readable, response: Writable,
|
||||
onError: (error: Error) => void): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
// Got copies every upstream header/status when piped to ServerResponse.
|
||||
// A plain stream keeps response policy with the caller's explicit allowlist.
|
||||
const body = new PassThrough();
|
||||
const cleanup = () => {
|
||||
response.removeListener("close", done);
|
||||
response.removeListener("finish", done);
|
||||
response.removeListener("error", failed);
|
||||
source.removeListener("error", upstreamFailed);
|
||||
source.unpipe(response);
|
||||
source.unpipe(body);
|
||||
body.unpipe(response);
|
||||
body.destroy();
|
||||
source.destroy();
|
||||
};
|
||||
const done = () => { cleanup(); resolve(); };
|
||||
@@ -22,6 +27,6 @@ export function streamResponse(source: Readable, response: Writable,
|
||||
if (response.destroyed || response.writableEnded) done();
|
||||
else if (source.errored) upstreamFailed(source.errored);
|
||||
else if (source.destroyed || source.readableEnded) upstreamFailed(new Error("upstream_stream_closed"));
|
||||
else source.pipe(response);
|
||||
else { body.pipe(response); source.pipe(body); }
|
||||
});
|
||||
}
|
||||
@@ -1,9 +1,10 @@
|
||||
import { rejectStreamerRedirects } from "./streamer-response";
|
||||
import got, { Response } from "got";
|
||||
|
||||
/** Wait only for headers; the body remains a backpressured stream. */
|
||||
export async function probeStreamerCache(url: string, json: Record<string, unknown>, headers: Record<string, string>, signal?: AbortSignal) {
|
||||
const stream = got.stream(url, { method: "POST", json: { ...json, token: "", cacheOnly: true },
|
||||
headers, timeout: { response: 15000 }, decompress: false });
|
||||
const stream = rejectStreamerRedirects(got.stream(url, { followRedirect: false, method: "POST", json: { ...json, token: "", cacheOnly: true },
|
||||
headers, timeout: { response: 15000 }, decompress: false }));
|
||||
const responseReady = new Promise<Response>((resolve, reject) => {
|
||||
stream.once("response", resolve);
|
||||
stream.once("error", reject);
|
||||
@@ -14,6 +15,7 @@ export async function probeStreamerCache(url: string, json: Record<string, unkno
|
||||
if (signal?.aborted) abort();
|
||||
try {
|
||||
const response = await responseReady;
|
||||
if (stream.errored) throw stream.errored;
|
||||
return { stream, response };
|
||||
} catch (error) {
|
||||
stream.destroy();
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
import { Readable } from "stream";
|
||||
import AnonymousError from "./AnonymousError";
|
||||
|
||||
/** Internal streamer endpoints return content, never browser redirects. */
|
||||
export function rejectStreamerRedirects<T extends Readable>(stream: T): T {
|
||||
stream.on("response", (response: { statusCode: number }) => {
|
||||
if (response.statusCode >= 300 && response.statusCode < 400) {
|
||||
stream.destroy(new AnonymousError("streamer_upstream_error", { httpStatus: 502 }));
|
||||
}
|
||||
});
|
||||
return stream;
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
const { expect } = require("chai");
|
||||
const http = require("node:http");
|
||||
const got = require("got");
|
||||
require("ts-node/register/transpile-only");
|
||||
const { streamResponse } = require("../src/core/response-stream");
|
||||
const { rejectStreamerRedirects } = require("../src/core/streamer-response");
|
||||
const { probeStreamerCache } = require("../src/core/streamer-cache-probe");
|
||||
|
||||
describe("streamer response isolation", function () {
|
||||
const servers = [];
|
||||
async function serve(handler) {
|
||||
const server = http.createServer(handler);
|
||||
servers.push(server);
|
||||
await new Promise(resolve => server.listen(0, "127.0.0.1", resolve));
|
||||
return `http://127.0.0.1:${server.address().port}`;
|
||||
}
|
||||
afterEach(async () => {
|
||||
await Promise.all(servers.splice(0).map(server => new Promise(resolve => {
|
||||
server.close(resolve);
|
||||
server.closeAllConnections();
|
||||
})));
|
||||
});
|
||||
|
||||
it("does not let Got copy identity-bearing upstream headers or status", async () => {
|
||||
const upstream = await serve((_req, res) => {
|
||||
res.writeHead(201, {
|
||||
Location: "https://github.com/private-owner/private-repository",
|
||||
Refresh: "0; url=https://github.com/private-owner/private-repository",
|
||||
"Content-Location": "/private-owner/private-repository",
|
||||
Link: "<https://github.com/private-owner/private-repository>; rel=canonical",
|
||||
"Set-Cookie": "owner=private-owner",
|
||||
"Content-Disposition": "attachment; filename=private-owner.zip",
|
||||
});
|
||||
res.end("anonymized content");
|
||||
});
|
||||
const proxy = await serve((_req, res) => {
|
||||
res.setHeader("Content-Type", "text/plain");
|
||||
void streamResponse(got.stream(upstream), res, () => res.end("error"));
|
||||
});
|
||||
const result = await got(proxy);
|
||||
expect(result.statusCode).to.equal(200);
|
||||
expect(result.body).to.equal("anonymized content");
|
||||
for (const header of ["location", "refresh", "content-location", "link", "set-cookie", "content-disposition"]) {
|
||||
expect(result.headers).not.to.have.property(header);
|
||||
}
|
||||
});
|
||||
|
||||
for (const status of [301, 302, 303, 307, 308]) {
|
||||
it(`rejects ${status} without forwarding its body or contacting the destination`, async () => {
|
||||
let contacted = 0;
|
||||
const target = await serve((_req, res) => { contacted++; res.end("private-owner"); });
|
||||
const upstream = await serve((_req, res) => {
|
||||
res.writeHead(status, { Location: target + "/private-owner/private-repository" });
|
||||
res.end("private-owner/private-repository");
|
||||
});
|
||||
const proxy = await serve((_req, res) => {
|
||||
const source = rejectStreamerRedirects(got.stream(upstream, { followRedirect: false }));
|
||||
void streamResponse(source, res, () => { res.statusCode = 502; res.end("upstream_error"); });
|
||||
});
|
||||
const result = await got(proxy, { throwHttpErrors: false, retry: 0 });
|
||||
expect(result.statusCode).to.equal(502);
|
||||
expect(result.body).to.equal("upstream_error");
|
||||
expect(result.headers).not.to.have.property("location");
|
||||
let failure;
|
||||
try { await probeStreamerCache(upstream, {}, {}); } catch (error) { failure = error; }
|
||||
expect(failure).to.be.instanceOf(Error);
|
||||
expect(contacted).to.equal(0);
|
||||
});
|
||||
}
|
||||
});
|
||||
Reference in new issue
Block a user