anonymous_github/test/shared-cache-http.test.js

const { strict: assert } = require("node:assert");
const { promises: fs } = require("node:fs");
const { tmpdir } = require("node:os");
const { join } = require("node:path");
const http = require("node:http");
const { Readable } = require("node:stream");
const { createHash } = require("node:crypto");
const { gunzipSync } = require("node:zlib");
const express = require("express");
require("ts-node/register/transpile-only");
const config = require("../src/config").default;
const { sharedBlobCache, resultBlobCache, closeBlobCaches } = require("../src/core/blob-cache");
const { sendSharedContent } = require("../src/core/cached-response");
const { closeCacheMetrics, cacheMetricsReport, observeCache } = require("../src/core/cache-observability");
const { monitorRequests } = require("../src/core/request-monitoring");
const GitHubStream = require("../src/core/source/GitHubStream").default;
const sha = b => createHash("sha1").update(`blob ${b.length}\0`).update(b).digest("hex");
const consume = async s => { const chunks = []; for await (const b of s) chunks.push(Buffer.from(b)); return Buffer.concat(chunks); };
function request(port, path, encoding = "gzip", extra = {}) {
  return new Promise((resolve, reject) => {
    const req = http.get({ host: "127.0.0.1", port, path, headers: { "accept-encoding": encoding, ...extra } }, res => {
      consume(res).then(body => resolve({ status: res.statusCode, headers: res.headers, body }), reject);
    }); req.on("error", reject);
  });
}
describe("shared cache HTTP and telemetry integration", function () {
  this.timeout(15000);
  let root, old, server, port, fills, body;
  beforeEach(async () => {
    old = { ...config }; closeBlobCaches(); closeCacheMetrics();
    root = await fs.mkdtemp(join(tmpdir(), "shared-http-"));
    Object.assign(config, { FOLDER: root, SHARED_CACHE_PATH: join(root, "cache"), SHARED_CACHE_ENABLED: true,
      SHARED_CACHE_BYTES: 16 * 1024 ** 2, SHARED_CACHE_RESULT_BYTES: 8 * 1024 ** 2, SHARED_CACHE_MAX_FILE_BYTES: 1024 ** 2,
      SHARED_CACHE_FREE_BYTES: 0, CACHE_MONITORING_ENABLED: true });
    fills = 0; body = Buffer.from("Public documentation with no identifying names.\n".repeat(100));
    const app = express(); app.use(monitorRequests("api", undefined, { info() {}, warn() {} }));
    app.get("/:project/:mode", async (req, res) => {
      if (req.headers.authorization !== "test-access") return res.status(403).end();
      const source = new GitHubStream({ repoId: req.params.project, organization: "org", repoName: "repo", commit: "commit", getToken: async () => "" });
      source.downloadWithFallback = async () => { fills++; return Readable.from(body); };
      source.downloadFile = () => Readable.from("invalid pointer");
      res.type("text/plain");
      try {
        const input = await source.getFileContentCache("README.md", req.params.project, () => ({ sha: sha(body), size: body.length }));
        const handled = await sendSharedContent(input, res, { filePath: "README.md", repoId: req.params.project,
          terms: req.params.mode === "changed" ? ["Public=>Private"] : ["Alice"] });
        if (!handled) { input.destroy(); res.status(500).end("unexpected bypass"); }
      } catch (e) { if (!res.headersSent) res.status(500).end(e.message); else res.destroy(e); }
    });
    await new Promise(resolve => { server = app.listen(0, "127.0.0.1", resolve); }); port = server.address().port;
  });
  afterEach(async () => {
    await new Promise(resolve => server.close(resolve)); closeBlobCaches(); closeCacheMetrics(); Object.assign(config, old);
    await fs.rm(root, { recursive: true, force: true });
  });
  const headers = { authorization: "test-access" };
  it("shares source content across projects without bypassing authorization", async () => {
    assert.equal((await request(port, "/a/plain")).status, 403); assert.equal(fills, 0);
    const a = await request(port, "/a/plain", "gzip", headers);
    assert.equal(a.status, 200); assert.equal(a.headers["content-encoding"], "gzip"); assert.deepEqual(gunzipSync(a.body), body);
    const b = await request(port, "/b/plain", "identity", headers);
    assert.deepEqual(b.body, body); assert.equal(b.headers["content-encoding"], undefined); assert.equal(fills, 1);
    assert.equal(sharedBlobCache().stats().blobs, 1); assert.equal(resultBlobCache().stats().blobs, 0);
    assert.equal((await request(port, "/b/plain", "gzip")).status, 403);
  });
  it("fetches public submodule content from its origin without forwarding parent credentials", async () => {
    const original = GitHubStream.prototype.downloadWithFallback;
    const upstream = { organization: "public-org", repoName: "dependency", commit: "a".repeat(40), path: "src/data.txt" };
    let calls = 0;
    GitHubStream.prototype.downloadWithFallback = async function (token, digest, path) {
      calls++;
      assert.equal(token, ""); assert.equal(digest, sha(body)); assert.equal(path, upstream.path);
      assert.equal(this.data.organization, upstream.organization); assert.equal(this.data.repoName, upstream.repoName);
      assert.equal(this.data.commit, upstream.commit);
      return Readable.from(body);
    };
    try {
      const source = new GitHubStream({ repoId: "parent", organization: "private-org", repoName: "parent",
        commit: "parent-commit", getToken: async () => "parent-secret" });
      const meta = () => ({ sha: sha(body), size: body.length, upstream });
      assert.deepEqual(await consume(await source.getFileContentCache("vendor/src/data.txt", "parent", meta)), body);
      assert.deepEqual(await consume(await source.getFileContentCache("vendor/src/data.txt", "parent", meta)), body);
      assert.equal(calls, 1);
    } finally { GitHubStream.prototype.downloadWithFallback = original; }
  });
  it("reuses unchanged decisions, separates changed rules, and caches small changed output", async () => {
    const first = await request(port, "/a/plain", "gzip", headers);
    const second = await request(port, "/a/plain", "gzip", headers);
    assert.deepEqual(second.body, first.body);
    const changed = await request(port, "/a/changed", "gzip", headers);
    assert.equal(gunzipSync(changed.body).toString(), body.toString().replaceAll("Public", "Private"));
    const repeated = await request(port, "/a/changed", "identity", headers);
    assert.equal(repeated.body.toString(), body.toString().replaceAll("Public", "Private"));
    assert.equal(resultBlobCache().stats().blobs, 1);
    const report = cacheMetricsReport(60); assert.equal(report.available, true);
    const count = name => report.rows.filter(r => r.name === name).reduce((n, r) => n + r.count, 0);
    assert.equal(count("rewrite_unchanged"), 1); assert.equal(count("rewrite_changed"), 1); assert.equal(count("decision_hit"), 2);
  });
  it("respects gzip q=0, rejects all forbidden encodings, and gzip-encodes small identity blobs when required", async () => {
    let r = await request(port, "/a/plain", "gzip;q=0, identity;q=1", headers);
    assert.equal(r.headers["content-encoding"], undefined); assert.deepEqual(r.body, body);
    r = await request(port, "/a/plain", "gzip;q=0, identity;q=0", headers); assert.equal(r.status, 406);
    body = Buffer.from("tiny text");
    r = await request(port, "/a/plain", "gzip, identity;q=0", headers);
    assert.equal(r.headers["content-encoding"], "gzip"); assert.deepEqual(gunzipSync(r.body), body);
    assert.ok(r.headers.vary.includes("Accept-Encoding"));
  });
  it("verifies the Git LFS pointer and resolved payload before sharing the payload", async () => {
    const payload = Buffer.from("large file payload");
    const pointer = Buffer.from(`version https://git-lfs.github.com/spec/v1\noid sha256:${createHash("sha256").update(payload).digest("hex")}\nsize ${payload.length}\n`);
    const source = new GitHubStream({ repoId: "lfs", organization: "org", repoName: "repo", commit: "commit", getToken: async () => "" });
    source.downloadWithFallback = async () => Readable.from(payload);
    source.downloadFile = () => Readable.from(pointer);
    assert.deepEqual(await consume(await source.getFileContentCache("data.bin", "lfs", () => ({ sha: sha(pointer), size: pointer.length }))), payload);
    const badPointer = Buffer.from(pointer.toString().replace(`size ${payload.length}`, "size 999"));
    source.downloadFile = () => Readable.from(badPointer);
    await assert.rejects(source.getFileContentCache("bad.bin", "lfs", () => ({ sha: sha(badPointer), size: badPointer.length })), /invalid_blob_hash/);
    assert.equal(sharedBlobCache().stats().blobs, 1);
  });
  it("uses bounded persistent metrics with separate monitoring traffic and no file identifiers", async () => {
    await request(port, "/a/plain", "identity", { ...headers, "user-agent": "Uptime-Kuma/1" });
    observeCache("compress", 100, 1, "zip");
    const report = cacheMetricsReport(60);
    assert.ok(report.rows.some(r => r.traffic === "monitor")); assert.ok(report.rows.some(r => r.traffic === "zip"));
    assert.ok(!JSON.stringify(report).includes("README")); assert.ok(!JSON.stringify(report).includes(sha(body)));
    config.CACHE_MONITORING_ENABLED = false; assert.equal(cacheMetricsReport().available, false);
  });
});