Merge remote-tracking branch 'origin/main' into integrate/diagnostics

This commit is contained in:
Thomas Durieux committed 2026-11-14 23:19:23 +00:00
commit 8eb5097070
9 files changed
+118 -14

No files matched your search

+10 -3
View File
@@ -22,9 +22,16 @@ on `fix/remaining-production-findings`; do not deploy it without that base.
in the repository. Export `GRAFANA_PASSWORD_FILE` as its absolute path and
`GRAFANA_ADMIN_USER` as a non-default username. No password value belongs in
a shell command, Compose file, issue, or PR.
3. Set `APP_RELEASE` to the deployed 7 to 40 character Git SHA. Build the app through
the existing deployment workflow. Add the override when recreating services:
`docker compose -f docker-compose.yml -f deploy/logging/app.compose.yml up -d`.
3. Set `APP_RELEASE` to the deployed 7 to 40 character Git SHA. Append the logging
overrides to the deployment's existing `COMPOSE_FILE`; do not replace existing
overrides. In particular, GitHub App deployments need
`docker-compose.github-app.yml` to retain the read-only signing-key mount in
both the API and streamers. For a GitHub App deployment with admin logging,
the complete list is
`COMPOSE_FILE=docker-compose.yml:docker-compose.github-app.yml:deploy/logging/app.compose.yml:deploy/logging/admin.compose.yml`.
Preserve any other site-specific overrides too. Run `scripts/deploy.sh`, which
checks that each replacement service can sign with the configured key before
stopping live services. Never print the key or expanded Compose environment.
This applies explicit service tags and nonblocking 10 MiB × 3 local buffers to
the API, all four streamers, MongoDB, Redis and the backup process. BullMQ
sandbox workers log through the API container. Existing containers need
+1 -1
View File
@@ -91,7 +91,7 @@
"readme_not_available": "No README for the repository is found.",
"page_not_supported_on_different_branch": "GitHub Pages is served from a different branch than the one selected. Pick the branch that GitHub Pages is configured to use.",
"page_not_activated": "GitHub Pages is not enabled on this repository. Enable it in the repository settings on GitHub before anonymizing.",
"is_removed": "This resource has been removed and is no longer available.",
"is_removed": "This resource has been removed. If you own it, open My work and select Removed to review its settings and available recovery options.",
"conf_name_missing": "A conference name is required.",
"conf_id_missing": "A conference ID is required.",
"conf_id_used": "This conference ID is already in use. Please choose a different one.",
+7
View File
@@ -55,6 +55,13 @@ before="$(image_id "$IMAGE")"
docker compose build anonymous_github
after="$(image_id "$IMAGE")"
# Validate replacement configuration before stopping any working container.
# A healthy HTTP endpoint does not prove that the GitHub App key is mounted.
for service in anonymous_github streamer; do
log "Checking GitHub App configuration in ${service}"
docker compose run --rm --no-deps --entrypoint node "$service" ./build/scripts/check-github-app-key.js
done
if [[ $legacy_alerts == 1 ]]; then
protocol="$(docker image inspect -f '{{index .Config.Labels "science.4open.owner-alert-protocol"}}' "$IMAGE")"
[[ $protocol == recipient-v2 ]] || { warn "replacement image lacks recipient alert migration support"; exit 1; }
+4 -6
View File
@@ -561,12 +561,10 @@ export default class Repository {
await this.markExpired();
}
}
if (
this.status == RepositoryStatus.EXPIRED ||
this.status == RepositoryStatus.EXPIRING ||
this.status == RepositoryStatus.REMOVING ||
this.status == RepositoryStatus.REMOVED
) {
if (this.status == RepositoryStatus.REMOVING || this.status == RepositoryStatus.REMOVED) {
throw new AnonymousError("is_removed", { object: this, httpStatus: 410 });
}
if (this.status == RepositoryStatus.EXPIRED || this.status == RepositoryStatus.EXPIRING) {
throw new AnonymousError("repository_expired", {
object: this,
httpStatus: 410,
+31
View File
@@ -0,0 +1,31 @@
import { createPrivateKey, createSign } from "crypto";
import { readFileSync } from "fs";
type AppKeyConfig = {
GITHUB_APP_ENABLED: boolean;
GITHUB_APP_PRIVATE_KEY: string;
GITHUB_APP_PRIVATE_KEY_FILE: string;
};
// Run in each replacement service, with its actual user, environment and mounts.
// Never include the key, its path, or crypto error details in deployment output.
export function checkGitHubAppKey(config: AppKeyConfig): void {
if (!config.GITHUB_APP_ENABLED) return;
try {
const key = createPrivateKey(config.GITHUB_APP_PRIVATE_KEY || readFileSync(config.GITHUB_APP_PRIVATE_KEY_FILE, "utf8"));
if (key.asymmetricKeyType !== "rsa") throw new Error("Not an RSA key");
createSign("RSA-SHA256").update("anonymous-github-deployment-preflight").sign(key);
} catch {
throw new Error("GitHub App signing key is missing, unreadable or invalid. Preserve the GitHub App Compose override and verify the replacement service's key configuration.");
}
}
if (require.main === module) {
try {
checkGitHubAppKey(require("../config").default);
} catch {
// Configuration validation can also fail. Do not echo environment values.
process.stderr.write("GitHub App preflight failed. Check required configuration and the read-only signing-key mount before deploying.\n");
process.exitCode = 1;
}
}
+1 -1
View File
@@ -155,7 +155,7 @@ describe("removal recovery", function () {
expect(repo.status).to.equal("removing");
let publicError;
try { await repo.check(); } catch (error) { publicError = error; }
expect(publicError.message).to.equal("repository_expired");
expect(publicError.message).to.equal("is_removed");
AnonymizedRepositoryModel.find = (filter) => ({
lean: async () => repo.status === filter.status ? [repo.model] : [],
+20 -3
View File
@@ -30,6 +30,7 @@ if (args[0] === "inspect") {
if (args[0] === "compose") {
if (args[1] === "build") { if (state.buildFails) finish("", 1); state.built = true; finish(); }
if (args[1] === "stop") { state.stopped = true; finish(); }
if (args[1] === "run" && args.at(-1).endsWith("check-github-app-key.js")) { finish("", args.includes(state.preflightFails) ? 1 : 0); }
if (args[1] === "run") { if (state.reconcileFails) finish("", 1); state.reconciled = true; finish(); }
if (args[1] === "up") { if (args.includes("streamer")) state.streamers = true; else state.api = true; finish(); }
if (args[1] === "ps") {
@@ -50,6 +51,22 @@ finish("unexpected docker call", 2);
} finally { fs.rmSync(directory, { recursive: true, force: true }); }
}
for (const preflightFails of ["anonymous_github", "streamer"]) it(`keeps working containers when ${preflightFails} key preflight fails`, () => {
const result = deploy({ preflightFails });
expect(result.status).not.to.equal(0);
expect(result.calls.some(args => args[0] === "compose" && ["stop", "up"].includes(args[1]))).to.equal(false);
expect(result.calls.some(args => args.includes("./build/scripts/reconcile-legacy-owner-alerts.js"))).to.equal(false);
});
it("checks both replacement services before stopping working instances", () => {
const result = deploy();
expect(result.status, result.stderr).to.equal(0);
const stop = result.calls.findIndex(args => args[1] === "stop");
for (const service of ["anonymous_github", "streamer"]) {
const check = result.calls.findIndex(args => args.includes(service) && args.includes("./build/scripts/check-github-app-key.js"));
expect(check).to.be.greaterThan(-1); expect(check).to.be.lessThan(stop);
expect(result.calls[check]).to.include.members(["--no-deps", "--rm", "--entrypoint", "node"]);
}
});
for (const sharedCache of [true, false]) it(`coordinates cache rollout with ${sharedCache ? "existing" : "new"} caching`, () => {
const result = deploy({ oldProtocol: "recipient-v2", sharedCache, coordinated: sharedCache ? 0 : 1 });
expect(result.status, result.stderr).to.equal(0);
@@ -70,7 +87,7 @@ finish("unexpected docker call", 2);
const stop = result.calls.findIndex(args => args[0] === "compose" && args[1] === "stop");
const up = result.calls.findIndex(args => args[0] === "compose" && args[1] === "up");
expect(stop).to.be.greaterThan(-1); expect(up).to.be.greaterThan(stop);
const reconcile = result.calls.findIndex(args => args[0] === "compose" && args[1] === "run");
const reconcile = result.calls.findIndex(args => args.includes("./build/scripts/reconcile-legacy-owner-alerts.js"));
expect(reconcile).to.be.greaterThan(stop); expect(reconcile).to.be.lessThan(up);
expect(result.calls[reconcile]).to.include("./build/scripts/reconcile-legacy-owner-alerts.js");
const checked = result.calls.slice(stop + 1, up).filter(args => args[0] === "inspect").map(args => args.at(-1));
@@ -87,7 +104,7 @@ finish("unexpected docker call", 2);
const result = deploy({ oldProtocol: "recipient-v1" });
expect(result.status, result.stderr).to.equal(0);
const stop = result.calls.findIndex(args => args[0] === "compose" && args[1] === "stop");
const reconcile = result.calls.findIndex(args => args[0] === "compose" && args[1] === "run");
const reconcile = result.calls.findIndex(args => args.includes("./build/scripts/reconcile-legacy-owner-alerts.js"));
const up = result.calls.findIndex(args => args[0] === "compose" && args[1] === "up");
expect(stop).to.be.greaterThan(-1); expect(reconcile).to.be.greaterThan(stop); expect(up).to.be.greaterThan(reconcile);
});
@@ -116,7 +133,7 @@ finish("unexpected docker call", 2);
for (const stopped of [true, false]) it(`reconciles legacy ${stopped ? "stopped" : "running"} containers when the replacement was already built`, () => {
const result = deploy({ built: true, stopped, force: 0 });
expect(result.status, result.stderr).to.equal(0);
const reconcile = result.calls.findIndex(args => args[0] === "compose" && args[1] === "run");
const reconcile = result.calls.findIndex(args => args.includes("./build/scripts/reconcile-legacy-owner-alerts.js"));
const up = result.calls.findIndex(args => args[0] === "compose" && args[1] === "up");
expect(reconcile).to.be.greaterThan(-1); expect(up).to.be.greaterThan(reconcile);
expect(result.stdout).not.to.include("nothing to deploy");
+35
View File
@@ -0,0 +1,35 @@
require("ts-node/register/transpile-only");
const { expect } = require("chai");
const { generateKeyPairSync } = require("crypto");
const fs = require("fs"), os = require("os"), path = require("path");
const { checkGitHubAppKey } = require("../src/scripts/check-github-app-key");
describe("GitHub App replacement key preflight", () => {
const pem = generateKeyPairSync("rsa", { modulusLength: 2048 }).privateKey.export({ type: "pkcs8", format: "pem" });
const config = (extra = {}) => ({ GITHUB_APP_ENABLED: true, GITHUB_APP_PRIVATE_KEY: "", GITHUB_APP_PRIVATE_KEY_FILE: "/missing/key.pem", ...extra });
it("allows App-disabled deployments without a key", () => {
expect(() => checkGitHubAppKey(config({ GITHUB_APP_ENABLED: false }))).not.to.throw();
});
it("fails a missing mount without disclosing the path", () => {
let error;
try { checkGitHubAppKey(config()); } catch (caught) { error = caught; }
expect(error.message).to.include("signing key is missing").and.not.to.include("/missing/key.pem");
});
it("checks mounted key contents and accepts inline key precedence", () => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "app-key-check-"));
const file = path.join(dir, "key.pem");
try {
fs.writeFileSync(file, pem, { mode: 0o600 });
expect(() => checkGitHubAppKey(config({ GITHUB_APP_PRIVATE_KEY_FILE: file }))).not.to.throw();
fs.writeFileSync(file, "secret-invalid-key");
let error;
try { checkGitHubAppKey(config({ GITHUB_APP_PRIVATE_KEY_FILE: file })); } catch (caught) { error = caught; }
expect(error.message).to.include("invalid").and.not.to.include("secret-invalid-key");
expect(() => checkGitHubAppKey(config({ GITHUB_APP_PRIVATE_KEY: pem }))).not.to.throw();
} finally { fs.rmSync(dir, { recursive: true, force: true }); }
});
it("rejects a non-RSA key that cannot produce GitHub RS256 JWTs", () => {
const key = generateKeyPairSync("ec", { namedCurve: "prime256v1" }).privateKey.export({ type: "pkcs8", format: "pem" });
expect(() => checkGitHubAppKey(config({ GITHUB_APP_PRIVATE_KEY: key }))).to.throw("invalid");
});
});
+9
View File
@@ -120,6 +120,15 @@ describe("Runtime regression fixes", function () {
});
describe("Repository.check()", function () {
for (const status of ["removed", "removing", "expired", "expiring"]) it(`reports ${status} without claiming every inactive repository expired`, async () => {
const repo = new Repository({ owner: "507f191e810c19729de860ea", status,
options: { expirationMode: "remove", expirationDate: new Date(Date.now() + 86400000) }, source: {} });
let error;
try { await repo.check(); } catch (caught) { error = caught; }
expect(error?.message).to.equal(status.startsWith("remov") ? "is_removed" : "repository_expired");
expect(error?.httpStatus).to.equal(410);
expect(repo.status).to.equal(status);
});
it("records logical expiration before throwing repository_expired", async function () {
const repo = new Repository({
owner: "507f191e810c19729de860ea",